As I've observed the DDL worker being a bottleneck of operations when working with replicas I've decided to give the `pool_size` option a try since the operations queued are related to different tables (and usually even different databases too) to see how things go.

Apparently things work fine (since the caller avoids sending related changes) but today I've seen this exception that could be related:

CH version: `version 22.3.2.1`

Query (not really important since those are tests):
```
ALTER TABLE d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 DELETE WHERE 1 = 1
```

Exception:
```
2022.05.06 09:01:01.303172 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Error> executeQuery: Code: 341. DB::Exception: Mutation 0000000000 was killed. (UNFINISHED) (version 22.3.2.1) (from 127.0.0.1:46444) (in query:  ALTER TABLE d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 DELETE WHERE 1 = 1 ), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0xa4dde1a in /usr/bin/clickhouse
1. DB::Exception::Exception<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&>(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) @ 0xa54a504 in /usr/bin/clickhouse
2. DB::checkMutationStatus(std::__1::optional<DB::MergeTreeMutationStatus>&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) @ 0x14f8f610 in /usr/bin/clickhouse
3. DB::StorageReplicatedMergeTree::waitMutationToFinishOnReplicas(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) const @ 0x14b851ad in /usr/bin/clickhouse
4. DB::StorageReplicatedMergeTree::waitMutation(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned long) const @ 0x14bf86ca in /usr/bin/clickhouse
5. DB::StorageReplicatedMergeTree::mutate(DB::MutationCommands const&, std::__1::shared_ptr<DB::Context const>) @ 0x14c1a986 in /usr/bin/clickhouse
6. DB::InterpreterAlterQuery::executeToTable(DB::ASTAlterQuery const&) @ 0x144034bc in /usr/bin/clickhouse
7. DB::InterpreterAlterQuery::execute() @ 0x14401b92 in /usr/bin/clickhouse
8. ? @ 0x148d0c9a in /usr/bin/clickhouse
9. DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, std::__1::optional<DB::FormatSettings> const&) @ 0x148d3fca in /usr/bin/clickhouse
10. DB::HTTPHandler::processQuery(DB::HTTPServerRequest&, DB::HTMLForm&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&, std::__1::optional<DB::CurrentThread::QueryScope>&) @ 0x151cdae7 in /usr/bin/clickhouse
11. DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x151d2292 in /usr/bin/clickhouse
12. DB::HTTPServerConnection::run() @ 0x1545979b in /usr/bin/clickhouse
13. Poco::Net::TCPServerConnection::start() @ 0x164b264f in /usr/bin/clickhouse
14. Poco::Net::TCPServerDispatcher::run() @ 0x164b4aa1 in /usr/bin/clickhouse
15. Poco::PooledThread::run() @ 0x16671e49 in /usr/bin/clickhouse
16. Poco::ThreadImpl::runnableEntry(void*) @ 0x1666f1a0 in /usr/bin/clickhouse
17. ? @ 0x7efcecf5f609 in ?
18. __clone @ 0x7efcece84163 in ?

2022.05.06 09:01:01.303294 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Error> DynamicQueryHandler: Code: 341. DB::Exception: Mutation 0000000000 was killed. (UNFINISHED), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0xa4dde1a in /usr/bin/clickhouse
1. DB::Exception::Exception<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&>(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) @ 0xa54a504 in /usr/bin/clickhouse
2. DB::checkMutationStatus(std::__1::optional<tor<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) @ 0x14f8f610 in /usr/bin/clickhouse
3. DB::StorageReplicatedMergeTree::waitMutationTDB::MergeTreeMutationStatus>&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocaoFinishOnReplicas(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) const @ 0x14b851ad in /usr/bin/clickhouse
4. DB::StorageReplicatedMergeTree::waitMutation(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned long) const @ 0x14bf86ca in /usr/bin/clickhouse
5. DB::StorageReplicatedMergeTree::mutate(DB::MutationCommands const&, std::__1::shared_ptr<DB::Context const>) @ 0x14c1a986 in /usr/bin/clickhouse
6. DB::InterpreterAlterQuery::executeToTable(DB::ASTAlterQuery const&) @ 0x144034bc in /usr/bin/clickhouse
7. DB::InterpreterAlterQuery::execute() @ 0x14401b92 in /usr/bin/clickhouse
8. ? @ 0x148d0c9a in /usr/bin/clickhouse
9. DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, std::__1::optional<DB::FormatSettings> const&) @ 0x148d3fca in /usr/bin/clickhouse
10. DB::HTTPHandler::processQuery(DB::HTTPServerRequest&, DB::HTMLForm&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&, std::__1::optional<DB::CurrentThread::QueryScope>&) @ 0x151cdae7 in /usr/bin/clickhouse
11. DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x151d2292 in /usr/bin/clickhouse
12. DB::HTTPServerConnection::run() @ 0x1545979b in /usr/bin/clickhouse
13. Poco::Net::TCPServerConnection::start() @ 0x164b264f in /usr/bin/clickhouse
14. Poco::Net::TCPServerDispatcher::run() @ 0x164b4aa1 in /usr/bin/clickhouse
15. Poco::PooledThread::run() @ 0x16671e49 in /usr/bin/clickhouse
16. Poco::ThreadImpl::runnableEntry(void*) @ 0x1666f1a0 in /usr/bin/clickhouse
17. ? @ 0x7efcecf5f609 in ?
18. __clone @ 0x7efcece84163 in ?
 (version 22.3.2.1)
```



Log in the replica that received the query:

```
2022.05.06 09:01:01.283220 [ 9825 ] {} <Trace> DynamicQueryHandler: Request URI: /?database=d_test_815b7009495344f7ab177b17c6250b5c&query_id=e1da4a60-2788-43ea-94f1-642b6955dbc1&max_result_bytes=104857600&log_queries=1&optimize_throw_if_noop=1&output_format_json_quote_64bit_integers=0&lock_acquire_timeout=120&wait_end_of_query=1&buffer_size=104857600&max_execution_time=3600&insert_deduplicate=1&mutations_sync=2
2022.05.06 09:01:01.283229 [ 9825 ] {} <Debug> HTTP-Session: 7bee1174-2cac-4d72-bb15-a8262f26cef0 Authenticating user 'default' from 127.0.0.1:46444
2022.05.06 09:01:01.283240 [ 9825 ] {} <Debug> HTTP-Session: 7bee1174-2cac-4d72-bb15-a8262f26cef0 Authenticated with global context as user 94309d50-4f52-5250-31bd-74fecac179db
2022.05.06 09:01:01.283247 [ 9825 ] {} <Debug> HTTP-Session: 7bee1174-2cac-4d72-bb15-a8262f26cef0 Creating query context from global context, user_id: 94309d50-4f52-5250-31bd-74fecac179db, parent context user: <NOT SET>
2022.05.06 09:01:01.283441 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Debug> executeQuery: (from 127.0.0.1:46444)  ALTER TABLE d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 DELETE WHERE 1 = 1 
2022.05.06 09:01:01.283484 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Trace> ContextAccess (default): Access granted: ALTER DELETE ON d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0
2022.05.06 09:01:01.283773 [ 296 ] {} <Trace> DDLWorker: scheduleTasks: initialized=true, size_before_filtering=643, queue_size=643, entries=query-0000000000..query-0000000642, first_failed_task_name=none, current_tasks_size=1, last_current_task=query-0000000641, last_skipped_entry_name=none
2022.05.06 09:01:01.283783 [ 296 ] {} <Debug> DDLWorker: Will schedule 1 tasks starting from query-0000000642
2022.05.06 09:01:01.283788 [ 296 ] {} <Trace> DDLWorker: Checking task query-0000000642
2022.05.06 09:01:01.283856 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Trace> InterpreterSelectQuery: Running 'analyze' second time
2022.05.06 09:01:01.284091 [ 163 ] {} <Debug> test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980 (ReplicatedMergeTreeQueue): Pulling 1 entries to queue: log-0000002648 - log-0000002648
2022.05.06 09:01:01.284880 [ 296 ] {} <Debug> DDLWorker: Waiting for queue updates
2022.05.06 09:01:01.284886 [ 1513 ] {} <Debug> DDLWorker: Processing task query-0000000642 (DROP DATABASE IF EXISTS d_test_506dac9b765a492a9da234ef1721ee7e ON CLUSTER tinybird)
2022.05.06 09:01:01.286009 [ 163 ] {} <Debug> test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980 (ReplicatedMergeTreeQueue): Pulled 1 entries to queue.
2022.05.06 09:01:01.286065 [ 1513 ] {} <Debug> DDLWorker: Executing query: DROP DATABASE IF EXISTS d_test_506dac9b765a492a9da234ef1721ee7e
2022.05.06 09:01:01.286111 [ 131 ] {} <Trace> MergeFromLogEntryTask: Executing log entry to merge parts 202205_2026_2026_0, 202205_2027_2027_0, 202205_2028_2028_0, 202205_2029_2029_0, 202205_2030_2030_0, 202205_2031_2031_0 to 202205_2026_2031_1
2022.05.06 09:01:01.286145 [ 1513 ] {180a0f02-1ed9-41bf-bdc6-5fa9fdb16800} <Debug> executeQuery: (from 0.0.0.0:0, user: ) /* ddl_entry=query-0000000642 */ DROP DATABASE IF EXISTS d_test_506dac9b765a492a9da234ef1721ee7e
2022.05.06 09:01:01.286159 [ 131 ] {} <Debug> DiskLocal: Reserving 1.00 MiB on disk `default`, having unreserved 80.96 GiB.
2022.05.06 09:01:01.286207 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTask::PrepareStage: Merging 6 parts: from 202205_2026_2026_0 to 202205_2031_2031_0 into Compact
2022.05.06 09:01:01.286304 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTask::PrepareStage: Selected MergeAlgorithm: Horizontal
2022.05.06 09:01:01.286354 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTreeSequentialSource: Reading 2 marks from part 202205_2026_2026_0, total 1 rows starting from the beginning of the part
2022.05.06 09:01:01.286412 [ 1513 ] {180a0f02-1ed9-41bf-bdc6-5fa9fdb16800} <Debug> DDLWorker: Executed query: DROP DATABASE IF EXISTS d_test_506dac9b765a492a9da234ef1721ee7e
2022.05.06 09:01:01.286423 [ 1513 ] {180a0f02-1ed9-41bf-bdc6-5fa9fdb16800} <Debug> MemoryTracker: Peak memory usage (for query): 0.00 B.
2022.05.06 09:01:01.286538 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTreeSequentialSource: Reading 2 marks from part 202205_2027_2027_0, total 1 rows starting from the beginning of the part
2022.05.06 09:01:01.286699 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTreeSequentialSource: Reading 2 marks from part 202205_2028_2028_0, total 1 rows starting from the beginning of the part
2022.05.06 09:01:01.286862 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTreeSequentialSource: Reading 2 marks from part 202205_2029_2029_0, total 1 rows starting from the beginning of the part
2022.05.06 09:01:01.286991 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTreeSequentialSource: Reading 2 marks from part 202205_2030_2030_0, total 1 rows starting from the beginning of the part
2022.05.06 09:01:01.287125 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTreeSequentialSource: Reading 2 marks from part 202205_2031_2031_0, total 1 rows starting from the beginning of the part
2022.05.06 09:01:01.288639 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Debug> MergeTask::MergeProjectionsStage: Merge sorted 6 rows, containing 18 columns (18 merged, 0 gathered) in 0.002465423 sec., 2433.6594572209315 rows/sec., 1.25 MiB/sec.
2022.05.06 09:01:01.288838 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Trace> MergedBlockOutputStream: filled checksums 202205_2026_2031_1 (state Temporary)
2022.05.06 09:01:01.289180 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Trace> test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980 (ee0155d3-b36c-48e6-ae71-90a3dc39f414): Renaming temporary part tmp_merge_202205_2026_2031_1 to 202205_2026_2031_1.
2022.05.06 09:01:01.289289 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Trace> test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980 (ee0155d3-b36c-48e6-ae71-90a3dc39f414) (MergerMutator): Merged 6 parts: from 202205_2026_2026_0 to 202205_2031_2031_0
2022.05.06 09:01:01.296565 [ 9868 ] {} <Trace> HTTPHandler-factory: HTTP Request for HTTPHandler-factory. Method: POST, Address: 127.0.0.1:46448, User-Agent: tb-internal-query, Length: 51, Content Type: application/x-www-form-urlencoded, Transfer Encoding: identity, X-Forwarded-For: (none)
2022.05.06 09:01:01.296662 [ 9868 ] {} <Trace> DynamicQueryHandler: Request URI: /?database=default&query_id=28907f64-b1e0-42c9-beef-bab7b1fbe45f&max_result_bytes=104857600&log_queries=1&optimize_throw_if_noop=1&output_format_json_quote_64bit_integers=0&lock_acquire_timeout=10&wait_end_of_query=1&buffer_size=104857600&max_execution_time=10
2022.05.06 09:01:01.296743 [ 195 ] {} <Information> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Loading 1 mutation entries: 0000000000 - 0000000000
2022.05.06 09:01:01.296775 [ 9868 ] {} <Debug> HTTP-Session: 0d0eb2ac-64d0-42b7-9041-1321c689a575 Authenticating user 'default' from 127.0.0.1:46448
2022.05.06 09:01:01.296801 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Trace> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (9931c694-ee0f-4f91-9547-e2d6d13f4511): Created mutation with ID 0000000000
2022.05.06 09:01:01.296825 [ 9868 ] {} <Debug> HTTP-Session: 0d0eb2ac-64d0-42b7-9041-1321c689a575 Authenticated with global context as user 94309d50-4f52-5250-31bd-74fecac179db
2022.05.06 09:01:01.296841 [ 9868 ] {} <Debug> HTTP-Session: 0d0eb2ac-64d0-42b7-9041-1321c689a575 Creating query context from global context, user_id: 94309d50-4f52-5250-31bd-74fecac179db, parent context user: <NOT SET>
2022.05.06 09:01:01.297059 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Debug> executeQuery: (from 127.0.0.1:46448) create database IF NOT EXISTS d_f02c61 FORMAT JSON
2022.05.06 09:01:01.297093 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Trace> ContextAccess (default): Access granted: CREATE DATABASE ON d_f02c61.*
2022.05.06 09:01:01.297354 [ 131 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Information> test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980 (ee0155d3-b36c-48e6-ae71-90a3dc39f414): The part /clickhouse/tables/01-01/test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980/replicas/clickhouse-02/parts/202205_2026_2031_1 on a replica suddenly appeared, will recheck checksums
2022.05.06 09:01:01.299880 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Information> DatabaseAtomic (d_f02c61): Metadata processed, database d_f02c61 has 0 tables and 0 dictionaries in total.
2022.05.06 09:01:01.299892 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Information> TablesLoader: Parsed metadata of 0 tables in 1 databases in 6.4513e-05 sec
2022.05.06 09:01:01.299898 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Information> TablesLoader: Loading 0 tables with 0 dependency level
2022.05.06 09:01:01.299904 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Information> DatabaseAtomic (d_f02c61): Starting up tables.
2022.05.06 09:01:01.299915 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Debug> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (9931c694-ee0f-4f91-9547-e2d6d13f4511): Waiting for clickhouse-02 to apply mutation 0000000000
2022.05.06 09:01:01.300081 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Debug> DynamicQueryHandler: Done processing query
2022.05.06 09:01:01.300098 [ 9868 ] {28907f64-b1e0-42c9-beef-bab7b1fbe45f} <Debug> MemoryTracker: Peak memory usage (for query): 0.00 B.
2022.05.06 09:01:01.300107 [ 9868 ] {} <Debug> HTTP-Session: 0d0eb2ac-64d0-42b7-9041-1321c689a575 Destroying unnamed session of user 94309d50-4f52-5250-31bd-74fecac179db
2022.05.06 09:01:01.301523 [ 131 ] {} <Debug> MemoryTracker: Peak memory usage Mutate/Merge: 4.06 MiB.
2022.05.06 09:01:01.303172 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Error> executeQuery: Code: 341. DB::Exception: Mutation 0000000000 was killed. (UNFINISHED) (version 22.3.2.1) (from 127.0.0.1:46444) (in query:  ALTER TABLE d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 DELETE WHERE 1 = 1 ), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0xa4dde1a in /usr/bin/clickhouse
1. DB::Exception::Exception<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&>(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) @ 0xa54a504 in /usr/bin/clickhouse
2. DB::checkMutationStatus(std::__1::optional<DB::MergeTreeMutationStatus>&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) @ 0x14f8f610 in /usr/bin/clickhouse
3. DB::StorageReplicatedMergeTree::waitMutationToFinishOnReplicas(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) const @ 0x14b851ad in /usr/bin/clickhouse
4. DB::StorageReplicatedMergeTree::waitMutation(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned long) const @ 0x14bf86ca in /usr/bin/clickhouse
5. DB::StorageReplicatedMergeTree::mutate(DB::MutationCommands const&, std::__1::shared_ptr<DB::Context const>) @ 0x14c1a986 in /usr/bin/clickhouse
6. DB::InterpreterAlterQuery::executeToTable(DB::ASTAlterQuery const&) @ 0x144034bc in /usr/bin/clickhouse
7. DB::InterpreterAlterQuery::execute() @ 0x14401b92 in /usr/bin/clickhouse
8. ? @ 0x148d0c9a in /usr/bin/clickhouse
9. DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, std::__1::optional<DB::FormatSettings> const&) @ 0x148d3fca in /usr/bin/clickhouse
10. DB::HTTPHandler::processQuery(DB::HTTPServerRequest&, DB::HTMLForm&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&, std::__1::optional<DB::CurrentThread::QueryScope>&) @ 0x151cdae7 in /usr/bin/clickhouse
11. DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x151d2292 in /usr/bin/clickhouse
12. DB::HTTPServerConnection::run() @ 0x1545979b in /usr/bin/clickhouse
13. Poco::Net::TCPServerConnection::start() @ 0x164b264f in /usr/bin/clickhouse
14. Poco::Net::TCPServerDispatcher::run() @ 0x164b4aa1 in /usr/bin/clickhouse
15. Poco::PooledThread::run() @ 0x16671e49 in /usr/bin/clickhouse
16. Poco::ThreadImpl::runnableEntry(void*) @ 0x1666f1a0 in /usr/bin/clickhouse
17. ? @ 0x7efcecf5f609 in ?
18. __clone @ 0x7efcece84163 in ?

2022.05.06 09:01:01.303294 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Error> DynamicQueryHandler: Code: 341. DB::Exception: Mutation 0000000000 was killed. (UNFINISHED), Stack trace (when copying this message, always include the lines below):

0. DB::Exception::Exception(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, int, bool) @ 0xa4dde1a in /usr/bin/clickhouse
1. DB::Exception::Exception<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&>(int, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) @ 0xa54a504 in /usr/bin/clickhouse
2. DB::checkMutationStatus(std::__1::optional<tor<char> > >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) @ 0x14f8f610 in /usr/bin/clickhouse
3. DB::StorageReplicatedMergeTree::waitMutationTDB::MergeTreeMutationStatus>&, std::__1::set<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::less<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocaoFinishOnReplicas(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&) const @ 0x14b851ad in /usr/bin/clickhouse
4. DB::StorageReplicatedMergeTree::waitMutation(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, unsigned long) const @ 0x14bf86ca in /usr/bin/clickhouse
5. DB::StorageReplicatedMergeTree::mutate(DB::MutationCommands const&, std::__1::shared_ptr<DB::Context const>) @ 0x14c1a986 in /usr/bin/clickhouse
6. DB::InterpreterAlterQuery::executeToTable(DB::ASTAlterQuery const&) @ 0x144034bc in /usr/bin/clickhouse
7. DB::InterpreterAlterQuery::execute() @ 0x14401b92 in /usr/bin/clickhouse
8. ? @ 0x148d0c9a in /usr/bin/clickhouse
9. DB::executeQuery(DB::ReadBuffer&, DB::WriteBuffer&, bool, std::__1::shared_ptr<DB::Context>, std::__1::function<void (std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&)>, std::__1::optional<DB::FormatSettings> const&) @ 0x148d3fca in /usr/bin/clickhouse
10. DB::HTTPHandler::processQuery(DB::HTTPServerRequest&, DB::HTMLForm&, DB::HTTPServerResponse&, DB::HTTPHandler::Output&, std::__1::optional<DB::CurrentThread::QueryScope>&) @ 0x151cdae7 in /usr/bin/clickhouse
11. DB::HTTPHandler::handleRequest(DB::HTTPServerRequest&, DB::HTTPServerResponse&) @ 0x151d2292 in /usr/bin/clickhouse
12. DB::HTTPServerConnection::run() @ 0x1545979b in /usr/bin/clickhouse
13. Poco::Net::TCPServerConnection::start() @ 0x164b264f in /usr/bin/clickhouse
14. Poco::Net::TCPServerDispatcher::run() @ 0x164b4aa1 in /usr/bin/clickhouse
15. Poco::PooledThread::run() @ 0x16671e49 in /usr/bin/clickhouse
16. Poco::ThreadImpl::runnableEntry(void*) @ 0x1666f1a0 in /usr/bin/clickhouse
17. ? @ 0x7efcecf5f609 in ?
18. __clone @ 0x7efcece84163 in ?
 (version 22.3.2.1)
2022.05.06 09:01:01.303377 [ 9825 ] {e1da4a60-2788-43ea-94f1-642b6955dbc1} <Debug> MemoryTracker: Peak memory usage (for query): 0.00 B.
2022.05.06 09:01:01.303384 [ 9825 ] {} <Debug> HTTP-Session: 7bee1174-2cac-4d72-bb15-a8262f26cef0 Destroying unnamed session of user 94309d50-4f52-5250-31bd-74fecac179db
2022.05.06 09:01:01.303809 [ 195 ] {} <Trace> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Adding mutation 0000000000 for partition ffc2b90782a6ac80cd2606b219db12bb for all block numbers less than 1
2022.05.06 09:01:01.303860 [ 234 ] {} <Trace> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Will check if mutation 0000000000 is done
2022.05.06 09:01:01.303870 [ 234 ] {} <Debug> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Trying to finalize 1 mutations
2022.05.06 09:01:01.305385 [ 234 ] {} <Trace> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Mutation 0000000000 is done
2022.05.06 09:01:01.308236 [ 9870 ] {} <Trace> HTTPHandler-factory: HTTP Request for HTTPHandler-factory. Method: POST, Address: 127.0.0.1:46452, User-Agent: tb-insert-chunk, Length: 450, Content Type: , Transfer Encoding: identity, X-Forwarded-For: (none)
``` 

Log in the second replica:
```
2022.05.06 09:01:01.288083 [ 344 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Trace> MergedBlockOutputStream: filled checksums 202205_2026_2031_1 (state Temporary)
2022.05.06 09:01:01.288437 [ 344 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Trace> test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980 (ee0155d3-b36c-48e6-ae71-90a3dc39f414): Renaming temporary part tmp_merge_202205_2026_2031_1 to 202205_2026_2031_1.
2022.05.06 09:01:01.288496 [ 344 ] {ee0155d3-b36c-48e6-ae71-90a3dc39f414::202205_2026_2031_1} <Trace> test_public_3e7de17c83d442809f79016c72ebc19f.t_ea74588f21024ac3b024641ebe98a980 (ee0155d3-b36c-48e6-ae71-90a3dc39f414) (MergerMutator): Merged 6 parts: from 202205_2026_2026_0 to 202205_2031_2031_0
2022.05.06 09:01:01.288662 [ 365 ] {} <Debug> test_public_3e7de17c83d442809f79016c72ebc19f.t_0d8fea409ba04b458b1399fdf65ce74e (e9a32087-6071-47eb-9ce7-e99d325535c0): Fetched part 2022_132_132_0 from /clickhouse/tables/01-01/test_public_3e7de17c83d442809f79016c72ebc19f.t_0d8fea409ba04b458b1399fdf65ce74e/replicas/clickhouse-01
2022.05.06 09:01:01.296685 [ 390 ] {} <Information> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Loading 1 mutation entries: 0000000000 - 0000000000
2022.05.06 09:01:01.297382 [ 390 ] {} <Trace> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Adding mutation 0000000000 for partition ffc2b90782a6ac80cd2606b219db12bb for all block numbers less than 1
2022.05.06 09:01:01.297442 [ 430 ] {} <Trace> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Will check if mutation 0000000000 is done
2022.05.06 09:01:01.297451 [ 430 ] {} <Debug> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Trying to finalize 1 mutations
2022.05.06 09:01:01.299869 [ 344 ] {} <Debug> MemoryTracker: Peak memory usage Mutate/Merge: 4.06 MiB.
2022.05.06 09:01:01.302271 [ 430 ] {} <Trace> d_test_815b7009495344f7ab177b17c6250b5c.t_afc78aa78e174881a648c09695b6f436_replaced_t_02ca728dfed941c28d362a7fd74385b0 (ReplicatedMergeTreeQueue): Mutation 0000000000 is done
2022.05.06 09:01:01.317085 [ 508 ] {} <Debug> DDLWorker: Scheduling tasks
2022.05.06 09:01:01.317089 [ 509 ] {} <Trace> DDLWorker: Too early to clean queue, will do it later.
2022.05.06 09:01:01.323894 [ 508 ] {} <Trace> DDLWorker: scheduleTasks: initialized=true, size_before_filtering=644, queue_size=644, entries=query-0000000000..query-0000000643, first_failed_task_name=none, current_tasks_size=1, last_current_task=query-0000000642, last_skipped_entry_name=none
```

I don't know if it's related, but I find it odd that `StorageReplicatedMergeTree::waitMutation` tries to set up the current replica as the first one to check but the logs say it's doing the check over the other replica instead (first `clickhouse-02`, then `clickhouse-01`).
```
$ egrep "Waiting for clickhouse.*to apply mutation" clickhouse-server-*.log | tail
clickhouse-server-1.log:2022.05.06 09:04:03.304003 [ 20287 ] {df7e366f-f5c9-4fd8-b16b-1d9022bfc946} <Debug> d_test_5c90353d4a684cf39731ffdbc0acc690.t_a40167a708f44058910a9aba12181e51_replaced_t_3848d2f366704d7785856f145782f2d2 (c9fad2c2-3df8-4d5e-abe1-180b57d4a25c): Waiting for clickhouse-02 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:03.315757 [ 20287 ] {df7e366f-f5c9-4fd8-b16b-1d9022bfc946} <Debug> d_test_5c90353d4a684cf39731ffdbc0acc690.t_a40167a708f44058910a9aba12181e51_replaced_t_3848d2f366704d7785856f145782f2d2 (c9fad2c2-3df8-4d5e-abe1-180b57d4a25c): Waiting for clickhouse-01 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:07.111596 [ 19391 ] {a2411e67-2e86-423e-8402-236d228df988} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_37c6a4091c5f46b59268c75d624029c3_replaced_t_52f26a10a6c748ae83a0c377a1d98a73 (76b91791-f1cc-4a38-8c24-7bbbdce2f738): Waiting for clickhouse-02 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:07.140792 [ 19391 ] {a2411e67-2e86-423e-8402-236d228df988} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_37c6a4091c5f46b59268c75d624029c3_replaced_t_52f26a10a6c748ae83a0c377a1d98a73 (76b91791-f1cc-4a38-8c24-7bbbdce2f738): Waiting for clickhouse-01 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:10.241893 [ 19391 ] {72a5fa63-1138-413c-af87-49ad12eb8f43} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_cc363d4dd26a40deaf54c9bb04db36b5_replaced_t_b2816294fccd42169093ece32c157569 (deb5a762-f20a-411f-a3d4-c050b0dfb541): Waiting for clickhouse-02 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:10.257769 [ 19391 ] {72a5fa63-1138-413c-af87-49ad12eb8f43} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_cc363d4dd26a40deaf54c9bb04db36b5_replaced_t_b2816294fccd42169093ece32c157569 (deb5a762-f20a-411f-a3d4-c050b0dfb541): Waiting for clickhouse-01 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:13.460351 [ 19391 ] {de6ca97b-74db-493a-8ce3-ae202cb740a0} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_ea82ddaa34fe48fb9a8b73667a99c83b_replaced_t_95ee3204e3d14330aa8909fe62e6ceb0 (bf906c86-e1b4-40f5-a536-b6a9f37efa61): Waiting for clickhouse-02 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:13.472835 [ 19391 ] {de6ca97b-74db-493a-8ce3-ae202cb740a0} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_ea82ddaa34fe48fb9a8b73667a99c83b_replaced_t_95ee3204e3d14330aa8909fe62e6ceb0 (bf906c86-e1b4-40f5-a536-b6a9f37efa61): Waiting for clickhouse-01 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:16.617920 [ 19391 ] {593ba3d2-3c5f-4b53-8274-d35c8cc28cdf} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_d07f9b4f7b784e2a8abea4effe90dbdf_replaced_t_3b83ca40996b443b90a6ebe7b2947035 (4c4aa3b7-67cd-43a7-976a-6e18554f869f): Waiting for clickhouse-02 to apply mutation 0000000000
clickhouse-server-1.log:2022.05.06 09:04:16.629736 [ 19391 ] {593ba3d2-3c5f-4b53-8274-d35c8cc28cdf} <Debug> d_test_cb6415836ba34f3c96d29ebdf9159b70.t_d07f9b4f7b784e2a8abea4effe90dbdf_replaced_t_3b83ca40996b443b90a6ebe7b2947035 (4c4aa3b7-67cd-43a7-976a-6e18554f869f): Waiting for clickhouse-01 to apply mutation 0000000000
```

I've only seen this once (over hundreds on runs) so it might not be related to `distributed_ddl.pool_size` and instead might be a different bug.

Any ideas on how to reproduce / debug this further are really appreciated.
ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/temp", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/replicas", "last added replica: " + replica_name,
            zkutil::CreateMode::Persistent));
        
        ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/quorum", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/quorum/last_part", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/quorum/failed_parts", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/mutations", "",
            zkutil::CreateMode::Persistent));
        
        ops.emplace_back(zkutil::makeCreateRequest(replica_path, "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/host", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/log_pointer", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/queue", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/parts", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/flags", "",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/is_lost", "0",
            zkutil::CreateMode::Persistent));
        ops.emplace_back(zkutil::makeCreateRequest(replica_path + "/metadata", metadata_str,
            zkutil::CreateMode::Persistent));