**Describe the bug**
```
5828589:2021.04.27 03:13:08.824665 [ 6101 ] {} <Trace> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94): Renaming temporary part tmp_merge_all_0_4_7_17 to all_0_4_7_17.
5828590:2021.04.27 03:13:08.824778 [ 3358 ] {} <Trace> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461): Renaming temporary part tmp_merge_all_0_4_7_17 to all_0_4_7_17.
5828591:2021.04.27 03:13:08.824823 [ 6101 ] {} <Warning> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94): Tried to add obsolete part all_0_4_7_17 covered by all_0_4_13_34 (state Committed)
5828623:2021.04.27 03:13:08.824951 [ 3358 ] {} <Warning> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461): Tried to add obsolete part all_0_4_7_17 covered by all_0_4_13_34 (state Committed)
5828624:2021.04.27 03:13:08.824995 [ 6101 ] {} <Warning> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94) (MergerMutator): Unexpected number of parts removed when adding all_0_4_7_17: 0 instead of 1
5828625:2021.04.27 03:13:08.825147 [ 3358 ] {} <Warning> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461) (MergerMutator): Unexpected number of parts removed when adding all_0_4_7_17: 0 instead of 1
5828643:2021.04.27 03:13:08.831273 [ 3358 ] {} <Trace> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461): Trying to immediately remove part all_0_4_7_17 (state Temporary)
5828644:2021.04.27 03:13:08.831658 [ 3358 ] {} <Fatal> : Logical error: 'Part all_0_4_7_17 doesn't exist'.
5828648:2021.04.27 03:13:08.832767 [ 6101 ] {} <Trace> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94): Trying to immediately remove part all_0_4_7_17 (state Temporary)
5828649:2021.04.27 03:13:08.833210 [ 6101 ] {} <Fatal> : Logical error: 'Part all_0_4_7_17 doesn't exist'.

```

https://clickhouse-test-reports.s3.yandex.net/0/9bb4d8769f1064ada4cce6b26d8ff61c11356965/stress_test_(debug).html#fail1
const StorageInMemoryMetadata & new_metadata, const StorageInMemoryMetadata & old_metadata, bool attach) const
{
    if (!new_metadata.sorting_key.definition_ast)
        throw Exception("ORDER BY cannot be empty", ErrorCodes::BAD_ARGUMENTS);
    KeyDescription new_sorting_key = new_metadata.sorting_key;
    KeyDescription new_primary_key = new_metadata.primary_key;
    size_t sorting_key_size = new_sorting_key.column_names.size();
    size_t primary_key_size = new_primary_key.column_names.size();
    if (primary_key_size > sorting_key_size)
        throw Exception("Primary key must be a prefix of the sorting key, but its length: "
            + toString(primary_key_size) + " is greater than the sorting key length: " + toString(sorting_key_size),
            ErrorCodes::BAD_ARGUMENTS);
    NameSet primary_key_columns_set;
    for (size_t i = 0; i < sorting_key_size; ++i)
    {
        const String & sorting_key_column = new_sorting_key.column_names[i];
        if (i < primary_key_size)
        {
            const String & pk_column = new_primary_key.column_names[i];
            if (pk_column != sorting_key_column)
                throw Exception("Primary key must be a prefix of the sorting key, but the column in the position "
                    + toString(i) + " is " + sorting_key_column +", not " + pk_column,
                    ErrorCodes::BAD_ARGUMENTS);
            if (!primary_key_columns_set.emplace(pk_column).second)
                throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS);
        }
    }
    auto all_columns = new_metadata.columns.getAllPhysical();