**Describe the bug**
```
5828589:2021.04.27 03:13:08.824665 [ 6101 ] {} <Trace> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94): Renaming temporary part tmp_merge_all_0_4_7_17 to all_0_4_7_17.
5828590:2021.04.27 03:13:08.824778 [ 3358 ] {} <Trace> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461): Renaming temporary part tmp_merge_all_0_4_7_17 to all_0_4_7_17.
5828591:2021.04.27 03:13:08.824823 [ 6101 ] {} <Warning> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94): Tried to add obsolete part all_0_4_7_17 covered by all_0_4_13_34 (state Committed)
5828623:2021.04.27 03:13:08.824951 [ 3358 ] {} <Warning> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461): Tried to add obsolete part all_0_4_7_17 covered by all_0_4_13_34 (state Committed)
5828624:2021.04.27 03:13:08.824995 [ 6101 ] {} <Warning> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94) (MergerMutator): Unexpected number of parts removed when adding all_0_4_7_17: 0 instead of 1
5828625:2021.04.27 03:13:08.825147 [ 3358 ] {} <Warning> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461) (MergerMutator): Unexpected number of parts removed when adding all_0_4_7_17: 0 instead of 1
5828643:2021.04.27 03:13:08.831273 [ 3358 ] {} <Trace> test_27.concurrent_alter_add_drop_2 (f0a064d0-dc89-4710-9781-a49bb0c16461): Trying to immediately remove part all_0_4_7_17 (state Temporary)
5828644:2021.04.27 03:13:08.831658 [ 3358 ] {} <Fatal> : Logical error: 'Part all_0_4_7_17 doesn't exist'.
5828648:2021.04.27 03:13:08.832767 [ 6101 ] {} <Trace> test_27.concurrent_alter_add_drop_3 (825021f7-129f-4841-ad6b-92215e3f1a94): Trying to immediately remove part all_0_4_7_17 (state Temporary)
5828649:2021.04.27 03:13:08.833210 [ 6101 ] {} <Fatal> : Logical error: 'Part all_0_4_7_17 doesn't exist'.

```

https://clickhouse-test-reports.s3.yandex.net/0/9bb4d8769f1064ada4cce6b26d8ff61c11356965/stress_test_(debug).html#fail1
if (columns_alter_type_metadata_only.count(command.column_name))
                {
                    auto it = old_types.find(command.column_name);
                    assert(it != old_types.end());
                    if (!isSafeForKeyConversion(it->second, command.data_type.get()))
                        throw Exception("ALTER of key column " + backQuoteIfNeed(command.column_name) + " from type "
                                    + it->second->getName() + " to type " + command.data_type->getName()
                                    + " is not safe because it can change the representation of primary key",
                            ErrorCodes::ALTER_OF_COLUMN_IS_FORBIDDEN);
                }
                if (old_metadata.getColumns().has(command.column_name))
                {
                    columns_to_check_conversion.push_back(
                        new_metadata.getColumns().getPhysical(command.column_name));
                }
            }
        }
    }
    checkProperties(new_metadata, old_metadata);
    checkTTLExpressions(new_metadata, old_metadata);
    if (!columns_to_check_conversion.empty())
    {
        auto old_header = old_metadata.getSampleBlock();
        performRequiredConversions(old_header, columns_to_check_conversion, getContext());
    }
    if (old_metadata.hasSettingsChanges())
    {
        const auto current_changes = old_metadata.getSettingsChanges()->as<const ASTSetQuery &>().changes;
        const auto & new_changes = new_metadata.settings_changes->as<const ASTSetQuery &>().changes;
        for (const auto & changed_setting : new_changes)