**How to reproduce**
Clickhouse server 20.13.1.5273
```
CREATE TABLE default.test
(
`key` UInt32,
`ts` DateTime CODEC(DoubleDelta, LZ4),
`db_time` DateTime DEFAULT now() COMMENT 'spout-ignore' CODEC(DoubleDelta, LZ4)
)
ENGINE = MergeTree
PARTITION BY toStartOfTenMinutes(db_time)
ORDER BY (key, ts)
TTL db_time + toIntervalHour(3)
SETTINGS index_granularity = 8192, merge_with_ttl_timeout = 3600, min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = 200000000, in_memory_parts_enable_wal = 0;


INSERT INTO test(key, ts) SELECT number % 1000, now() + intDiv(number,1000) FROM numbers(500);

SELECT * FROM test;
500 rows in set. Elapsed: 0.003 sec.

DETACH TABLE test;

ATTACH TABLE test;

SELECT * FROM test;

0 rows in set. Elapsed: 0.002 sec.
```

**Expected behavior**
Clickhouse would flush parts on disk.
{
    auto lock = lockParts();
    for (const auto & part : remove)
    {
        auto it_part = data_parts_by_info.find(part->info);
        if (it_part == data_parts_by_info.end())
            throw Exception("Part " + part->getNameWithState() + " not found in data_parts", ErrorCodes::LOGICAL_ERROR);
        modifyPartState(part, IMergeTreeDataPart::State::Temporary);
        
        data_parts_indexes.erase(it_part);
    }
}
void MergeTreeData::removePartsFromWorkingSet(const DataPartsVector & remove, bool clear_without_timeout, DataPartsLock * acquired_lock)