**How to reproduce**
Clickhouse server 20.13.1.5273
```
CREATE TABLE default.test
(
`key` UInt32,
`ts` DateTime CODEC(DoubleDelta, LZ4),
`db_time` DateTime DEFAULT now() COMMENT 'spout-ignore' CODEC(DoubleDelta, LZ4)
)
ENGINE = MergeTree
PARTITION BY toStartOfTenMinutes(db_time)
ORDER BY (key, ts)
TTL db_time + toIntervalHour(3)
SETTINGS index_granularity = 8192, merge_with_ttl_timeout = 3600, min_rows_for_compact_part = 1000000, min_bytes_for_compact_part = 200000000, in_memory_parts_enable_wal = 0;


INSERT INTO test(key, ts) SELECT number % 1000, now() + intDiv(number,1000) FROM numbers(500);

SELECT * FROM test;
500 rows in set. Elapsed: 0.003 sec.

DETACH TABLE test;

ATTACH TABLE test;

SELECT * FROM test;

0 rows in set. Elapsed: 0.002 sec.
```

**Expected behavior**
Clickhouse would flush parts on disk.
{
    auto remove_time = clear_without_timeout ? 0 : time(nullptr);
    for (const DataPartPtr & part : remove)
    {
        if (part->getState() == IMergeTreeDataPart::State::Committed)
        {
            removePartContributionToColumnAndSecondaryIndexSizes(part);
            removePartContributionToDataVolume(part);
        }
        if (part->getState() == IMergeTreeDataPart::State::Committed || clear_without_timeout)
            part->remove_time.store(remove_time, std::memory_order_relaxed);
        if (part->getState() != IMergeTreeDataPart::State::Outdated)
            modifyPartState(part, IMergeTreeDataPart::State::Outdated);
        if (isInMemoryPart(part) && getSettings()->in_memory_parts_enable_wal)
            getWriteAheadLog()->dropPart(part->name);
    }
}
void MergeTreeData::removePartsFromWorkingSetImmediatelyAndSetTemporaryState(const DataPartsVector & remove)