**Describe the bug**

I have a table like:

```
CREATE TABLE test (
    time DateTime64(3)
    ...
) ENGINE = MergeTree()
PARTITION BY toStartOfInterval(time, INTERVAL 1 HOUR)
...
```

But when I search in `system.parts` it does not appear to have the `min/max` columns appropriately set?

```
partition:                             2020-06-01 01:00:00
min_date:                              1970-01-01
max_date:                              1970-01-01
min_time:                              1970-01-01 00:00:00
max_time:                              1970-01-01 00:00:00
```

I assume that if these were correctly set, doing a `select toStartOfInterval(time, interval 1 day), count()` type query would be pretty instant rather than trying to scan everything? I am guessing this is due to the `DateTime64` type in use?

Clickhouse 20.11.3.3

{
    auto remove_time = clear_without_timeout ? 0 : time(nullptr);
    for (const DataPartPtr & part : remove)
    {
        if (part->getState() == IMergeTreeDataPart::State::Committed)
        {
            removePartContributionToColumnSizes(part);
            removePartContributionToDataVolume(part);
        }
        if (part->getState() == IMergeTreeDataPart::State::Committed || clear_without_timeout)
            part->remove_time.store(remove_time, std::memory_order_relaxed);
        if (part->getState() != IMergeTreeDataPart::State::Outdated)
            modifyPartState(part, IMergeTreeDataPart::State::Outdated);
        if (isInMemoryPart(part) && getSettings()->in_memory_parts_enable_wal)
            getWriteAheadLog()->dropPart(part->name);
    }
}
void MergeTreeData::removePartsFromWorkingSetImmediatelyAndSetTemporaryState(const DataPartsVector & remove)