**Describe the bug**
https://clickhouse-test-reports.s3.yandex.net/22289/c71da4a5c8e655f4bdfaa33b92ab022b97dfdf1a/fuzzer_ubsan/server.log

`SELECT * FROM (SELECT number AS l FROM system.numbers LIMIT 3) AS s1 ANY LEFT JOIN (SELECT toLowCardinality(number) AS r FROM system.numbers LIMIT 1024) AS s2 ON (l + 1) = (r * 1)`

```
../src/Common/assert_cast.h:50:12: runtime error: downcast of address 0x7f986c30b6c0 which does not point to an object of type 'const DB::ColumnVector<DB::UInt64>'
0x7f986c30b6c0: note: object is of type 'DB::ColumnLowCardinality'
 00 00 00 00  88 20 96 09 00 00 00 00  01 00 00 00 98 7f 00 00  60 8a aa ae 9a 7f 00 00  00 28 72 2c
              ^~~~~~~~~~~~~~~~~~~~~~~
              vptr for 'DB::ColumnLowCardinality'

    #0 0x1af6b158 in DB::ColumnVector<unsigned long> const& assert_cast<DB::ColumnVector<unsigned long> const&, DB::IColumn const&>(DB::IColumn const&) obj-x86_64-linux-gnu/../src/Common/assert_cast.h:50:12
    #1 0x1af6b158 in DB::ColumnVector<unsigned long>::insertFrom(DB::IColumn const&, unsigned long) obj-x86_64-linux-gnu/../src/Columns/ColumnVector.h:130:24
    #2 0x1abcde40 in DB::(anonymous namespace)::extractMinMax(DB::Block const&, DB::Block const&) obj-x86_64-linux-gnu/../src/Interpreters/MergeJoin.cpp:91:21
    #3 0x1abd172e in DB::MergeJoin::mergeFlushedRightBlocks()::$_0::operator()(DB::Block const&) const obj-x86_64-linux-gnu/../src/Interpreters/MergeJoin.cpp:548:47
    #4 0x1abd172e in decltype(std::__1::forward<DB::MergeJoin::mergeFlushedRightBlocks()::$_0&>(fp)(std::__1::forward<DB::Block const&>(fp0))) std::__1::__invoke<DB::MergeJoin::mergeFlushedRightBlocks()::$_0&, DB::Block const&>(DB::MergeJoin::mergeFlushedRightBlocks()::$_0&, DB::Block const&) obj-x86_64-linux-gnu/../contrib/libcxx/include/type_traits:3676:1
    #5 0x1ac8bd8f in std::__1::function<void (DB::Block const&)>::operator()(DB::Block const&) const obj-x86_64-linux-gnu/../contrib/libcxx/include/functional:2560:12
    #6 0x1ac8bd8f in DB::(anonymous namespace)::flushToManyFiles(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, DB::Block const&, DB::IBlockInputStream&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::function<void (DB::Block const&)>) obj-x86_64-linux-gnu/../src/Interpreters/SortedBlocksWriter.cpp:42:9
    #7 0x1ac8bb3f in DB::SortedBlocksWriter::finishMerge(std::__1::function<void (DB::Block const&)>) obj-x86_64-linux-gnu/../src/Interpreters/SortedBlocksWriter.cpp:197:12
    #8 0x1abcd751 in DB::MergeJoin::mergeFlushedRightBlocks() obj-x86_64-linux-gnu/../src/Interpreters/MergeJoin.cpp:552:41
    #9 0x1abcc5f5 in DB::MergeJoin::setTotals(DB::Block const&) obj-x86_64-linux-gnu/../src/Interpreters/MergeJoin.cpp:491:5
    #10 0x1bb7026a in DB::CreatingSetsTransform::finishSubquery() obj-x86_64-linux-gnu/../src/Processors/Transforms/CreatingSetsTransform.cpp:89:18
    #11 0x1bb719cf in DB::CreatingSetsTransform::generate() obj-x86_64-linux-gnu/../src/Processors/Transforms/CreatingSetsTransform.cpp:145:5
    #12 0x1b8e2e65 in DB::IAccumulatingTransform::work() obj-x86_64-linux-gnu/../src/Processors/IAccumulatingTransform.cpp:102:32
```

```
Changed settings: 
max_joined_block_size_rows = '2000', 
max_threads = '16', 
distributed_directory_monitor_sleep_time_ms = '10', 
distributed_directory_monitor_max_sleep_time_ms = '100', 
distributed_directory_monitor_batch_inserts = '1', 
allow_suspicious_low_cardinality_types = '1', 
optimize_skip_unused_shards = '1', 
log_queries = '1', 
log_queries_min_type = 'EXCEPTION_WHILE_PROCESSING', 
join_use_nulls = '0', 
join_default_strictness = 'ALL',                     <---
max_rows_to_read = '100000', 
max_rows_in_join = '1000', 
join_algorithm = 'partial_merge',                    <---
max_memory_usage = '50000000', 
send_logs_level = 'fatal', 
prefer_localhost_replica = '1', 
optimize_move_functions_out_of_any = '0', 
optimize_on_insert = '0', 
optimize_aggregators_of_group_by_keys = '0', 
output_format_pretty_max_column_pad_width = '250'
```

Similar to #21619 but different.
}
    
    if (!not_processed && left_blocks_buffer)
        not_processed = std::make_shared<NotProcessed>(NotProcessed{{}, 0, 0, 0});
    JoinCommon::restoreLowCardinalityInplace(block);
}
template <bool in_memory, bool is_all>
void MergeJoin::joinSortedBlock(Block & block, ExtraBlockPtr & not_processed)
{
    
    size_t rows_to_reserve = is_left ? block.rows() : 0;
    MutableColumns left_columns = makeMutableColumns(block, (is_all ? rows_to_reserve : 0));
    MutableColumns right_columns = makeMutableColumns(right_columns_to_add, rows_to_reserve);
    MergeJoinCursor left_cursor(block, left_merge_description);
    size_t left_key_tail = 0;
    size_t skip_right = 0;
    size_t right_blocks_count = rightBlocksCount<in_memory>();
    size_t starting_right_block = 0;
    if (not_processed)
    {
        auto & continuation = static_cast<NotProcessed &>(*not_processed);
        left_cursor.nextN(continuation.left_position);
        skip_right = continuation.right_position;
        starting_right_block = continuation.right_block;
        not_processed.reset();
    }
    bool with_left_inequals = (is_left && !is_semi_join) || is_full;
    if (with_left_inequals)
    {
        for (size_t i = starting_right_block; i < right_blocks_count; ++i)