**Describe the bug**
https://clickhouse-test-reports.s3.yandex.net/22328/43d9972f18d43c6736c08e67c01deb64b733109a/fuzzer_asan/server.log

**How to reproduce**
`SELECT * FROM (SELECT number AS l FROM system.numbers LIMIT 3) ANY LEFT JOIN (SELECT toLowCardinality(number) AS r FROM system.numbers LIMIT 1025) ON (l + 1025) = (r * 1)`

```
Changed settings: min_insert_block_size_rows = '0', min_insert_block_size_bytes = '0', max_threads = '16', totals_mode = 'after_having_auto', log_queries = '1', join_use_nulls = '0', joined_subquery_requires_alias = '0', max_rows_to_group_by = '100000', group_by_overflow_mode = 'any', max_bytes_in_join = '100', 

join_algorithm = 'auto', 

max_partitions_per_insert_block = '100', mutations_sync = '1', aggregate_functions_null_for_empty = '1'
```

Maybe the reason is the same as #22386.

PS. We should not encourage people to use merge join algorithm until the issue will be fixed.
{
    if constexpr (!in_memory)
        return flushed_right_blocks.size();
    else
        return loaded_right_blocks.size();
}
template <bool in_memory>
std::shared_ptr<Block> MergeJoin::loadRightBlock(size_t pos) const
{
    if constexpr (!in_memory)
    {
        auto load_func = [&]() -> std::shared_ptr<Block>
        {
            TemporaryFileStream input(flushed_right_blocks[pos]->path(), materializeBlock(right_sample_block));
            return std::make_shared<Block>(input.block_in->read());
        };
        return cached_right_blocks->getOrSet(pos, load_func).first;
    }
    else
        return loaded_right_blocks[pos];
}
void MergeJoin::initRightTableWriter()
{
    disk_writer = std::make_unique<SortedBlocksWriter>(size_limits, table_join->getTemporaryVolume(),
                    right_sample_block, right_sort_description, max_rows_in_right_block, max_files_to_merge,
                    table_join->temporaryFilesCodec());
    disk_writer->addBlocks(right_blocks);
    right_blocks.clear();
}
class NonMergeJoinedBlockInputStream : private NotJoined, public IBlockInputStream