Reproducer: 
``` sql
SELECT count() FROM (SELECT queryID() AS t FROM remote('127.0.0.{1..3}', numbers(10)) WITH TOTALS HAVING t = initialQueryID())

Code: 49. DB::Exception: Received from localhost:9000. DB::Exception: Chunk info was not set for chunk in GroupingAggregatedTransform.. (LOGICAL_ERROR)
``` 
std::shared_ptr<TableJoin> analyzed_join, const ColumnsWithTypeAndName & left_sample_columns, std::unique_ptr<QueryPlan> & joined_plan, ContextPtr context)
{
    const auto & settings = context->getSettings();
    Block left_sample_block(left_sample_columns);
    for (auto & column : left_sample_block)
    {
        if (!column.column)
            column.column = column.type->createColumn();
    }
    Block right_sample_block = joined_plan->getCurrentDataStream().header;
    std::vector<String> tried_algorithms;
    if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::DIRECT))
    {
        tried_algorithms.push_back(toString(JoinAlgorithm::DIRECT));
        JoinPtr direct_join = tryKeyValueJoin(analyzed_join, right_sample_block);
        if (direct_join)
        {
            
            joined_plan.reset();
            return direct_join;
        }
    }
    if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::PARTIAL_MERGE) ||
        analyzed_join->isEnabledAlgorithm(JoinAlgorithm::PREFER_PARTIAL_MERGE))
    {
        tried_algorithms.push_back(toString(JoinAlgorithm::PARTIAL_MERGE));
        if (MergeJoin::isSupported(analyzed_join))
            return std::make_shared<MergeJoin>(analyzed_join, right_sample_block);
    }
    if (analyzed_join->isEnabledAlgorithm(JoinAlgorithm::HASH) ||