### Apache Airflow version

2.3.4

### What happened

n the graph view of the dag, after clearing the task (and its downstream tasks) in a task group and refreshing the page the browser the task group becomes disconnected from the dag. See attached gif.
![airflow_2_3_4_task_group_bug](https://user-images.githubusercontent.com/6542519/187409008-767e13e6-ab91-4875-9f3e-bd261b346d0f.gif)
The issue is not persistent and consistent. The graph view becomes disconnected from time to time as you can see on the attached video.

### What you think should happen instead

The graph should be rendered properly and consistently.

### How to reproduce

1. Add the following dag to the dag folder:
```
import logging
import time
from typing import List

import pendulum
from airflow import DAG
from airflow.operators.python import PythonOperator
from airflow.utils.task_group import TaskGroup


def log_function(message: str, **kwargs):
    logging.info(message)
    time.sleep(3)


def create_file_handling_task_group(supplier):
    with TaskGroup(group_id=f"file_handlig_task_group_{supplier}", ui_color='#666666') as file_handlig_task_group:
        entry = PythonOperator(
            task_id='entry',
            python_callable=log_function,
            op_kwargs={'message': 'create_file_handlig_task_group-Entry-task'}
        )
        with TaskGroup(group_id=f"file_handling_task_sub_group-{supplier}",
                       ui_color='#666666') as file_handlig_task_sub_group:
            sub_group_submit = PythonOperator(
                task_id='sub_group_submit',
                python_callable=log_function,
                op_kwargs={'message': 'create_file_handlig_sub_group_submit'}
            )
            sub_group_monitor = PythonOperator(
                task_id='sub_group_monitor',
                python_callable=log_function,
                op_kwargs={'message': 'create_file_handlig_sub_group_monitor'}
            )
            sub_group_submit >> sub_group_monitor
        entry >> file_handlig_task_sub_group
    return file_handlig_task_group


def get_stage_1_taskgroups(supplierlist: List) -> List[TaskGroup]:
    return [create_file_handling_task_group(supplier) for supplier in supplierlist]


def connect_stage1_to_stage2(self, stage1_tasks: List[TaskGroup], stage2_tasks: List[TaskGroup]) -> None:
    if stage2_tasks:
        for stage1_task in stage1_tasks:
            supplier_code: str = self.get_supplier_code(stage1_task)
            stage2_task = self.get_suppliers_tasks(supplier_code, stage2_tasks)
            stage1_task >> stage2_task


def get_stage_2_taskgroup(taskgroup_id: str):
    with TaskGroup(group_id=taskgroup_id, ui_color='#666666') as stage_2_taskgroup:
        sub_group_submit = PythonOperator(
            task_id='sub_group_submit',
            python_callable=log_function,
            op_kwargs={'message': 'create_file_handlig_sub_group_submit'}
        )
        sub_group_monitor = PythonOperator(
            task_id='sub_group_monitor',
            python_callable=log_function,
            op_kwargs={'message': 'create_file_handlig_sub_group_monitor'}
        )
        sub_group_submit >> sub_group_monitor
    return stage_2_taskgroup


def create_dag():
    with DAG(
            dag_id="horizon-task-group-bug",
            start_date=pendulum.datetime(2021, 1, 1, tz="UTC"),
            catchup=False,
            description="description"
    ) as dag:
        start = PythonOperator(
            task_id='start_main',
            python_callable=log_function,
            op_kwargs={'message': 'Entry-task'}
        )
        end = PythonOperator(
            task_id='end_main',
            python_callable=log_function,
            op_kwargs={'message': 'End-task'}
        )
        with TaskGroup(group_id=f"main_file_task_group", ui_color='#666666') as main_file_task_group:
            end_main_file_task_stage_1 = PythonOperator(
                task_id='end_main_file_task_stage_1',
                python_callable=log_function,
                op_kwargs={'message': 'end_main_file_task_stage_1'}
            )
            first_stage = get_stage_1_taskgroups(['9001', '9002'])
            first_stage >> get_stage_2_taskgroup("stage_2_1_taskgroup")
            first_stage >> get_stage_2_taskgroup("stage_2_2_taskgroup")
            first_stage >> end_main_file_task_stage_1
        start >> main_file_task_group >> end
        return dag


dag = create_dag()

```
2. Go to de graph view of the dag.
3. Run the dag.
4. After the dag run has finished. Clear the "sub_group_submit" task within the "stage_2_1_taskgroup" with downstream tasks.
5. Refresh the page multiple times and notice how from time to time the "stage_2_1_taskgroup" becomes disconnected from the dag.
6.  Clear the "sub_group_submit" task within the "stage_2_2_taskgroup" with downstream tasks.
7.  Refresh the page multiple times and notice how from time to time the "stage_2_2_taskgroup" becomes disconnected from the dag.

### Operating System

Mac OS, Linux

### Versions of Apache Airflow Providers

_No response_

### Deployment

Other Docker-based deployment

### Deployment details

Custom docker image based on apache/airflow:2.3.4-python3.10

### Anything else

_No response_

### Are you willing to submit PR?

- [ ] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)

DAG("DAG", default_args={"start_date": "2019-06-01"})
    def test_dag_naive_start_end_dates_strings(self):
        DAG("DAG", default_args={"start_date": "2019-06-01", "end_date": "2019-06-05"})
    def test_dag_start_date_propagates_to_end_date(self):
        """
        Tests that a start_date string with a timezone and an end_date string without a timezone
        are accepted and that the timezone from the start carries over the end
        This test is a little indirect, it works by setting start and end equal except for the
        timezone and then testing for equality after the DAG construction.  They'll be equal
        only if the same timezone was applied to both.
        An explicit check the `tzinfo` attributes for both are the same is an extra check.
        """
        dag = DAG(
            "DAG", default_args={"start_date": "2019-06-05T00:00:00+05:00", "end_date": "2019-06-05T00:00:00"}
        )
        assert dag.default_args["start_date"] == dag.default_args["end_date"]
        assert dag.default_args["start_date"].tzinfo == dag.default_args["end_date"].tzinfo
    def test_dag_naive_default_args_start_date(self):
        dag = DAG("DAG", default_args={"start_date": datetime.datetime(2018, 1, 1)})
        assert dag.timezone == settings.TIMEZONE
        dag = DAG("DAG", start_date=datetime.datetime(2018, 1, 1))
        assert dag.timezone == settings.TIMEZONE
    def test_dag_none_default_args_start_date(self):
        """
        Tests if a start_date of None in default_args
        works.
        """
        dag = DAG("DAG", default_args={"start_date": None})
        assert dag.timezone == settings.TIMEZONE
    def test_dag_task_priority_weight_total(self):