As discussed in  https://github.com/apache/airflow/pull/15534#discussion_r620500075, we could possibly rewrite Beam Hook to remove the need of storing state in it.
#
#
from __future__ import annotations
import copy
import os
import re
import subprocess