<!--

Welcome to Apache Airflow!  For a smooth issue process, try to answer the following questions.
Don't worry if they're not all applicable; just try to include what you can :-)

If you need to include code snippets or logs, please put them in fenced code
blocks.  If they're super-long, please use the details tag like
<details><summary>super-long log</summary> lots of stuff </details>

Please delete these comment blocks before submitting the issue.

-->

**Description**

Related to the discussion here: https://github.com/apache/airflow/pull/8553#discussion_r416136954

Allow user to define expected terminal state. e.g. `JOB_STATE_CANCELLED` could be not treated as failure but expected terminal state.


<!-- A short description of your feature -->


poll_sleep=self.dataflow_hook.poll_sleep,
            num_retries=self.dataflow_hook.num_retries,
            cancel_timeout=DEFAULT_CANCEL_TIMEOUT,
            wait_until_finished=self.dataflow_hook.wait_until_finished,
        )
        mock_controller.return_value.get_jobs.assert_called_once_with(refresh=True)
        assert result == {"id": TEST_JOB_ID}
    @mock.patch(DATAFLOW_STRING.format("_DataflowJobsController"))
    @mock.patch(DATAFLOW_STRING.format("DataflowHook.get_conn"))
    def test_cancel_job(self, mock_get_conn, jobs_controller):
        self.dataflow_hook.cancel_job(
            job_name=UNIQUE_JOB_NAME, job_id=TEST_JOB_ID, project_id=TEST_PROJECT, location=TEST_LOCATION
        )
        jobs_controller.assert_called_once_with(
            dataflow=mock_get_conn.return_value,
            job_id=TEST_JOB_ID,
            location=TEST_LOCATION,
            name=UNIQUE_JOB_NAME,
            poll_sleep=10,
            project_number=TEST_PROJECT,
            num_retries=5,
            drain_pipeline=False,
            cancel_timeout=DEFAULT_CANCEL_TIMEOUT,
        )
        jobs_controller.cancel()
    @mock.patch(DATAFLOW_STRING.format("_DataflowJobsController"))
    @mock.patch(DATAFLOW_STRING.format("DataflowHook.provide_authorized_gcloud"))
    @mock.patch(DATAFLOW_STRING.format("DataflowHook.get_conn"))
    @mock.patch(DATAFLOW_STRING.format("subprocess.run"))
    def test_start_sql_job_failed_to_run(