def create_fake_clusters(self):
        self.now = datetime.utcnow().replace(microsecond=0)
        self.add_mock_s3_data({'my_bucket': {}})

        # create a timestamp the given number of *hours*, *minutes*, etc.
        # in the past. If any *kwargs* are None, return None.
        def ago(**kwargs):
            if any(v is None for v in kwargs.values()):
                return None
            return to_iso8601(self.now - timedelta(**kwargs))

        # Build a step object easily
        # also make it respond to .args()
        def step(jar='/home/hadoop/contrib/streaming/hadoop-streaming.jar',
                 args=self._DEFAULT_STEP_ARGS,
                 state='COMPLETED',
                 create_hours_ago=None,
                 start_hours_ago=None,
                 end_hours_ago=None,
                 name='Streaming Step',
                 action_on_failure='TERMINATE_CLUSTER',
                 **kwargs):

            return MockEmrObject(
                config=MockEmrObject(
                    actiononfailure=action_on_failure,
                    args=[MockEmrObject(value=a) for a in args],
                    jar=jar,
                ),
                status=MockEmrObject(
                    state=state,
                    timeline=MockEmrObject(
                        creationdatetime=ago(hours=create_hours_ago),
                        enddatetime=ago(hours=end_hours_ago),
                        startdatetime=ago(hours=start_hours_ago),
                    ),
                )
            )


        # empty job
        self.add_mock_emr_cluster(MockEmrObject(
            id = 'j-EMPTY',
            status=MockEmrObject(
                state='STARTING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=10)
                ),
            ),
        ))

        # job that's bootstrapping
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-BOOTSTRAPPING',
            status=MockEmrObject(
                state='BOOTSTRAPPING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=10),
                ),
            ),
            _steps=[step(create_hours_ago=10, state='PENDING')],
        ))

        # currently running job
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-CURRENTLY_RUNNING',
            status=MockEmrObject(
                state='RUNNING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=4, minutes=15),
                    readydatetime=ago(hours=4, minutes=10))
                ),
            _steps=[step(start_hours_ago=4, state='RUNNING')],
            ),
        )

        # finished cluster
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-DONE',
            status=MockEmrObject(
                state='TERMINATED',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=10),
                    readydatetime=ago(hours=8),
                    enddatetime=ago(hours=5),
                ),
            ),
            _steps=[step(start_hours_ago=8, end_hours_ago=6)],
        ))

        # idle cluster
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-DONE_AND_IDLE',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[step(start_hours_ago=4, end_hours_ago=2)],
        ))

        # idle cluster with 4.x step format. should still be
        # recognizable as a streaming step
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-DONE_AND_IDLE_4_X',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[step(start_hours_ago=4, end_hours_ago=2,
                         jar='command-runner.jar',
                         args=['hadoop-streaming'] + self._DEFAULT_STEP_ARGS)],
        ))

        # idle cluster with an active lock
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-IDLE_AND_LOCKED',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[step(start_hours_ago=4, end_hours_ago=2)],
        ))
        self.add_mock_s3_data({
            'my_bucket': {
                'locks/j-IDLE_AND_LOCKED/2': b'not_you',
            },
        }, time_modified=self.now)

        # idle cluster with an expired lock
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-IDLE_AND_EXPIRED',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[step(start_hours_ago=4, end_hours_ago=2)],
        ))
        self.add_mock_s3_data({
            'my_bucket': {
                'locks/j-IDLE_AND_EXPIRED/2': b'not_you',
            },
        }, time_modified=(self.now - timedelta(minutes=5)))

        # idle cluster with an expired lock
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-IDLE_BUT_INCOMPLETE_STEPS',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[step(start_hours_ago=4, end_hours_ago=None)],
        ))

        # hive cluster (looks completed but isn't)
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-HIVE',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[step(
                start_hours_ago=4,
                end_hours_ago=4,
                jar=('s3://us-east-1.elasticmapreduce/libs/script-runner/'
                     'script-runner.jar'),
                args=[],
            )],
        ))

        # custom hadoop streaming jar
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-CUSTOM_DONE_AND_IDLE',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[step(
                start_hours_ago=4,
                end_hours_ago=4,
                jar=('s3://my_bucket/tmp/somejob/files/'
                     'oddjob-0.0.3-SNAPSHOT-standalone.jar'),
                args=[],
            )],
        ))

        mock_emr_conn = self.connect_emr()

        # hadoop debugging without any other steps
        mock_emr_conn.run_jobflow(_id='j-DEBUG_ONLY',
                                  name='DEBUG_ONLY',
                                  enable_debugging=True,
                                  now=self.now - timedelta(hours=3),
                                  job_flow_role='fake-instance-profile',
                                  service_role='fake-service-role',
        )
        j_debug_only = self.mock_emr_clusters['j-DEBUG_ONLY']
        j_debug_only.status.state = 'WAITING'
        j_debug_only.status.timeline.readydatetime = ago(hours=2, minutes=55)
        j_debug_only._steps[0].status.state = 'COMPLETED'
        j_debug_only._steps[0].status.timeline.enddatetime = ago(hours=2)

        # hadoop debugging + actual job
        # same jar as hive but with different args
        mock_emr_conn.run_jobflow(_id='j-HADOOP_DEBUGGING',
                                  name='HADOOP_DEBUGGING',
                                  enable_debugging=True,
                                  now=self.now - timedelta(hours=6),
                                  job_flow_role='fake-instance-profile',
                                  service_role='fake-service-role',
        )
        j_hadoop_debugging = self.mock_emr_clusters['j-HADOOP_DEBUGGING']
        j_hadoop_debugging._steps.append(step())
        j_hadoop_debugging.status.state = 'WAITING'
        j_hadoop_debugging.status.timeline.readydatetime = ago(
            hours=4, minutes=55)

        # Need to reset times manually because mockboto resets them
        j_hadoop_debugging._steps[0].status.state = 'COMPLETED'
        j_hadoop_debugging._steps[0].status.timeline.enddatetime = ago(hours=5)
        j_hadoop_debugging._steps[1].status.timeline.startdatetime = ago(
            hours=4)
        j_hadoop_debugging._steps[1].status.timeline.enddatetime = ago(hours=2)

        # should skip cancelled steps
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-IDLE_AND_FAILED',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=6),
                    readydatetime=ago(hours=5, minutes=5),
                ),
            ),
            _steps=[
                step(start_hours_ago=4, end_hours_ago=3, state='FAILED'),
                step(
                    state='CANCELLED',
                )
            ],
        ))

        # pooled cluster reaching end of full hour
        self.add_mock_emr_cluster(MockEmrObject(
            _bootstrapactions=[
                MockEmrObject(args=[], name='action 0'),
                MockEmrObject(args=[
                    MockEmrObject(
                        value='pool-0123456789abcdef0123456789abcdef'),
                    MockEmrObject(value='reflecting'),
                ], name='master'),
            ],
            id='j-POOLED',
            status=MockEmrObject(
                state='WAITING',
                timeline=MockEmrObject(
                    creationdatetime=ago(minutes=55),
                    readydatetime=ago(minutes=50),
                ),
            ),
        ))

        # cluster that has had pending jobs but hasn't run them
        self.add_mock_emr_cluster(MockEmrObject(
            id='j-PENDING_BUT_IDLE',
            status=MockEmrObject(
                state='RUNNING',
                timeline=MockEmrObject(
                    creationdatetime=ago(hours=3),
                    readydatetime=ago(hours=2, minutes=50),
                ),
            ),
            _steps=[step(create_hours_ago=3, state='PENDING')],
        ))
dataset/ETHPy150Open Yelp/mrjob/tests/tools/emr/test_terminate_idle_clusters.py/ClusterTerminationTestCase.create_fake_clusters/original