


'''simple docstring'''





import argparse
import logging
import os
import sys
import time

import tensorflow as tf
from datasets import load_dataset
from tqdm import tqdm

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification
from transformers.utils import is_sagemaker_dp_enabled


if os.environ.get('''SDP_ENABLED''') or is_sagemaker_dp_enabled():
    __lowerCamelCase       				=	True
    __lowerCamelCase       				=	'''p3dn.24xlarge'''
    import smdistributed.dataparallel.tensorflow as sdp
else:
    __lowerCamelCase       				=	False


def 					a__    (				UpperCamelCase_	:			Optional[int],      UpperCamelCase_	:			List[str],      UpperCamelCase_	:			Optional[int],      UpperCamelCase_	:			int,      UpperCamelCase_	:			Union[str, Any],      UpperCamelCase_	:			List[Any],      UpperCamelCase_	:			Tuple=None	):
   UpperCAmelCase__  :int            =			tqdm(UpperCamelCase_	)
   for i, batch in enumerate(UpperCamelCase_	):
      with tf.GradientTape() as tape:
         UpperCAmelCase__      ,	UpperCAmelCase__  :Union[str, Any]            =			batch
         UpperCAmelCase__  :int            =			model(UpperCamelCase_	)
         UpperCAmelCase__  :Tuple            =			loss(UpperCamelCase_,      outputs.logits	)

      if SDP_ENABLED:
         UpperCAmelCase__  :Any            =			sdp.DistributedGradientTape(UpperCamelCase_,      sparse_as_dense=UpperCamelCase_	)

      UpperCAmelCase__  :Dict            =			tape.gradient(UpperCamelCase_,      model.trainable_variables	)
      opt.apply_gradients(zip(UpperCamelCase_,      model.trainable_variables	)	)

      pbar.set_description(F'''Loss: {loss_value:.4f}'''	)

      if SDP_ENABLED and i == 0:
         sdp.broadcast_variables(model.variables,      root_rank=0	)
         sdp.broadcast_variables(opt.variables(),      root_rank=0	)

      if max_steps and i >= max_steps:
         break

   UpperCAmelCase__  :Optional[int]            =			{'''loss''': loss_value.numpy()}
   return train_results


def 					a__    (				UpperCamelCase_	:			int,      UpperCamelCase_	:			List[str],      UpperCamelCase_	:			Optional[Any]	):
   # Load dataset
   UpperCAmelCase__      ,	UpperCAmelCase__  :Optional[Any]            =			load_dataset('''imdb''',      split=['''train''', '''test''']	)

   # Preprocess train dataset
   UpperCAmelCase__  :Optional[int]            =			train_dataset.map(
       lambda UpperCamelCase_	: tokenizer(e['''text'''],      truncation=UpperCamelCase_,      padding='''max_length'''	),      batched=UpperCamelCase_	)
   train_dataset.set_format(type='''tensorflow''',      columns=['''input_ids''', '''attention_mask''', '''label''']	)

   UpperCAmelCase__  :Dict            =			{
       x: train_dataset[x].to_tensor(default_value=0,      shape=[None, tokenizer.model_max_length]	)
       for x in ['''input_ids''', '''attention_mask''']
   }
   UpperCAmelCase__  :Any            =			tf.data.Dataset.from_tensor_slices((train_features, train_dataset['''label'''])	)

   # Preprocess test dataset
   UpperCAmelCase__  :str            =			test_dataset.map(
       lambda UpperCamelCase_	: tokenizer(e['''text'''],      truncation=UpperCamelCase_,      padding='''max_length'''	),      batched=UpperCamelCase_	)
   test_dataset.set_format(type='''tensorflow''',      columns=['''input_ids''', '''attention_mask''', '''label''']	)

   UpperCAmelCase__  :Union[str, Any]            =			{
       x: test_dataset[x].to_tensor(default_value=0,      shape=[None, tokenizer.model_max_length]	)
       for x in ['''input_ids''', '''attention_mask''']
   }
   UpperCAmelCase__  :Optional[int]            =			tf.data.Dataset.from_tensor_slices((test_features, test_dataset['''label'''])	)

   if SDP_ENABLED:
      UpperCAmelCase__  :int            =			tf_train_dataset.shard(sdp.size(),      sdp.rank()	)
      UpperCAmelCase__  :Optional[Any]            =			tf_test_dataset.shard(sdp.size(),      sdp.rank()	)
   UpperCAmelCase__  :int            =			tf_train_dataset.batch(UpperCamelCase_,      drop_remainder=UpperCamelCase_	)
   UpperCAmelCase__  :Optional[int]            =			tf_test_dataset.batch(UpperCamelCase_,      drop_remainder=UpperCamelCase_	)

   return tf_train_dataset, tf_test_dataset


if __name__ == "__main__":
    __lowerCamelCase       				=	argparse.ArgumentParser()

    # Hyperparameters sent by the client are passed as command-line arguments to the script.
    parser.add_argument('''--epochs''', type=int, default=3)
    parser.add_argument('''--per_device_train_batch_size''', type=int, default=16)
    parser.add_argument('''--per_device_eval_batch_size''', type=int, default=8)
    parser.add_argument('''--model_name_or_path''', type=str)
    parser.add_argument('''--learning_rate''', type=str, default=5E-5)
    parser.add_argument('''--do_train''', type=bool, default=True)
    parser.add_argument('''--do_eval''', type=bool, default=True)
    parser.add_argument('''--output_dir''', type=str)
    parser.add_argument('''--max_steps''', type=int, default=None)

    # Data, model, and output directories
    parser.add_argument('''--output_data_dir''', type=str, default=os.environ['''SM_OUTPUT_DATA_DIR'''])
    parser.add_argument('''--model_dir''', type=str, default=os.environ['''SM_MODEL_DIR'''])
    parser.add_argument('''--n_gpus''', type=str, default=os.environ['''SM_NUM_GPUS'''])

    __lowerCamelCase				,		__lowerCamelCase       				=	parser.parse_known_args()

    # Set up logging
    __lowerCamelCase       				=	logging.getLogger(__name__)

    logging.basicConfig(
        level=logging.getLevelName('''INFO'''),
        handlers=[logging.StreamHandler(sys.stdout)],
        format='''%(asctime)s - %(name)s - %(levelname)s - %(message)s''',
    )

    if SDP_ENABLED:
        sdp.init()

        __lowerCamelCase       				=	tf.config.experimental.list_physical_devices('''GPU''')
        for gpu in gpus:
            tf.config.experimental.set_memory_growth(gpu, True)
        if gpus:
            tf.config.experimental.set_visible_devices(gpus[sdp.local_rank()], '''GPU''')

    # Load model and tokenizer
    __lowerCamelCase       				=	TFAutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
    __lowerCamelCase       				=	AutoTokenizer.from_pretrained(args.model_name_or_path)

    # get datasets
    __lowerCamelCase				,		__lowerCamelCase       				=	get_datasets(
        tokenizer=tokenizer,
        train_batch_size=args.per_device_train_batch_size,
        eval_batch_size=args.per_device_eval_batch_size,
    )

    # fine optimizer and loss
    __lowerCamelCase       				=	tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
    __lowerCamelCase       				=	tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
    __lowerCamelCase       				=	[tf.keras.metrics.SparseCategoricalAccuracy()]
    model.compile(optimizer=optimizer, loss=loss, metrics=metrics)

    # Training
    if args.do_train:
        # train_results = model.fit(tf_train_dataset, epochs=args.epochs, batch_size=args.train_batch_size)
        __lowerCamelCase       				=	time.time()
        __lowerCamelCase       				=	fit(
            model,
            loss,
            optimizer,
            tf_train_dataset,
            args.epochs,
            args.per_device_train_batch_size,
            max_steps=args.max_steps,
        )
        __lowerCamelCase       				=	time.time() - start_train_time
        logger.info('''*** Train ***''')
        logger.info(F'''train_runtime = {end_train_time}''')

        __lowerCamelCase       				=	os.path.join(args.output_dir, '''train_results.txt''')

        if not SDP_ENABLED or sdp.rank() == 0:
            with open(output_eval_file, '''w''') as writer:
                logger.info('''***** Train results *****''')
                logger.info(train_results)
                for key, value in train_results.items():
                    logger.info(F'''  {key} = {value}''')
                    writer.write(F'''{key} = {value}\n''')

    # Evaluation
    if args.do_eval and (not SDP_ENABLED or sdp.rank() == 0):
        __lowerCamelCase       				=	model.evaluate(tf_test_dataset, batch_size=args.per_device_eval_batch_size, return_dict=True)
        logger.info('''*** Evaluate ***''')

        __lowerCamelCase       				=	os.path.join(args.output_dir, '''eval_results.txt''')

        with open(output_eval_file, '''w''') as writer:
            logger.info('''***** Eval results *****''')
            logger.info(result)
            for key, value in result.items():
                logger.info(F'''  {key} = {value}''')
                writer.write(F'''{key} = {value}\n''')

    # Save result
    if SDP_ENABLED:
        if sdp.rank() == 0:
            model.save_pretrained(args.output_dir)
            tokenizer.save_pretrained(args.output_dir)
    else:
        model.save_pretrained(args.output_dir)
        tokenizer.save_pretrained(args.output_dir)







'''simple docstring'''





import gc
import unittest

import numpy as np
import torch
from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer

from diffusers import (
    AutoencoderKL,
    DDIMParallelScheduler,
    DDPMParallelScheduler,
    StableDiffusionParadigmsPipeline,
    UNetaDConditionModel,
)
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import (
    enable_full_determinism,
    require_torch_gpu,
)

from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS
from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin


enable_full_determinism()




class 							UpperCAmelCase ( _snake_case	,   _snake_case	,   unittest.TestCase	):
 UpperCAmelCase				=				StableDiffusionParadigmsPipeline
 UpperCAmelCase				=				TEXT_TO_IMAGE_PARAMS
 UpperCAmelCase				=				TEXT_TO_IMAGE_BATCH_PARAMS
 UpperCAmelCase				=				TEXT_TO_IMAGE_IMAGE_PARAMS
 UpperCAmelCase				=				TEXT_TO_IMAGE_IMAGE_PARAMS


 def 					__SCREAMING_SNAKE_CASE				(			self       :  int   ):
    torch.manual_seed(0   )
    UpperCAmelCase__  :Optional[int]            =			UNetaDConditionModel(
        block_out_channels=(3_2, 6_4) ,					layers_per_block=2 ,					sample_size=3_2 ,					in_channels=4 ,					out_channels=4 ,					down_block_types=('''DownBlock2D''', '''CrossAttnDownBlock2D''') ,					up_block_types=('''CrossAttnUpBlock2D''', '''UpBlock2D''') ,					cross_attention_dim=3_2 ,					attention_head_dim=(2, 4) ,					use_linear_projection=__lowerCamelCase ,					)
    UpperCAmelCase__  :Union[str, Any]            =			DDIMParallelScheduler(
        beta_start=0.0_00_85 ,					beta_end=0.0_12 ,					beta_schedule='''scaled_linear''' ,					clip_sample=__lowerCamelCase ,					set_alpha_to_one=__lowerCamelCase ,					)
    torch.manual_seed(0   )
    UpperCAmelCase__  :Tuple            =			AutoencoderKL(
        block_out_channels=[3_2, 6_4] ,					in_channels=3 ,					out_channels=3 ,					down_block_types=['''DownEncoderBlock2D''', '''DownEncoderBlock2D'''] ,					up_block_types=['''UpDecoderBlock2D''', '''UpDecoderBlock2D'''] ,					latent_channels=4 ,					sample_size=1_2_8 ,					)
    torch.manual_seed(0   )
    UpperCAmelCase__  :Tuple            =			CLIPTextConfig(
        bos_token_id=0 ,					eos_token_id=2 ,					hidden_size=3_2 ,					intermediate_size=3_7 ,					layer_norm_eps=1e-05 ,					num_attention_heads=4 ,					num_hidden_layers=5 ,					pad_token_id=1 ,					vocab_size=1_0_0_0 ,					hidden_act='''gelu''' ,					projection_dim=5_1_2 ,					)
    UpperCAmelCase__  :List[Any]            =			CLIPTextModel(__lowerCamelCase   )
    UpperCAmelCase__  :Tuple            =			CLIPTokenizer.from_pretrained('''hf-internal-testing/tiny-random-clip'''   )

    UpperCAmelCase__  :int            =			{
        '''unet''': unet,
        '''scheduler''': scheduler,
        '''vae''': vae,
        '''text_encoder''': text_encoder,
        '''tokenizer''': tokenizer,
        '''safety_checker''': None,
        '''feature_extractor''': None,
    }
    return components


 def 					__SCREAMING_SNAKE_CASE				(			self       :  str ,					__lowerCamelCase       :  List[Any] ,					__lowerCamelCase       :  Any=0   ):
    if str(__lowerCamelCase   ).startswith('''mps'''   ):
       UpperCAmelCase__  :str            =			torch.manual_seed(__lowerCamelCase   )
    else:
       UpperCAmelCase__  :Union[str, Any]            =			torch.Generator(device=__lowerCamelCase   ).manual_seed(__lowerCamelCase   )
    UpperCAmelCase__  :Union[str, Any]            =			{
        '''prompt''': '''a photograph of an astronaut riding a horse''',
        '''generator''': generator,
        '''num_inference_steps''': 1_0,
        '''guidance_scale''': 6.0,
        '''output_type''': '''numpy''',
        '''parallel''': 3,
        '''debug''': True,
    }
    return inputs


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Dict   ):
    UpperCAmelCase__  :Dict            =			'''cpu'''  # ensure determinism for the device-dependent torch.Generator
    UpperCAmelCase__  :Dict            =			self.get_dummy_components()
    UpperCAmelCase__  :str            =			StableDiffusionParadigmsPipeline(**__lowerCamelCase   )
    UpperCAmelCase__  :List[Any]            =			sd_pipe.to(__lowerCamelCase   )
    sd_pipe.set_progress_bar_config(disable=__lowerCamelCase   )

    UpperCAmelCase__  :Optional[int]            =			self.get_dummy_inputs(__lowerCamelCase   )
    UpperCAmelCase__  :Any            =			sd_pipe(**__lowerCamelCase   ).images
    UpperCAmelCase__  :Union[str, Any]            =			image[0, -3:, -3:, -1]
    assert image.shape == (1, 6_4, 6_4, 3)

    UpperCAmelCase__  :List[Any]            =			np.array([0.47_73, 0.54_17, 0.47_23, 0.49_25, 0.56_31, 0.47_52, 0.52_40, 0.49_35, 0.50_23]   )

    assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1e-2


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Dict   ):
    UpperCAmelCase__  :Any            =			'''cpu'''  # ensure determinism for the device-dependent torch.Generator
    UpperCAmelCase__  :Dict            =			self.get_dummy_components()
    torch.manual_seed(0   )
    UpperCAmelCase__  :Tuple            =			DDPMParallelScheduler()
    torch.manual_seed(0   )
    UpperCAmelCase__  :List[str]            =			StableDiffusionParadigmsPipeline(**__lowerCamelCase   )
    UpperCAmelCase__  :Optional[int]            =			sd_pipe.to(__lowerCamelCase   )
    sd_pipe.set_progress_bar_config(disable=__lowerCamelCase   )

    UpperCAmelCase__  :Optional[int]            =			self.get_dummy_inputs(__lowerCamelCase   )
    UpperCAmelCase__  :Optional[Any]            =			sd_pipe(**__lowerCamelCase   ).images
    UpperCAmelCase__  :Dict            =			image[0, -3:, -3:, -1]
    assert image.shape == (1, 6_4, 6_4, 3)

    UpperCAmelCase__  :List[str]            =			np.array([0.35_73, 0.44_20, 0.49_60, 0.47_99, 0.37_96, 0.38_79, 0.48_19, 0.43_65, 0.44_68]   )

    assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1e-2


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Dict   ):
    super().test_inference_batch_consistent(batch_sizes=[1, 2]   )


 def 					__SCREAMING_SNAKE_CASE				(			self       :  List[str]   ):
    super().test_inference_batch_single_identical(batch_size=2 ,					expected_max_diff=3e-3   )





 def 					__SCREAMING_SNAKE_CASE				(			self       :  Dict   ):
    UpperCAmelCase__  :int            =			'''cpu'''  # ensure determinism for the device-dependent torch.Generator
    UpperCAmelCase__  :str            =			self.get_dummy_components()
    UpperCAmelCase__  :Any            =			StableDiffusionParadigmsPipeline(**__lowerCamelCase   )
    UpperCAmelCase__  :List[str]            =			sd_pipe.to(__lowerCamelCase   )
    sd_pipe.set_progress_bar_config(disable=__lowerCamelCase   )

    UpperCAmelCase__  :Tuple            =			self.get_dummy_inputs(__lowerCamelCase   )
    UpperCAmelCase__  :Optional[int]            =			'''french fries'''
    UpperCAmelCase__  :Any            =			sd_pipe(**__lowerCamelCase ,					negative_prompt=__lowerCamelCase   )
    UpperCAmelCase__  :int            =			output.images
    UpperCAmelCase__  :Tuple            =			image[0, -3:, -3:, -1]

    assert image.shape == (1, 6_4, 6_4, 3)

    UpperCAmelCase__  :Tuple            =			np.array([0.47_71, 0.54_20, 0.46_83, 0.49_18, 0.56_36, 0.47_25, 0.52_30, 0.49_23, 0.50_15]   )

    assert np.abs(image_slice.flatten() - expected_slice   ).max() < 1e-2




@slow
@require_torch_gpu
class 							UpperCAmelCase ( unittest.TestCase	):


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Dict   ):
    super().tearDown()
    gc.collect()
    torch.cuda.empty_cache()


 def 					__SCREAMING_SNAKE_CASE				(			self       :  Optional[Any] ,					__lowerCamelCase       :  Tuple=0   ):
    UpperCAmelCase__  :List[Any]            =			torch.Generator(device=__lowerCamelCase   ).manual_seed(__lowerCamelCase   )
    UpperCAmelCase__  :List[str]            =			{
        '''prompt''': '''a photograph of an astronaut riding a horse''',
        '''generator''': generator,
        '''num_inference_steps''': 1_0,
        '''guidance_scale''': 7.5,
        '''output_type''': '''numpy''',
        '''parallel''': 3,
        '''debug''': True,
    }
    return inputs





 def 					__SCREAMING_SNAKE_CASE				(			self       :  int   ):
    UpperCAmelCase__  :Dict            =			'''stabilityai/stable-diffusion-2-base'''
    UpperCAmelCase__  :Union[str, Any]            =			DDIMParallelScheduler.from_pretrained(__lowerCamelCase ,					subfolder='''scheduler'''   )
    UpperCAmelCase__  :Dict            =			StableDiffusionParadigmsPipeline.from_pretrained(__lowerCamelCase ,					scheduler=__lowerCamelCase ,					safety_checker=__lowerCamelCase   )
    pipe.to(__lowerCamelCase   )
    pipe.set_progress_bar_config(disable=__lowerCamelCase   )
    pipe.enable_attention_slicing()

    UpperCAmelCase__  :Dict            =			self.get_inputs()
    UpperCAmelCase__  :int            =			pipe(**__lowerCamelCase   ).images
    UpperCAmelCase__  :List[str]            =			image[0, -3:, -3:, -1].flatten()

    assert image.shape == (1, 5_1_2, 5_1_2, 3)

    UpperCAmelCase__  :Optional[int]            =			np.array([0.96_22, 0.96_02, 0.97_48, 0.95_91, 0.96_30, 0.96_91, 0.96_61, 0.96_31, 0.97_41]   )

    assert np.abs(expected_slice - image_slice   ).max() < 1e-2



