




'''simple docstring'''




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation

snake_case__          =						logging.get_logger(__name__)

snake_case__          =						{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

snake_case__          =						{
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
}

snake_case__          =						{
    """gpt2""": 10_24,
    """gpt2-medium""": 10_24,
    """gpt2-large""": 10_24,
    """gpt2-xl""": 10_24,
    """distilgpt2""": 10_24,
}




@lru_cache()
def 						snake_case__    ( )   ->   List[str]:
					A_ :     Tuple =							(
					    list(range(ord('''!'''			)			,		ord('''~'''			) + 1			)			) + list(range(ord('''¡'''			)			,		ord('''¬'''			) + 1			)			) + list(range(ord('''®'''			)			,		ord('''ÿ'''			) + 1			)			)
					)
					A_ :     int =							bs[:]
					A_ :     Optional[Any] =							0
					for b in range(2**8			):
										if b not in bs:
															bs.append(lowerCamelCase__			)
															cs.append(2**8 + n			)
															n += 1
					A_ :     Tuple =							[chr(lowerCamelCase__			) for n in cs]
					return dict(zip(lowerCamelCase__			,		lowerCamelCase__			)			)




def 						snake_case__    ( lowerCamelCase__				:   List[str]			)   ->   Optional[int]:
					A_ :     Dict =							set()
					A_ :     Any =							word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)			)
										A_ :     Optional[int] =							char
					return pairs






class     UpperCamelCase_     (a__     ):







						"""simple docstring"""


						_lowerCAmelCase    						=					VOCAB_FILES_NAMES
						_lowerCAmelCase    						=					PRETRAINED_VOCAB_FILES_MAP
						_lowerCAmelCase    						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						_lowerCAmelCase    						=					['input_ids', 'attention_mask']







						def __init__(     self      :       int							, _lowerCamelCase      :       Optional[Any]							, _lowerCamelCase      :       List[Any]							, _lowerCamelCase      :       Any="replace"							, _lowerCamelCase      :       int="<|endoftext|>"							, _lowerCamelCase      :       Optional[int]="<|endoftext|>"							, _lowerCamelCase      :       Tuple="<|endoftext|>"							, _lowerCamelCase      :       Tuple=None							, _lowerCamelCase      :       Optional[Any]=False							, _lowerCamelCase      :       Optional[Any]=False							, **_lowerCamelCase      :       Tuple							, ):





											"""simple docstring"""







											A_ :     Union[str, Any] =							AddedToken(_lowerCamelCase							, lstrip=_lowerCamelCase							, rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase							, _lowerCamelCase    ) else bos_token
											A_ :     str =							AddedToken(_lowerCamelCase							, lstrip=_lowerCamelCase							, rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase							, _lowerCamelCase    ) else eos_token
											A_ :     Any =							AddedToken(_lowerCamelCase							, lstrip=_lowerCamelCase							, rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase							, _lowerCamelCase    ) else unk_token
											A_ :     str =							AddedToken(_lowerCamelCase							, lstrip=_lowerCamelCase							, rstrip=_lowerCamelCase    ) if isinstance(_lowerCamelCase							, _lowerCamelCase    ) else pad_token
											super().__init__(
											    errors=_lowerCamelCase							, unk_token=_lowerCamelCase							, bos_token=_lowerCamelCase							, eos_token=_lowerCamelCase							, pad_token=_lowerCamelCase							, add_prefix_space=_lowerCamelCase							, add_bos_token=_lowerCamelCase							, **_lowerCamelCase							, )
											A_ :     Optional[Any] =							add_bos_token

											with open(_lowerCamelCase							, encoding='''utf-8'''    ) as vocab_handle:
																A_ :     int =							json.load(_lowerCamelCase    )
											A_ :     Dict =							{v: k for k, v in self.encoder.items()}
											A_ :     Dict =							errors  # how to handle errors in decoding
											A_ :     List[Any] =							bytes_to_unicode()
											A_ :     str =							{v: k for k, v in self.byte_encoder.items()}
											with open(_lowerCamelCase							, encoding='''utf-8'''    ) as merges_handle:
																A_ :     Tuple =							merges_handle.read().split('''\n'''    )[1:-1]
											A_ :     List[Any] =							[tuple(merge.split()    ) for merge in bpe_merges]
											A_ :     List[Any] =							dict(zip(_lowerCamelCase							, range(len(_lowerCamelCase    )    )    )    )
											A_ :     Dict =							{}
											A_ :     Optional[int] =							add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											A_ :     Dict =							re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''    )







						@property
						def       _a    (     self      :       List[Any]    ):





											"""simple docstring"""







											return len(self.encoder    )







						def       _a    (     self      :       Dict    ):





											"""simple docstring"""







											return dict(self.encoder							, **self.added_tokens_encoder    )







						def       _a    (     self      :       str							, _lowerCamelCase      :       Optional[Any]    ):





											"""simple docstring"""







											if token in self.cache:
																return self.cache[token]
											A_ :     Any =							tuple(_lowerCamelCase    )
											A_ :     List[str] =							get_pairs(_lowerCamelCase    )

											if not pairs:
																return token

											while True:
																A_ :     Dict =							min(_lowerCamelCase							, key=lambda _lowerCamelCase    : self.bpe_ranks.get(_lowerCamelCase							, float('''inf'''    )    )    )
																if bigram not in self.bpe_ranks:
																					break
																A_ ,A_ :     List[Any] =							bigram
																A_ :     int =							[]
																A_ :     Dict =							0
																while i < len(_lowerCamelCase    ):
																					try:
																										A_ :     Optional[int] =							word.index(_lowerCamelCase							, _lowerCamelCase    )
																					except ValueError:
																										new_word.extend(word[i:]    )
																										break
																					else:
																										new_word.extend(word[i:j]    )
																										A_ :     Optional[Any] =							j

																					if word[i] == first and i < len(_lowerCamelCase    ) - 1 and word[i + 1] == second:
																										new_word.append(first + second    )
																										i += 2
																					else:
																										new_word.append(word[i]    )
																										i += 1
																A_ :     Dict =							tuple(_lowerCamelCase    )
																A_ :     Union[str, Any] =							new_word
																if len(_lowerCamelCase    ) == 1:
																					break
																else:
																					A_ :     Tuple =							get_pairs(_lowerCamelCase    )
											A_ :     List[str] =							''' '''.join(_lowerCamelCase    )
											A_ :     int =							word
											return word







						def       _a    (     self      :       List[str]							, _lowerCamelCase      :       Any							, _lowerCamelCase      :       int=None    ):





											"""simple docstring"""







											if self.add_bos_token:
																A_ :     List[str] =							[self.bos_token_id]
											else:
																A_ :     Union[str, Any] =							[]

											A_ :     Optional[int] =							bos_token_ids + token_ids_a

											if token_ids_a is None:
																return output

											return output + bos_token_ids + token_ids_a







						def       _a    (     self      :       Dict							, _lowerCamelCase      :       List[int]							, _lowerCamelCase      :       Optional[List[int]] = None							, _lowerCamelCase      :       bool = False    ):





											"""simple docstring"""







											if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowerCamelCase							, token_ids_a=_lowerCamelCase							, already_has_special_tokens=_lowerCamelCase    )

											if not self.add_bos_token:
																return super().get_special_tokens_mask(
																    token_ids_a=_lowerCamelCase							, token_ids_a=_lowerCamelCase							, already_has_special_tokens=_lowerCamelCase    )

											if token_ids_a is None:
																return [1] + ([0] * len(_lowerCamelCase    ))
											return [1] + ([0] * len(_lowerCamelCase    )) + [1] + ([0] * len(_lowerCamelCase    ))







						def       _a    (     self      :       Optional[Any]							, _lowerCamelCase      :       Any    ):





											"""simple docstring"""







											A_ :     Optional[int] =							[]
											for token in re.findall(self.pat							, _lowerCamelCase    ):
																A_ :     str =							''''''.join(
																    self.byte_encoder[b] for b in token.encode('''utf-8'''    )    )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_lowerCamelCase    ).split(''' '''    )    )
											return bpe_tokens







						def       _a    (     self      :       Any							, _lowerCamelCase      :       Optional[Any]    ):





											"""simple docstring"""







											return self.encoder.get(_lowerCamelCase							, self.encoder.get(self.unk_token    )    )







						def       _a    (     self      :       Any							, _lowerCamelCase      :       Optional[int]    ):





											"""simple docstring"""







											return self.decoder.get(_lowerCamelCase    )







						def       _a    (     self      :       List[Any]							, _lowerCamelCase      :       Optional[int]    ):





											"""simple docstring"""







											A_ :     Optional[Any] =							''''''.join(_lowerCamelCase    )
											A_ :     List[Any] =							bytearray([self.byte_decoder[c] for c in text]    ).decode('''utf-8'''							, errors=self.errors    )
											return text







						def       _a    (     self      :       Dict							, _lowerCamelCase      :       str							, _lowerCamelCase      :       Optional[str] = None    ):





											"""simple docstring"""







											if not os.path.isdir(_lowerCamelCase    ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
																return
											A_ :     Any =							os.path.join(
											    _lowerCamelCase							, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']    )
											A_ :     Union[str, Any] =							os.path.join(
											    _lowerCamelCase							, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']    )

											with open(_lowerCamelCase							, '''w'''							, encoding='''utf-8'''    ) as f:
																f.write(json.dumps(self.encoder							, indent=2							, sort_keys=_lowerCamelCase							, ensure_ascii=_lowerCamelCase    ) + '''\n'''    )

											A_ :     List[str] =							0
											with open(_lowerCamelCase							, '''w'''							, encoding='''utf-8'''    ) as writer:
																writer.write('''#version: 0.2\n'''    )
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()							, key=lambda _lowerCamelCase    : kv[1]    ):
																					if index != token_index:
																										logger.warning(
																										    f'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
																										    ''' Please check that the tokenizer is not corrupted!'''    )
																										A_ :     str =							token_index
																					writer.write(''' '''.join(_lowerCamelCase    ) + '''\n'''    )
																					index += 1

											return vocab_file, merge_file







						def       _a    (     self      :       str							, _lowerCamelCase      :       Optional[int]							, _lowerCamelCase      :       Optional[Any]=False							, **_lowerCamelCase      :       Optional[Any]    ):





											"""simple docstring"""







											A_ :     Tuple =							kwargs.pop('''add_prefix_space'''							, self.add_prefix_space    )
											if is_split_into_words or add_prefix_space:
																A_ :     List[Any] =							''' ''' + text
											return (text, kwargs)







						def       _a    (     self      :       List[Any]							, _lowerCamelCase      :       "Conversation"    ):





											"""simple docstring"""







											A_ :     Dict =							[]
											for is_user, text in conversation.iter_texts():
																input_ids.extend(self.encode(_lowerCamelCase							, add_special_tokens=_lowerCamelCase    ) + [self.eos_token_id]    )
											if len(_lowerCamelCase    ) > self.model_max_length:
																A_ :     List[Any] =							input_ids[-self.model_max_length :]
											return input_ids









'''simple docstring'''




import argparse
import logging
import sys
import time

import tensorflow as tf
from datasets import load_dataset

from transformers import AutoTokenizer, TFAutoModelForSequenceClassification


if __name__ == "__main__":
							snake_case__          =						argparse.ArgumentParser()

							# Hyperparameters sent by the client are passed as command-line arguments to the script.
							parser.add_argument("""--epochs""", type=int, default=1)
							parser.add_argument("""--per_device_train_batch_size""", type=int, default=16)
							parser.add_argument("""--per_device_eval_batch_size""", type=int, default=8)
							parser.add_argument("""--model_name_or_path""", type=str)
							parser.add_argument("""--learning_rate""", type=str, default=5e-5)
							parser.add_argument("""--do_train""", type=bool, default=True)
							parser.add_argument("""--do_eval""", type=bool, default=True)
							parser.add_argument("""--output_dir""", type=str)

							snake_case__       ,						snake_case__          =						parser.parse_known_args()

							# overwrite batch size until we have tf_glue.py
							snake_case__          =						16
							snake_case__          =						16

							# Set up logging
							snake_case__          =						logging.getLogger(__name__)

							logging.basicConfig(
							    level=logging.getLevelName("""INFO"""),
							    handlers=[logging.StreamHandler(sys.stdout)],
							    format="""%(asctime)s - %(name)s - %(levelname)s - %(message)s""",
							)

							# Load model and tokenizer
							snake_case__          =						TFAutoModelForSequenceClassification.from_pretrained(args.model_name_or_path)
							snake_case__          =						AutoTokenizer.from_pretrained(args.model_name_or_path)

							# Load dataset
							snake_case__       ,						snake_case__          =						load_dataset("""imdb""", split=["""train""", """test"""])
							snake_case__          =						train_dataset.shuffle().select(range(50_00))  # smaller the size for train dataset to 5k
							snake_case__          =						test_dataset.shuffle().select(range(5_00))  # smaller the size for test dataset to 500

							# Preprocess train dataset
							snake_case__          =						train_dataset.map(
							    lambda e: tokenizer(e["""text"""], truncation=True, padding="""max_length"""), batched=True
							)
							train_dataset.set_format(type="""tensorflow""", columns=["""input_ids""", """attention_mask""", """label"""])

							snake_case__          =						{
							    x: train_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
							    for x in ["""input_ids""", """attention_mask"""]
							}
							snake_case__          =						tf.data.Dataset.from_tensor_slices((train_features, train_dataset["""label"""])).batch(
							    args.per_device_train_batch_size
							)

							# Preprocess test dataset
							snake_case__          =						test_dataset.map(
							    lambda e: tokenizer(e["""text"""], truncation=True, padding="""max_length"""), batched=True
							)
							test_dataset.set_format(type="""tensorflow""", columns=["""input_ids""", """attention_mask""", """label"""])

							snake_case__          =						{
							    x: test_dataset[x].to_tensor(default_value=0, shape=[None, tokenizer.model_max_length])
							    for x in ["""input_ids""", """attention_mask"""]
							}
							snake_case__          =						tf.data.Dataset.from_tensor_slices((test_features, test_dataset["""label"""])).batch(
							    args.per_device_eval_batch_size
							)

							# fine optimizer and loss
							snake_case__          =						tf.keras.optimizers.Adam(learning_rate=args.learning_rate)
							snake_case__          =						tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
							snake_case__          =						[tf.keras.metrics.SparseCategoricalAccuracy()]
							model.compile(optimizer=optimizer, loss=loss, metrics=metrics)

							snake_case__          =						time.time()
							snake_case__          =						model.fit(tf_train_dataset, epochs=args.epochs, batch_size=args.per_device_train_batch_size)
							snake_case__          =						time.time() - start_train_time

							logger.info("""*** Train ***""")
							logger.info(F'train_runtime = {end_train_time}')
							for key, value in train_results.history.items():
														logger.info(F'  {key} = {value}')



