





"""simple docstring"""







import os
from shutil import copyfile
from typing import TYPE_CHECKING, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging
from ...utils.versions import require_version


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

require_version('tokenizers>=0.13.3')

if is_sentencepiece_available():
  from .tokenization_llama import LlamaTokenizer
else:
  SCREAMING_SNAKE_CASE_      =					None

SCREAMING_SNAKE_CASE_      =					logging.get_logger(__name__)
SCREAMING_SNAKE_CASE_      =					{'vocab_file': 'tokenizer.model', 'tokenizer_file': 'tokenizer.json'}

SCREAMING_SNAKE_CASE_      =					'[INST]', '[/INST]'
SCREAMING_SNAKE_CASE_      =					'<<SYS>>\n', '\n<</SYS>>\n\n'

# fmt: off
SCREAMING_SNAKE_CASE_      =					'You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. Your\\nanswers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. Please ensure\\nthat your responses are socially unbiased and positive in nature.\n\nIf a question does not make any sense, or is not factually coherent, explain why instead of answering something not\\ncorrect. If you don\'t know the answer to a question, please don\'t share false information.'

class 							snake_case_ (				lowercase_ ):







   """simple docstring"""




   A_										=					VOCAB_FILES_NAMES
   A_										=					LlamaTokenizer
   A_										=					"""left"""
   A_										=					["""input_ids""", """attention_mask"""]







   def __init__(		self					,				lowerCamelCase_=None					,				lowerCamelCase_=None					,				lowerCamelCase_=False					,				lowerCamelCase_="<unk>"					,				lowerCamelCase_="<s>"					,				lowerCamelCase_="</s>"					,				lowerCamelCase_=True					,				lowerCamelCase_=False					,				**lowerCamelCase_					,				)							->      Dict:
       super().__init__(
           vocab_file=lowerCamelCase_					,				tokenizer_file=lowerCamelCase_					,				clean_up_tokenization_spaces=lowerCamelCase_					,				unk_token=lowerCamelCase_					,				bos_token=lowerCamelCase_					,				eos_token=lowerCamelCase_					,				**lowerCamelCase_					,				)
       UpperCamelCase										=  add_bos_token
       UpperCamelCase										=  add_eos_token
       self.update_post_processor()

       UpperCamelCase										=  vocab_file
       UpperCamelCase										=  False if not self.vocab_file else True







   def      UpperCAmelCase__ (		self)							->      List[Any]:
       UpperCamelCase										=  self.bos_token
       UpperCamelCase										=  self.bos_token_id

       UpperCamelCase										=  self.eos_token
       UpperCamelCase										=  self.eos_token_id

       UpperCamelCase										=  F'{(bos+":0 ") * self.add_bos_token}$A:0{(" "+eos+":0") * self.add_eos_token}'
       UpperCamelCase										=  F'{single}{(" "+bos+":1") * self.add_bos_token} $B:1{(" "+eos+":1") * self.add_eos_token}'

       UpperCamelCase										=  []
       if self.add_bos_token:
           special_tokens.append((bos, bos_token_id))
       if self.add_eos_token:
           special_tokens.append((eos, eos_token_id))
       UpperCamelCase										=  processors.TemplateProcessing(
           single=lowerCamelCase_					,				pair=lowerCamelCase_					,				special_tokens=lowerCamelCase_)







   @property
   def      UpperCAmelCase__ (		self)							->      str:
       return self._add_eos_token







   @property
   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       return self._add_bos_token







   @add_eos_token.setter
   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      Tuple:
       UpperCamelCase										=  value
       self.update_post_processor()







   @add_bos_token.setter
   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      str:
       UpperCamelCase										=  value
       self.update_post_processor()







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_ = None)							->      Optional[Any]:







       if not self.can_save_slow_tokenizer:
           raise ValueError(
               '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
               '''tokenizer.''')

       if not os.path.isdir(lowerCamelCase_):
           logger.error(F'Vocabulary path ({save_directory}) should be a directory')
           return
       UpperCamelCase										=  os.path.join(
           lowerCamelCase_					,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

       if os.path.abspath(self.vocab_file) != os.path.abspath(lowerCamelCase_):
           copyfile(self.vocab_file					,				lowerCamelCase_)

       return (out_vocab_file,)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_)							->      Optional[Any]:
       UpperCamelCase										=  list(conversation.iter_texts())
       if not all([is_user for is_user, msg in dialogue[::2]]) or not all(
           [not is_user for is_user, msg in dialogue[1::2]]):
           raise ValueError(
               '''The model only supports \'user\' and \'assistant\' roles, starting with user and alternating (u/a/u/a/u...)''')

       UpperCamelCase										=  []
       if len(conversation.past_user_inputs) > 0:
           if not conversation.past_user_inputs[0].startswith(lowerCamelCase_) or E_SYS not in conversation.past_user_inputs[0]:
               UpperCamelCase										=  (
                   B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + conversation.past_user_inputs[0]
               )
       elif not dialogue[0][1].startswith(lowerCamelCase_) or E_SYS not in dialogue[0][1]:
           UpperCamelCase										=  (dialogue[0][0], B_SYS + DEFAULT_SYSTEM_PROMPT + E_SYS + dialogue[0][1])

       dialog_tokens += sum(
           [
               [self.bos_token_id]
               + self.encode(
                   F'{B_INST} {(prompt[1]).strip()} {E_INST} {(answer[1]).strip()} '					,				add_special_tokens=lowerCamelCase_)
               + [self.eos_token_id]
               for prompt, answer in zip(dialogue[::2]					,				dialogue[1::2])
           ]					,				[]					,				)
       if not (dialogue[-1][0]):
           raise ValueError(F'Last message must be from user, got {dialogue[-1]["role"]}')
       dialog_tokens += [self.bos_token_id] + self.encode(
           F'{B_INST} {(dialogue[-1][1]).strip()} {E_INST}'					,				add_special_tokens=lowerCamelCase_)
       return dialog_tokens





"""simple docstring"""







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import VideoMAEImageProcessor

class 							snake_case_ (				unittest.TestCase ):







   """simple docstring"""




   def __init__(		self					,				lowerCamelCase_					,				lowerCamelCase_=7					,				lowerCamelCase_=3					,				lowerCamelCase_=1_0					,				lowerCamelCase_=1_8					,				lowerCamelCase_=3_0					,				lowerCamelCase_=4_0_0					,				lowerCamelCase_=True					,				lowerCamelCase_=None					,				lowerCamelCase_=True					,				lowerCamelCase_=[0.5, 0.5, 0.5]					,				lowerCamelCase_=[0.5, 0.5, 0.5]					,				lowerCamelCase_=None					,				)							->      Optional[Any]:
       UpperCamelCase										=  size if size is not None else {'''shortest_edge''': 1_8}
       UpperCamelCase										=  crop_size if crop_size is not None else {'''height''': 1_8, '''width''': 1_8}

       UpperCamelCase										=  parent
       UpperCamelCase										=  batch_size
       UpperCamelCase										=  num_channels
       UpperCamelCase										=  num_frames
       UpperCamelCase										=  image_size
       UpperCamelCase										=  min_resolution
       UpperCamelCase										=  max_resolution
       UpperCamelCase										=  do_resize
       UpperCamelCase										=  size
       UpperCamelCase										=  do_normalize
       UpperCamelCase										=  image_mean
       UpperCamelCase										=  image_std
       UpperCamelCase										=  crop_size







   def      UpperCAmelCase__ (		self)							->      Tuple:
       return {
           "image_mean": self.image_mean,
           "image_std": self.image_std,
           "do_normalize": self.do_normalize,
           "do_resize": self.do_resize,
           "size": self.size,
           "crop_size": self.crop_size,
       }

@require_torch
@require_vision
class 							snake_case_ (				lowerCamelCase_  ,   unittest.TestCase ):







   """simple docstring"""




   A_										=					VideoMAEImageProcessor if is_vision_available() else None







   def      UpperCAmelCase__ (		self)							->      Optional[Any]:
       UpperCamelCase										=  VideoMAEImageProcessingTester(self)







   @property
   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       return self.image_processor_tester.prepare_image_processor_dict()







   def      UpperCAmelCase__ (		self)							->      List[Any]:
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       self.assertTrue(hasattr(lowerCamelCase_					,				'''image_mean'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''image_std'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''do_normalize'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''do_resize'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''do_center_crop'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''size'''))







   def      UpperCAmelCase__ (		self)							->      Dict:
       UpperCamelCase										=  self.image_processing_class.from_dict(self.image_processor_dict)
       self.assertEqual(image_processor.size					,				{'''shortest_edge''': 1_8})
       self.assertEqual(image_processor.crop_size					,				{'''height''': 1_8, '''width''': 1_8})

       UpperCamelCase										=  self.image_processing_class.from_dict(self.image_processor_dict					,				size=4_2					,				crop_size=8_4)
       self.assertEqual(image_processor.size					,				{'''shortest_edge''': 4_2})
       self.assertEqual(image_processor.crop_size					,				{'''height''': 8_4, '''width''': 8_4})







   def      UpperCAmelCase__ (		self)							->      List[str]:
       pass







   def      UpperCAmelCase__ (		self)							->      Dict:
       # Initialize image_processing
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       # create random PIL videos
       UpperCamelCase										=  prepare_video_inputs(self.image_processor_tester					,				equal_resolution=lowerCamelCase_)
       for video in video_inputs:
           self.assertIsInstance(lowerCamelCase_					,				lowerCamelCase_)
           self.assertIsInstance(video[0]					,				Image.Image)

       # Test not batched input
       UpperCamelCase										=  image_processing(video_inputs[0]					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)

       # Test batched
       UpperCamelCase										=  image_processing(lowerCamelCase_					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)







   def      UpperCAmelCase__ (		self)							->      Dict:
       # Initialize image_processing
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       # create random numpy tensors
       UpperCamelCase										=  prepare_video_inputs(self.image_processor_tester					,				equal_resolution=lowerCamelCase_					,				numpify=lowerCamelCase_)
       for video in video_inputs:
           self.assertIsInstance(lowerCamelCase_					,				lowerCamelCase_)
           self.assertIsInstance(video[0]					,				np.ndarray)

       # Test not batched input
       UpperCamelCase										=  image_processing(video_inputs[0]					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)

       # Test batched
       UpperCamelCase										=  image_processing(lowerCamelCase_					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)







   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       # Initialize image_processing
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       # create random PyTorch tensors
       UpperCamelCase										=  prepare_video_inputs(self.image_processor_tester					,				equal_resolution=lowerCamelCase_					,				torchify=lowerCamelCase_)
       for video in video_inputs:
           self.assertIsInstance(lowerCamelCase_					,				lowerCamelCase_)
           self.assertIsInstance(video[0]					,				torch.Tensor)

       # Test not batched input
       UpperCamelCase										=  image_processing(video_inputs[0]					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)

       # Test batched
       UpperCamelCase										=  image_processing(lowerCamelCase_					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)