
import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
					import importlib.resources as pkg_resources
except ImportError:
					# Try backported to PY<37 `importlib_resources`.
					import importlib_resources as pkg_resources

from . import resources


SCREAMING_SNAKE_CASE							:Optional[Any]											=       'https://github.com/huggingface/datasets/tree/main/src/datasets/utils'
SCREAMING_SNAKE_CASE							:List[Any]											=       f'''{BASE_REF_URL}/{__file__}'''
SCREAMING_SNAKE_CASE							:Tuple											=       logging.getLogger(__name__)


def 		UpperCAmelCase    (			a_		)	->			Tuple[Any, str]:







				"""simple docstring"""



				__A         =				pkg_resources.read_text(SCREAMING_SNAKE_CASE__	,     SCREAMING_SNAKE_CASE__		)
				return yaml.safe_load(SCREAMING_SNAKE_CASE__		), F'''{BASE_REF_URL}/resources/{resource}'''


SCREAMING_SNAKE_CASE				,					SCREAMING_SNAKE_CASE							:int											=       load_yaml_resource('readme_structure.yaml')

SCREAMING_SNAKE_CASE							:Tuple											=       [
    '[Needs More Information]',
    '[More Information Needed]',
    '(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)',
]

# Dictionary representation of section/readme, error_list, warning_list
SCREAMING_SNAKE_CASE							:int											=       Tuple[dict, List[str], List[str]]





class 							UpperCAmelCase   :




	'''simple docstring'''





	def __init__( self      :				int							,A      :				str							,A      :				str							,A      :				List[str] = None							,A      :				bool = False				):
					__A         =				name
					__A         =				level
					__A         =				lines
					__A         =				''''''
					__A         =				True
					__A         =				{}
					__A         =				[]
					__A         =				[]
					if self.lines is not None:
									self.parse(suppress_parsing_errors=UpperCamelCase__				)






	def        UpperCamelCase_						( self      :				str							,A      :				bool = False				):
					__A         =				''''''
					__A         =				[]
					__A         =				False
					for line in self.lines:
									if line.strip(" \n"				) == "":
													continue
									elif line.strip(" \n"				)[:3] == "```":
													__A         =				not code_start
									elif line.split()[0] == self.level + "#" and not code_start:
													if current_sub_level != "":
																	__A         =				Section(UpperCamelCase__							,self.level + "#"							,UpperCamelCase__				)
																	__A         =				[]
													else:
																	if current_lines != []:
																					self.text += "".join(UpperCamelCase__				).strip()
																					if self.text != "" and self.text not in FILLER_TEXT:
																									__A         =				False
																					__A         =				[]

													__A         =				''' '''.join(line.split()[1:]				).strip(" \n"				)
									else:
													current_lines.append(UpperCamelCase__				)
					else:
									if current_sub_level != "":
													if current_sub_level in self.content:
																	self.parsing_error_list.append(
																	    f'''Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'''				)
													__A         =				Section(UpperCamelCase__							,self.level + "#"							,UpperCamelCase__				)
									else:
													if current_lines != []:
																	self.text += "".join(UpperCamelCase__				).strip()
																	if self.text != "" and self.text not in FILLER_TEXT:
																					__A         =				False

					if self.level == "" and not suppress_parsing_errors:
									if self.parsing_error_list != [] or self.parsing_warning_list != []:
													__A         =				'''\n'''.join("-\t" + x for x in self.parsing_error_list + self.parsing_warning_list				)
													__A         =				f'''The following issues were found while parsing the README at `{self.name}`:\n''' + errors
													raise ValueError(UpperCamelCase__				)






	def        UpperCamelCase_						( self      :				Any							,A      :				dict				):
					__A         =				[]
					__A         =				[]
					if structure["allow_empty"] is False:
									# If content is expected
									if self.is_empty_text and self.content == {}:
													# If no content is found, mention it in the error_list
													error_list.append(f'''Expected some content in section `{self.name}` but it is empty.'''				)

					if structure["allow_empty_text"] is False:
									# If some text is expected
									if self.is_empty_text:
													# If no text is found, mention it in the error_list
													error_list.append(
													    f'''Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'''				)
        # Subsections Validation
					if structure["subsections"] is not None:
									# If subsections are expected
									if self.content == {}:
													# If no subsections are present
													__A         =				[subsection['''name'''] for subsection in structure['''subsections''']]
													# Mention the expected values in the error_list
													error_list.append(
													    f'''Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]				)}. Found \'None\'.'''				)
									else:
													# If some subsections are present
													__A         =				[subsection['''name'''] for subsection in structure['''subsections''']]
													__A         =				False
													for idx, name in enumerate(UpperCamelCase__				):
																	if name not in self.content:
																					# If the expected subsection is not present
																					error_list.append(f'''Section `{self.name}` is missing subsection: `{name}`.'''				)
																					__A         =				True
																	else:
																					# If the subsection is present, validate subsection, return the result
																					# and concat the errors from subsection to section error_list

																					# Skip sublevel validation if current level is `###`
																					if self.level == "###":
																									continue
																					else:
																									__A         =				self.content[name].validate(
																									    structure["subsections"][idx]				)
																					error_list += subsec_error_list
																					warning_list += subsec_warning_list

													if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
																	for name in self.content:
																					if name not in structure_names:
																									# If an extra subsection is present
																									warning_list.append(
																									    f'''`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'''				)
					if error_list:
									# If there are errors, do not return the dictionary as it is invalid
									return {}, error_list, warning_list
					else:
									return self.to_dict(), error_list, warning_list







	def        UpperCamelCase_						( self      :				Any				):
					return {
					    "name": self.name,
					    "text": self.text,
					    "is_empty_text": self.is_empty_text,
					    "subsections": [value.to_dict() for value in self.content.values()],
					}





class 							UpperCAmelCase   (						__SCREAMING_SNAKE_CASE						):  # Level 0




	'''simple docstring'''





	def __init__( self      :				Tuple							,A      :				str							,A      :				List[str]							,A      :				dict = None							,A      :				bool = False				):
					super().__init__(name=UpperCamelCase__							,level=""				)  # Not using lines here as we need to use a child class parse
					__A         =				structure
					__A         =				-2
					__A         =				0
					__A         =				lines
					if self.lines is not None:
									self.parse(suppress_parsing_errors=UpperCamelCase__				)






	def        UpperCamelCase_						( self      :				Union[str, Any]				):







					if self.structure is None:
									__A         =				self._validate(UpperCamelCase__				)
					else:
									__A         =				self._validate(self.structure				)
					if error_list != [] or warning_list != []:
									__A         =				'''\n'''.join(["-\t" + x for x in error_list + warning_list]				)
									__A         =				f'''The following issues were found for the README at `{self.name}`:\n''' + errors
									raise ValueError(UpperCamelCase__				)






	@classmethod
	def        UpperCamelCase_						( cls      :				Union[str, Any]							,A      :				Path							,A      :				dict = None							,A      :				bool = False				):







					with open(UpperCamelCase__							,encoding="utf-8"				) as f:
									__A         =				f.readlines()
					return cls(UpperCamelCase__							,UpperCamelCase__							,UpperCamelCase__							,suppress_parsing_errors=UpperCamelCase__				)






	@classmethod
	def        UpperCamelCase_						( cls      :				Optional[int]							,A      :				str							,A      :				dict = None							,A      :				str = "root"							,A      :				bool = False				):
					__A         =				string.split("\n"				)
					return cls(UpperCamelCase__							,UpperCamelCase__							,UpperCamelCase__							,suppress_parsing_errors=UpperCamelCase__				)






	def        UpperCamelCase_						( self      :				Union[str, Any]							,A      :				bool = False				):
					__A         =				0

					for line in self.lines:
									self.yaml_tags_line_count += 1
									if line.strip(" \n"				) == "---":
													self.tag_count += 1
													if self.tag_count == 2:
																	break
									line_count += 1
					if self.tag_count == 2:
									__A         =				self.lines[line_count + 1 :]  # Get the last + 1 th item.
					else:
									__A         =				self.lines[self.tag_count :]
					super().parse(suppress_parsing_errors=UpperCamelCase__				)






	def __str__( self      :				Optional[int]				):
					return str(self.to_dict()				)







	def        UpperCamelCase_						( self      :				int							,A      :				List[str]				):
					__A         =				[]
					__A         =				[]
					if self.yaml_tags_line_count == 0:
									warning_list.append("Empty YAML markers are present in the README."				)
					elif self.tag_count == 0:
									warning_list.append("No YAML markers are present in the README."				)
					elif self.tag_count == 1:
									warning_list.append("Only the start of YAML tags present in the README."				)
					# Check how many first level sections are present.
					__A         =				len(self.content.keys()				)
					if num_first_level_keys > 1:
									# If more than one, add to the error list, continue
									error_list.append(
									    f'''The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()				)]				)}. Only one heading is expected. Skipping further validation for this README.'''				)
					elif num_first_level_keys < 1:
									# If less than one, append error.
									error_list.append(
									    "The README has no first-level headings. One heading is expected. Skipping further validation for this README."				)

					else:
									# If one exactly
									__A         =				list(self.content.keys()				)[0]  # Get the key
									if start_key.startswith("Dataset Card for"				):  # Check correct start
													# If the starting is correct, validate all the sections
													__A         =				self.content[start_key].validate(
													    readme_structure["subsections"][0]				)
													error_list += sec_error_list
													warning_list += sec_warning_list
									else:
													# If not found, append error
													error_list.append(
													    "No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README."				)
					if error_list:
									# If there are errors, do not return the dictionary as it is invalid
									return {}, error_list, warning_list
					else:
									return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
					from argparse import ArgumentParser

					SCREAMING_SNAKE_CASE							:Union[str, Any]											=       ArgumentParser(usage='Validate the content (excluding YAML tags) of a README.md file.')
					ap.add_argument('readme_filepath')
					SCREAMING_SNAKE_CASE							:Tuple											=       ap.parse_args()
					SCREAMING_SNAKE_CASE							:Dict											=       Path(args.readme_filepath)
					SCREAMING_SNAKE_CASE							:Any											=       ReadMe.from_readme(readme_filepath)


import unittest

from transformers import SpeechaTextaConfig
from transformers.testing_utils import is_torch_available, require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers.models.speech_to_text_a.modeling_speech_to_text_a import (
					    SpeechaTextaDecoder,
					    SpeechaTextaForCausalLM,
					)





@require_torch
class 							UpperCAmelCase   :




	'''simple docstring'''





	def __init__( self      :				Dict							,A      :				Tuple							,A      :				Union[str, Any]=99							,A      :				Dict=13							,A      :				int=16							,A      :				int=7							,A      :				Optional[int]=True							,A      :				str=True							,A      :				Optional[Any]=True							,A      :				Optional[int]=False							,A      :				Dict=True							,A      :				Any=2							,A      :				List[Any]=32							,A      :				Optional[Any]=4							,A      :				Tuple=4							,A      :				List[Any]=30							,A      :				Optional[Any]=0							,A      :				str=1							,A      :				List[Any]=2							,A      :				List[Any]=None							,):
					__A         =				parent
					__A         =				batch_size
					__A         =				decoder_seq_length
					# For common tests
					__A         =				self.decoder_seq_length
					__A         =				is_training
					__A         =				use_attention_mask
					__A         =				use_labels

					__A         =				vocab_size
					__A         =				d_model
					__A         =				d_model
					__A         =				decoder_layers
					__A         =				decoder_layers
					__A         =				decoder_ffn_dim
					__A         =				decoder_attention_heads
					__A         =				decoder_attention_heads
					__A         =				eos_token_id
					__A         =				bos_token_id
					__A         =				pad_token_id
					__A         =				decoder_start_token_id
					__A         =				use_cache
					__A         =				max_position_embeddings

					__A         =				None
					__A         =				decoder_seq_length
					__A         =				2
					__A         =				1






	def        UpperCamelCase_						( self      :				Tuple				):
					__A         =				ids_tensor([self.batch_size, self.decoder_seq_length]							,self.vocab_size				)

					__A         =				None
					if self.use_attention_mask:
									__A         =				ids_tensor([self.batch_size, self.decoder_seq_length]							,vocab_size=2				)

					__A         =				None
					if self.use_labels:
									__A         =				ids_tensor([self.batch_size, self.decoder_seq_length]							,self.vocab_size				)

					__A         =				SpeechaTextaConfig(
					    vocab_size=self.vocab_size							,d_model=self.d_model							,decoder_layers=self.decoder_layers							,decoder_ffn_dim=self.decoder_ffn_dim							,decoder_attention_heads=self.decoder_attention_heads							,eos_token_id=self.eos_token_id							,bos_token_id=self.bos_token_id							,use_cache=self.use_cache							,pad_token_id=self.pad_token_id							,decoder_start_token_id=self.decoder_start_token_id							,max_position_embeddings=self.max_position_embeddings							,)

					return (
					    config,
					    input_ids,
					    attention_mask,
					    lm_labels,
					)






	def        UpperCamelCase_						( self      :				Tuple							,A      :				List[str]							,A      :				Tuple							,A      :				Optional[int]							,A      :				List[str]							,):
					__A         =				True
					__A         =				SpeechaTextaDecoder(config=A				).to(A				).eval()
					__A         =				input_ids[:2]

					input_ids[input_ids == 0] += 1
					# first forward pass
					__A         =				model(A							,use_cache=A				)
					__A         =				model(A				)
					__A         =				model(A							,use_cache=A				)

					self.parent.assertTrue(len(A				) == len(A				)				)
					self.parent.assertTrue(len(A				) == len(A				) + 1				)

					__A         =				outputs["past_key_values"]

					# create hypothetical next token and extent to next_input_ids
					__A         =				ids_tensor((2, 1)							,config.vocab_size - 1				) + 1

					# append to next input_ids and
					__A         =				torch.cat([input_ids, next_tokens]							,dim=-1				)

					print(A				)

					__A         =				model(A				)["last_hidden_state"]
					__A         =				model(A							,past_key_values=A				)["last_hidden_state"]

					# select random slice
					__A         =				ids_tensor((1,)							,output_from_past.shape[-1]				).item()
					__A         =				output_from_no_past[:, next_input_ids.shape[-1] - 1, random_slice_idx].detach()
					__A         =				output_from_past[:, 0, random_slice_idx].detach()

					# test that outputs are equal for slice
					assert torch.allclose(A							,A							,atol=1E-3				)







	def        UpperCamelCase_						( self      :				Any				):
					__A         =				self.prepare_config_and_inputs()
					(
					    (
					    __A
					)    ,							(
					    __A
					)    ,							(
					    __A
					)    ,							(
					    __A
					)    ,							
					)         =				config_and_inputs

					__A         =				{
					    "input_ids": input_ids,
					    "attention_mask": attention_mask,
					}
					return config, inputs_dict





@require_torch
class 							UpperCAmelCase   (						__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE    ,    unittest.TestCase						):




	'''simple docstring'''





	snake_case_      					=  (SpeechaTextaDecoder, SpeechaTextaForCausalLM) if is_torch_available() else ()
	snake_case_      					=  (SpeechaTextaForCausalLM,) if is_torch_available() else ()
	snake_case_      					=  {"text-generation": SpeechaTextaForCausalLM} if is_torch_available() else {}
	snake_case_      					=  True
	snake_case_      					=  False






	def        UpperCamelCase_						( self      :				Any							,):
					__A         =				SpeechaTextaStandaloneDecoderModelTester(self							,is_training=A				)
					__A         =				ConfigTester(self							,config_class=A				)






	def        UpperCamelCase_						( self      :				Optional[Any]				):
					pass






	def        UpperCamelCase_						( self      :				int				):
					pass






	def        UpperCamelCase_						( self      :				Tuple				):
					pass






	def        UpperCamelCase_						( self      :				str				):
					self.config_tester.run_common_tests()






	def        UpperCamelCase_						( self      :				Dict				):
					__A         =				self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_decoder_model_past(*A				)







	def        UpperCamelCase_						( self      :				str				):
					return
