

'''simple docstring'''




import logging
from pathlib import Path
from typing import Any, List, Tuple

import yaml


# loading package files: https://stackoverflow.com/a/20885799
try:
  import importlib.resources as pkg_resources
except ImportError:
  # Try backported to PY<37 `importlib_resources`.
  import importlib_resources as pkg_resources

from . import resources


_a				: int 							=						'https://github.com/huggingface/datasets/tree/main/src/datasets/utils'
_a				: Any 							=						F'''{BASE_REF_URL}/{__file__}'''
_a				: List[Any] 							=						logging.getLogger(__name__)


def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   str						):
  UpperCAmelCase													=	pkg_resources.read_text(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						)
  return yaml.safe_load(SCREAMING_SNAKE_CASE						), f'''{BASE_REF_URL}/resources/{resource}'''


_a							,      _a				: Any 							=						load_yaml_resource('readme_structure.yaml')

_a				: Any 							=						[
    '[Needs More Information]',
    '[More Information Needed]',
    '(https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)',
]

# Dictionary representation of section/readme, error_list, warning_list
_a				: Tuple 							=						Tuple[dict, List[str], List[str]]



class        lowercase_	:

  '''simple docstring'''





  def __init__(     self   , a_   , a_   , a_ = None   , a_ = False     )							->				Dict:


    """simple docstring"""
    UpperCAmelCase													=	name
    UpperCAmelCase													=	level
    UpperCAmelCase													=	lines
    UpperCAmelCase													=	''
    UpperCAmelCase													=	True
    UpperCAmelCase													=	{}
    UpperCAmelCase													=	[]
    UpperCAmelCase													=	[]
    if self.lines is not None:
      self.parse(suppress_parsing_errors=a_     )
  def 						snake_case_			(     self   , a_ = False     )							->				List[Any]:


    """simple docstring"""
    UpperCAmelCase													=	''
    UpperCAmelCase													=	[]
    UpperCAmelCase													=	False
    for line in self.lines:
      if line.strip(' \n'     ) == "":
        continue
      elif line.strip(' \n'     )[:3] == "```":
        UpperCAmelCase													=	not code_start
      elif line.split()[0] == self.level + "#" and not code_start:
        if current_sub_level != "":
          UpperCAmelCase													=	Section(a_   , self.level + '#'   , a_     )
          UpperCAmelCase													=	[]
        else:
          if current_lines != []:
            self.text += "".join(a_     ).strip()
            if self.text != "" and self.text not in FILLER_TEXT:
              UpperCAmelCase													=	False
            UpperCAmelCase													=	[]

        UpperCAmelCase													=	' '.join(line.split()[1:]     ).strip(' \n'     )
      else:
        current_lines.append(a_     )
    else:
      if current_sub_level != "":
        if current_sub_level in self.content:
          self.parsing_error_list.append(
              F'''Multiple sections with the same heading `{current_sub_level}` have been found. Please keep only one of these sections.'''     )
        UpperCAmelCase													=	Section(a_   , self.level + '#'   , a_     )
      else:
        if current_lines != []:
          self.text += "".join(a_     ).strip()
          if self.text != "" and self.text not in FILLER_TEXT:
            UpperCAmelCase													=	False

    if self.level == "" and not suppress_parsing_errors:
      if self.parsing_error_list != [] or self.parsing_warning_list != []:
        UpperCAmelCase													=	UpperCAmelCase													=	'\n'.join('-\t' + x for x in self.parsing_error_list + self.parsing_warning_list     )
        UpperCAmelCase													=	F'''The following issues were found while parsing the README at `{self.name}`:\n''' + errors
        raise ValueError(a_     )
  def 						snake_case_			(     self   , a_     )							->				ReadmeValidatorOutput:


    """simple docstring"""
    UpperCAmelCase													=	[]
    UpperCAmelCase													=	[]
    if structure["allow_empty"] is False:
      # If content is expected
      if self.is_empty_text and self.content == {}:
        # If no content is found, mention it in the error_list
        error_list.append(F'''Expected some content in section `{self.name}` but it is empty.'''     )

    if structure["allow_empty_text"] is False:
      # If some text is expected
      if self.is_empty_text:
        # If no text is found, mention it in the error_list
        error_list.append(
            F'''Expected some text in section `{self.name}` but it is empty (text in subsections are ignored).'''     )
        # Subsections Validation
    if structure["subsections"] is not None:
      # If subsections are expected
      if self.content == {}:
        # If no subsections are present
        UpperCAmelCase													=	[subsection['name'] for subsection in structure['subsections']]
        # Mention the expected values in the error_list
        error_list.append(
            F'''Section `{self.name}` expected the following subsections: {', '.join(['`'+x+'`' for x in values]     )}. Found \'None\'.'''     )
      else:
        # If some subsections are present
        UpperCAmelCase													=	[subsection['name'] for subsection in structure['subsections']]
        UpperCAmelCase													=	False
        for idx, name in enumerate(a_     ):
          if name not in self.content:
            # If the expected subsection is not present
            error_list.append(F'''Section `{self.name}` is missing subsection: `{name}`.'''     )
            UpperCAmelCase													=	True
          else:
            # If the subsection is present, validate subsection, return the result
            # and concat the errors from subsection to section error_list

            # Skip sublevel validation if current level is `###`
            if self.level == "###":
              continue
            else:
              UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase													=	self.content[name].validate(
                  structure['subsections'][idx]     )
            error_list += subsec_error_list
            warning_list += subsec_warning_list

        if has_missing_subsections:  # we only allow to have extra subsections if all the other ones are here
          for name in self.content:
            if name not in structure_names:
              # If an extra subsection is present
              warning_list.append(
                  F'''`{self.name}` has an extra subsection: `{name}`. Skipping further validation checks for this subsection as expected structure is unknown.'''     )
    if error_list:
      # If there are errors, do not return the dictionary as it is invalid
      return {}, error_list, warning_list
    else:
      return self.to_dict(), error_list, warning_list





  def 						snake_case_			(     self     )							->				dict:


    """simple docstring"""
    return {
        "name": self.name,
        "text": self.text,
        "is_empty_text": self.is_empty_text,
        "subsections": [value.to_dict() for value in self.content.values()],
    }




class        lowercase_	(		a ):  # Level 0

  '''simple docstring'''





  def __init__(     self   , a_   , a_   , a_ = None   , a_ = False     )							->				Any:


    """simple docstring"""
    super().__init__(name=a_   , level=''     )  # Not using lines here as we need to use a child class parse
    UpperCAmelCase													=	structure
    UpperCAmelCase													=	-2
    UpperCAmelCase													=	0
    UpperCAmelCase													=	lines
    if self.lines is not None:
      self.parse(suppress_parsing_errors=a_     )
  def 						snake_case_			(     self     )							->				Union[str, Any]:


    """simple docstring"""
    if self.structure is None:
      UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase													=	self._validate(a_     )
    else:
      UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase													=	self._validate(self.structure     )
    if error_list != [] or warning_list != []:
      UpperCAmelCase													=	'\n'.join(['-\t' + x for x in error_list + warning_list]     )
      UpperCAmelCase													=	F'''The following issues were found for the README at `{self.name}`:\n''' + errors
      raise ValueError(a_     )
  @classmethod
  def 						snake_case_			(     cls   , a_   , a_ = None   , a_ = False     )							->				Dict:


    """simple docstring"""
    with open(a_   , encoding='utf-8'     ) as f:
      UpperCAmelCase													=	f.readlines()
    return cls(a_   , a_   , a_   , suppress_parsing_errors=a_     )
  @classmethod
  def 						snake_case_			(     cls   , a_   , a_ = None   , a_ = "root"   , a_ = False     )							->				Tuple:


    """simple docstring"""
    UpperCAmelCase													=	string.split('\n'     )
    return cls(a_   , a_   , a_   , suppress_parsing_errors=a_     )
  def 						snake_case_			(     self   , a_ = False     )							->				Optional[int]:


    """simple docstring"""
    # Skip Tags
    UpperCAmelCase													=	0

    for line in self.lines:
      self.yaml_tags_line_count += 1
      if line.strip(' \n'     ) == "---":
        self.tag_count += 1
        if self.tag_count == 2:
          break
      line_count += 1
    if self.tag_count == 2:
      UpperCAmelCase													=	self.lines[line_count + 1 :]  # Get the last + 1 th item.
    else:
      UpperCAmelCase													=	self.lines[self.tag_count :]
    super().parse(suppress_parsing_errors=a_     )
  def __str__(     self     )							->				str:


    """simple docstring"""
    return str(self.to_dict()     )





  def 						snake_case_			(     self   , a_     )							->				str:


    """simple docstring"""
    UpperCAmelCase													=	[]
    UpperCAmelCase													=	[]
    if self.yaml_tags_line_count == 0:
      warning_list.append('Empty YAML markers are present in the README.'     )
    elif self.tag_count == 0:
      warning_list.append('No YAML markers are present in the README.'     )
    elif self.tag_count == 1:
      warning_list.append('Only the start of YAML tags present in the README.'     )
    # Check how many first level sections are present.
    UpperCAmelCase													=	len(self.content.keys()     )
    if num_first_level_keys > 1:
      # If more than one, add to the error list, continue
      error_list.append(
          F'''The README has several first-level headings: {', '.join(['`'+x+'`' for x in list(self.content.keys()     )]     )}. Only one heading is expected. Skipping further validation for this README.'''     )
    elif num_first_level_keys < 1:
      # If less than one, append error.
      error_list.append(
          'The README has no first-level headings. One heading is expected. Skipping further validation for this README.'     )

    else:
      # If one exactly
      UpperCAmelCase													=	list(self.content.keys()     )[0]  # Get the key
      if start_key.startswith('Dataset Card for'     ):  # Check correct start
        # If the starting is correct, validate all the sections
        UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase													=	self.content[start_key].validate(
            readme_structure['subsections'][0]     )
        error_list += sec_error_list
        warning_list += sec_warning_list
      else:
        # If not found, append error
        error_list.append(
            'No first-level heading starting with `Dataset Card for` found in README. Skipping further validation for this README.'     )
    if error_list:
      # If there are errors, do not return the dictionary as it is invalid
      return {}, error_list, warning_list
    else:
      return self.to_dict(), error_list, warning_list


if __name__ == "__main__":
  from argparse import ArgumentParser

  _a				: Union[str, Any] 							=						ArgumentParser(usage='Validate the content (excluding YAML tags) of a README.md file.')
  ap.add_argument('readme_filepath')
  _a				: List[str] 							=						ap.parse_args()
  _a				: Dict 							=						Path(args.readme_filepath)
  _a				: List[Any] 							=						ReadMe.from_readme(readme_filepath)





'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_a				: Optional[int] 							=						logging.get_logger(__name__)

_a				: Dict 							=						{
    'mnaylor/mega-base-wikitext': 'https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json',
}



class        lowercase_	(		a ):

  '''simple docstring'''





  __lowerCAmelCase					: List[str]      =   "mega"
  def __init__(     self   , a_=3_0_5_2_2   , a_=1_2_8   , a_=4   , a_=2_5_6   , a_=1_6   , a_=True   , a_=6_4   , a_=False   , a_=-1   , a_=None   , a_=True   , a_="scalenorm"   , a_=True   , a_="silu"   , a_="softmax"   , a_=0.1   , a_=0.1   , a_=0.1   , a_=False   , a_=True   , a_=2_5_6   , a_=True   , a_=0.1   , a_=2_0_4_8   , a_=False   , a_=2   , a_=0.02   , a_=0.2   , a_=0.02   , a_=1.0   , a_=1   , a_=0   , a_=2   , a_="rotary"   , a_=None   , a_=True   , a_=True   , **a_   , )							->				Any:


    """simple docstring"""
    super().__init__(pad_token_id=a_   , bos_token_id=a_   , eos_token_id=a_   , **a_     )

    UpperCAmelCase													=	vocab_size
    UpperCAmelCase													=	hidden_size
    UpperCAmelCase													=	num_hidden_layers
    UpperCAmelCase													=	activation
    UpperCAmelCase													=	attention_activation
    UpperCAmelCase													=	intermediate_size
    UpperCAmelCase													=	ema_projection_size
    UpperCAmelCase													=	bidirectional
    UpperCAmelCase													=	shared_representation_size
    UpperCAmelCase													=	use_chunking
    UpperCAmelCase													=	chunk_size
    UpperCAmelCase													=	truncation
    UpperCAmelCase													=	normalize_before_mega
    UpperCAmelCase													=	normalization_type
    UpperCAmelCase													=	norm_affine
    UpperCAmelCase													=	dropout_prob
    UpperCAmelCase													=	hidden_dropout_prob
    UpperCAmelCase													=	attention_probs_dropout_prob
    UpperCAmelCase													=	use_feature_dropout
    UpperCAmelCase													=	use_normalized_ffn
    UpperCAmelCase													=	nffn_hidden_size
    UpperCAmelCase													=	normalize_before_ffn
    UpperCAmelCase													=	nffn_activation_dropout_prob
    UpperCAmelCase													=	max_positions
    UpperCAmelCase													=	add_token_type_embeddings
    UpperCAmelCase													=	type_vocab_size
    UpperCAmelCase													=	initializer_range
    UpperCAmelCase													=	ema_delta_alpha_range
    UpperCAmelCase													=	ema_beta_range
    UpperCAmelCase													=	ema_gamma_omega_range
    UpperCAmelCase													=	relative_positional_bias
    UpperCAmelCase													=	use_cache
    UpperCAmelCase													=	classifier_dropout
    UpperCAmelCase													=	add_lm_hidden_dense_layer
    UpperCAmelCase													=	1  # not used but required by Hugging Face




class        lowercase_	(		a ):

  '''simple docstring'''





  @property
  def 						snake_case_			(     self     )							->				Mapping[str, Mapping[int, str]]:


    """simple docstring"""
    if self.task == "multiple-choice":
      UpperCAmelCase													=	{0: 'batch', 1: 'choice', 2: 'sequence'}
    else:
      UpperCAmelCase													=	{0: 'batch', 1: 'sequence'}
    return OrderedDict(
        [
            ('input_ids', dynamic_axis),
            ('attention_mask', dynamic_axis),
        ]     )


