


'''simple docstring'''







import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


A   :  Optional[int]       		=				get_logger(__name__)


if TYPE_CHECKING:
   from .builder import DatasetBuilder






def    lowerCAmelCase__ (       lowerCamelCase  :    List[Any]						,lowerCamelCase  :    Optional[Union[str, bool]] = None     ):
       _A  :			int											=      importlib.import_module(lowerCamelCase     )

       # TODO(QL): always update the module to add subsequent new authentication
       if hasattr(lowerCamelCase						,'_patched_for_streaming'     ) and module._patched_for_streaming:
              return

       def wrap_auth(lowerCamelCase  :    str     ):
              @wraps(lowerCamelCase     )
              def wrapper(*lowerCamelCase  :    Dict						,**lowerCamelCase  :    List[str]     ):
                     return function(*lowerCamelCase						,use_auth_token=lowerCamelCase						,**lowerCamelCase     )

              _A  :			Union[str, Any]											=      'wrap_auth'
              return wrapper

       # open files in a streaming fashion
       patch_submodule(lowerCamelCase						,'open'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'os.listdir'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'os.walk'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'glob.glob'						,wrap_auth(lowerCamelCase     )     ).start()
       # allow to navigate in remote zip files
       patch_submodule(lowerCamelCase						,'os.path.join'						,lowerCamelCase     ).start()
       patch_submodule(lowerCamelCase						,'os.path.dirname'						,lowerCamelCase     ).start()
       patch_submodule(lowerCamelCase						,'os.path.basename'						,lowerCamelCase     ).start()
       patch_submodule(lowerCamelCase						,'os.path.relpath'						,lowerCamelCase     ).start()
       patch_submodule(lowerCamelCase						,'os.path.split'						,lowerCamelCase     ).start()
       patch_submodule(lowerCamelCase						,'os.path.splitext'						,lowerCamelCase     ).start()
       # allow checks on paths
       patch_submodule(lowerCamelCase						,'os.path.exists'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'os.path.isdir'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'os.path.isfile'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'os.path.getsize'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'pathlib.Path'						,lowerCamelCase     ).start()
       # file readers
       patch_submodule(lowerCamelCase						,'gzip.open'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'numpy.load'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'pandas.read_csv'						,wrap_auth(lowerCamelCase     )						,attrs=['__version__']     ).start()
       patch_submodule(lowerCamelCase						,'pandas.read_excel'						,wrap_auth(lowerCamelCase     )						,attrs=['__version__']     ).start()
       patch_submodule(lowerCamelCase						,'scipy.io.loadmat'						,wrap_auth(lowerCamelCase     )						,attrs=['__version__']     ).start()
       patch_submodule(lowerCamelCase						,'xml.etree.ElementTree.parse'						,wrap_auth(lowerCamelCase     )     ).start()
       patch_submodule(lowerCamelCase						,'xml.dom.minidom.parse'						,wrap_auth(lowerCamelCase     )     ).start()
       _A  :			Dict											=      True






def    lowerCAmelCase__ (       lowerCamelCase  :    "DatasetBuilder"     ):
       extend_module_for_streaming(builder.__module__						,use_auth_token=builder.use_auth_token     )
       # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
       if not builder.__module__.startswith('datasets.'     ):  # check that it's not a packaged builder like csv
              for imports in get_imports(inspect.getfile(builder.__class__     )     ):
                     if imports[0] == "internal":
                            _A  :			Tuple											=      imports[1]
                            _A  :			List[str]											=      '.'.join(builder.__module__.split('.'     )[:-1] + [internal_import_name]     )
                            extend_module_for_streaming(lowerCamelCase						,use_auth_token=builder.use_auth_token     )

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
       from .builder import DatasetBuilder

       _A  :			Optional[Any]											=      [
           cls.__module__
           for cls in type(lowerCamelCase     ).__mro__[1:]  # make sure it's not the same module we've already patched
           if issubclass(cls						,lowerCamelCase     ) and cls.__module__ != DatasetBuilder.__module__
       ]  # check it's not a standard builder from datasets.builder
       for module in parent_builder_modules:
              extend_module_for_streaming(lowerCamelCase						,use_auth_token=builder.use_auth_token     )










'''simple docstring'''







import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    SummarizationPipeline,
    TFPreTrainedModel,
    pipeline,
)
from transformers.testing_utils import get_gpu_count, is_pipeline_test, require_tf, require_torch, slow, torch_device
from transformers.tokenization_utils import TruncationStrategy

from .test_pipelines_common import ANY


A   :  Optional[Any]       		=				-1 if torch_device == '''cpu''' else 0


@is_pipeline_test
class 				__lowerCamelCase				( unittest.TestCase ):
    """simple docstring"""
    a									=					MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
    a									=					TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING


    def      A					(						self    :							Union[str, Any]   ,							SCREAMING_SNAKE_CASE    :							Tuple   ,							SCREAMING_SNAKE_CASE    :							Optional[Any]   ,							SCREAMING_SNAKE_CASE    :							Dict):
           _A  :			str											=      SummarizationPipeline(model=SCREAMING_SNAKE_CASE   ,							tokenizer=SCREAMING_SNAKE_CASE)
           return summarizer, ["(CNN)The Palestinian Authority officially became", "Some other text"]


    def      A					(						self    :							Optional[int]   ,							SCREAMING_SNAKE_CASE    :							Tuple   ,							SCREAMING_SNAKE_CASE    :							Optional[int]):
           _A  :			str											=      summarizer.model

           _A  :			Tuple											=      summarizer('(CNN)The Palestinian Authority officially became')
           self.assertEqual(SCREAMING_SNAKE_CASE   ,							[{'summary_text': ANY(SCREAMING_SNAKE_CASE)}])

           _A  :			Optional[Any]											=      summarizer(
               '(CNN)The Palestinian Authority officially became '   ,							num_beams=2   ,							min_length=2   ,							max_length=5   ,							)
           self.assertEqual(SCREAMING_SNAKE_CASE   ,							[{'summary_text': ANY(SCREAMING_SNAKE_CASE)}])

           # Some models (Switch Transformers, LED, T5, LongT5, etc) can handle long sequences.
           _A  :			Union[str, Any]											=      [
               'SwitchTransformersConfig',
               'T5Config',
               'LongT5Config',
               'LEDConfig',
               'PegasusXConfig',
               'FSMTConfig',
               'M2M100Config',
               'ProphetNetConfig',  # positional embeddings up to a fixed maximum size (otherwise clamping the values)
           ]
           if model.config.__class__.__name__ not in model_can_handle_longer_seq:
                  # Too long and exception is expected.
                  # For TF models, if the weights are initialized in GPU context, we won't get expected index error from
                  # the embedding layer.
                  if not (
                      isinstance(SCREAMING_SNAKE_CASE   ,							SCREAMING_SNAKE_CASE)
                      and get_gpu_count() > 0
                      and len(summarizer.model.trainable_weights) > 0
                  ):
                         with self.assertRaises(SCREAMING_SNAKE_CASE):
                                _A  :			Any											=      summarizer('This ' * 1000)
           _A  :			int											=      summarizer('This ' * 1000   ,							truncation=TruncationStrategy.ONLY_FIRST)


    @require_torch
    def      A					(						self    :							int):
           _A  :			Tuple											=      pipeline(task='summarization'   ,							model='sshleifer/tiny-mbart'   ,							framework='pt')
           _A  :			List[str]											=      summarizer('This is a small test')
           self.assertEqual(
               SCREAMING_SNAKE_CASE   ,							[
                   {
                       'summary_text': 'เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป'
                   }
               ]   ,							)


    @require_tf
    def      A					(						self    :							int):
           _A  :			Optional[Any]											=      pipeline(task='summarization'   ,							model='sshleifer/tiny-mbart'   ,							framework='tf')
           _A  :			Any											=      summarizer('This is a small test')
           self.assertEqual(
               SCREAMING_SNAKE_CASE   ,							[
                   {
                       'summary_text': 'เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป'
                   }
               ]   ,							)


    @require_torch
    @slow
    def      A					(						self    :							List[Any]):
           _A  :			int											=      pipeline(task='summarization'   ,							device=SCREAMING_SNAKE_CASE)
           _A  :			int											=      (
               ' (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on'
               ' Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The'
               ' formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based.'
               ' The Palestinians signed the ICC\'s founding Rome Statute in January, when they also accepted its'
               ' jurisdiction over alleged crimes committed "in the occupied Palestinian territory, including East'
               ' Jerusalem, since June 13, 2014." Later that month, the ICC opened a preliminary examination into the'
               ' situation in Palestinian territories, paving the way for possible war crimes investigations against'
               ' Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and'
               ' the United States, neither of which is an ICC member, opposed the Palestinians\' efforts to join the'
               ' body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday\'s ceremony, said it was a'
               ' move toward greater justice. "As Palestine formally becomes a State Party to the Rome Statute today, the'
               ' world is also a step closer to ending a long era of impunity and injustice," he said, according to an'
               ' ICC news release. "Indeed, today brings us closer to our shared goals of justice and peace." Judge'
               ' Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the'
               ' Palestinians. "As the Rome Statute today enters into force for the State of Palestine, Palestine'
               ' acquires all the rights as well as responsibilities that come with being a State Party to the Statute.'
               ' These are substantive commitments, which cannot be taken lightly," she said. Rights group Human Rights'
               ' Watch welcomed the development. "Governments seeking to penalize Palestine for joining the ICC should'
               ' immediately end their pressure, and countries that support universal acceptance of the court\'s treaty'
               ' should speak out to welcome its membership," said Balkees Jarrah, international justice counsel for the'
               ' group. \"What\'s objectionable is the attempts to undermine international justice, not Palestine\'s'
               ' decision to join a treaty to which over 100 countries around the world are members." In January, when'
               ' the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an'
               ' outrage, saying the court was overstepping its boundaries. The United States also said it "strongly"'
               ' disagreed with the court\'s decision. \"As we have said repeatedly, we do not believe that Palestine is a'
               ' state and therefore we do not believe that it is eligible to join the ICC," the State Department said in'
               ' a statement. It urged the warring sides to resolve their differences through direct negotiations. "We'
               ' will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace,"'
               ' it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the'
               ' territories as "Palestine." While a preliminary examination is not a formal investigation, it allows the'
               ' court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou'
               ' Bensouda said her office would "conduct its analysis in full independence and impartiality." The war'
               ' between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry'
               ' will include alleged war crimes committed since June. The International Criminal Court was set up in'
               ' 2002 to prosecute genocide, crimes against humanity and war crimes. CNN\'s Vasco Cotovio, Kareem Khadder'
               ' and Faith Karimi contributed to this report.'
           )
           _A  :			Optional[Any]											=      (
               ' The Palestinian Authority becomes the 123rd member of the International Criminal Court . The move gives'
               ' the court jurisdiction over alleged crimes in Palestinian territories . Israel and the United States'
               ' opposed the Palestinians\' efforts to join the court . Rights group Human Rights Watch welcomes the move,'
               ' says governments seeking to penalize Palestine should end pressure .'
           )
           _A  :			Union[str, Any]											=      summarizer(SCREAMING_SNAKE_CASE)
           self.assertEqual(result[0]['summary_text']   ,							SCREAMING_SNAKE_CASE)






