





'''simple docstring'''





from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy



class 				__snake_case  (     _SCREAMING_SNAKE_CASE						):


    '''simple docstring'''







    lowerCamelCase__						:		Tuple							=  'MarkupLMFeatureExtractor'
    lowerCamelCase__						:		Dict							=  ('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
    lowerCamelCase__						:		str							=  True



    def __call__(					self      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_=None      ,							A_ = True      ,							A_ = False      ,							A_ = None      ,							A_ = None      ,							A_ = 0      ,							A_ = None      ,							A_ = None      ,							A_ = None      ,							A_ = False      ,							A_ = False      ,							A_ = False      ,							A_ = False      ,							A_ = True      ,							A_ = None      ,							**A_      ,							):





        '''simple docstring'''

        if self.parse_html:
            if html_strings is None:
                raise ValueError('''Make sure to pass HTML strings in case `parse_html` is set to `True`'''    )

            if nodes is not None or xpaths is not None or node_labels is not None:
                raise ValueError(
                    '''Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'''    )

            SCREAMING_SNAKE_CASE__       							= self.feature_extractor(UpperCamelCase__    )
            SCREAMING_SNAKE_CASE__       							= features['''nodes''']
            SCREAMING_SNAKE_CASE__       							= features['''xpaths''']
        else:
            if html_strings is not None:
                raise ValueError('''You have passed HTML strings but `parse_html` is set to `False`.'''    )
            if nodes is None or xpaths is None:
                raise ValueError('''Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'''    )

        # # second, apply the tokenizer
        if questions is not None and self.parse_html:
            if isinstance(UpperCamelCase__      ,							UpperCamelCase__    ):
                SCREAMING_SNAKE_CASE__       							= [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

        SCREAMING_SNAKE_CASE__       							= self.tokenizer(
            text=questions if questions is not None else nodes      ,							text_pair=nodes if questions is not None else None      ,							xpaths=UpperCamelCase__      ,							node_labels=UpperCamelCase__      ,							add_special_tokens=UpperCamelCase__      ,							padding=UpperCamelCase__      ,							truncation=UpperCamelCase__      ,							max_length=UpperCamelCase__      ,							stride=UpperCamelCase__      ,							pad_to_multiple_of=UpperCamelCase__      ,							return_token_type_ids=UpperCamelCase__      ,							return_attention_mask=UpperCamelCase__      ,							return_overflowing_tokens=UpperCamelCase__      ,							return_special_tokens_mask=UpperCamelCase__      ,							return_offsets_mapping=UpperCamelCase__      ,							return_length=UpperCamelCase__      ,							verbose=UpperCamelCase__      ,							return_tensors=UpperCamelCase__      ,							**UpperCamelCase__      ,							)

        return encoded_inputs



    def    lowercase_				(					self      ,							*A_      ,							**A_    ):





        '''simple docstring'''

        return self.tokenizer.batch_decode(*UpperCamelCase__      ,							**UpperCamelCase__    )



    def    lowercase_				(					self      ,							*A_      ,							**A_    ):





        '''simple docstring'''

        return self.tokenizer.decode(*UpperCamelCase__      ,							**UpperCamelCase__    )




    @property
    def    lowercase_				(					self    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= self.tokenizer.model_input_names
        return tokenizer_input_names







import torch

from transformers import AutoTokenizer



class 				__snake_case  (     __SCREAMING_SNAKE_CASE						):


    '''simple docstring'''







    def __init__(					self      ,							A_    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= AutoTokenizer.from_pretrained(A_    )



    def    lowercase_				(					self      ,							A_    ):





        '''simple docstring'''

        if isinstance(A_      ,							A_    ) and all(isinstance(_x      ,							A_    ) for _x in x    ):
            SCREAMING_SNAKE_CASE__       							= None
            for l in x:
                SCREAMING_SNAKE_CASE__       							= self.tokenizer(
                    A_      ,							padding='''max_length'''      ,							max_length=3_84      ,							truncation=A_      ,							return_tensors='''pt'''      ,							)
                SCREAMING_SNAKE_CASE__       							= torch.tensor([len(A_    )]    )
                if d is not None:
                    for k in d.keys():
                        SCREAMING_SNAKE_CASE__       							= torch.cat((d[k], t[k])      ,							0    )
                else:
                    SCREAMING_SNAKE_CASE__       							= t

            SCREAMING_SNAKE_CASE__       							= torch.tensor(self.tokenizer.convert_tokens_to_ids('''[E]'''    )    )
            SCREAMING_SNAKE_CASE__       							= torch.tensor(self.tokenizer.convert_tokens_to_ids('''[/E]'''    )    )

        elif isinstance(A_      ,							A_    ) and all(isinstance(_x      ,							A_    ) for _x in x    ):
            SCREAMING_SNAKE_CASE__       							= self.tokenizer(
                A_      ,							padding='''max_length'''      ,							max_length=3_84      ,							truncation=A_      ,							return_tensors='''pt'''      ,							)

        else:
            raise Exception(
                '''Type of parameter x was not recognized! Only `list of strings` for query or `list of lists of'''
                ''' strings` for supports are supported.'''    )

        return d




    def    lowercase_				(					self      ,							A_      ,							A_      ,							A_      ,							A_      ,							A_=0.70    ):





        '''simple docstring'''

        SCREAMING_SNAKE_CASE__       							= []
        for idx in range(len(W_query['''input_ids''']    )    ):
            SCREAMING_SNAKE_CASE__       							= SCREAMING_SNAKE_CASE__       							= range(p_start.shape[1]    )

            SCREAMING_SNAKE_CASE__       							= []
            for start_id in start_indexes:
                for end_id in end_indexes:
                    if start_id < end_id:
                        output.append(
                            (
                                start_id,
                                end_id,
                                p_start[idx][start_id].item(),
                                p_end[idx][end_id].item(),
                            )    )

            output.sort(key=lambda A_    : (tup[2] * tup[3])      ,							reverse=A_    )
            SCREAMING_SNAKE_CASE__       							= []
            for k in range(len(A_    )    ):
                if output[k][2] * output[k][3] >= thresh:
                    SCREAMING_SNAKE_CASE__							,		SCREAMING_SNAKE_CASE__       							= output[k][0], output[k][1]
                    SCREAMING_SNAKE_CASE__       							= self.tokenizer.decode(W_query['''input_ids'''][idx][c_start_pos:c_end_pos]    )
                    temp.append((decoded, output[k][2] * output[k][3])    )

            final_outputs.append(A_    )

        return final_outputs
