


import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings('''ignore''')

snake_case       :					Optional[int]        =  """K-MEANS-CLUST/ """
def        __lowerCamelCase		(				UpperCAmelCase_		:	int				,						UpperCAmelCase_		:	Optional[Any]				,						UpperCAmelCase_		:	List[Any]=None		):




      """simple docstring"""





      if seed is not None:  # useful for obtaining consistent results
            np.random.seed(UpperCAmelCase_		)
      a		:Optional[Any]											= data.shape[0]  # number of data points

      # Pick K indices from range [0, N).
      a		:Optional[int]											= np.random.randint(0				,						UpperCAmelCase_				,						UpperCAmelCase_		)

      # Keep centroids as dense format, as many entries will be nonzero due to averaging.
      # As long as at least one document in a cluster contains a word,
      # it will carry a nonzero weight in the TF-IDF vector of the centroid.
      a		:Dict											= data[rand_indices, :]

      return centroids
def        __lowerCamelCase		(				UpperCAmelCase_		:	str				,						UpperCAmelCase_		:	Union[str, Any]		):




      """simple docstring"""





      return pairwise_distances(UpperCAmelCase_				,						UpperCAmelCase_				,						metric='''euclidean'''		)
def        __lowerCamelCase		(				UpperCAmelCase_		:	Dict				,						UpperCAmelCase_		:	str		):




      """simple docstring"""





      a		:str											= centroid_pairwise_dist(UpperCAmelCase_				,						UpperCAmelCase_		)

      # Compute cluster assignments for each data point:
      # Fill in the blank (RHS only)
      a		:Optional[Any]											= np.argmin(UpperCAmelCase_				,						axis=1		)

      return cluster_assignment
def        __lowerCamelCase		(				UpperCAmelCase_		:	str				,						UpperCAmelCase_		:	Optional[int]				,						UpperCAmelCase_		:	Union[str, Any]		):




      """simple docstring"""





      a		:Tuple											= []
      for i in range(UpperCAmelCase_		):
            # Select all data points that belong to cluster i. Fill in the blank (RHS only)
            a		:Any											= data[cluster_assignment == i]
            # Compute the mean of the data points. Fill in the blank (RHS only)
            a		:List[str]											= member_data_points.mean(axis=0		)
            new_centroids.append(UpperCAmelCase_		)
      a		:Union[str, Any]											= np.array(UpperCAmelCase_		)

      return new_centroids
def        __lowerCamelCase		(				UpperCAmelCase_		:	Optional[int]				,						UpperCAmelCase_		:	Dict				,						UpperCAmelCase_		:	Dict				,						UpperCAmelCase_		:	int		):




      """simple docstring"""





      a		:Optional[Any]											= 0.0
      for i in range(UpperCAmelCase_		):
            # Select all data points that belong to cluster i. Fill in the blank (RHS only)
            a		:Tuple											= data[cluster_assignment == i, :]

            if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
                  # Compute distances from centroid to data points (RHS only)
                  a		:Union[str, Any]											= pairwise_distances(
                      UpperCAmelCase_				,						[centroids[i]]				,						metric='''euclidean'''		)
                  a		:int											= distances**2
                  heterogeneity += np.sum(UpperCAmelCase_		)

      return heterogeneity
def        __lowerCamelCase		(				UpperCAmelCase_		:	Tuple				,						UpperCAmelCase_		:	Union[str, Any]		):




      """simple docstring"""





      plt.figure(figsize=(7, 4)		)
      plt.plot(UpperCAmelCase_				,						linewidth=4		)
      plt.xlabel('''# Iterations'''		)
      plt.ylabel('''Heterogeneity'''		)
      plt.title(F'''Heterogeneity of clustering over time, K={k:d}'''		)
      plt.rcParams.update({'''font.size''': 16}		)
      plt.show()
def        __lowerCamelCase		(				UpperCAmelCase_		:	List[str]				,						UpperCAmelCase_		:	Union[str, Any]				,						UpperCAmelCase_		:	Tuple				,						UpperCAmelCase_		:	str=500				,						UpperCAmelCase_		:	Any=None				,						UpperCAmelCase_		:	Union[str, Any]=False		):




      """simple docstring"""





      a		:Dict											= initial_centroids[:]
      a		:Union[str, Any]											= None

      for itr in range(UpperCAmelCase_		):
            if verbose:
                  print(UpperCAmelCase_				,						end=''''''		)

            # 1. Make cluster assignments using nearest centroids
            a		:List[str]											= assign_clusters(UpperCAmelCase_				,						UpperCAmelCase_		)

            # 2. Compute a new centroid for each of the k clusters, averaging all data
            #    points assigned to that cluster.
            a		:int											= revise_centroids(UpperCAmelCase_				,						UpperCAmelCase_				,						UpperCAmelCase_		)

            # Check for convergence: if none of the assignments changed, stop
            if (
                prev_cluster_assignment is not None
                and (prev_cluster_assignment == cluster_assignment).all()
            ):
                  break

            # Print number of new assignments
            if prev_cluster_assignment is not None:
                  a		:Union[str, Any]											= np.sum(prev_cluster_assignment != cluster_assignment		)
                  if verbose:
                        print(
                            F'''    {num_changed:5d} elements changed their cluster assignment.'''		)

        # Record heterogeneity convergence metric
            if record_heterogeneity is not None:
                  # YOUR CODE HERE
                  a		:Union[str, Any]											= compute_heterogeneity(UpperCAmelCase_				,						UpperCAmelCase_				,						UpperCAmelCase_				,						UpperCAmelCase_		)
                  record_heterogeneity.append(UpperCAmelCase_		)

            a		:Union[str, Any]											= cluster_assignment[:]

      return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
    from sklearn import datasets as ds

    snake_case       :					Optional[Any]        =  ds.load_iris()
    snake_case       :					Optional[Any]        =  3
    snake_case       :					int        =  []
    snake_case       :					int        =  get_initial_centroids(dataset['''data'''], k, seed=0)
    snake_case       :					Any        =  kmeans(
        dataset['''data'''],
        k,
        initial_centroids,
        maxiter=4_00,
        record_heterogeneity=heterogeneity,
        verbose=True,
    )
    plot_heterogeneity(heterogeneity, k)
def        __lowerCamelCase		(				UpperCAmelCase_		:	int				,						UpperCAmelCase_		:	List[str]				,						UpperCAmelCase_		:	Tuple=None		):




      """simple docstring"""





      if fill_missing_report:
            a		:Union[str, Any]											= df.fillna(value=UpperCAmelCase_		)
      a		:Optional[Any]											= 1
      a		:Tuple											= df.select_dtypes(np.number		).columns
      a		:Any											= (
          df.groupby(['''Cluster''']		)[  # construct report dataframe
              numeric_cols
          ]  # group by cluster number
          .agg(
              [
                  ('''sum''', np.sum),
                  ('''mean_with_zeros''', lambda UpperCAmelCase_		: np.mean(np.nan_to_num(UpperCAmelCase_		)		)),
                  ('''mean_without_zeros''', lambda UpperCAmelCase_		: x.replace(0				,						np.NaN		).mean()),
                  (
                      '''mean_25-75''',
                      lambda UpperCAmelCase_		: np.mean(
                          np.nan_to_num(
                              sorted(UpperCAmelCase_		)[
                                  round(len(UpperCAmelCase_		) * 25 / 100		) : round(len(UpperCAmelCase_		) * 75 / 100		)
                              ]		)		),
                  ),
                  ('''mean_with_na''', np.mean),
                  ('''min''', lambda UpperCAmelCase_		: x.min()),
                  ('''5%''', lambda UpperCAmelCase_		: x.quantile(0.05		)),
                  ('''25%''', lambda UpperCAmelCase_		: x.quantile(0.25		)),
                  ('''50%''', lambda UpperCAmelCase_		: x.quantile(0.50		)),
                  ('''75%''', lambda UpperCAmelCase_		: x.quantile(0.75		)),
                  ('''95%''', lambda UpperCAmelCase_		: x.quantile(0.95		)),
                  ('''max''', lambda UpperCAmelCase_		: x.max()),
                  ('''count''', lambda UpperCAmelCase_		: x.count()),
                  ('''stdev''', lambda UpperCAmelCase_		: x.std()),
                  ('''mode''', lambda UpperCAmelCase_		: x.mode()[0]),
                  ('''median''', lambda UpperCAmelCase_		: x.median()),
                  ('''# > 0''', lambda UpperCAmelCase_		: (x > 0).sum()),
              ]		)
          .T.reset_index()
          .rename(index=UpperCAmelCase_				,						columns={'''level_0''': '''Features''', '''level_1''': '''Type'''}		)
      )  # rename columns
      # calculate the size of cluster(count of clientID's)
      a		:Optional[int]											= report[
          (report['''Features'''] == '''dummy''') & (report['''Type'''] == '''count''')
      ].copy()  # avoid SettingWithCopyWarning
      a		:int											= (
          '''ClusterSize'''  # rename created cluster df to match report column names
      )
      a		:str											= '''# of Customers'''
      a		:Optional[Any]											= pd.DataFrame(
          clustersize.iloc[:, 2:].values
          / clustersize.iloc[:, 2:].values.sum()		)
      a		:int											= '''% of Customers'''  # rename created cluster df to match report column names
      a		:List[str]											= '''ClusterProportion'''
      a		:List[Any]											= clusterproportion.columns.tolist()
      a		:Dict											= cols[-2:] + cols[:-2]
      a		:Dict											= clusterproportion[cols]  # rearrange columns to match report
      a		:Union[str, Any]											= report.columns
      a		:List[Any]											= pd.DataFrame(
          abs(
              report[report['''Type'''] == '''count'''].iloc[:, 2:].values
              - clustersize.iloc[:, 2:].values		)		)  # generating df with count of nan values
      a		:List[Any]											= 0
      a		:Tuple											= '''# of nan'''
      a		:Optional[Any]											= report[
          report['''Type'''] == '''count'''
      ].Features.tolist()  # filling values in order to match report
      a		:str											= a.columns.tolist()
      a		:List[Any]											= cols[-2:] + cols[:-2]
      a		:Dict											= a[cols]  # rearrange columns to match report
      a		:str											= report.columns  # rename columns to match report
      a		:int											= report.drop(
          report[report.Type == '''count'''].index		)  # drop count values except cluster size
      a		:List[str]											= pd.concat(
          [report, a, clustersize, clusterproportion]				,						axis=0		)  # concat report with clustert size and nan values
      a		:Tuple											= report['''Features'''].isin(UpperCAmelCase_		)
      a		:Union[str, Any]											= report.columns.tolist()
      a		:Union[str, Any]											= cols[0:2] + cols[-1:] + cols[2:-1]
      a		:str											= report[cols]
      a		:List[str]											= {
          '''ClusterSize''': 9,
          '''ClusterProportion''': 8,
          '''mean_with_zeros''': 7,
          '''mean_with_na''': 6,
          '''max''': 5,
          '''50%''': 4,
          '''min''': 3,
          '''25%''': 2,
          '''75%''': 1,
          '''# of nan''': 0,
          '''# > 0''': -1,
          '''sum_with_na''': -2,
      }
      a		:Union[str, Any]											= (
          report.assign(
              Sortera=lambda UpperCAmelCase_		: x.Type.map(UpperCAmelCase_		)				,						Sortera=lambda UpperCAmelCase_		: list(reversed(range(len(UpperCAmelCase_		)		)		)		)				,						)
          .sort_values(['''Sorter1''', '''Mark''', '''Sorter2''']				,						ascending=UpperCAmelCase_		)
          .drop(['''Sorter1''', '''Sorter2''']				,						axis=1		)
      )
      a		:str											= ''''''
      a		:List[str]											= report.reset_index()
      a		:Tuple											= report.drop(columns=['''index''']		)
      return report


if __name__ == "__main__":
    import doctest

    doctest.testmod()









from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy







class 	_snake_case						(   _snake_case							):
 SCREAMING_SNAKE_CASE__   = 'MarkupLMFeatureExtractor'
 SCREAMING_SNAKE_CASE__   = ('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
 SCREAMING_SNAKE_CASE__   = True




 def __call__(	self      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase = True      ,						_lowerCamelCase = False      ,						_lowerCamelCase = None      ,						_lowerCamelCase = None      ,						_lowerCamelCase = 0      ,						_lowerCamelCase = None      ,						_lowerCamelCase = None      ,						_lowerCamelCase = None      ,						_lowerCamelCase = False      ,						_lowerCamelCase = False      ,						_lowerCamelCase = False      ,						_lowerCamelCase = False      ,						_lowerCamelCase = True      ,						_lowerCamelCase = None      ,						**_lowerCamelCase      ,						):
       # first, create nodes and xpaths
       if self.parse_html:
             if html_strings is None:
                   raise ValueError('''Make sure to pass HTML strings in case `parse_html` is set to `True`'''					)

             if nodes is not None or xpaths is not None or node_labels is not None:
                   raise ValueError(
                       '''Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'''					)

             a		:int											= self.feature_extractor(_lowerCamelCase					)
             a		:Tuple											= features['''nodes''']
             a		:Dict											= features['''xpaths''']
       else:
             if html_strings is not None:
                   raise ValueError('''You have passed HTML strings but `parse_html` is set to `False`.'''					)
             if nodes is None or xpaths is None:
                   raise ValueError('''Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'''					)

        # # second, apply the tokenizer
       if questions is not None and self.parse_html:
             if isinstance(_lowerCamelCase      ,						_lowerCamelCase					):
                   a		:Tuple											= [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

       a		:Optional[int]											= self.tokenizer(
           text=questions if questions is not None else nodes      ,						text_pair=nodes if questions is not None else None      ,						xpaths=_lowerCamelCase      ,						node_labels=_lowerCamelCase      ,						add_special_tokens=_lowerCamelCase      ,						padding=_lowerCamelCase      ,						truncation=_lowerCamelCase      ,						max_length=_lowerCamelCase      ,						stride=_lowerCamelCase      ,						pad_to_multiple_of=_lowerCamelCase      ,						return_token_type_ids=_lowerCamelCase      ,						return_attention_mask=_lowerCamelCase      ,						return_overflowing_tokens=_lowerCamelCase      ,						return_special_tokens_mask=_lowerCamelCase      ,						return_offsets_mapping=_lowerCamelCase      ,						return_length=_lowerCamelCase      ,						verbose=_lowerCamelCase      ,						return_tensors=_lowerCamelCase      ,						**_lowerCamelCase      ,						)

       return encoded_inputs




 def  SCREAMING_SNAKE_CASE__       (	self      ,						*_lowerCamelCase      ,						**_lowerCamelCase					):
       return self.tokenizer.batch_decode(*_lowerCamelCase      ,						**_lowerCamelCase					)




 def  SCREAMING_SNAKE_CASE__       (	self      ,						*_lowerCamelCase      ,						**_lowerCamelCase					):
       return self.tokenizer.decode(*_lowerCamelCase      ,						**_lowerCamelCase					)




 @property
 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Any											= self.tokenizer.model_input_names
       return tokenizer_input_names





