

import warnings

import numpy as np
import pandas as pd
from matplotlib import pyplot as plt
from sklearn.metrics import pairwise_distances

warnings.filterwarnings('''ignore''')

__A    	=			'''K-MEANS-CLUST/ '''



def    snake_case_(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase=None  )							->			Union[str, Any]:



  """simple docstring"""






  if seed is not None:  # useful for obtaining consistent results
    np.random.seed(_UpperCamelCase  )
  _snake_case		=							data.shape[0]  # number of data points

  # Pick K indices from range [0, N).
  _snake_case		=							np.random.randint(0				, _UpperCamelCase				, _UpperCamelCase  )

  # Keep centroids as dense format, as many entries will be nonzero due to averaging.
  # As long as at least one document in a cluster contains a word,
  # it will carry a nonzero weight in the TF-IDF vector of the centroid.
  _snake_case		=							data[rand_indices, :]

  return centroids



def    snake_case_(_UpperCamelCase				, _UpperCamelCase  )							->			Optional[int]:



  """simple docstring"""






  return pairwise_distances(_UpperCamelCase				, _UpperCamelCase				, metric='''euclidean'''  )



def    snake_case_(_UpperCamelCase				, _UpperCamelCase  )							->			int:



  """simple docstring"""






  _snake_case		=							centroid_pairwise_dist(_UpperCamelCase				, _UpperCamelCase  )

  # Compute cluster assignments for each data point:
  # Fill in the blank (RHS only)
  _snake_case		=							np.argmin(_UpperCamelCase				, axis=1  )

  return cluster_assignment



def    snake_case_(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase  )							->			List[str]:



  """simple docstring"""






  _snake_case		=							[]
  for i in range(_UpperCamelCase  ):
    # Select all data points that belong to cluster i. Fill in the blank (RHS only)
    _snake_case		=							data[cluster_assignment == i]
    # Compute the mean of the data points. Fill in the blank (RHS only)
    _snake_case		=							member_data_points.mean(axis=0  )
    new_centroids.append(_UpperCamelCase  )
  _snake_case		=							np.array(_UpperCamelCase  )

  return new_centroids



def    snake_case_(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase  )							->			Union[str, Any]:



  """simple docstring"""






  _snake_case		=							0.0
  for i in range(_UpperCamelCase  ):
    # Select all data points that belong to cluster i. Fill in the blank (RHS only)
    _snake_case		=							data[cluster_assignment == i, :]

    if member_data_points.shape[0] > 0:  # check if i-th cluster is non-empty
      # Compute distances from centroid to data points (RHS only)
      _snake_case		=							pairwise_distances(
          _UpperCamelCase				, [centroids[i]]				, metric='''euclidean'''  )
      _snake_case		=							distances**2
      heterogeneity += np.sum(_UpperCamelCase  )

  return heterogeneity



def    snake_case_(_UpperCamelCase				, _UpperCamelCase  )							->			str:



  """simple docstring"""






  plt.figure(figsize=(7, 4)  )
  plt.plot(_UpperCamelCase				, linewidth=4  )
  plt.xlabel('''# Iterations'''  )
  plt.ylabel('''Heterogeneity'''  )
  plt.title(F"""Heterogeneity of clustering over time, K={k:d}"""  )
  plt.rcParams.update({'''font.size''': 16}  )
  plt.show()



def    snake_case_(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase=500				, _UpperCamelCase=None				, _UpperCamelCase=False  )							->			Optional[Any]:



  """simple docstring"""






  _snake_case		=							initial_centroids[:]
  _snake_case		=							None

  for itr in range(_UpperCamelCase  ):
    if verbose:
      print(_UpperCamelCase				, end=''''''  )

    # 1. Make cluster assignments using nearest centroids
    _snake_case		=							assign_clusters(_UpperCamelCase				, _UpperCamelCase  )

    # 2. Compute a new centroid for each of the k clusters, averaging all data
    #    points assigned to that cluster.
    _snake_case		=							revise_centroids(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase  )

    # Check for convergence: if none of the assignments changed, stop
    if (
        prev_cluster_assignment is not None
        and (prev_cluster_assignment == cluster_assignment).all()
    ):
      break

    # Print number of new assignments
    if prev_cluster_assignment is not None:
      _snake_case		=							np.sum(prev_cluster_assignment != cluster_assignment  )
      if verbose:
        print(
            F"""    {num_changed:5d} elements changed their cluster assignment."""  )

        # Record heterogeneity convergence metric
    if record_heterogeneity is not None:
      # YOUR CODE HERE
      _snake_case		=							compute_heterogeneity(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase  )
      record_heterogeneity.append(_UpperCamelCase  )

    _snake_case		=							cluster_assignment[:]

  return centroids, cluster_assignment


# Mock test below
if False:  # change to true to run this test case.
 from sklearn import datasets as ds

 __A    	=			ds.load_iris()
 __A    	=			3
 __A    	=			[]
 __A    	=			get_initial_centroids(dataset['''data'''], k, seed=0)
 __A	,  __A    	=			kmeans(
     dataset['''data'''],
     k,
     initial_centroids,
     maxiter=4_00,
     record_heterogeneity=heterogeneity,
     verbose=True,
 )
 plot_heterogeneity(heterogeneity, k)



def    snake_case_(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase=None  )							->			pd.DataFrame:



  """simple docstring"""






  if fill_missing_report:
    _snake_case		=							df.fillna(value=_UpperCamelCase  )
  _snake_case		=							1
  _snake_case		=							df.select_dtypes(np.number  ).columns
  _snake_case		=							(
      df.groupby(['''Cluster''']  )[  # construct report dataframe
          numeric_cols
      ]  # group by cluster number
      .agg(
          [
              ('''sum''', np.sum),
              ('''mean_with_zeros''', lambda _UpperCamelCase  : np.mean(np.nan_to_num(_UpperCamelCase  )  )),
              ('''mean_without_zeros''', lambda _UpperCamelCase  : x.replace(0				, np.NaN  ).mean()),
              (
                  '''mean_25-75''',
                  lambda _UpperCamelCase  : np.mean(
                      np.nan_to_num(
                          sorted(_UpperCamelCase  )[
                              round(len(_UpperCamelCase  ) * 25 / 100  ) : round(len(_UpperCamelCase  ) * 75 / 100  )
                          ]  )  ),
              ),
              ('''mean_with_na''', np.mean),
              ('''min''', lambda _UpperCamelCase  : x.min()),
              ('''5%''', lambda _UpperCamelCase  : x.quantile(0.05  )),
              ('''25%''', lambda _UpperCamelCase  : x.quantile(0.25  )),
              ('''50%''', lambda _UpperCamelCase  : x.quantile(0.50  )),
              ('''75%''', lambda _UpperCamelCase  : x.quantile(0.75  )),
              ('''95%''', lambda _UpperCamelCase  : x.quantile(0.95  )),
              ('''max''', lambda _UpperCamelCase  : x.max()),
              ('''count''', lambda _UpperCamelCase  : x.count()),
              ('''stdev''', lambda _UpperCamelCase  : x.std()),
              ('''mode''', lambda _UpperCamelCase  : x.mode()[0]),
              ('''median''', lambda _UpperCamelCase  : x.median()),
              ('''# > 0''', lambda _UpperCamelCase  : (x > 0).sum()),
          ]  )
      .T.reset_index()
      .rename(index=_UpperCamelCase				, columns={'''level_0''': '''Features''', '''level_1''': '''Type'''}  )
  )  # rename columns
  # calculate the size of cluster(count of clientID's)
  _snake_case		=							report[
      (report['''Features'''] == '''dummy''') & (report['''Type'''] == '''count''')
  ].copy()  # avoid SettingWithCopyWarning
  _snake_case		=							(
      '''ClusterSize'''  # rename created cluster df to match report column names
  )
  _snake_case		=							'''# of Customers'''
  _snake_case		=							pd.DataFrame(
      clustersize.iloc[:, 2:].values
      / clustersize.iloc[:, 2:].values.sum()  )
  _snake_case		=							'''% of Customers'''  # rename created cluster df to match report column names
  _snake_case		=							'''ClusterProportion'''
  _snake_case		=							clusterproportion.columns.tolist()
  _snake_case		=							cols[-2:] + cols[:-2]
  _snake_case		=							clusterproportion[cols]  # rearrange columns to match report
  _snake_case		=							report.columns
  _snake_case		=							pd.DataFrame(
      abs(
          report[report['''Type'''] == '''count'''].iloc[:, 2:].values
          - clustersize.iloc[:, 2:].values  )  )  # generating df with count of nan values
  _snake_case		=							0
  _snake_case		=							'''# of nan'''
  _snake_case		=							report[
      report['''Type'''] == '''count'''
  ].Features.tolist()  # filling values in order to match report
  _snake_case		=							a.columns.tolist()
  _snake_case		=							cols[-2:] + cols[:-2]
  _snake_case		=							a[cols]  # rearrange columns to match report
  _snake_case		=							report.columns  # rename columns to match report
  _snake_case		=							report.drop(
      report[report.Type == '''count'''].index  )  # drop count values except cluster size
  _snake_case		=							pd.concat(
      [report, a, clustersize, clusterproportion]				, axis=0  )  # concat report with clustert size and nan values
  _snake_case		=							report['''Features'''].isin(_UpperCamelCase  )
  _snake_case		=							report.columns.tolist()
  _snake_case		=							cols[0:2] + cols[-1:] + cols[2:-1]
  _snake_case		=							report[cols]
  _snake_case		=							{
      '''ClusterSize''': 9,
      '''ClusterProportion''': 8,
      '''mean_with_zeros''': 7,
      '''mean_with_na''': 6,
      '''max''': 5,
      '''50%''': 4,
      '''min''': 3,
      '''25%''': 2,
      '''75%''': 1,
      '''# of nan''': 0,
      '''# > 0''': -1,
      '''sum_with_na''': -2,
  }
  _snake_case		=							(
      report.assign(
          Sortera=lambda _UpperCamelCase  : x.Type.map(_UpperCamelCase  )				, Sortera=lambda _UpperCamelCase  : list(reversed(range(len(_UpperCamelCase  )  )  )  )				, )
      .sort_values(['''Sorter1''', '''Mark''', '''Sorter2''']				, ascending=_UpperCamelCase  )
      .drop(['''Sorter1''', '''Sorter2''']				, axis=1  )
  )
  _snake_case		=							''''''
  _snake_case		=							report.reset_index()
  _snake_case		=							report.drop(columns=['''index''']  )
  return report


if __name__ == "__main__":
 import doctest

 doctest.testmod()




import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_url
from PIL import Image

from transformers import DeformableDetrConfig, DeformableDetrForObjectDetection, DeformableDetrImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
__A    	=			logging.get_logger(__name__)



def    snake_case_(_UpperCamelCase  )							->			Tuple:



  """simple docstring"""






  if "backbone.0.body" in orig_key:
    _snake_case		=							orig_key.replace('''backbone.0.body'''				, '''backbone.conv_encoder.model'''  )
  if "transformer" in orig_key:
    _snake_case		=							orig_key.replace('''transformer.'''				, ''''''  )
  if "norm1" in orig_key:
    if "encoder" in orig_key:
      _snake_case		=							orig_key.replace('''norm1'''				, '''self_attn_layer_norm'''  )
    else:
      _snake_case		=							orig_key.replace('''norm1'''				, '''encoder_attn_layer_norm'''  )
  if "norm2" in orig_key:
    if "encoder" in orig_key:
      _snake_case		=							orig_key.replace('''norm2'''				, '''final_layer_norm'''  )
    else:
      _snake_case		=							orig_key.replace('''norm2'''				, '''self_attn_layer_norm'''  )
  if "norm3" in orig_key:
    _snake_case		=							orig_key.replace('''norm3'''				, '''final_layer_norm'''  )
  if "linear1" in orig_key:
    _snake_case		=							orig_key.replace('''linear1'''				, '''fc1'''  )
  if "linear2" in orig_key:
    _snake_case		=							orig_key.replace('''linear2'''				, '''fc2'''  )
  if "query_embed" in orig_key:
    _snake_case		=							orig_key.replace('''query_embed'''				, '''query_position_embeddings'''  )
  if "cross_attn" in orig_key:
    _snake_case		=							orig_key.replace('''cross_attn'''				, '''encoder_attn'''  )

  return orig_key



def    snake_case_(_UpperCamelCase  )							->			Any:



  """simple docstring"""






  for i in range(6  ):
    # read in weights + bias of input projection layer of self-attention
    _snake_case		=							state_dict.pop(F"""decoder.layers.{i}.self_attn.in_proj_weight"""  )
    _snake_case		=							state_dict.pop(F"""decoder.layers.{i}.self_attn.in_proj_bias"""  )
    # next, add query, keys and values (in that order) to the state dict
    _snake_case		=							in_proj_weight[:256, :]
    _snake_case		=							in_proj_bias[:256]
    _snake_case		=							in_proj_weight[256:512, :]
    _snake_case		=							in_proj_bias[256:512]
    _snake_case		=							in_proj_weight[-256:, :]
    _snake_case		=							in_proj_bias[-256:]



def    snake_case_()							->			int:



  """simple docstring"""






  _snake_case		=							'''http://images.cocodataset.org/val2017/000000039769.jpg'''
  _snake_case		=							Image.open(requests.get(_UpperCamelCase				, stream=_UpperCamelCase  ).raw  )

  return im



@torch.no_grad()
def    snake_case_(_UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, _UpperCamelCase				, )							->			Dict:



  """simple docstring"""






  _snake_case		=							DeformableDetrConfig()
  # set config attributes
  if single_scale:
    _snake_case		=							1
  _snake_case		=							dilation
  _snake_case		=							with_box_refine
  _snake_case		=							two_stage
  # set labels
  _snake_case		=							91
  _snake_case		=							'''huggingface/label-files'''
  _snake_case		=							'''coco-detection-id2label.json'''
  _snake_case		=							json.load(open(cached_download(hf_hub_url(_UpperCamelCase				, _UpperCamelCase				, repo_type='''dataset'''  )  )				, '''r'''  )  )
  _snake_case		=							{int(_UpperCamelCase  ): v for k, v in idalabel.items()}
  _snake_case		=							idalabel
  _snake_case		=							{v: k for k, v in idalabel.items()}

  # load image processor
  _snake_case		=							DeformableDetrImageProcessor(format='''coco_detection'''  )

  # prepare image
  _snake_case		=							prepare_img()
  _snake_case		=							image_processor(images=_UpperCamelCase				, return_tensors='''pt'''  )
  _snake_case		=							encoding['''pixel_values''']

  logger.info('''Converting model...'''  )

  # load original state dict
  _snake_case		=							torch.load(_UpperCamelCase				, map_location='''cpu'''  )['''model''']
  # rename keys
  for key in state_dict.copy().keys():
    _snake_case		=							state_dict.pop(_UpperCamelCase  )
    _snake_case		=							val
  # query, key and value matrices need special treatment
  read_in_q_k_v(_UpperCamelCase  )
  # important: we need to prepend a prefix to each of the base model keys as the head models use different attributes for them
  _snake_case		=							'''model.'''
  for key in state_dict.copy().keys():
    if not key.startswith('''class_embed'''  ) and not key.startswith('''bbox_embed'''  ):
      _snake_case		=							state_dict.pop(_UpperCamelCase  )
      _snake_case		=							val
    # finally, create HuggingFace model and load state dict
  _snake_case		=							DeformableDetrForObjectDetection(_UpperCamelCase  )
  model.load_state_dict(_UpperCamelCase  )
  model.eval()

  _snake_case		=							'''cuda''' if torch.cuda.is_available() else '''cpu'''
  model.to(_UpperCamelCase  )
  # verify our conversion
  _snake_case		=							model(pixel_values.to(_UpperCamelCase  )  )

  _snake_case		=							torch.tensor(
      [[-9.6645, -4.3449, -5.8705], [-9.7035, -3.8504, -5.0724], [-10.5634, -5.3379, -7.5116]]  )
  _snake_case		=							torch.tensor([[0.8693, 0.2289, 0.2492], [0.3150, 0.5489, 0.5845], [0.5563, 0.7580, 0.8518]]  )

  if single_scale:
    _snake_case		=							torch.tensor(
        [[-9.9051, -4.2541, -6.4852], [-9.6947, -4.0854, -6.8033], [-10.0665, -5.8470, -7.7003]]  )
    _snake_case		=							torch.tensor([[0.7292, 0.4991, 0.5532], [0.7959, 0.2426, 0.4236], [0.7582, 0.3518, 0.4451]]  )

  if single_scale and dilation:
    _snake_case		=							torch.tensor(
        [[-8.9652, -4.1074, -5.6635], [-9.0596, -4.9447, -6.6075], [-10.1178, -4.5275, -6.2671]]  )
    _snake_case		=							torch.tensor([[0.7665, 0.4130, 0.4769], [0.8364, 0.1841, 0.3391], [0.6261, 0.3895, 0.7978]]  )

  if with_box_refine:
    _snake_case		=							torch.tensor(
        [[-8.8895, -5.4187, -6.8153], [-8.4706, -6.1668, -7.6184], [-9.0042, -5.5359, -6.9141]]  )
    _snake_case		=							torch.tensor([[0.7828, 0.2208, 0.4323], [0.0892, 0.5996, 0.1319], [0.5524, 0.6389, 0.8914]]  )

  if with_box_refine and two_stage:
    _snake_case		=							torch.tensor(
        [[-6.7108, -4.3213, -6.3777], [-8.9014, -6.1799, -6.7240], [-6.9315, -4.4735, -6.2298]]  )
    _snake_case		=							torch.tensor([[0.2583, 0.5499, 0.4683], [0.7652, 0.9068, 0.4882], [0.5490, 0.2763, 0.0564]]  )

  print('''Logits:'''				, outputs.logits[0, :3, :3]  )

  assert torch.allclose(outputs.logits[0, :3, :3]				, expected_logits.to(_UpperCamelCase  )				, atol=1E-4  )
  assert torch.allclose(outputs.pred_boxes[0, :3, :3]				, expected_boxes.to(_UpperCamelCase  )				, atol=1E-4  )

  print('''Everything ok!'''  )

  # Save model and image processor
  logger.info(F"""Saving PyTorch model and image processor to {pytorch_dump_folder_path}..."""  )
  Path(_UpperCamelCase  ).mkdir(exist_ok=_UpperCamelCase  )
  model.save_pretrained(_UpperCamelCase  )
  image_processor.save_pretrained(_UpperCamelCase  )

  # Push to hub
  if push_to_hub:
    _snake_case		=							'''deformable-detr'''
    model_name += "-single-scale" if single_scale else ""
    model_name += "-dc5" if dilation else ""
    model_name += "-with-box-refine" if with_box_refine else ""
    model_name += "-two-stage" if two_stage else ""
    print('''Pushing model to hub...'''  )
    model.push_to_hub(repo_path_or_name=_UpperCamelCase				, organization='''nielsr'''				, commit_message='''Add model'''  )


if __name__ == "__main__":
 __A    	=			argparse.ArgumentParser()

 parser.add_argument(
     '''--checkpoint_path''',
     type=str,
     default='''/home/niels/checkpoints/deformable_detr/r50_deformable_detr-checkpoint.pth''',
     help='''Path to Pytorch checkpoint (.pth file) you\'d like to convert.''',
 )
 parser.add_argument('''--single_scale''', action='''store_true''', help='''Whether to set config.num_features_levels = 1.''')
 parser.add_argument('''--dilation''', action='''store_true''', help='''Whether to set config.dilation=True.''')
 parser.add_argument('''--with_box_refine''', action='''store_true''', help='''Whether to set config.with_box_refine=True.''')
 parser.add_argument('''--two_stage''', action='''store_true''', help='''Whether to set config.two_stage=True.''')
 parser.add_argument(
     '''--pytorch_dump_folder_path''',
     default=None,
     type=str,
     required=True,
     help='''Path to the folder to output PyTorch model.''',
 )
 parser.add_argument(
     '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
 )
 __A    	=			parser.parse_args()
 convert_deformable_detr_checkpoint(
     args.checkpoint_path,
     args.single_scale,
     args.dilation,
     args.with_box_refine,
     args.two_stage,
     args.pytorch_dump_folder_path,
     args.push_to_hub,
 )

