app.py

import os
os.system('pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu')
import glob, fitz
import PIL
import re
import torch
import cv2
import pytesseract
import pandas as pd
import numpy as np
import gradio as gr
from PIL import Image
from tqdm import tqdm
from difflib import SequenceMatcher
from itertools import groupby
from scipy import ndimage
from scipy.ndimage import interpolation as inter
from datasets import load_metric
from datasets import load_dataset
from datasets.features import ClassLabel
from transformers import AutoProcessor
from PIL import Image, ImageDraw, ImageFont
from transformers import AutoModelForTokenClassification
from transformers.data.data_collator import default_data_collator
from datasets import Features, Sequence, ClassLabel, Value, Array2D, Array3D
from transformers import LayoutLMv3ForTokenClassification,LayoutLMv3FeatureExtractor,LayoutLMv3ImageProcessor
import io
# import paddleocr
# from paddleocr import PaddleOCR
auth_token = os.environ.get("HUGGING_FACE_HUB_TOKEN")
import warnings
# Ignore warning messages
warnings.filterwarnings("ignore")

id2label= {0: 'others', 1: 'issuer_name', 2: 'issuer_addr', 3: 'issuer_cap', 4: 'issuer_city', 5: 'issuer_prov', 6: 'issuer_state', 7: 'issuer_tel', 8: 'issuer_id', 9: 'issuer_fax', 10: 'issuer_vat', 11: 'issuer_contact', 12: 'issuer_contact_email', 13: 'issuer_contact_phone', 14: 'receiver_name', 15: 'receiver_addr', 16: 'receiver_cap', 17: 'receiver_city', 18: 'receiver_prov', 19: 'receiver_state', 20: 'receiver_tel', 21: 'receiver_fax', 22: 'receiver_vat', 23: 'receiver_id', 24: 'receiver_contact', 25: 'dest_name', 26: 'dest_addr', 27: 'dest_cap', 28: 'dest_city', 29: 'dest_prov', 30: 'dest_state', 31: 'dest_tel', 32: 'dest_fax', 33: 'dest_vat', 34: 'doc_type', 35: 'doc_nr', 36: 'doc_date', 37: 'order_nr', 38: 'order_date', 39: 'service_order', 40: 'shipment_nr', 41: 'client_reference', 42: 'client_vat', 43: 'client_id', 44: 'client_code', 45: 'time', 46: 'notes', 47: 'client_tel', 48: 'art_code', 49: 'ref_code', 50: 'order_reason', 51: 'order_ref', 52: 'order_ref_date', 53: 'detail_desc', 54: 'lot_id', 55: 'lot_qty', 56: 'detail_um', 57: 'detail_qty', 58: 'detail_tare', 59: 'detail_grossw', 60: 'detail_packages', 61: 'detail_netw', 62: 'detail_origin', 63: 'payment_bank', 64: 'payment_terms', 65: 'tot_qty', 66: 'tot_grossw', 67: 'tot_netw', 68: 'tot_volume', 69: 'shipment_reason', 70: 'package_type', 71: 'transport_respons', 72: 'transport_vectors', 73: 'transport_terms', 74: 'transport_datetime', 75: 'return_plt', 76: 'nonreturn_plt', 77: 'dest_signature', 78: 'driver_signature', 79: 'transport_signature', 80: 'page', 81: 'varieta', 82: 'raccolta', 83: 'detail_volume'}
custom_config = r'--oem 3 --psm 6'
lang='eng'


#Google Vision OCR 
from google.cloud import vision_v1p3beta1 as vision
from google.cloud import vision_v1p3beta1 as vision
from google.cloud import vision
# os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "test-apikey.json"

processor = AutoProcessor.from_pretrained("microsoft/layoutlmv3-base", apply_ocr=False)
model = AutoModelForTokenClassification.from_pretrained("sxandie/doc-ai-information-extraction",use_auth_token=auth_token)

from tabulate import tabulate
def print_df(df):
  print(tabulate(df, headers = df.columns, tablefmt = 'psql'))


def process_image_pytesseract(image,width,height):
    width, height = image.size
    feature_extractor = LayoutLMv3ImageProcessor(apply_ocr=True,lang=lang)
    encoding_feature_extractor = feature_extractor(image, return_tensors="pt",truncation=True)
    words, boxes = encoding_feature_extractor.words, encoding_feature_extractor.boxes
    return words,boxes

def create_bounding_box5(vertices, width_scale, height_scale):

  # Get the x, y coordinates
  x1 = int(vertices[0].x * width_scale)
  y1 = int(vertices[0].y * height_scale)

  x2 = int(vertices[2].x * width_scale)
  y2 = int(vertices[2].y * height_scale)

  # Validate x1 < x2
  if x1 > x2:
    x1, x2 = x2, x1

  # Validate y1 < y2
  if y1 > y2:
    y1, y2 = y2, y1

  # Return valid bounding box
  return [x1, y1, x2, y2]

#Google Vision OCR
def process_image_GoogleVision(image, width, height):
    inference_image = [image.convert("RGB")]
    client = vision.ImageAnnotatorClient()
    with io.BytesIO() as output:
        image.save(output, format='JPEG')
        content = output.getvalue()
    image = vision.Image(content=content)

    response = client.text_detection(image=image)
    texts = response.text_annotations

    # Get the bounding box vertices and remove the first item
    bboxes = [text.bounding_poly.vertices[1:] for text in texts]
    # Create the list of words and boxes
    words = [text.description for text in texts]
    boxes = [create_bounding_box5(bbox, 1000/width, 1000/height) for bbox in bboxes]
    return words,boxes


def generate_unique_colors(id2label):
     # Generate unique colors
     label_ints = np.random.choice(len(PIL.ImageColor.colormap), len(id2label), replace=False)
     label_color_pil = list(PIL.ImageColor.colormap.values())
     label_color = [label_color_pil[i] for i in label_ints]

     color = {}
     for k, v in id2label.items():
         if v[:2] == '':
             color['o'] = label_color[k]
         else:
             color[v[0:]] = label_color[k]

     return color

def create_bounding_box1(bbox_data, width_scale: float, height_scale: float):
    xs = []
    ys = []
    for x, y in bbox_data:
        xs.append(x)
        ys.append(y)

    left = int(max(0, min(xs) * width_scale))
    top = int(max(0, min(ys) * height_scale))
    right = int(min(1000, max(xs) * width_scale))
    bottom = int(min(1000, max(ys) * height_scale))

    return [left, top, right, bottom]



def unnormalize_box(bbox, width, height):
     return [
         width * (bbox[0] / 1000),
         height * (bbox[1] / 1000),
         width * (bbox[2] / 1000),
         height * (bbox[3] / 1000),
     ]


def iob_to_label(label):
    return id2label.get(label, 'others')

def process_image(image):
    custom_config = r'--oem 3 --psm 6'
    # lang='eng+deu+ita+chi_sim'
    lang='eng'
    width, height = image.size
    feature_extractor = LayoutLMv3FeatureExtractor(apply_ocr=True)
    encoding_feature_extractor = feature_extractor(image, return_tensors="pt",truncation=True)
    words, boxes = encoding_feature_extractor.words, encoding_feature_extractor.boxes

    custom_config = r'--oem 3 --psm 6'
    # encode
    inference_image = [image.convert("RGB")]
    encoding = processor(inference_image , truncation=True, return_offsets_mapping=True, return_tensors="pt", padding="max_length", stride =128, max_length=512, return_overflowing_tokens=True)
    offset_mapping = encoding.pop('offset_mapping')
    overflow_to_sample_mapping = encoding.pop('overflow_to_sample_mapping')

    # change the shape of pixel values
    x = []
    for i in range(0, len(encoding['pixel_values'])):
      x.append(encoding['pixel_values'][i])
    x = torch.stack(x)
    encoding['pixel_values'] = x

    # forward pass
    outputs = model(**encoding)

    # get predictions
    predictions = outputs.logits.argmax(-1).squeeze().tolist()
    token_boxes = encoding.bbox.squeeze().tolist()

    # only keep non-subword predictions
    preds = []
    l_words = []
    bboxes = []
    token_section_num = []

    if (len(token_boxes) == 512):
      predictions = [predictions]
      token_boxes = [token_boxes]


    for i in range(0, len(token_boxes)):
      for j in range(0, len(token_boxes[i])):
        unnormal_box = unnormalize_box(token_boxes[i][j], width, height)
        if (np.asarray(token_boxes[i][j]).shape != (4,)):
          continue
        elif (token_boxes[i][j] == [0, 0, 0, 0] or token_boxes[i][j] == 0):
          #print('zero found!')
          continue
        # if bbox is available in the list, just we need to update text
        elif (unnormal_box not in bboxes):
          preds.append(predictions[i][j])
          l_words.append(processor.tokenizer.decode(encoding["input_ids"][i][j]))
          bboxes.append(unnormal_box)
          token_section_num.append(i)
        else:
          # we have to update the word
          _index = bboxes.index(unnormal_box)
          if (token_section_num[_index] == i):
            # check if they're in a same section or not (documents with more than 512 tokens will divide to seperate
            # parts, so it's possible to have a word in both of the pages and we have to control that repetetive words
            # HERE: because they're in a same section, so we can merge them safely
            l_words[_index] = l_words[_index] + processor.tokenizer.decode(encoding["input_ids"][i][j])

          else:
            continue

    return bboxes, preds, l_words, image



def process_image_encoding(model, processor, image, words, boxes,width,height):
    # encode
    inference_image = [image.convert("RGB")]
    encoding = processor(inference_image ,words,boxes=boxes, truncation=True, return_offsets_mapping=True, return_tensors="pt",
                     padding="max_length", stride =128, max_length=512, return_overflowing_tokens=True)
    offset_mapping = encoding.pop('offset_mapping')
    overflow_to_sample_mapping = encoding.pop('overflow_to_sample_mapping')

    # change the shape of pixel values
    x = []
    for i in range(0, len(encoding['pixel_values'])):
      x.append(encoding['pixel_values'][i])
    x = torch.stack(x)
    encoding['pixel_values'] = x

    # forward pass
    outputs = model(**encoding)

    # get predictions
    predictions = outputs.logits.argmax(-1).squeeze().tolist()
    token_boxes = encoding.bbox.squeeze().tolist()

    # only keep non-subword predictions
    preds = []
    l_words = []
    bboxes = []
    token_section_num = []

    if (len(token_boxes) == 512):
      predictions = [predictions]
      token_boxes = [token_boxes]

    for i in range(0, len(token_boxes)):
      for j in range(0, len(token_boxes[i])):
        unnormal_box = unnormalize_box(token_boxes[i][j], width, height)
        if (np.asarray(token_boxes[i][j]).shape != (4,)):
          continue
        elif (token_boxes[i][j] == [0, 0, 0, 0] or token_boxes[i][j] == 0):
          #print('zero found!')
          continue
        # if bbox is available in the list, just we need to update text
        elif (unnormal_box not in bboxes):
          preds.append(predictions[i][j])
          l_words.append(processor.tokenizer.decode(encoding["input_ids"][i][j]))
          bboxes.append(unnormal_box)
          token_section_num.append(i)
        else:
          # we have to update the word
          _index = bboxes.index(unnormal_box)
          if (token_section_num[_index] == i):
            # check if they're in a same section or not (documents with more than 512 tokens will divide to seperate
            # parts, so it's possible to have a word in both of the pages and we have to control that repetetive words
            # HERE: because they're in a same section, so we can merge them safely
            l_words[_index] = l_words[_index] + processor.tokenizer.decode(encoding["input_ids"][i][j])
          else:
            continue

    return bboxes, preds, l_words, image


def process_form_(json_df):

  labels = [x['LABEL'] for x in json_df]
  texts = [x['TEXT'] for x in json_df]
  cmb_list = []
  for i, j in enumerate(labels):
    cmb_list.append([labels[i], texts[i]])

  grouper = lambda l: [[k] + sum((v[1::] for v in vs), []) for k, vs in groupby(l, lambda x: x[0])]

  list_final = grouper(cmb_list)
  lst_final = []
  for x in list_final:
    json_dict = {}
    json_dict[x[0]] = (' ').join(x[1:])
    lst_final.append(json_dict)

  return lst_final
    

def createExcel(maindf, detailsdf, pdffile):
  outputPath = f'{pdffile}.xlsx'
  with pd.ExcelWriter(outputPath, engine='xlsxwriter') as writer:
    maindf.to_excel(writer, sheet_name='headers', index=False)
    detailsdf.to_excel(writer, sheet_name='details', index=False)
    worksheet1 = writer.sheets["headers"]
    for idx, col in enumerate(maindf):
      series = maindf[col]
      max_len = max((
        series.astype(str).map(len).max(),
        len(str(series.name))
      )) + 1
      worksheet1.set_column(idx, idx, max_len)
    worksheet2 = writer.sheets["details"]
    for idx, col in enumerate(detailsdf):
      series = detailsdf[col]
      max_len = max((
        series.astype(str).map(len).max(),
        len(str(series.name))
      )) + 1
      worksheet2.set_column(idx, idx, max_len)
  return outputPath


def visualize_image(final_bbox, final_preds, l_words, image,label2color):

      draw = ImageDraw.Draw(image)
      font = ImageFont.load_default()
      json_df = []

      for ix, (prediction, box) in enumerate(zip(final_preds, final_bbox)):
        if prediction is not None:
          predicted_label = iob_to_label(prediction).lower()
        if predicted_label not in ["others"]:
          draw.rectangle(box, outline=label2color[predicted_label])
          draw.text((box[0]+10, box[1]-10), text=predicted_label, fill=label2color[predicted_label], font=font)
        json_dict = {}
        json_dict['TEXT'] = l_words[ix]
        json_dict['LABEL'] = label2color[predicted_label]
        json_df.append(json_dict)
      return image, json_df

def rotate_image(image):
    extracted_text = pytesseract.image_to_string(image)
    # check if the image contains any text
    if not extracted_text:
        print("The image does not contain any text.")
        return None
    elif extracted_text.isspace():
        print("The image contains only spaces.")
        return None
    text = pytesseract.image_to_osd(image)
    angle = int(re.search('(?<=Rotate: )\d+', text).group(0))
    angle = 360 - angle
    rotated = ndimage.rotate(image, angle)
    data = Image.fromarray(rotated)
    return data


# correct the skewness of images
def correct_skew(image, delta=1, limit=5):
    def determine_score(arr, angle):
        data = inter.rotate(arr, angle, reshape=False, order=0)
        histogram = np.sum(data, axis=1, dtype=float)
        score = np.sum((histogram[1:] - histogram[:-1]) ** 2, dtype=float)
        return histogram, score

    # Convert the PIL Image object to a numpy array
    image = np.asarray(image.convert('L'), dtype=np.uint8)

    # Apply thresholding
    thresh = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]

    scores = []
    angles = np.arange(-limit, limit + delta, delta)
    for angle in angles:
        histogram, score = determine_score(thresh, angle)
        scores.append(score)
    best_angle = angles[scores.index(max(scores))]

    (h, w) = image.shape[:2]
    center = (w // 2, h // 2)
    M = cv2.getRotationMatrix2D(center, best_angle, 1.0)
    corrected = cv2.warpAffine(image, M, (w, h), flags=cv2.INTER_CUBIC, \
            borderMode=cv2.BORDER_REPLICATE)
    return best_angle, corrected


def removeBorders(img):
  result = img.copy()

  if len(result.shape) == 2:
      # if the input image is grayscale, convert it to BGR format
      result = cv2.cvtColor(result, cv2.COLOR_GRAY2BGR)

  gray = cv2.cvtColor(result, cv2.COLOR_BGR2GRAY) # convert to grayscale
  thresh = cv2.threshold(gray, 0, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)[1]

  # Remove horizontal lines
  horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (40,1))
  remove_horizontal = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, horizontal_kernel, iterations=2)
  cnts = cv2.findContours(remove_horizontal, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
  cnts = cnts[0] if len(cnts) == 2 else cnts[1]
  for c in cnts:
      cv2.drawContours(result, [c], -1, (255,255,255), 5)

  # Remove vertical lines
  vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,40))
  remove_vertical = cv2.morphologyEx(thresh, cv2.MORPH_OPEN, vertical_kernel, iterations=2)
  cnts = cv2.findContours(remove_vertical, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
  cnts = cnts[0] if len(cnts) == 2 else cnts[1]
  for c in cnts:
      cv2.drawContours(result, [c], -1, (255,255,255), 5)

  return result

def color2label_except(label2color, excluded_labels):
    """
    Inversely maps colors to labels based on the provided label2color dictionary,
    excluding the specified labels.

    Args:
        label2color (dict): Dictionary mapping labels to colors.
        excluded_labels (list): List of labels to exclude.

    Returns:
        dict: Dictionary mapping colors to labels, excluding the specified labels.
    """
    # Filter out excluded labels from label2color dictionary
    filtered_label2color = {label: color for label, color in label2color.items() if label not in excluded_labels}

    # Invert the filtered label2color dictionary to create color2label mapping
    return {v: k for k, v in filtered_label2color.items()}


def add_dataframe(df_main,labels_repeating,label2color):
  col_name_map =color2label_except(label2color,labels_repeating)

  columns = list(col_name_map.values())
  data = {col:[] for col in columns}
  for i in df_main:
      for k, v in i.items():
          if k in col_name_map:
              data[col_name_map[k]].append(v)

  # join the list of strings for each column and convert to a dataframe
  for col in columns:
      data[col] = [' '.join(data[col])]
  df_upper = pd.DataFrame(data)
  key_value_pairs = []
  for col in df_upper.columns:
      key_value_pairs.append({'key': col, 'value': df_upper[col][0]})
  df_key_value = pd.DataFrame(key_value_pairs)
  # Extract the value from the containertype column
  # container_quantity = int(df_key_value[df_key_value['key'] == 'containertype']['value'].str.split("x").str[0])

  # # Add a new row to the DataFrame
  # df_key_value = df_key_value.append({'key': 'containerquantity', 'value': container_quantity}, ignore_index=True)

  # # Extract the desired value from the containertype column
  # df_key_value.loc[df_key_value['key'] == 'containertype', 'value'] = df_key_value.loc[df_key_value['key'] == 'containertype', 'value'].str.split("x").str[1]
  return df_key_value


import statistics

def id2label_row(s, id2label):
  if s in id2label.values():
      return s
  return id2label[s]

def dist_height(y1,y2):
  return abs(int(y1)- int(y2))


def mergeBoxes(df):
  xmin, ymin, xmax, ymax = [], [], [], []
  for i in range(df.shape[0]):
    box = df['bbox_column'].iloc[i]
    xmin.append(box[0])
    ymin.append(box[1])
    xmax.append(box[2])
    ymax.append(box[3])
  return [min(xmin), min(ymin), max(xmax), max(ymax)]


def transform_dataset(df, merge_labels):
  df_temp = df.iloc[merge_labels] # a duplicate df with only concerned rows
  df_temp.reset_index(drop = True, inplace = True)
  text = ' '.join(df_temp['scr_column'])
  bbox = mergeBoxes(df_temp)
  retain_index = merge_labels[0] #the first index is parent row
  df['scr_column'].iloc[retain_index] = text
  df['bbox_column'].iloc[retain_index] = bbox
  # keeping the first & removing rest
  df = df.loc[~df.index.isin(merge_labels[1:]), :]
  df.reset_index(drop = True, inplace = True)
  return df


def box_overlap(box1, box2, horizontal_vertical):
     # Extract coordinates of box1
    x1_box1, y1_box1, x2_box1, y2_box1 = box1
    # Extract coordinates of box2
    x1_box2, y1_box2, x2_box2, y2_box2 = box2

    # Check if boxes overlap horizontally and vertically
    if horizontal_vertical == "H":
      if x1_box1 <= x2_box2 and x2_box1 >= x1_box2:
        return True
      else:
        return False
    if horizontal_vertical == "V":
      if y1_box1 <= y2_box2 and y2_box1 >= y1_box2:
        return True
      else:
        return False


def horizonatal_merging(df, font_length, perform_overlapping =False, x_change = 0, y_change = 0):
  fat_df = df.copy()
  for i in range(df.shape[0]):
    box = fat_df['bbox_column'].iloc[i]
    fat_df['bbox_column'].iloc[i] = [box[0]-x_change, box[1]-y_change, box[2]+x_change, box[3] + y_change]
  if perform_overlapping == True:
    redundant_rows = []
    for i in range(fat_df.shape[0]):
      box_i = fat_df.bbox_column[i]
      indices2merge = []

      for j in range(i+1, fat_df.shape[0]):
        if fat_df.preds_column[j] == fat_df.preds_column[i]: # if labels are same
          box_j = fat_df.bbox_column[j]
          if abs(box_i[1]-box_j[3])<font_length*1.5: # if the boxes are at height within 50% more range of font size
            # Check if boxes overlap horizontally
            if box_overlap(box_i, box_j, 'H'):
              indices2merge.append(j)
              df.scr_column[i] += df.scr_column[j]
              box_i = fat_df.bbox_column[j]  # finding the next connected word

      #once we have all indices that belong to a particular category
      # merging the boundong boxes, keeping them in 1st note/row.
      if len(indices2merge)!=0:
        df['bbox_column'].iloc[i] = mergeBoxes(df.loc[indices2merge])
      redundant_rows.extend(indices2merge)

    #now since all the transformation is done, lets remove the redundant rows
    return df.drop(redundant_rows)


def mergeLabelsExtensive_repeating(df_grouped, repeating_label):
  df_grouped.reset_index(inplace = True, drop = True)
  # this function merges same label entities together in a single instance.
  df_grouped = df_grouped[df_grouped['preds_column'].isin(repeating_label)]
  font_length =0
  count = 0
  while count<5 and count<df_grouped.shape[0]:
    box_i = df_grouped['bbox_column'].iloc[count] # box of current label contains [x1,y1,x3,y3]
    font_length += box_i[3]-box_i[1]
    count +=1
  font_length = font_length/5

  df_grouped = horizonatal_merging(df_grouped, font_length, True, 30, 0)
  return df_grouped



def group_labels_wrt_height(df):
  """
  This function groups the labels based on the height of the bounding box.
  """
  #sorting the lines based on heights using column 'y_axis'
  df = df.sort_values(by='y_axis')
  df.reset_index(inplace = True, drop = True)
  print("entering: group_labels_wrt_height ")

  final_yaxis = []
  final_scr = []
  final_pred = []

  current_group = []
  current_scr = []
  current_pred = []


  # Iterate through the column values
  for i, (value,scr,preds ) in enumerate(zip(df['y_axis'], df['scr_column'], df['preds_column'])):
      if i == 0:
          # Start a new group with the first value
          current_group.append(value)
          current_scr.append(scr)
          current_pred.append(preds)
      else:
          # Check if the difference between the current value and the previous value is <= 20
          if abs(value - df['y_axis'][i - 1]) <= 35:
              # Add the value to the current group
              current_group.append(value)
              current_scr.append(scr)
              current_pred.append(preds)
          else:
              # Start a new group with the current value
              final_yaxis.append(current_group)
              final_scr.append(current_scr)
              final_pred.append(current_pred)

              current_group = [value]
              current_scr = [scr]
              current_pred = [preds]


  # Add the last group
  final_yaxis.append(current_group)
  final_scr.append(current_scr)
  final_pred.append(current_pred)

  final_grouped_df = pd.DataFrame({'y_axis': final_yaxis, 'scr_column': final_scr, 'preds_column': final_pred})

  print("Grouped df after sorting based on height")
  print_df(final_grouped_df)

  return final_grouped_df



# searches the set of labels in the whole range
def search_labelSet_height_range(df, d, keyList):
  print("search_labelSet_height_range")
  keyDict = dict.fromkeys(keyList, []) #stores the required information as dictonary, then coverted to df
  print("Dataframe from extraction is going to happen: ")

  for i in range(df.shape[0]): # search df for right-bottom y axis value and check if it lies within the range d.
    box = df['bbox_column'].iloc[i]
    if dist_height(box[1], d)<50:
      key = df['preds_column'].iloc[i]
      keyDict[key] = df['scr_column'].iloc[i]
  return keyDict


def clean_colText(df, column):
  for i in range(df.shape[0]):
    df[column].iloc[i] = df[column].iloc[i].replace('[', '').replace('|', '').replace('+', '')
  return df


def find_repeatingLabels(df, labels_repeating):
  print("In find_repeatingLabels: ")
  row2drop = [] # dropping the rows that have been covered in previous dataframe
  for i in range(df.shape[0]):
    df['preds_column'].iloc[i] = id2label_row(df['preds_column'].iloc[i], id2label)
    if df['preds_column'].iloc[i] not in labels_repeating:
      row2drop.append(i)
  df.drop(index = row2drop, inplace = True)
  df = clean_colText(df, 'scr_column')

  print("removing non-tabular labels.")

  df = mergeLabelsExtensive_repeating(df,labels_repeating)
  print('after merging non-tabular labels: ')

  labels_repeating = list(set(list(df["preds_column"])))
  print("labels_repeating in this document are: ",labels_repeating)
  # adding extra column that contains the Y-axis information (Height)
  df['y_axis'] = np.NaN
  for i in range(df.shape[0]):
    box = df['bbox_column'].iloc[i]
    df['y_axis'].iloc[i] = box[1]

  print("After adding y-axis data in the dataframes: ")
  df = mergeLabelsExtensive(df)
  print("aftermerging the df extensively")
  print("Grouping the labels wrt heights: ")
  grouped_df = group_labels_wrt_height(df)

  #once labels are grouped, now we will create dictionaries for labels and values occuring in single line
  row_dicts = [] # will contains each row of df as single dictionary.
  for _, row in grouped_df.iterrows():
      row_dict = {}
      for preds, scr in zip(row['preds_column'], row['scr_column']):
          row_dict[preds] = scr
      row_dicts.append(row_dict)

  #creating new
  final_df = pd.DataFrame(columns=labels_repeating)
  for d in row_dicts:
    final_df = final_df.append(d, ignore_index=True)
  final_df = final_df.fillna('')
  return final_df


def mergeImageVertical(images):
   # pick the image which is the smallest, and resize the others to match it (can be arbitrary image shape here)
  min_shape = sorted( [(np.sum(i.size), i.size ) for i in images])[0][1]
  imgs_comb = np.hstack([i.resize(min_shape) for i in images])
  # for a vertical stacking it is simple: use vstack
  imgs_comb = np.vstack([i.resize(min_shape) for i in images])
  imgs_comb = Image.fromarray(imgs_comb)
  return imgs_comb

def perform_erosion(img):
    # Check if the image is already in grayscale
    if len(img.shape) == 2:
        gray = img
    else:
        # Convert the image to grayscale
        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # Define the kernel for erosion and dilation
    kernel = np.ones((3, 3), np.uint8)

    # Perform erosion followed by dilation
    erosion = cv2.erode(gray, kernel, iterations=1)
    dilation = cv2.dilate(erosion, kernel, iterations=1)

    # Double the size of the image
    double_size = cv2.resize(gray, None, fx=2, fy=2, interpolation=cv2.INTER_LINEAR)

    # Perform erosion on the doubled image
    double_erosion = cv2.erode(double_size, kernel, iterations=1)

    return double_erosion



def remove_leading_trailing_special_characters(input_string):
    cleaned_string = re.sub(r'^[^A-Za-z0-9]+|[^A-Za-z0-9]+$', '', str(input_string))
    return cleaned_string

def clean_dataframe(df):
    # Apply the remove_leading_trailing_special_characters function to all string columns
    for column in df.select_dtypes(include='object').columns:
        df[column] = df[column].apply(remove_leading_trailing_special_characters)

    # Remove rows with all NaN or blank values
    df = df.fillna('')  # Replace NaN values with blank
    return df

def mergeLabelsExtensive(df_grouped):
  i = 0
  while i < df_grouped.shape[0]:
    merge_labels = [i] # collects indices whose data has been merged, so we need to delete it now.
    label = df_grouped['preds_column'].iloc[i]
    box1 = df_grouped['bbox_column'].iloc[i]

    for j in range(i+1, df_grouped.shape[0]):
      box2 = df_grouped['bbox_column'].iloc[j]
      if label == df_grouped['preds_column'].iloc[j] and dist_height(box1[3], box2[3])<20: # which are in the vicinity of 20 pixels.
        merge_labels.append(j)
    print_df(df_grouped)
    df_grouped = transform_dataset(df_grouped, merge_labels)
    i = i+1
  return df_grouped

def multilabelsHandle(df, thermo_details):
  # Since 0 is assigned to 'others' and these values are not so important. We delete these values.
  df = df[df.preds_column != 0]
  df.reset_index(drop=True, inplace=True)
  for i in range(df.shape[0]):
    df['preds_column'].iloc[i] = id2label.get(df['preds_column'].iloc[i])
  df['preds_column'].unique()
  df_grouped = df.copy() #stores the index of relevant labels.
  df_grouped.shape[0]
  for i in range(df.shape[0]):
    if df['preds_column'].iloc[i] not in thermo_details:
      df_grouped.drop(i, inplace = True)
  df_grouped.reset_index(drop=True, inplace=True)

  keyList = df_grouped['preds_column'].unique()
  df_grouped = mergeLabelsExtensive(df_grouped)

  # extract the height of boxes
  df_grouped = extract_yaxis(df_grouped)
  shipment_labels = ['delivery_name','delivery_address','contact_phone']
  # shipment
  heights_shipment = get_heights(df_grouped, shipment_labels)

  # now segregating the other repeating values in df like measiure, weight, volume etc.
  # they will be containeed within the heights, as they act as boudaries.
  df_labelSet = pd.DataFrame(columns= thermo_details)
  for i in range(len(heights_shipment)):
    if i == len(heights_shipment)-1:
      new_df = search_labelSet_between_h1_h2(df_grouped, heights_shipment[i],  5000, keyList)
    else:
      new_df = search_labelSet_between_h1_h2(df_grouped, heights_shipment[i],  heights_shipment[i+1], keyList)
    df_labelSet = df_labelSet.append(new_df, ignore_index=True)
  return df_labelSet


def completepreprocess(pdffile,ocr_type):
  myDataFrame = pd.DataFrame()
  myDataFrame2 = pd.DataFrame()
  merge_pages=[]
  doc = fitz.open(pdffile)
  for i in range(0, len(doc)):
    page = doc.load_page(i)
    zoom = 2
    mat = fitz.Matrix(zoom, zoom)
    pix = page.get_pixmap(matrix = mat, dpi = 300)
    image = Image.frombytes("RGB", [pix.width, pix.height], pix.samples)
    ro_image = rotate_image(image)
    if ro_image is None:
      return None
    angle, skewed_image = correct_skew(ro_image)
    if skewed_image is None:
      return None
    remove_border = removeBorders(skewed_image)
    image = Image.fromarray(remove_border)
    width,height=image.size
    label2color = generate_unique_colors(id2label)
    width,height=image.size
    if ocr_type == "GoogleVisionOCR":
        words, boxes = process_image_GoogleVision(image, width, height)
    else:
        words, boxes = process_image_pytesseract(image, width, height)

    bbox, preds, words, image = process_image_encoding(model, processor, image, words, boxes,width,height)
    im, df_visualize = visualize_image(bbox, preds, words, image,label2color)
    df_main = process_form_(df_visualize)

    bbox_column = bbox
    preds_column = preds
    scr_column = words

    # dictionary of lists
    dict = {'bbox_column': bbox_column, 'preds_column': preds_column, 'scr_column': scr_column}
    df_single_page = pd.DataFrame(dict)
    labels_repeating = ['art_code', 'ref_code', 'detail_desc','lot_id','detail_qty','detail_um','detail_tare','detail_grossw','detail_netw','detail_origin','varieta','raccolta']
    df_repeating_page = find_repeatingLabels(df_single_page, labels_repeating)
    myDataFrame2= myDataFrame2.append(df_repeating_page,sort=False)

    df1=add_dataframe(df_main,labels_repeating,label2color).astype(str)
    myDataFrame= myDataFrame.append(df1,sort=False).reset_index(drop = True)
    myDataFrame['value'].apply(len)
    row2drop = []
    for i in range(myDataFrame.shape[0]):
      if len( myDataFrame['value'].iloc[i]) ==0:
        row2drop.append(i)
    myDataFrame.drop(index = row2drop, inplace = True)
    myDataFrame.reset_index(drop = True, inplace = True)
    myDataFrame = myDataFrame[myDataFrame["value"].notnull()]
    myDataFrame.drop_duplicates(subset=["key"],inplace=True)
    myDataFrame2 = myDataFrame2.loc[:, ~(myDataFrame2.apply(lambda x: all(isinstance(val, list) and len(val) == 0 for val in x)))]
    merge_pages.append(im)
  im2=mergeImageVertical(merge_pages)
  myDataFrame2 = clean_dataframe(myDataFrame2)
  myDataFrame = clean_dataframe(myDataFrame)
  myDataFrame = myDataFrame[myDataFrame['key'] != 'others']
  output_excel_path = createExcel(myDataFrame, myDataFrame2, pdffile.name)
  return im2,myDataFrame,myDataFrame2,output_excel_path


title = "Interactive demo: Transport Document Information Extraction model PDF/Images"
description = "Results will show up in a few seconds. This model is trained on only 1326 Images whereas 226 images are used for testing purposes. The annotated image can be opened in a new window for a better view."

css = """.output_image, .input_image {height: 600px !important}"""
examples = [["sample_doc.pdf"]]
           
iface = gr.Interface(
    fn=completepreprocess,
    inputs=[
        gr.components.File(label="PDF"),
        gr.components.Dropdown(label="Select the OCR", choices=["Pytesseract","GoogleVisionOCR"]),
    ],
    outputs=[
        gr.components.Image(type="pil", label="annotated image"),
        "dataframe",
        "dataframe",
        gr.File(label="Excel output")
    ],
    title=title,
    description=description,
    examples=examples,
    css=css
)

iface.launch(inline=True, debug=True)