Update app.py

app.py

@@ -1,17 +1,31 @@
 from transformers import AutoTokenizer
 import gradio as gr
+import os
 
-
-tokenizer = AutoTokenizer.from_pretrained("kiranr/gpt2-tokenizer")
+# Retrieve the Hugging Face token from secrets
+huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
 
 def tokenize(input_text):
-    tokens = tokenizer(input_text)["input_ids"]
-    return f"Number of tokens: {len(tokens)}"
+    palmyra_x_003_tokens = len(palmyra_x_003_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    gpt2_tokens = len(gpt2_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+    palmyra_x_004_tokens = len(palmyra_x_004_tokenizer(input_text, add_special_tokens=True)["input_ids"])
+
+    results = {
+        "Palmyra-X-004": palmyra_x_004_tokens,
+        "Palmyra-Fin & Med": palmyra_x_003_tokens,
+        "Palmyra-X-003": gpt2_tokens
+    }
+
+    # Sort the results in descending order based on token length
+    sorted_results = sorted(results.items(), key=lambda x: x[1], reverse=True)
+
+    return "\n".join([f"{model}: {tokens}" for model, tokens in sorted_results])
+
 
+if __name__ == "__main__":
+    palmyra_x_003_tokenizer = AutoTokenizer.from_pretrained("wassemgtk/palmyra-x-003-tokenizer", token=huggingface_token)
+    gpt2_tokenizer = AutoTokenizer.from_pretrained("gpt2")
+    palmyra_x_004_tokenizer = AutoTokenizer.from_pretrained("wassemgtk/palmyra-x-004-tokenizer", token=huggingface_token)
 
-demo = gr.Interface(
-    fn=tokenize,
-    inputs=gr.Textbox(lines=7),
-    outputs="text",
-)
-demo.launch()
+    iface = gr.Interface(fn=tokenize, inputs=gr.Textbox(label="Input Text", lines=19), outputs="text")
+    iface.launch()