Hugging Face's logo

Spaces:
Writer
/
token-counter
Running on CPU Upgrade

App Files Community
kiranr commited on
Commit
97b1935
·
1 Parent(s): cdd4984

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -1,8 +1,8 @@
1
- from transformers import AutoTokenizer
2
  import gradio as gr
3
 
4
 
5
- tokenizer = AutoTokenizer.from_pretrained("kiranr/gpt2-tokenizer")
6
 
7
  def tokenize(input_text):
8
  tokens = tokenizer(input_text)["input_ids"]
 
1
+ from transformers import GPT2Tokenizer
2
  import gradio as gr
3
 
4
 
5
+ tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
6
 
7
  def tokenize(input_text):
8
  tokens = tokenizer(input_text)["input_ids"]