Migrate to tokenizers.js

script.js

@@ -1,16 +1,21 @@
-import { AutoTokenizer } from "https://nobsdelivr.private.coffee/npm/@huggingface/transformers@4.0.0";
+import { Tokenizer } from "https://nobsdelivr.private.coffee/npm/@huggingface/tokenizers@0.1.3";
 
 let tokenizeButton = document.querySelector("#tokenize");
 tokenizeButton.addEventListener("click", async () => {
   tokenizeButton.disabled = true;
-  let tokenizer = await AutoTokenizer.from_pretrained(document.querySelector("#model").value);
+  let modelId = document.querySelector("#model").value;
+  let tokenizerJSONSocket = await fetch(`https://huggingface.co/${modelId}/resolve/main/tokenizer.json`);
+  let tokenizerJSON = await tokenizerJSONSocket.json();
+  let tokenizerConfigSocket = await fetch(`https://huggingface.co/${modelId}/resolve/main/tokenizer_config.json`);
+  let tokenizerConfig = await tokenizerConfigSocket.json();
+  let tokenizer = new Tokenizer(tokenizerJSON, tokenizerConfig);
   let text = document.querySelector("#text").value.replace(/[^\S ]/g, "");
   let tokenized = tokenizer.encode(text);
   document.querySelector("#totalLength").textContent = `Total length: ${tokenized.length}`;
   let output = document.querySelector("#output");
   output.contentDocument.body.innerHTML = "";
   let colors = ["#ff0000", "#ff8700", "#ffd300", "#deff0a", "#a1ff0a", "#0aff99", "#0aefff", "#147df5", "#580aff", "#be0aff"].sort(() => Math.random() - 0.5);
-  tokenized.forEach((token, index) => {
+  tokenized.ids.forEach((token, index) => {
     let content = document.createElement("span");
     content.textContent = tokenizer.decode([token]);
     content.style.backgroundColor = colors[index % colors.length];