Upload gemma_2_gptq_inference.ipynb with huggingface_hub

gemma_2_gptq_inference.ipynb

@@ -0,0 +1,304 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "abba1d18-4a4d-4978-9766-82f3940e411e",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Requirement already satisfied: auto-gptq in /opt/conda/lib/python3.10/site-packages (0.7.1)\n",
+      "Requirement already satisfied: accelerate>=0.26.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.31.0)\n",
+      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (2.20.0)\n",
+      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.2.0)\n",
+      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (1.26.3)\n",
+      "Requirement already satisfied: rouge in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (1.0.1)\n",
+      "Requirement already satisfied: gekko in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (1.1.3)\n",
+      "Requirement already satisfied: torch>=1.13.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (2.2.0)\n",
+      "Requirement already satisfied: safetensors in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.4.3)\n",
+      "Requirement already satisfied: transformers>=4.31.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (4.43.0.dev0)\n",
+      "Requirement already satisfied: peft>=0.5.0 in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (0.11.1)\n",
+      "Requirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from auto-gptq) (4.66.4)\n",
+      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (23.1)\n",
+      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (5.9.0)\n",
+      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (6.0.1)\n",
+      "Requirement already satisfied: huggingface-hub in /opt/conda/lib/python3.10/site-packages (from accelerate>=0.26.0->auto-gptq) (0.23.4)\n",
+      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (3.13.1)\n",
+      "Requirement already satisfied: typing-extensions>=4.8.0 in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (4.9.0)\n",
+      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (1.12)\n",
+      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (3.1)\n",
+      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (3.1.2)\n",
+      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from torch>=1.13.0->auto-gptq) (2023.12.2)\n",
+      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.31.0->auto-gptq) (2024.5.15)\n",
+      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from transformers>=4.31.0->auto-gptq) (2.32.3)\n",
+      "Requirement already satisfied: tokenizers<0.20,>=0.19 in /opt/conda/lib/python3.10/site-packages (from transformers>=4.31.0->auto-gptq) (0.19.1)\n",
+      "Requirement already satisfied: pyarrow>=15.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (16.1.0)\n",
+      "Requirement already satisfied: pyarrow-hotfix in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (0.6)\n",
+      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (0.3.8)\n",
+      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (2.2.2)\n",
+      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (3.4.1)\n",
+      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (0.70.16)\n",
+      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets->auto-gptq) (3.9.5)\n",
+      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from rouge->auto-gptq) (1.16.0)\n",
+      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (1.3.1)\n",
+      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (23.1.0)\n",
+      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (1.4.1)\n",
+      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (6.0.5)\n",
+      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (1.9.4)\n",
+      "Requirement already satisfied: async-timeout<5.0,>=4.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets->auto-gptq) (4.0.3)\n",
+      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (2.0.4)\n",
+      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (3.4)\n",
+      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (1.26.18)\n",
+      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->transformers>=4.31.0->auto-gptq) (2023.11.17)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.13.0->auto-gptq) (2.1.3)\n",
+      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->auto-gptq) (2.9.0.post0)\n",
+      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->auto-gptq) (2023.3.post1)\n",
+      "Requirement already satisfied: tzdata>=2022.7 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets->auto-gptq) (2024.1)\n",
+      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.13.0->auto-gptq) (1.3.0)\n",
+      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
+      "\u001b[0mERROR: unknown command \"insall\" - maybe you meant \"install\"\n"
+     ]
+    }
+   ],
+   "source": [
+    "!pip install auto-gptq\n",
+    "!pip insall --upgrade transformers optimum accelerate"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6cefbcaa-9c28-4d68-b15e-9889f73332dd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!pip install parallelformers"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "id": "bf65bf24-e67b-4af7-8890-fba4182584e0",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "c7abb0ecf7aa42b3a5ade8aa6644cbbc",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from huggingface_hub import HfApi, notebook_login\n",
+    "\n",
+    "notebook_login()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "id": "fe3c29b7-1946-4f44-9d07-e9798e656bc2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
+    "import torch"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "id": "1bff45ce-9a64-4ceb-97e0-0e2ab5f15c45",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Hyperparams\n",
+    "\n",
+    "quant_model_repo = \"Granther/Gemma-2-9B-Instruct-4Bit-GPTQ\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "id": "5b060dc1-4a95-476f-8707-412a4452dd89",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/opt/conda/lib/python3.10/site-packages/transformers/modeling_utils.py:4565: FutureWarning: `_is_quantized_training_enabled` is going to be deprecated in transformers 4.39.0. Please use `model.hf_quantizer.is_trainable` instead\n",
+      "  warnings.warn(\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "fceb82427c58488da421d6415e0df516",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "from optimum.gptq import load_quantized_model\n",
+    "\n",
+    "model = AutoModelForCausalLM.from_pretrained(quant_model_repo, device_map='auto')\n",
+    "tokenizer = AutoTokenizer.from_pretrained(quant_model_repo)\n",
+    "\n",
+    "#model = load_quantized_model(model, device_map=\"auto\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "id": "440a9d11-8fb0-4ded-885f-52a226e326d1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "tensor([[    2,   106,  1645,   108, 27445,   682,  1105, 20731,   107,   108,\n",
+       "           106,  2516,   108]])"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "input_text = [\n",
+    "    {\"role\": \"user\", \"content\": \"Tell me about paris\"},\n",
+    "]\n",
+    "\n",
+    "prompt = tokenizer.apply_chat_template(input_text, tokenize=True, add_generation_prompt=True, return_tensors='pt')\n",
+    "# >>> '<bos><start_of_turn>user\\nTell me about paris<end_of_turn>\\n<start_of_turn>model\\n' \n",
+    "\n",
+    "prompt\n",
+    "\n",
+    "#prompt = {key: value.to(model.device) for key, value in prompt}\n",
+    "\n",
+    "# with torch.no_grad():\n",
+    "#     outputs = model.generate(**inputs, max_length=100, num_return_sequences=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "id": "9ed28a1b-2c65-4762-8b7c-4338195729af",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\"Tell me about python \\n\\nLet's talk about Python!\\n\\n**What is Python?**\\n\\nPython is a high-level, interpreted, general-purpose programming language.  Let's break down what that means:\\n\\n* **High-level:** Python is designed to be easy for humans to read and write. It uses words and symbols that are closer to natural language than the low-level instructions computers directly understand.\\n* **Interpreted:** Python code is executed line\""
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "outputs = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
+    "outputs"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8f040240-5bb0-41a1-a823-678f5ffb3dc2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "chat = [\n",
+    "    { \"role\": \"user\", \"content\": \"Write a hello world program\" },\n",
+    "]\n",
+    "prompt = tokenizer.apply_chat_template(chat, tokenize=False, add_generation_prompt=True)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "id": "9cf56d96-bf6c-43cd-a062-aa8368b3406b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "\n",
+    "pipe = pipeline('text-generation', model=model, tokenizer=quant_model_repo)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "id": "eb34f0bc-9a4b-4f8b-baf9-9869e2d1bef1",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "[{'generated_text': 'hello, how are you?\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n'}]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "pipe(\"hello, how are you\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "52a00f3b-e601-4800-9708-5f656c54ffc8",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}