Update app.py

app.py

@@ -1,13 +1,22 @@
 import gradio as gr
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
+import bitsandbytes as bnb
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 
-# Define the BLOOM model name
+# Define the model name
 model_name = "CreitinGameplays/ConvAI-9b"
 
+# Quantization configuration with bitsandbytes settings
+bnb_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_use_double_quant=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype=torch.bfloat16
+)
+
 # Load tokenizer and model
 tokenizer = AutoTokenizer.from_pretrained(model_name)
-model = AutoModelForCausalLM.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, quantization_config=bnb_config, low_cpu_mem_usage=True)
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 model.to(device)