Update README.md to include GPTQModel usage.
Browse files
README.md
CHANGED
|
@@ -83,17 +83,3 @@ model = GPTQModel.load(model_id)
|
|
| 83 |
result = model.generate("Uncovering deep insights")[0] # tokens
|
| 84 |
print(model.tokenizer.decode(result)) # string output
|
| 85 |
```
|
| 86 |
-
|
| 87 |
-
### Run the model with GPTQModel
|
| 88 |
-
|
| 89 |
-
GPTQModel package: https://github.com/ModelCloud/GPTQModel
|
| 90 |
-
|
| 91 |
-
```
|
| 92 |
-
pip install -v gptqmodel=="1.8.0" --no-build-isolation
|
| 93 |
-
from gptqmodel import GPTQModel
|
| 94 |
-
|
| 95 |
-
model_id = 'iproskurina/Mistral-7B-v0.3-GPTQ-8bit-g128'
|
| 96 |
-
model = GPTQModel.load(model_id)
|
| 97 |
-
result = model.generate("Uncovering deep insights")[0] # tokens
|
| 98 |
-
print(model.tokenizer.decode(result)) # string output
|
| 99 |
-
```
|
|
|
|
| 83 |
result = model.generate("Uncovering deep insights")[0] # tokens
|
| 84 |
print(model.tokenizer.decode(result)) # string output
|
| 85 |
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|