Delete README.md

README.md

@@ -1,239 +0,0 @@
----
-library_name: peft
-license: apache-2.0
-base_model: mistralai/Mistral-Small-24B-Instruct-2501
-tags:
-- axolotl
-- generated_from_trainer
-datasets:
-- ToastyPigeon/some-rp-extended
-- ToastyPigeon/gutenberg-sft
-- ToastyPigeon/fujin-filtered-instruct
-- ToastyPigeon/adventure-combined-no-slop-matches-instruct
-- ToastyPigeon/nifty-multi-filtered
-model-index:
-- name: new-ms-rp-test-ws
-  results: []
----
-
-<!-- This model card has been generated automatically according to the information the Trainer had access to. You
-should probably proofread and complete it, then remove this comment. -->
-
-[<img src="https://raw.githubusercontent.com/axolotl-ai-cloud/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/axolotl-ai-cloud/axolotl)
-<details><summary>See axolotl config</summary>
-
-axolotl version: `0.6.0`
-```yaml
-# git clone https://github.com/axolotl-ai-cloud/axolotl
-# cd axolotl
-# git checkout 844331005c1ef45430ff26b9f42f757dce6ee66a
-# pip3 install packaging ninja huggingface_hub[cli]
-# pip3 install -e '.[flash-attn,deepspeed]'
-# huggingface-cli login --token $hf_key && wandb login $wandb_key
-# python -m axolotl.cli.preprocess new-ms-rp-test.yml
-# accelerate launch -m axolotl.cli.train new-ms-rp-test.yml
-# python -m axolotl.cli.merge_lora new-ms-rp-test.yml
-# huggingface-cli upload ToastyPigeon/new-ms-rp-test train-workspace/merged . --exclude "*.md"
-
-# git clone https://github.com/axolotl-ai-cloud/axolotl && cd axolotl && pip3 install packaging ninja huggingface_hub[cli] && pip3 install cut-cross-entropy[transformers] && pip3 install -e '.[flash-attn,deepspeed]' && cd .. && huggingface-cli login --token $hf_key && wandb login $wandb_key
-
-# Model
-base_model: mistralai/Mistral-Small-24B-Instruct-2501
-model_type: AutoModelForCausalLM
-tokenizer_type: AutoTokenizer
-
-load_in_8bit: false
-load_in_4bit: true
-strict: false
-bf16: true
-fp16:
-tf32: false
-flash_attention: true
-special_tokens:
-
-# Output
-output_dir: ./train-workspace
-hub_model_id: ToastyPigeon/new-ms-rp-test-ws
-hub_strategy: "all_checkpoints"
-resume_from_checkpoint:
-saves_per_epoch: 2
-
-# Data
-sequence_len: 4096 # fits
-min_sample_len: 128
-dataset_prepared_path: last_run_prepared
-datasets:
-  - path: ToastyPigeon/some-rp-extended
-    type: chat_template
-    field_messages: conversations
-    message_field_role: from
-    message_field_content: value
-  - path: ToastyPigeon/gutenberg-sft
-    type: chat_template
-    field_messages: conversations
-    message_field_role: from
-    message_field_content: value
-  - path: ToastyPigeon/fujin-filtered-instruct
-    type: chat_template
-    field_messages: conversations
-    message_field_role: from
-    message_field_content: value
-  - path: ToastyPigeon/adventure-combined-no-slop-matches-instruct
-    type: chat_template
-    field_messages: conversations
-    message_field_role: from
-    message_field_content: value
-    split: train[:1000]
-  - path: ToastyPigeon/nifty-multi-filtered
-    type: completion
-    field: text
-    split: train[:1000]
-warmup_steps: 5
-shuffle_merged_datasets: true
-sample_packing: true
-pad_to_sequence_len: true
-chat_template: jinja
-chat_template_jinja: "{%- set default_system_message = \"You are an unbiased, unfiltered, and unaligned AI.\" %}\n\n{{- bos_token }}\n\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content'] %}\n    {%- set loop_messages = messages[1:] %}\n{%- else %}\n    {%- set system_message = default_system_message %}\n    {%- set loop_messages = messages %}\n{%- endif %}\n{{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}\n\n{%- for message in loop_messages %}\n    {%- if message['role'] == 'user' %}\n        {{- '[INST]' + message['content'] + '[/INST]' }}\n    {%- elif message['role'] == 'system' %}\n        {{- '[SYSTEM_PROMPT]' + message['content'] + '[/SYSTEM_PROMPT]' }}\n    {%- elif message['role'] == 'assistant' %}\n        {{- message['content'] + eos_token }}\n    {%- else %}\n        {{- raise_exception('Only user, system and assistant roles are supported!') }}\n    {%- endif %}\n{%- endfor %}"
-
-# Batching
-num_epochs: 1
-gradient_accumulation_steps: 1
-micro_batch_size: 4
-eval_batch_size: 1
-
-# Evaluation
-val_set_size: 100
-evals_per_epoch: 10
-eval_table_size:
-eval_max_new_tokens: 256
-eval_sample_packing: false
-
-save_safetensors: true
-
-# WandB
-wandb_project: MS-Rp-Test
-#wandb_entity:
-
-gradient_checkpointing: 'unsloth'
-#gradient_checkpointing_kwargs:
-#  use_reentrant: false
-
-unsloth_cross_entropy_loss: true
-#unsloth_lora_mlp: true
-#unsloth_lora_qkv: true
-#unsloth_lora_o: true
-
-# LoRA
-adapter: qlora
-lora_model_dir:
-lora_r: 16
-lora_alpha: 32
-lora_dropout: 0.5
-lora_target_linear: 
-lora_fan_in_fan_out:
-lora_target_modules:
-  - gate_proj
-  - down_proj
-  - up_proj
-  - q_proj
-  - v_proj
-  - k_proj
-  - o_proj
-lora_modules_to_save:
-
-# Optimizer
-optimizer: paged_ademamix_8bit # adamw_8bit
-lr_scheduler: cosine
-learning_rate: 5e-5
-cosine_min_lr_ratio: 0.1
-weight_decay: 0.01
-max_grad_norm: 1.0
-
-# Misc
-train_on_inputs: false
-group_by_length: false
-early_stopping_patience:
-local_rank:
-logging_steps: 1
-xformers_attention:
-debug:
-deepspeed: /workspace/axolotl/deepspeed_configs/zero3_bf16.json # previously blank
-fsdp:
-fsdp_config:
-
-plugins:
-  - axolotl.integrations.liger.LigerPlugin
-#  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
-#cut_cross_entropy: true
-liger_rope: true
-liger_rms_norm: true
-liger_layer_norm: true
-liger_glu_activation: true
-liger_fused_linear_cross_entropy: true
-
-gc_steps: 10
-seed: 69
-```
-
-</details><br>
-
-# new-ms-rp-test-ws
-
-This model is a fine-tuned version of [mistralai/Mistral-Small-24B-Instruct-2501](https://huggingface.co/mistralai/Mistral-Small-24B-Instruct-2501) on the ToastyPigeon/some-rp-extended, the ToastyPigeon/gutenberg-sft, the ToastyPigeon/fujin-filtered-instruct, the ToastyPigeon/adventure-combined-no-slop-matches-instruct and the ToastyPigeon/nifty-multi-filtered datasets.
-It achieves the following results on the evaluation set:
-- Loss: 1.9832
-
-## Model description
-
-More information needed
-
-## Intended uses & limitations
-
-More information needed
-
-## Training and evaluation data
-
-More information needed
-
-## Training procedure
-
-### Training hyperparameters
-
-The following hyperparameters were used during training:
-- learning_rate: 5e-05
-- train_batch_size: 4
-- eval_batch_size: 1
-- seed: 69
-- distributed_type: multi-GPU
-- num_devices: 4
-- total_train_batch_size: 16
-- total_eval_batch_size: 4
-- optimizer: Use OptimizerNames.PAGED_ADEMAMIX_8BIT and the args are:
-No additional optimizer arguments
-- lr_scheduler_type: cosine
-- lr_scheduler_warmup_steps: 5
-- num_epochs: 1.0
-
-### Training results
-
-| Training Loss | Epoch  | Step | Validation Loss |
-|:-------------:|:------:|:----:|:---------------:|
-| 1.9624        | 0.0056 | 1    | 2.0950          |
-| 2.0035        | 0.1017 | 18   | 2.0137          |
-| 2.0876        | 0.2034 | 36   | 2.0019          |
-| 1.9243        | 0.3051 | 54   | 1.9959          |
-| 2.0775        | 0.4068 | 72   | 1.9916          |
-| 1.8296        | 0.5085 | 90   | 1.9892          |
-| 1.9782        | 0.6102 | 108  | 1.9868          |
-| 2.1268        | 0.7119 | 126  | 1.9848          |
-| 1.7833        | 0.8136 | 144  | 1.9841          |
-| 2.1287        | 0.9153 | 162  | 1.9832          |
-
-
-### Framework versions
-
-- PEFT 0.14.0
-- Transformers 4.48.1
-- Pytorch 2.5.1+cu124
-- Datasets 3.2.0
-- Tokenizers 0.21.0