From c63184a4de7287425e821954da43090c118a2022 Mon Sep 17 00:00:00 2001
From: Matt Williams <m@technovangelist.com>
Date: Fri, 24 Jan 2025 09:02:12 -0800
Subject: [PATCH] Add example for finetuning with unsloth

Signed-off-by: Matt Williams <m@technovangelist.com>
---
 2025-01-24-unslothfinetune/finetune.py      |  64 ++++++++++++
 2025-01-24-unslothfinetune/readme.md        | 103 ++++++++++++++++++++
 2025-01-24-unslothfinetune/requirements.txt |  10 ++
 3 files changed, 177 insertions(+)
 create mode 100644 2025-01-24-unslothfinetune/finetune.py
 create mode 100644 2025-01-24-unslothfinetune/readme.md
 create mode 100644 2025-01-24-unslothfinetune/requirements.txt

diff --git a/2025-01-24-unslothfinetune/finetune.py b/2025-01-24-unslothfinetune/finetune.py
new file mode 100644
index 0000000..12a3c42
--- /dev/null
+++ b/2025-01-24-unslothfinetune/finetune.py
@@ -0,0 +1,64 @@
+from unsloth import FastLanguageModel, is_bfloat16_supported
+import torch
+from unsloth.chat_templates import get_chat_template
+from datasets import load_dataset
+from trl import SFTTrainer
+from transformers import Trainer, TrainingArguments
+
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+model_name="mistralai/Mistral-7B-v0.1", 
+max_seq_length=2048
+)
+
+model = FastLanguageModel.get_peft_model(model)
+
+tokenizer = get_chat_template(
+tokenizer, 
+mapping = {"role" : "from", "content" : "value", "user" : "human", "assistant" : "gpt"}
+)
+
+origdataset = load_dataset("philschmid/guanaco-sharegpt-style", split="train")
+conversations_dataset = origdataset.select_columns(['conversations'])
+
+dataset = conversations_dataset.map(
+lambda x: {
+"text": tokenizer.apply_chat_template(
+x["conversations"],
+tokenize=False,
+add_generation_prompt=False
+)
+},
+batched=True,
+batch_size=100,
+desc="Formatting conversations"
+)
+
+trainer = SFTTrainer(
+model = model,
+tokenizer = tokenizer,
+train_dataset = dataset,
+dataset_text_field = "text",
+dataset_num_proc = 2,
+max_seq_length = 2048,
+packing = False, # Can make training 5x faster for short sequences.
+args = TrainingArguments(
+per_device_train_batch_size = 2,
+gradient_accumulation_steps = 4,
+warmup_steps = 5,
+max_steps = 60,
+learning_rate = 2e-4,
+fp16 = not is_bfloat16_supported(),
+bf16 = is_bfloat16_supported(),
+logging_steps = 1,
+optim = "adamw_8bit",
+weight_decay = 0.01,
+lr_scheduler_type = "linear",
+seed = 3407,
+output_dir = "outputs",
+report_to = "none", # Use this for WandB etc
+),
+)
+trainer.train()
+
+model.save_pretrained_gguf("ggufmodel", tokenizer, quantization_method = "q4_k_m")
diff --git a/2025-01-24-unslothfinetune/readme.md b/2025-01-24-unslothfinetune/readme.md
new file mode 100644
index 0000000..639fd38
--- /dev/null
+++ b/2025-01-24-unslothfinetune/readme.md
@@ -0,0 +1,103 @@
+# Unsloth Mistral-7B Finetuning Example
+
+This repository demonstrates how to finetune Mistral-7B using Unsloth, a library that optimizes LLM training. The example uses the Guanaco dataset in ShareGPT format for instruction tuning.
+
+## Installation
+
+1. Clone this repository:
+```bash
+git clone [your-repo-url]
+cd [repo-name]
+```
+
+2. Install dependencies from requirements.txt:
+```bash
+pip install -r requirements.txt
+```
+
+The requirements.txt includes all necessary packages for running the finetuning script.
+
+## Hardware Requirements
+
+- Minimum 16GB GPU VRAM
+- CUDA-compatible GPU with 7.0 or higher compute capability
+- Linux or Windows
+
+## Quick Start
+
+1. Clone this repository:
+```bash
+git clone https://github.com/technovangelist/videoprojects.git
+cd videoprojects/2025-01-24-unslothfinetune
+```
+
+2. Install dependencies:
+```bash
+pip install -r requirements.txt
+```
+
+3. Run the training script:
+```bash
+python train.py
+```
+
+## Code Explanation
+
+### Model Initialization
+```python
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name="mistralai/Mistral-7B-v0.1", 
+    max_seq_length=2048
+)
+```
+Loads the Mistral-7B model with Unsloth optimizations.
+
+### Chat Template Configuration
+```python
+tokenizer = get_chat_template(
+    tokenizer, 
+    mapping = {"role": "from", "content": "value", "user": "human", "assistant": "gpt"}
+)
+```
+Configures the chat template for proper conversation formatting.
+
+### Dataset Loading
+```python
+origdataset = load_dataset("philschmid/guanaco-sharegpt-style", split="train")
+conversations_dataset = origdataset.select_columns(['conversations'])
+```
+Loads the Guanaco dataset in ShareGPT format.
+
+### Training Configuration
+Key training parameters:
+- Batch size: 2
+- Gradient accumulation steps: 4
+- Learning rate: 2e-4
+- Maximum steps: 60
+- Sequence length: 2048
+
+### Model Export
+```python
+model.save_pretrained_gguf("ggufmodel", tokenizer, quantization_method = "q4_k_m")
+```
+Saves the model in GGUF format with q4_k_m quantization.
+
+## Training Parameters
+
+| Parameter | Value |
+|-----------|--------|
+| Learning Rate | 2e-4 |
+| Batch Size | 2 |
+| Gradient Accumulation | 4 |
+| Max Steps | 60 |
+| Warmup Steps | 5 |
+| Weight Decay | 0.01 |
+| Optimizer | AdamW 8-bit |
+| Scheduler | Linear |
+
+## Output
+
+The finetuned model will be saved in:
+- `outputs/` - Checkpoint files
+- `ggufmodel/` - GGUF format for inference
+
diff --git a/2025-01-24-unslothfinetune/requirements.txt b/2025-01-24-unslothfinetune/requirements.txt
new file mode 100644
index 0000000..4220e77
--- /dev/null
+++ b/2025-01-24-unslothfinetune/requirements.txt
@@ -0,0 +1,10 @@
+unsloth>=0.3.0
+torch>=2.0.0
+transformers>=4.36.0
+datasets>=2.14.0
+trl>=0.7.4
+accelerate>=0.24.0
+bitsandbytes>=0.41.0
+scipy>=1.11.0
+click>=8.0.0  # Required by wandb/trl
+wandb>=0.15.0  # Optional but good to have explicitly listed
\ No newline at end of file