add axolotl files

Signed-off-by: Matt Williams <m@technovangelist.com>

2025-01-30-axolotl/Modelfile

@@ -0,0 +1,18 @@
+FROM outputs/lora-out/merged
+
+PARAMETER stop "<|start_header_id|>"
+PARAMETER stop "<|end_header_id|>"
+PARAMETER stop "<|eot_id|>"
+PARAMETER stop "<|reserved_special_token"
+PARAMETER num_ctx 4096
+PARAMETER stop "Ahoy there"
+
+PARAMETER repeat_penalty 1.15
+PARAMETER temperature 1.2
+PARAMETER top_p 0.9
+PARAMETER frequency_penalty 0.2
+PARAMETER presence_penalty 0.1
+TEMPLATE """{{ if .System }}<|start_header_id|>system<|end_header_id|>
+{{ .System }}<|eot_id|>{{ end }}{{ if .Prompt }}<|start_header_id|>user<|end_header_id|>
+{{ .Prompt }}<|eot_id|>{{ end }}<|start_header_id|>assistant<|end_header_id|>
+{{ .Response }}<|eot_id|>"""

2025-01-30-axolotl/deepspeed_configs/zero2.json

@@ -0,0 +1,22 @@
+{
+  "zero_optimization": {
+    "stage": 2,
+    "allgather_partitions": true,
+    "allgather_bucket_size": 2e8,
+    "overlap_comm": true,
+    "reduce_scatter": true,
+    "reduce_bucket_size": 2e8,
+    "contiguous_gradients": true
+  },
+  "fp16": {
+    "enabled": false
+  },
+  "bf16": {
+    "enabled": true
+  },
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "zero_allow_untested_optimizer": true
+}

2025-01-30-axolotl/pirate.yml

@@ -4,8 +4,10 @@ strict: false
 
 chat_template: llama3
 datasets:
-  - path: prepared_dataset
+  - path: winglian/pirate-ultrachat-10k
     type: chat_template
+    message_field_role: role
+    message_field_content: content
 dataset_prepared_path: last_run_prepared
 val_set_size: 0.005
 output_dir: ./outputs/lora-out
@@ -30,7 +32,6 @@ gradient_accumulation_steps: 2
 micro_batch_size: 2
 num_epochs: 2
 optimizer: adamw_bnb_8bit
-lr_scheduler: cosine
 learning_rate: 0.0002
 
 train_on_inputs: false