hotdogs/gemma4-26b-python-18k-alpaca-lora

# v6_26b_pipeline.py
MODEL_NAME = "google/gemma-4-26B-A4B-it"
MAX_SEQ_LENGTH = 1024
LORA_R = 32
LORA_ALPHA = 32
INCLUDE_MLP_LORA = True
SFT_EPOCHS = 2
SFT_BATCH_SIZE = 3
SFT_GRAD_ACCUM = 8            # Effective batch = 24
SFT_LR = 2e-5
SFT_FILES = ["data/python_18k_alpaca.jsonl"]

→ Epoch 1 avg: 0.7003
Step 800: Loss 0.4429  (epoch 2)
Step 950: Loss 0.4298
Step 1100: Loss 0.4486
Step 1250: Loss 0.4409
Step 1400: Loss 0.4113
Step 1500: Loss 0.4309
→ Epoch 2 avg: 0.4330 🎯 Best!

from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import PeftModel
import torch

model = AutoModelForCausalLM.from_pretrained(
    "google/gemma-4-26B-A4B-it",
    torch_dtype=torch.bfloat16,
    device_map="auto"
)
model = PeftModel.from_pretrained(model, "hotdogs/gemma4-26b-python-18k-alpaca-lora")

tokenizer = AutoTokenizer.from_pretrained("google/gemma-4-26B-A4B-it")
messages = [
    {"role": "system", "content": "You are a Python programming assistant."},
    {"role": "user", "content": "Write a Python function to find all prime numbers up to N."}
]
inputs = tokenizer.apply_chat_template(messages, tokenize=True, return_tensors="pt", add_generation_prompt=True).to(model.device)
outputs = model.generate(inputs, max_new_tokens=1024, temperature=0.7, do_sample=True)
print(tokenizer.decode(outputs[0], skip_special_tokens=True))

adapter_model.safetensors   — LoRA weights (227 MB)
adapter_config.json         — r=32, alpha=32
tokenizer.json              — Gemma 4 tokenizer (31 MB)
v6_26b_pipeline.py          — Training script