5 เทคนิค Fine-Tuning LLM แบบประหยัดงบที่ใช้งานได้จริง

import torch
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer
from peft import LoraConfig, get_peft_model
from trl import SFTTrainer, SFTConfig
 
model_id = "meta-llama/Llama-3.1-8B-Instruct"
 
tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token
 
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    torch_dtype=torch.bfloat16,
    device_map="auto",
)
 
# A real, public instruction dataset, 15,011 human-written examples
dataset = load_dataset("databricks/databricks-dolly-15k", split="train")
 
def format_example(example):
    if example["context"]:
        text = (
            f"### Instruction:\n{example['instruction']}\n\n"
            f"### Context:\n{example['context']}\n\n"
            f"### Response:\n{example['response']}"
        )
    else:
        text = f"### Instruction:\n{example['instruction']}\n\n### Response:\n{example['response']}"
    return {"text": text}
 
dataset = dataset.map(format_example)
 
# LoRA config: r controls adapter capacity, all-linear targets every
# linear layer rather than hand-picking attention modules
lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules="all-linear",
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)
 
model = get_peft_model(model, lora_config)
model.print_trainable_parameters()  # confirms how few parameters actually train
 
training_args = SFTConfig(
    output_dir="./llama-3.1-8b-dolly-lora",
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    num_train_epochs=3,
    learning_rate=2e-4,
    logging_steps=10,
    save_strategy="epoch",
    bf16=True,
    dataset_text_field="text",
    max_length=1024,
)
 
trainer = SFTTrainer(model=model, args=training_args, train_dataset=dataset)
trainer.train()
trainer.save_model("./llama-3.1-8b-dolly-lora-final")

import torch
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
from trl import SFTTrainer, SFTConfig
 
model_id = "meta-llama/Llama-3.1-8B-Instruct"
 
# 4-bit NormalFloat quantization of the frozen base model
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
    bnb_4bit_use_double_quant=True,  # quantizes the quantization constants too
)
 
tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token
 
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    device_map="auto",
)
 
# Preps the quantized model for stable gradient flow through
# the LoRA adapters during training
model = prepare_model_for_kbit_training(model)
 
dataset = load_dataset("databricks/databricks-dolly-15k", split="train")
dataset = dataset.map(format_example)  # same formatting function as Technique 1
 
lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules="all-linear",
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)
model = get_peft_model(model, lora_config)
 
training_args = SFTConfig(
    output_dir="./llama-3.1-8b-dolly-qlora",
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    num_train_epochs=3,
    learning_rate=2e-4,
    logging_steps=10,
    save_strategy="epoch",
    bf16=True,
    dataset_text_field="text",
    max_length=1024,
    optim="paged_adamw_8bit",  # memory-efficient optimizer built for quantized training
)
 
trainer = SFTTrainer(model=model, args=training_args, train_dataset=dataset)
trainer.train()
trainer.save_model("./llama-3.1-8b-dolly-qlora-final")

from unsloth import FastLanguageModel
from datasets import load_dataset
from trl import SFTTrainer, SFTConfig
 
# Unsloth ships pre-quantized versions of popular models,
# which skips a separate quantization step at load time
model, tokenizer = FastLanguageModel.from_pretrained(
    model_name="unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit",
    max_seq_length=1024,
    load_in_4bit=True,
)
 
model = FastLanguageModel.get_peft_model(
    model,
    r=16,
    lora_alpha=32,
    target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
    lora_dropout=0.05,
    bias="none",
    use_gradient_checkpointing="unsloth",  # Unsloth's own memory-saving checkpointing
)
 
dataset = load_dataset("databricks/databricks-dolly-15k", split="train")
dataset = dataset.map(format_example)  # same formatting function as Technique 1
 
training_args = SFTConfig(
    output_dir="./llama-3.1-8b-dolly-unsloth",
    per_device_train_batch_size=4,
    gradient_accumulation_steps=4,
    num_train_epochs=3,
    learning_rate=2e-4,
    logging_steps=10,
    bf16=True,
    dataset_text_field="text",
    max_length=1024,
)
 
trainer = SFTTrainer(model=model, args=training_args, train_dataset=dataset)
trainer.train()
model.save_pretrained("./llama-3.1-8b-dolly-unsloth-final")

import torch
from datasets import load_dataset
from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
from peft import LoraConfig
from trl import DPOTrainer, DPOConfig
 
# Starting from the QLoRA checkpoint trained in Technique 2, adding
# a preference-alignment pass on top of the instruction-tuned model
model_id = "./llama-3.1-8b-dolly-qlora-final"
 
bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
)
 
tokenizer = AutoTokenizer.from_pretrained(model_id)
tokenizer.pad_token = tokenizer.eos_token
 
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    quantization_config=bnb_config,
    device_map="auto",
)
 
# A real, public preference dataset: prompts with a chosen and
# a rejected response, ranked by human or model judgment
dataset = load_dataset("Intel/orca_dpo_pairs", split="train").select(range(2000))
 
def format_dpo_example(example):
    return {
        "prompt": example["question"],
        "chosen": example["chosen"],
        "rejected": example["rejected"],
    }
 
dataset = dataset.map(format_dpo_example)
 
lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules="all-linear",
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM",
)
 
training_args = DPOConfig(
    output_dir="./llama-3.1-8b-dolly-dpo",
    per_device_train_batch_size=2,
    gradient_accumulation_steps=8,
    num_train_epochs=1,
    learning_rate=5e-5,
    beta=0.1,  # controls how strongly the model is pulled toward chosen responses
    logging_steps=10,
    bf16=True,
)
 
trainer = DPOTrainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    processing_class=tokenizer,
    peft_config=lora_config,
)
 
trainer.train()
trainer.save_model("./llama-3.1-8b-dolly-dpo-final")

def estimate_finetune_cost(
    num_examples: int,
    epochs: int,
    tokens_per_example: int = 512,
    tokens_per_second: int = 3000,   # rough throughput for an 8B QLoRA run
    hourly_rate: float = 0.34,       # adjust to the GPU and provider you're using
) -> dict:
    """A rough planning estimate, not a precise quote. Useful for
    deciding which GPU tier a job actually needs before renting one."""
    total_tokens = num_examples * tokens_per_example * epochs
    training_hours = (total_tokens / tokens_per_second) / 3600
    estimated_cost = training_hours * hourly_rate
 
return {
        "total_tokens": total_tokens,
        "estimated_hours": round(training_hours, 2),
        "estimated_cost_usd": round(estimated_cost, 2),
    }
 
# Full Dolly dataset, 3 epochs, on an RTX 4090
print(estimate_finetune_cost(num_examples=15011, epochs=3))