Large Language Model (LLM) Training

Overview

Large Language Models (LLMs) are transformer-based neural networks trained on massive text corpora to understand and generate human-like text. Modern LLMs range from 7B to over 1T parameters.

Training Pipeline

1. Data Collection & Preprocessing

Modern Data Pipeline:
1. Web scraping (Common Crawl, Reddit, etc.)
2. Quality filtering (CCNet, Gopher rules)
3. Deduplication (MinHash, SimHash)
4. Toxicity filtering (Perspective API)
5. Multi-lingual processing

2. Tokenization

Modern Tokenizers: SentencePiece, Tiktoken
Vocabulary Sizes: 32K-256K tokens
Special Tokens: BOS, EOS, padding, mask tokens

3. Pre-training

import torch
import torch.nn as nn
from transformers import (
    AutoTokenizer, 
    AutoModelForCausalLM,
    TrainingArguments,
    Trainer,
    DataCollatorForLanguageModeling
)
from datasets import load_dataset
import os

# Configuration
model_name = "microsoft/DialoGPT-small"
output_dir = "./llm-training-output"

# Load tokenizer and model
tokenizer = AutoTokenizer.from_pretrained(model_name)
tokenizer.pad_token = tokenizer.eos_token  # Set pad token

model = AutoModelForCausalLM.from_pretrained(
    model_name,
    torch_dtype=torch.float16,
    device_map="auto"
)

# Load and prepare dataset
dataset = load_dataset("wikitext", "wikitext-2-raw-v1")

def tokenize_function(examples):
    return tokenizer(
        examples["text"],
        truncation=True,
        padding=False,
        max_length=512,
        return_tensors=None
    )

tokenized_datasets = dataset.map(
    tokenize_function,
    batched=True,
    remove_columns=dataset["train"].column_names
)

# Data collator for language modeling
data_collator = DataCollatorForLanguageModeling(
    tokenizer=tokenizer,
    mlm=False,  # Causal language modeling
    return_tensors="pt"
)

# Training arguments
training_args = TrainingArguments(
    output_dir=output_dir,
    overwrite_output_dir=True,
    num_train_epochs=3,
    per_device_train_batch_size=4,
    per_device_eval_batch_size=4,
    gradient_accumulation_steps=4,
    warmup_steps=500,
    learning_rate=5e-5,
    weight_decay=0.01,
    logging_steps=100,
    eval_steps=500,
    save_steps=1000,
    evaluation_strategy="steps",
    save_strategy="steps",
    load_best_model_at_end=True,
    metric_for_best_model="eval_loss",
    greater_is_better=False,
    fp16=True,  # Mixed precision training
    dataloader_pin_memory=False,
    report_to=None  # Disable wandb/tensorboard if not needed
)

# Create trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["validation"],
    data_collator=data_collator,
    tokenizer=tokenizer,
)

# Start training
print("Starting training...")
trainer.train()

# Save the final model
trainer.save_model()
tokenizer.save_pretrained(output_dir)
print(f"Training completed. Model saved to {output_dir}")

# Example inference
def generate_text(prompt, max_length=100):
    inputs = tokenizer.encode(prompt, return_tensors="pt")
    with torch.no_grad():
        outputs = model.generate(
            inputs,
            max_length=max_length,
            num_return_sequences=1,
            temperature=0.7,
            do_sample=True,
            pad_token_id=tokenizer.eos_token_id
        )
    return tokenizer.decode(outputs[0], skip_special_tokens=True)

# Test the trained model
test_prompt = "The future of artificial intelligence"
generated_text = generate_text(test_prompt)
print(f"Generated: {generated_text}")

Key Components Explained: