Optimization Comparison Examples

This section provides comprehensive examples for comparing different optimizers and loss functions using Torchium’s benchmarking tools.

Quick Benchmarking

Simple Regression Benchmark

import torch
import torch.nn as nn
import torchium
from torchium.benchmarks import QuickBenchmark

# Initialize benchmark
benchmark = QuickBenchmark()

# Run simple regression benchmark
results = benchmark.simple_regression_benchmark()

# Print results
for optimizer_name, metrics in results.items():
    print(f"{optimizer_name}:")
    print(f"  Final Loss: {metrics['final_loss']:.6f}")
    print(f"  Convergence Time: {metrics['convergence_time']:.2f}s")
    print(f"  Memory Usage: {metrics['memory_usage']:.2f}MB")

# Compare specific optimizers
optimizers_to_test = ['adam', 'adamw', 'sam', 'ranger', 'lion', 'adabelief']
results = benchmark.compare_optimizers(optimizers_to_test)

# Print comparison results
print("\nOptimizer Comparison Results:")
for optimizer_name, metrics in results.items():
    print(f"{optimizer_name}:")
    print(f"  Final Loss: {metrics['final_loss']:.6f}")
    print(f"  Convergence Time: {metrics['convergence_time']:.2f}s")
    print(f"  Memory Usage: {metrics['memory_usage']:.2f}MB")

Comprehensive Benchmarking

Multi-Task Benchmarking

from torchium.benchmarks import OptimizerBenchmark

# Initialize comprehensive benchmark
benchmark = OptimizerBenchmark()

# Test on different tasks
tasks = ['regression', 'classification', 'computer_vision', 'nlp']

for task in tasks:
    print(f"\nBenchmarking {task}...")
    results = benchmark.benchmark_task(task)

    # Analyze results
    best_optimizer = min(results.items(), key=lambda x: x[1]['final_loss'])
    print(f"Best optimizer for {task}: {best_optimizer[0]}")
    print(f"Final loss: {best_optimizer[1]['final_loss']:.6f}")

# Compare all optimizers across all tasks
all_results = benchmark.benchmark_all_tasks()

# Print summary
print("\nOverall Results Summary:")
for task, task_results in all_results.items():
    print(f"\n{task.upper()}:")
    sorted_results = sorted(task_results.items(), key=lambda x: x[1]['final_loss'])
    for i, (optimizer, metrics) in enumerate(sorted_results[:5]):  # Top 5
        print(f"  {i+1}. {optimizer}: {metrics['final_loss']:.6f}")

Custom Benchmarking

Custom Model Benchmarking

class CustomModel(nn.Module):
    def __init__(self, input_dim=10, hidden_dim=64, output_dim=1):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(input_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, hidden_dim),
            nn.ReLU(),
            nn.Linear(hidden_dim, output_dim)
        )

    def forward(self, x):
        return self.net(x)

# Define custom benchmark
def custom_benchmark(model_class, optimizers, dataloader, num_epochs=100):
    results = {}

    for optimizer_name, optimizer_class in optimizers.items():
        print(f"Benchmarking {optimizer_name}...")

        # Create model and optimizer
        model = model_class()
        optimizer = optimizer_class(model.parameters(), lr=1e-3)
        criterion = nn.MSELoss()

        # Training loop
        start_time = time.time()
        model.train()

        for epoch in range(num_epochs):
            total_loss = 0
            for batch in dataloader:
                optimizer.zero_grad()
                output = model(batch.input)
                loss = criterion(output, batch.target)
                loss.backward()
                optimizer.step()
                total_loss += loss.item()

            if epoch % 20 == 0:
                print(f"  Epoch {epoch}, Loss: {total_loss/len(dataloader):.4f}")

        training_time = time.time() - start_time
        final_loss = total_loss / len(dataloader)

        results[optimizer_name] = {
            'final_loss': final_loss,
            'training_time': training_time
        }

    return results

# Define optimizers to test
optimizers = {
    'adam': torchium.optimizers.Adam,
    'adamw': torchium.optimizers.AdamW,
    'sam': torchium.optimizers.SAM,
    'ranger': torchium.optimizers.Ranger,
    'lion': torchium.optimizers.Lion,
    'adabelief': torchium.optimizers.AdaBelief,
    'lbfgs': torchium.optimizers.LBFGS
}

# Run custom benchmark
results = custom_benchmark(CustomModel, optimizers, dataloader)

# Print results
print("\nCustom Benchmark Results:")
sorted_results = sorted(results.items(), key=lambda x: x[1]['final_loss'])
for i, (optimizer, metrics) in enumerate(sorted_results):
    print(f"{i+1}. {optimizer}:")
    print(f"   Final Loss: {metrics['final_loss']:.6f}")
    print(f"   Training Time: {metrics['training_time']:.2f}s")

Loss Function Comparison

Classification Loss Comparison

def compare_classification_losses(model, dataloader, num_epochs=100):
    losses = {
        'cross_entropy': torchium.losses.CrossEntropyLoss(),
        'focal': torchium.losses.FocalLoss(alpha=0.25, gamma=2.0),
        'label_smoothing': torchium.losses.LabelSmoothingLoss(num_classes=10, smoothing=0.1),
        'class_balanced': torchium.losses.ClassBalancedLoss(num_classes=10)
    }

    results = {}

    for loss_name, criterion in losses.items():
        print(f"Testing {loss_name}...")

        # Create optimizer
        optimizer = torchium.optimizers.Adam(model.parameters(), lr=1e-3)

        # Training loop
        start_time = time.time()
        model.train()

        for epoch in range(num_epochs):
            total_loss = 0
            correct = 0
            total = 0

            for batch in dataloader:
                optimizer.zero_grad()
                output = model(batch.input)
                loss = criterion(output, batch.target)
                loss.backward()
                optimizer.step()

                total_loss += loss.item()
                _, predicted = torch.max(output.data, 1)
                total += batch.target.size(0)
                correct += (predicted == batch.target).sum().item()

            if epoch % 20 == 0:
                accuracy = 100 * correct / total
                print(f"  Epoch {epoch}, Loss: {total_loss/len(dataloader):.4f}, Accuracy: {accuracy:.2f}%")

        training_time = time.time() - start_time
        final_accuracy = 100 * correct / total

        results[loss_name] = {
            'final_loss': total_loss / len(dataloader),
            'final_accuracy': final_accuracy,
            'training_time': training_time
        }

    return results

# Run classification loss comparison
results = compare_classification_losses(model, dataloader)

# Print results
print("\nClassification Loss Comparison Results:")
sorted_results = sorted(results.items(), key=lambda x: x[1]['final_accuracy'], reverse=True)
for i, (loss_name, metrics) in enumerate(sorted_results):
    print(f"{i+1}. {loss_name}:")
    print(f"   Final Accuracy: {metrics['final_accuracy']:.2f}%")
    print(f"   Final Loss: {metrics['final_loss']:.6f}")
    print(f"   Training Time: {metrics['training_time']:.2f}s")

Segmentation Loss Comparison

def compare_segmentation_losses(model, dataloader, num_epochs=100):
    losses = {
        'dice': torchium.losses.DiceLoss(smooth=1e-5),
        'tversky': torchium.losses.TverskyLoss(alpha=0.3, beta=0.7),
        'focal_tversky': torchium.losses.FocalTverskyLoss(alpha=0.3, beta=0.7, gamma=2.0),
        'lovasz': torchium.losses.LovaszLoss(),
        'boundary': torchium.losses.BoundaryLoss()
    }

    results = {}

    for loss_name, criterion in losses.items():
        print(f"Testing {loss_name}...")

        # Create optimizer
        optimizer = torchium.optimizers.SAM(model.parameters(), lr=1e-3, rho=0.05)

        # Training loop
        start_time = time.time()
        model.train()

        for epoch in range(num_epochs):
            total_loss = 0

            for batch in dataloader:
                # First forward pass
                output = model(batch.images)
                loss = criterion(output, batch.masks)
                loss.backward()

                # SAM perturbation step
                optimizer.first_step(zero_grad=True)

                # Second forward pass
                output = model(batch.images)
                loss = criterion(output, batch.masks)
                loss.backward()

                # SAM update step
                optimizer.second_step(zero_grad=True)

                total_loss += loss.item()

            if epoch % 20 == 0:
                print(f"  Epoch {epoch}, Loss: {total_loss/len(dataloader):.4f}")

        training_time = time.time() - start_time

        results[loss_name] = {
            'final_loss': total_loss / len(dataloader),
            'training_time': training_time
        }

    return results

# Run segmentation loss comparison
results = compare_segmentation_losses(model, dataloader)

# Print results
print("\nSegmentation Loss Comparison Results:")
sorted_results = sorted(results.items(), key=lambda x: x[1]['final_loss'])
for i, (loss_name, metrics) in enumerate(sorted_results):
    print(f"{i+1}. {loss_name}:")
    print(f"   Final Loss: {metrics['final_loss']:.6f}")
    print(f"   Training Time: {metrics['training_time']:.2f}s")

Performance Analysis

Memory Usage Analysis

import psutil
import torch

def analyze_memory_usage(model, optimizers, dataloader, num_batches=100):
    process = psutil.Process()
    results = {}

    for optimizer_name, optimizer_class in optimizers.items():
        print(f"Analyzing memory usage for {optimizer_name}...")

        # Create model and optimizer
        model = model_class()
        optimizer = optimizer_class(model.parameters(), lr=1e-3)
        criterion = nn.MSELoss()

        # Measure initial memory
        initial_memory = process.memory_info().rss / 1024 / 1024  # MB

        # Training loop
        max_memory = initial_memory
        model.train()

        for i, batch in enumerate(dataloader):
            if i >= num_batches:
                break

            optimizer.zero_grad()
            output = model(batch.input)
            loss = criterion(output, batch.target)
            loss.backward()
            optimizer.step()

            # Measure current memory
            current_memory = process.memory_info().rss / 1024 / 1024  # MB
            max_memory = max(max_memory, current_memory)

        # Calculate memory usage
        memory_usage = max_memory - initial_memory

        results[optimizer_name] = {
            'memory_usage': memory_usage,
            'max_memory': max_memory
        }

    return results

# Run memory analysis
results = analyze_memory_usage(CustomModel, optimizers, dataloader)

# Print results
print("\nMemory Usage Analysis Results:")
sorted_results = sorted(results.items(), key=lambda x: x[1]['memory_usage'])
for i, (optimizer, metrics) in enumerate(sorted_results):
    print(f"{i+1}. {optimizer}:")
    print(f"   Memory Usage: {metrics['memory_usage']:.2f}MB")
    print(f"   Max Memory: {metrics['max_memory']:.2f}MB")

Convergence Analysis

def analyze_convergence(model, optimizers, dataloader, num_epochs=100):
    results = {}

    for optimizer_name, optimizer_class in optimizers.items():
        print(f"Analyzing convergence for {optimizer_name}...")

        # Create model and optimizer
        model = model_class()
        optimizer = optimizer_class(model.parameters(), lr=1e-3)
        criterion = nn.MSELoss()

        # Training loop
        losses = []
        model.train()

        for epoch in range(num_epochs):
            total_loss = 0
            for batch in dataloader:
                optimizer.zero_grad()
                output = model(batch.input)
                loss = criterion(output, batch.target)
                loss.backward()
                optimizer.step()
                total_loss += loss.item()

            avg_loss = total_loss / len(dataloader)
            losses.append(avg_loss)

        # Analyze convergence
        final_loss = losses[-1]
        convergence_epoch = None
        for i, loss in enumerate(losses):
            if loss < final_loss * 1.1:  # Within 10% of final loss
                convergence_epoch = i
                break

        results[optimizer_name] = {
            'final_loss': final_loss,
            'convergence_epoch': convergence_epoch,
            'losses': losses
        }

    return results

# Run convergence analysis
results = analyze_convergence(CustomModel, optimizers, dataloader)

# Print results
print("\nConvergence Analysis Results:")
sorted_results = sorted(results.items(), key=lambda x: x[1]['final_loss'])
for i, (optimizer, metrics) in enumerate(sorted_results):
    print(f"{i+1}. {optimizer}:")
    print(f"   Final Loss: {metrics['final_loss']:.6f}")
    print(f"   Convergence Epoch: {metrics['convergence_epoch']}")
    print(f"   Total Epochs: {len(metrics['losses'])}")

Visualization

Loss Curve Visualization

import matplotlib.pyplot as plt

def plot_loss_curves(results):
    plt.figure(figsize=(12, 8))

    for optimizer_name, metrics in results.items():
        losses = metrics['losses']
        plt.plot(losses, label=optimizer_name, linewidth=2)

    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.title('Loss Curves Comparison')
    plt.legend()
    plt.grid(True)
    plt.yscale('log')
    plt.show()

# Plot loss curves
plot_loss_curves(results)

Performance Comparison Table

def create_performance_table(results):
    import pandas as pd

    # Create DataFrame
    data = []
    for optimizer, metrics in results.items():
        data.append({
            'Optimizer': optimizer,
            'Final Loss': f"{metrics['final_loss']:.6f}",
            'Convergence Epoch': metrics['convergence_epoch'],
            'Memory Usage (MB)': f"{metrics.get('memory_usage', 0):.2f}",
            'Training Time (s)': f"{metrics.get('training_time', 0):.2f}"
        })

    df = pd.DataFrame(data)
    df = df.sort_values('Final Loss')

    print("\nPerformance Comparison Table:")
    print(df.to_string(index=False))

# Create performance table
create_performance_table(results)

Statistical Analysis

Statistical Significance Testing

from scipy import stats
import numpy as np

def statistical_analysis(results, num_runs=5):
    # Run multiple times for statistical significance
    all_results = {}

    for optimizer_name, optimizer_class in optimizers.items():
        print(f"Running statistical analysis for {optimizer_name}...")

        final_losses = []
        for run in range(num_runs):
            # Create model and optimizer
            model = model_class()
            optimizer = optimizer_class(model.parameters(), lr=1e-3)
            criterion = nn.MSELoss()

            # Training loop
            model.train()
            for epoch in range(100):
                total_loss = 0
                for batch in dataloader:
                    optimizer.zero_grad()
                    output = model(batch.input)
                    loss = criterion(output, batch.target)
                    loss.backward()
                    optimizer.step()
                    total_loss += loss.item()

                if epoch % 20 == 0:
                    print(f"    Run {run+1}, Epoch {epoch}, Loss: {total_loss/len(dataloader):.4f}")

            final_losses.append(total_loss / len(dataloader))

        all_results[optimizer_name] = {
            'mean': np.mean(final_losses),
            'std': np.std(final_losses),
            'values': final_losses
        }

    return all_results

# Run statistical analysis
statistical_results = statistical_analysis(results)

# Print statistical results
print("\nStatistical Analysis Results:")
for optimizer, metrics in statistical_results.items():
    print(f"{optimizer}:")
    print(f"  Mean: {metrics['mean']:.6f}")
    print(f"  Std: {metrics['std']:.6f}")
    print(f"  Values: {[f'{v:.6f}' for v in metrics['values']]}")

# Perform t-tests
print("\nT-test Results:")
optimizer_names = list(statistical_results.keys())
for i in range(len(optimizer_names)):
    for j in range(i+1, len(optimizer_names)):
        opt1, opt2 = optimizer_names[i], optimizer_names[j]
        values1 = statistical_results[opt1]['values']
        values2 = statistical_results[opt2]['values']

        t_stat, p_value = stats.ttest_ind(values1, values2)
        print(f"{opt1} vs {opt2}: t={t_stat:.4f}, p={p_value:.4f}")

These examples demonstrate comprehensive benchmarking and comparison methodologies for Torchium’s optimizers and loss functions. Use these tools to find the best combination for your specific use case.