深度学习优化：从梯度下降到Adam的理论与实践

1. 技术分析

1.1 优化算法分类

1.2 算法对比

2. 核心功能实现

2.1 PyTorch优化器使用

import torch import torch.nn as nn import torch.optim as optim class SimpleCNN(nn.Module): def __init__(self): super().__init__() self.features = nn.Sequential( nn.Conv2d(3, 32, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2), nn.Conv2d(32, 64, 3, padding=1), nn.ReLU(), nn.AdaptiveAvgPool2d(1) ) self.classifier = nn.Linear(64, 10) def forward(self, x): x = self.features(x) x = x.view(x.size(0), -1) return self.classifier(x) def create_optimizers(model, lr=0.001): optimizers = {} # SGD optimizers['sgd'] = optim.SGD( model.parameters(), lr=lr, momentum=0.9, weight_decay=1e-4 ) # Adam optimizers['adam'] = optim.Adam( model.parameters(), lr=lr, betas=(0.9, 0.999), weight_decay=1e-4 ) # AdamW optimizers['adamw'] = optim.AdamW( model.parameters(), lr=lr, betas=(0.9, 0.999), weight_decay=0.01 ) return optimizers

2.2 学习率调度器

class LearningRateScheduler: @staticmethod def create_scheduler(optimizer, scheduler_type, epochs): schedulers = { 'step': optim.lr_scheduler.StepLR( optimizer, step_size=30, gamma=0.1 ), 'exponential': optim.lr_scheduler.ExponentialLR( optimizer, gamma=0.95 ), 'cosine': optim.lr_scheduler.CosineAnnealingLR( optimizer, T_max=epochs ), 'plateau': optim.lr_scheduler.ReduceLROnPlateau( optimizer, mode='min', factor=0.5, patience=5 ), } return schedulers.get(scheduler_type) class WarmupScheduler: def __init__(self, optimizer, warmup_epochs, total_epochs, base_lr): self.optimizer = optimizer self.warmup_epochs = warmup_epochs self.total_epochs = total_epochs self.base_lr = base_lr def step(self, epoch): if epoch < self.warmup_epochs: lr = self.base_lr * (epoch + 1) / self.warmup_epochs else: progress = (epoch - self.warmup_epochs) / (self.total_epochs - self.warmup_epochs) lr = self.base_lr * 0.5 * (1 + math.cos(math.pi * progress)) for param_group in self.optimizer.param_groups: param_group['lr'] = lr return lr

2.3 混合精度训练

from torch.cuda.amp import autocast, GradScaler def train_amp(model, dataloader, optimizer, criterion): scaler = GradScaler() model.train() for inputs, targets in dataloader: inputs, targets = inputs.cuda(), targets.cuda() optimizer.zero_grad() # 自动混合精度 with autocast(): outputs = model(inputs) loss = criterion(outputs, targets) # 缩放损失并反向传播 scaler.scale(loss).backward() scaler.step(optimizer) scaler.update()

3. 实验对比

3.1 优化器性能对比

3.2 学习率调度对比

def benchmark_schedulers(): """学习率调度器性能对比""" results = {} for scheduler_type in ['step', 'cosine', 'exponential']: model = SimpleCNN() optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9) scheduler = LearningRateScheduler.create_scheduler( optimizer, scheduler_type, epochs=100 ) # 训练和评估 # ... results[scheduler_type] = best_accuracy return results

4. 最佳实践

4.1 优化器选择建议

4.2 学习率选择

5. 总结

深度学习优化要点：

1. SGD：泛化性能好，需要仔细调参
2. AdamW：收敛快、泛化好，是Transformer的首选
3. 学习率调度：配合调度器效果更佳