# 基礎(chǔ)安裝
pip install accelerate

# 安裝時(shí)包含所有依賴（推薦）
pip install 'accelerate[torch]'

# 從源碼安裝最新版本
pip install git+https://github.com/huggingface/accelerate.git

accelerate env

from accelerate import Accelerator
import torch
from torch.utils.data import DataLoader, Dataset

# 簡(jiǎn)單數(shù)據(jù)集
class MyDataset(Dataset):
    def __len__(self): return 1000
    def __getitem__(self, idx): return torch.tensor([idx]), torch.tensor([idx%2])

# 初始化加速器
accelerator = Accelerator()

# 模型、數(shù)據(jù)加載器、優(yōu)化器
model = torch.nn.Linear(1, 1)
dataloader = DataLoader(MyDataset(), batch_size=32)
optimizer = torch.optim.Adam(model.parameters())

# 準(zhǔn)備組件
model, optimizer, dataloader = accelerator.prepare(
    model, optimizer, dataloader
)

# 訓(xùn)練循環(huán)
for epoch in range(3):
    for inputs, targets in dataloader:
        outputs = model(inputs.float())
        loss = torch.nn.functional.mse_loss(outputs, targets.float())
        accelerator.backward(loss)
        optimizer.step()
        optimizer.zero_grad()
    print(f"Epoch {epoch} completed")

# 啟用FP16混合精度
accelerator = Accelerator(mixed_precision='fp16')

# 其余代碼與案例1相同，但會(huì)自動(dòng)使用混合精度

from accelerate import Accelerator
import torch

accelerator = Accelerator()

# 假設(shè)我們有模型和測(cè)試數(shù)據(jù)加載器
model, test_dataloader = accelerator.prepare(model, test_dataloader)

model.eval()
total_correct = 0
total = 0

with torch.no_grad():
    for inputs, labels in test_dataloader:
        outputs = model(inputs)
        predictions = outputs.argmax(dim=1)
        # 收集所有進(jìn)程的結(jié)果
        all_predictions = accelerator.gather(predictions)
        all_labels = accelerator.gather(labels)
        total_correct += (all_predictions == all_labels).sum().item()
        total += all_labels.size(0)

# 只在主進(jìn)程打印結(jié)果
if accelerator.is_main_process:
    print(f"Accuracy: {total_correct / total:.2f}")

accelerate launch --num_processes=2 train.py

# 保存訓(xùn)練狀態(tài)
accelerator.save_state('checkpoint')

# 加載訓(xùn)練狀態(tài)
accelerator.load_state('checkpoint')

from accelerate import Accelerator
from transformers import AutoModelForSequenceClassification, AutoTokenizer
from datasets import load_dataset

accelerator = Accelerator()

# 加載模型、分詞器和數(shù)據(jù)集
model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
dataset = load_dataset("imdb")

# 預(yù)處理數(shù)據(jù)
def preprocess_function(examples):
    return tokenizer(examples["text"], truncation=True, max_length=512)
tokenized_dataset = dataset.map(preprocess_function, batched=True)

# 準(zhǔn)備訓(xùn)練器組件
train_loader = DataLoader(tokenized_dataset["train"], batch_size=8)
optimizer = torch.optim.Adam(model.parameters(), lr=5e-5)

# 加速準(zhǔn)備
model, optimizer, train_loader = accelerator.prepare(model, optimizer, train_loader)

# 訓(xùn)練循環(huán)（簡(jiǎn)化版）
for batch in train_loader:
    outputs = model(**batch)
    loss = outputs.loss
    accelerator.backward(loss)
    optimizer.step()
    optimizer.zero_grad()