Optuna Hyperparameter Optimization Cheat Sheet

import optunadef objective(trial):    lr = trial.suggest_float("lr", 1e-5, 1e-1, log=True)    n_layers = trial.suggest_int("n_layers", 1, 4)    optimizer_name = trial.suggest_categorical("optimizer", ["Adam", "SGD"])    dropout = trial.suggest_float("dropout", 0.0, 0.5, step=0.1)    model = build_model(n_layers, dropout)    accuracy = train_and_eval(model, lr, optimizer_name)    return accuracy  # value optuna will maximize/minimizestudy = optuna.create_study(direction="maximize", study_name="mnist-cnn")study.optimize(objective, n_trials=100, timeout=600)print(study.best_params)print(study.best_value)

from optuna.samplers import TPESamplerfrom optuna.pruners import MedianPrunerstudy = optuna.create_study(    direction="minimize",    sampler=TPESampler(seed=42),    pruner=MedianPruner(n_startup_trials=5, n_warmup_steps=10),)def objective(trial):    for epoch in range(30):        loss = train_one_epoch()        trial.report(loss, step=epoch)        if trial.should_prune():            raise optuna.TrialPruned()    return loss

# create a persistent study backed by SQLite or PostgreSQLoptuna create-study --study-name "nlp-tuning" \  --storage "postgresql://user:pass@host/optuna_db"# each worker (on any machine) attaches to the same studypython train_worker.py  # internally calls optuna.load_study(...)# inspect results from the CLIoptuna trials --study-name "nlp-tuning" --storage "postgresql://user:pass@host/optuna_db"

import optunaimport lightgbm as lgbdef objective(trial):    params = {        "objective": "binary",        "metric": "auc",        "num_leaves": trial.suggest_int("num_leaves", 16, 256),        "learning_rate": trial.suggest_float("learning_rate", 1e-3, 0.3, log=True),        "feature_fraction": trial.suggest_float("feature_fraction", 0.5, 1.0),        "bagging_fraction": trial.suggest_float("bagging_fraction", 0.5, 1.0),    }    gbm = lgb.train(params, train_set, valid_sets=[valid_set],                     callbacks=[lgb.early_stopping(50)])    preds = gbm.predict(X_valid)    return roc_auc_score(y_valid, preds)study = optuna.create_study(direction="maximize")study.optimize(objective, n_trials=50)