# nolint start
library(mlexperiments)
library(mllrnrs)# nolint start
library(mlexperiments)
library(mllrnrs)See https://github.com/kapsner/mllrnrs/blob/main/R/learner_lightgbm.R for implementation details.
library(mlbench)
data("BreastCancer")
dataset <- BreastCancer |>
data.table::as.data.table() |>
na.omit()
feature_cols <- colnames(dataset)[2:10]
target_col <- "Class"seed <- 123
if (isTRUE(as.logical(Sys.getenv("_R_CHECK_LIMIT_CORES_")))) {
# on cran
ncores <- 2L
} else {
ncores <- ifelse(
test = parallel::detectCores() > 4,
yes = 4L,
no = ifelse(
test = parallel::detectCores() < 2L,
yes = 1L,
no = parallel::detectCores()
)
)
}
options("mlexperiments.bayesian.max_init" = 4L)
options("mlexperiments.optim.lgb.nrounds" = 20L)
options("mlexperiments.optim.lgb.early_stopping_rounds" = 5L)data_split <- splitTools::partition(
y = dataset[, get(target_col)],
p = c(train = 0.7, test = 0.3),
type = "stratified",
seed = seed
)
train_x <- model.matrix(
~ -1 + .,
dataset[data_split$train, .SD, .SDcols = feature_cols]
)
train_y <- as.integer(dataset[data_split$train, get(target_col)]) - 1L
test_x <- model.matrix(
~ -1 + .,
dataset[data_split$test, .SD, .SDcols = feature_cols]
)
test_y <- as.integer(dataset[data_split$test, get(target_col)]) - 1Lfold_list <- splitTools::create_folds(
y = train_y,
k = 3,
type = "stratified",
seed = seed
)# required learner arguments, not optimized
learner_args <- list(
max_depth = -1L,
verbose = -1L,
objective = "binary",
metric = "binary_logloss"
)
# set arguments for predict function and performance metric,
# required for mlexperiments::MLCrossValidation and
# mlexperiments::MLNestedCV
predict_args <- NULL
performance_metric <- metric("auc")
performance_metric_args <- list(positive = "1", negative = "0")
return_models <- FALSE
# required for grid search and initialization of bayesian optimization
parameter_grid <- expand.grid(
bagging_fraction = seq(0.6, 1, .2),
feature_fraction = seq(0.6, 1, .2),
min_data_in_leaf = seq(2, 10, 2),
learning_rate = seq(0.1, 0.2, 0.1),
num_leaves = seq(2, 20, 4)
)
# reduce to a maximum of 10 rows
if (nrow(parameter_grid) > 10) {
set.seed(123)
sample_rows <- sample(seq_len(nrow(parameter_grid)), 10, FALSE)
parameter_grid <- kdry::mlh_subset(parameter_grid, sample_rows)
}
# required for bayesian optimization
parameter_bounds <- list(
bagging_fraction = c(0.2, 1),
feature_fraction = c(0.2, 1),
min_data_in_leaf = c(2L, 12L),
learning_rate = c(0.1, 0.2),
num_leaves = c(2L, 20L)
)
optim_args <- list(
n_iter = ncores,
kappa = 3.5,
acq = "ucb"
)tuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$learner_args <- learner_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_grid <- tuner$execute(k = 3)
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
head(tuner_results_grid)
#> setting_id metric_optim_mean nrounds bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective
#> <int> <num> <int> <num> <num> <num> <num> <num> <int> <int> <char>
#> 1: 1 0.1413020 18 0.6 0.6 4 0.2 18 -1 -1 binary
#> 2: 2 0.1336577 18 0.8 1.0 10 0.2 6 -1 -1 binary
#> 3: 3 0.1959308 20 0.8 0.8 4 0.1 2 -1 -1 binary
#> 4: 4 0.1665362 20 1.0 0.8 4 0.1 10 -1 -1 binary
#> 5: 5 0.1325429 20 1.0 0.6 6 0.2 18 -1 -1 binary
#> 6: 6 0.1651197 20 1.0 1.0 8 0.1 14 -1 -1 binary
#> metric
#> <char>
#> 1: binary_logloss
#> 2: binary_logloss
#> 3: binary_logloss
#> 4: binary_logloss
#> 5: binary_logloss
#> 6: binary_loglosstuner <- mlexperiments::MLTuneParameters$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
ncores = ncores,
seed = seed
)
tuner$parameter_grid <- parameter_grid
tuner$parameter_bounds <- parameter_bounds
tuner$learner_args <- learner_args
tuner$optim_args <- optim_args
tuner$split_type <- "stratified"
tuner$set_data(
x = train_x,
y = train_y
)
tuner_results_bayesian <- tuner$execute(k = 3)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.032 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.1959308
#> elapsed = 0.04 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1288185
#> elapsed = 0.034 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1336577
#> elapsed = 0.038 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1622234
#> elapsed = 0.048 Round = 5 bagging_fraction = 0.8514907 feature_fraction = 0.5359993 min_data_in_leaf = 9.0000 learning_rate = 0.1801683 num_leaves = 20.0000 Value = -0.1392806
#> elapsed = 0.043 Round = 6 bagging_fraction = 0.2079895 feature_fraction = 0.385567 min_data_in_leaf = 3.0000 learning_rate = 0.1618143 num_leaves = 11.0000 Value = -0.137801
#> elapsed = 0.036 Round = 7 bagging_fraction = 0.8973112 feature_fraction = 0.2399434 min_data_in_leaf = 12.0000 learning_rate = 0.2000 num_leaves = 15.0000 Value = -0.1243682
#> elapsed = 0.044 Round = 8 bagging_fraction = 0.4285091 feature_fraction = 0.2176898 min_data_in_leaf = 12.0000 learning_rate = 0.1330769 num_leaves = 20.0000 Value = -0.1403338
#>
#> Best Parameters Found:
#> Round = 7 bagging_fraction = 0.8973112 feature_fraction = 0.2399434 min_data_in_leaf = 12.0000 learning_rate = 0.2000 num_leaves = 15.0000 Value = -0.1243682
head(tuner_results_bayesian)
#> setting_id bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves Value max_depth verbose objective metric
#> <int> <num> <num> <num> <num> <num> <num> <int> <int> <char> <char>
#> 1: 1 0.8000000 0.8000000 4 0.1000000 2 -0.1959308 -1 -1 binary binary_logloss
#> 2: 2 0.6000000 0.6000000 6 0.2000000 10 -0.1288185 -1 -1 binary binary_logloss
#> 3: 3 0.8000000 1.0000000 10 0.2000000 6 -0.1336577 -1 -1 binary binary_logloss
#> 4: 4 0.8000000 0.6000000 8 0.1000000 14 -0.1622234 -1 -1 binary binary_logloss
#> 5: 5 0.8514907 0.5359993 9 0.1801683 20 -0.1392806 -1 -1 binary binary_logloss
#> 6: 6 0.2079895 0.3855670 3 0.1618143 11 -0.1378010 -1 -1 binary binary_logloss
#> metric_optim_mean
#> <num>
#> 1: 0.1959308
#> 2: 0.1288185
#> 3: 0.1336577
#> 4: 0.1622234
#> 5: 0.1392806
#> 6: 0.1378010validator <- mlexperiments::MLCrossValidation$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
fold_list = fold_list,
ncores = ncores,
seed = seed
)
validator$learner_args <- tuner$results$best.setting[-1]
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> CV fold: Fold2
#>
#> CV fold: Fold3
head(validator_results)
#> fold performance feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective metric nrounds
#> <char> <num> <num> <num> <num> <num> <int> <int> <char> <char> <int>
#> 1: Fold1 0.9890556 0.2399434 12 0.2 15 -1 -1 binary binary_logloss 20
#> 2: Fold2 0.9932386 0.2399434 12 0.2 15 -1 -1 binary binary_logloss 20
#> 3: Fold3 0.9876374 0.2399434 12 0.2 15 -1 -1 binary binary_logloss 20validator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "grid",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- return_models
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Parameter settings [==================================================================>--------------------------------------------] 6/10 ( 60%)
#> Parameter settings [=============================================================================>---------------------------------] 7/10 ( 70%)
#> Parameter settings [========================================================================================>----------------------] 8/10 ( 80%)
#> Parameter settings [===================================================================================================>-----------] 9/10 ( 90%)
#> Parameter settings [==============================================================================================================] 10/10 (100%)
head(validator_results)
#> fold performance nrounds bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective
#> <char> <num> <int> <num> <num> <num> <num> <num> <int> <int> <char>
#> 1: Fold1 0.9902913 20 0.8 1.0 10 0.2 6 -1 -1 binary
#> 2: Fold2 0.9883842 19 0.8 1.0 10 0.2 6 -1 -1 binary
#> 3: Fold3 0.9862637 18 0.6 0.6 6 0.2 10 -1 -1 binary
#> metric
#> <char>
#> 1: binary_logloss
#> 2: binary_logloss
#> 3: binary_loglossvalidator <- mlexperiments::MLNestedCV$new(
learner = mllrnrs::LearnerLightgbm$new(
metric_optimization_higher_better = FALSE
),
strategy = "bayesian",
fold_list = fold_list,
k_tuning = 3L,
ncores = ncores,
seed = seed
)
validator$parameter_grid <- parameter_grid
validator$learner_args <- learner_args
validator$split_type <- "stratified"
validator$parameter_bounds <- parameter_bounds
validator$optim_args <- optim_args
validator$predict_args <- predict_args
validator$performance_metric <- performance_metric
validator$performance_metric_args <- performance_metric_args
validator$return_models <- TRUE
validator$set_data(
x = train_x,
y = train_y
)
validator_results <- validator$execute()
#>
#> CV fold: Fold1
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.027 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.2081399
#> elapsed = 0.035 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1575606
#> elapsed = 0.03 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1462635
#> elapsed = 0.036 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1733679
#> elapsed = 0.029 Round = 5 bagging_fraction = 0.5596134 feature_fraction = 0.9300915 min_data_in_leaf = 7.0000 learning_rate = 0.1800818 num_leaves = 2.0000 Value = -0.1645675
#> elapsed = 0.032 Round = 6 bagging_fraction = 0.8950514 feature_fraction = 0.2196616 min_data_in_leaf = 12.0000 learning_rate = 0.1620435 num_leaves = 20.0000 Value = -0.149935
#> elapsed = 0.025 Round = 7 bagging_fraction = 0.2000 feature_fraction = 0.991415 min_data_in_leaf = 12.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.210863
#> elapsed = 0.043 Round = 8 bagging_fraction = 0.2000 feature_fraction = 0.8495294 min_data_in_leaf = 2.0000 learning_rate = 0.1000 num_leaves = 20.0000 Value = -0.2048307
#>
#> Best Parameters Found:
#> Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1462635
#>
#> CV fold: Fold2
#> CV progress [==============================================================================>----------------------------------------] 2/3 ( 67%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.029 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.1968844
#> elapsed = 0.04 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1448359
#> elapsed = 0.032 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1431115
#> elapsed = 0.035 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1744139
#> elapsed = 0.037 Round = 5 bagging_fraction = 0.5157127 feature_fraction = 0.9291875 min_data_in_leaf = 9.0000 learning_rate = 0.1626326 num_leaves = 6.0000 Value = -0.1437751
#> elapsed = 0.042 Round = 6 bagging_fraction = 0.803404 feature_fraction = 0.8105688 min_data_in_leaf = 9.0000 learning_rate = 0.1840982 num_leaves = 6.0000 Value = -0.1449229
#> elapsed = 0.034 Round = 7 bagging_fraction = 0.3970573 feature_fraction = 0.271388 min_data_in_leaf = 9.0000 learning_rate = 0.16237 num_leaves = 10.0000 Value = -0.1432943
#> elapsed = 0.039 Round = 8 bagging_fraction = 0.3970573 feature_fraction = 0.2713869 min_data_in_leaf = 9.0000 learning_rate = 0.16237 num_leaves = 10.0000 Value = -0.1432943
#>
#> Best Parameters Found:
#> Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1431115
#>
#> CV fold: Fold3
#> CV progress [=======================================================================================================================] 3/3 (100%)
#>
#> Number of rows of initialization grid > than 'options("mlexperiments.bayesian.max_init")'...
#> ... reducing initialization grid to 4 rows.
#> elapsed = 0.026 Round = 1 bagging_fraction = 0.8000 feature_fraction = 0.8000 min_data_in_leaf = 4.0000 learning_rate = 0.1000 num_leaves = 2.0000 Value = -0.1935595
#> elapsed = 0.034 Round = 2 bagging_fraction = 0.6000 feature_fraction = 0.6000 min_data_in_leaf = 6.0000 learning_rate = 0.2000 num_leaves = 10.0000 Value = -0.1456779
#> elapsed = 0.03 Round = 3 bagging_fraction = 0.8000 feature_fraction = 1.0000 min_data_in_leaf = 10.0000 learning_rate = 0.2000 num_leaves = 6.0000 Value = -0.1592296
#> elapsed = 0.039 Round = 4 bagging_fraction = 0.8000 feature_fraction = 0.6000 min_data_in_leaf = 8.0000 learning_rate = 0.1000 num_leaves = 14.0000 Value = -0.1740933
#> elapsed = 0.029 Round = 5 bagging_fraction = 0.5157127 feature_fraction = 0.9291875 min_data_in_leaf = 9.0000 learning_rate = 0.1998166 num_leaves = 6.0000 Value = -0.1436106
#> elapsed = 0.042 Round = 6 bagging_fraction = 0.2000 feature_fraction = 0.4928526 min_data_in_leaf = 6.0000 learning_rate = 0.1736983 num_leaves = 20.0000 Value = -0.1451436
#> elapsed = 0.039 Round = 7 bagging_fraction = 0.8514907 feature_fraction = 0.5359993 min_data_in_leaf = 9.0000 learning_rate = 0.1068914 num_leaves = 20.0000 Value = -0.1693466
#> elapsed = 0.034 Round = 8 bagging_fraction = 0.2000 feature_fraction = 0.2000 min_data_in_leaf = 12.0000 learning_rate = 0.1000 num_leaves = 20.0000 Value = -0.1729361
#>
#> Best Parameters Found:
#> Round = 5 bagging_fraction = 0.5157127 feature_fraction = 0.9291875 min_data_in_leaf = 9.0000 learning_rate = 0.1998166 num_leaves = 6.0000 Value = -0.1436106
head(validator_results)
#> fold performance bagging_fraction feature_fraction min_data_in_leaf learning_rate num_leaves max_depth verbose objective metric
#> <char> <num> <num> <num> <num> <num> <num> <int> <int> <char> <char>
#> 1: Fold1 0.9902913 0.8000000 1.0000000 10 0.2000000 6 -1 -1 binary binary_logloss
#> 2: Fold2 0.9883842 0.8000000 1.0000000 10 0.2000000 6 -1 -1 binary binary_logloss
#> 3: Fold3 0.9869505 0.5157127 0.9291875 9 0.1998166 6 -1 -1 binary binary_logloss
#> nrounds
#> <int>
#> 1: 20
#> 2: 19
#> 3: 19preds_lightgbm <- mlexperiments::predictions(
object = validator,
newdata = test_x
)perf_lightgbm <- mlexperiments::performance(
object = validator,
prediction_results = preds_lightgbm,
y_ground_truth = test_y,
type = "binary"
)
perf_lightgbm
#> model performance AUC Brier BrierScaled BAC TP TN FP FN TPR TNR FPR FNR PPV
#> <char> <num> <num> <num> <num> <num> <int> <int> <int> <int> <num> <num> <num> <num> <num>
#> 1: Fold1 0.9764718 0.9764718 0.03740030 0.8354976 0.9407131 64 133 1 8 0.8888889 0.9925373 0.007462687 0.11111111 0.9846154
#> 2: Fold2 0.9906198 0.9906198 0.04184296 0.8159569 0.9268242 62 133 1 10 0.8611111 0.9925373 0.007462687 0.13888889 0.9841270
#> 3: Fold3 0.9811360 0.9811360 0.04052871 0.8217375 0.9401949 65 131 3 7 0.9027778 0.9776119 0.022388060 0.09722222 0.9558824
#> NPV FDR MCC F1 GMEAN GPR ACC MMCE BER
#> <num> <num> <num> <num> <num> <num> <num> <num> <num>
#> 1: 0.9432624 0.01538462 0.9043538 0.9343066 0.9392845 0.9355286 0.9563107 0.04368932 0.05928690
#> 2: 0.9300699 0.01587302 0.8834041 0.9185185 0.9244917 0.9205665 0.9466019 0.05339806 0.07317579
#> 3: 0.9492754 0.04411765 0.8926878 0.9285714 0.9394500 0.9289507 0.9514563 0.04854369 0.05980514