Can you find the mistake in my custom evaluation metric? XGBOOST R

Question

I am trying to add a custom evaluation metric in xgboost in R. In this example the predictions on the testing data are 0.4399617 and 0.2455548. The evaluation metric should say 0.3299713 at iteration 100, but it says 0 through all iterations. Here is my code. What am I missing?

require(xgboost)
set.seed(1)

training_vectors<-structure(c(0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 
                              1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 
                              0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0.941638125728341, 1.15364696297532, 
                              0.558399566424809, 0.559068309566054, 1.34949191060958, 0.614667592104398, 
                              0.207457370529451, 1.66408696198076, 0.76941954573777, -1.85691525645726, 
                              -0.0613798089093204, 0.20816125741963, -0.436302839464195, 0.574420472319906, 
                              1.24323759174665, -0.467671854259046, 0.585910761696343, 0.423800826967858, 
                              0.945893018087821, 0.634033886774145, -0.170806014074692, 1.20924568989354, 
                              -0.668327426122953, 1.89404791341356, 0.709921569900634, 1.4457264865345, 
                              1.22925123638156, -2.521332087981, 0.76941954573777, -0.267363051433809, 
                              -1.7861474451685, 0.869918459564304, 0.600571391571742, 1.13777309775596, 
                              0.313749021477565, 0.111930375282131, 0.295723261240941, -2.04696858752227, 
                              -0.97008985024718, 0.873483392785015, -1.0981420685219, 1.00467719588919, 
                              0.740785038669935, -0.677232827891266, -0.0766134135029085, -1.08143136489723, 
                              0.216661556244158, -1.1272647763345, -1.61180487843188, -1.63155599746285, 
                              2.15000886809406, 2.53192103573109, 0.70440536695558, 0.946339932753538, 
                              0.943524028634784, 0.530787783016417, -0.102653872127614, 0.613598009661027, 
                              0.583935505954246, -1.51512017663048, -0.0706365387617147, 0.0125717436939713, 
                              -0.352542515747031, 0.395209376148268, 0.517331305821383, -0.394815389936023, 
                              1.10604808665121, 0.203372942327374, -0.0224412487659761, 0.295119053511649, 
                              -0.210267894496878, -0.140104931734263, -0.567507664011455, 2.55041654544799, 
                              0.106082577868771, 0.575002046761972, 1.05666473460522, -1.2591919695921, 
                              0.583935505954246, -0.333784770325339, -0.932439256507077, 0.943415608073274, 
                              1.11486885051983, 0.911690809656619, 1.01832626935543, -0.269520909228016, 
                              0.524767862503835, -2.57382748554509, -0.854732466657237, 0.162199692992326, 
                              -0.897155532683592, 0.847232738311629, 1.39468973142335, -0.538626895200607, 
                              -0.0526162811083247, -0.949650083397134, 0.123673629179791, -0.964074555799597, 
                              -0.466180877467377, -0.889901074779966, -0.877961787746227, -0.440551755277531, 
                              -0.437089625668648, 0.145465596919057, 1.97946702545303, 0.705881542891751, 
                              -0.652983064702968, -1.29750592515877, -0.666105460592316, 0.695743768828895, 
                              1.31499420613652, -0.489916890095383, 0.746697613155754, -1.80939055796183, 
                              -0.683018482311327, -0.689851440901128, 0.639573132787043, -0.501302309205543, 
                              -1.37282606099636, 0.577185268899497, 1.24513622074992, 0.728305296064452, 
                              -0.860670337014914, 1.13993892317613, -0.0055917224731583, -0.772799096757393, 
                              0.152452209168004, -0.333258731135924, -0.666105460592316, 0.34332315765547, 
                              -1.41585084443192, -0.585375321510597, -0.453792414724353, 1.0884216380393, 
                              0.011045277153211, -0.447629850436624, 0.879135472478845, 0.659622183932429, 
                              0.362355620754987, -2.01778316638862, 1.02943598434488, -2.26969617624586, 
                              -0.589076976166479, 1.14150317555875, -0.0163468910582442, 0.559033964119561, 
                              -3.30870348382885, 0.599666170996592, -1.64650347971611, -0.480310994666598
), .Dim = c(50L, 4L), .Dimnames = list(c(1:50), c("feature_1", "feature_2", 
                                                  "feature_3", "feature_4")))
testing_vectors<-structure(c(1, 1, 1.78188666352518, 0.501744180397169, 1.948022033796133, 
                             0.565917845278659, -1.208314471487369, -0.624564113268186), .Dim = c(2L, 
                                                                                                  4L), .Dimnames = list(c("1", "2"), c("feature_1", "feature_2", 
                                                                                                                                           "feature_3", "feature_4")))
training_vectors<-xgb.DMatrix(data=as.matrix(training_vectors),label=as.matrix(c(0L, 1L, 0L, 0L, 1L, 1L, 1L, 1L, 0L, 0L, 0L, 1L, 0L, 1L, 0L, 
                                                                                 0L, 0L, 0L, 0L, 1L, 1L, 1L, 0L, 0L, 1L, 1L, 1L, 1L, 0L, 1L, 0L, 
                                                                                 1L, 1L, 0L, 0L, 1L, 0L, 0L, 1L, 0L, 0L, 0L, 1L, 1L, 0L, 0L, 0L, 
                                                                                 0L, 1L, 1L)))
testing_vectors<-xgb.DMatrix(data=as.matrix(testing_vectors))
watchlist <- list(test=testing_vectors)
my_metric <- function(preds,testing_vectors){
  g<-c(0.75,1.5)
  c<-ifelse(preds-0.05<0.20,0,preds)
  d<-c*g
  result_matrix<-sum(d)
  return(list(metric = "result", value = result_matrix))
}   
trained_models<-xgb.train(data=training_vectors,gamma=0,nrounds=100,max_depth=2,objective="binary:logistic", verbose = 0,feval=my_metric,watchlist = watchlist)
print(predict(trained_models,testing_vectors))
print(trained_models$evaluation_log$test_result)

user_15851978 · Answer

One empty list can be created to store the value of every evaluation metric. And then we can run a loop.

eval_metric <- c()

for (i in 1:100) {
  
  trained_models<-xgb.train(data=training_vectors,gamma=0,nrounds=i,max_depth=2,objective="binary:logistic", verbose = 0,feval=my_metric,watchlist = watchlist)
  eval_metric[i] <- my_metric(predict(trained_models,testing_vectors), testing_vectors)$value
}

The output:

eval_metric

  [1] 1.0833762 1.0332087 1.0702217 1.1165583 0.9980249 1.0447095 0.9964721 0.9674231 0.8648293
 [10] 0.9044608 0.8724537 0.9304222 0.8491665 0.8829176 0.9304336 0.9221882 0.8533177 0.8376518
 [19] 0.7965470 0.8284276 0.8067912 0.7947449 0.7577542 0.7864774 0.7560513 0.7355429 0.7609600
 [28] 0.7640666 0.7101464 0.7291165 0.7655773 0.7347603 0.6886943 0.7110074 0.6942958 0.6838692
 [37] 0.2975801 0.3121724 0.6874055 0.3178953 0.3018035 0.3133702 0.6857661 0.6927544 0.3043382
 [46] 0.2982567 0.2908952 0.2772635 0.2722214 0.2677541 0.2610758 0.2715461 0.2818424 0.3041806
 [55] 0.3227641 0.3138340 0.3105319 0.3045225 0.3009517 0.3114915 0.3061301 0.3169128 0.3118879
 [64] 0.3083425 0.3185155 0.3115889 0.3202170 0.3141242 0.3115893 0.3265834 0.3178155 0.3211948
 [73] 0.3145838 0.3232811 0.3168709 0.3215020 0.3140709 0.3214312 0.3146561 0.3219147 0.3156422
 [82] 0.3099746 0.3176437 0.3261342 0.3212111 0.3146619 0.3215416 0.3296011 0.3362954 0.3328568
 [91] 0.3266897 0.3216920 0.3297096 0.3246411 0.3192709 0.3235182 0.6988236 0.3270507 0.7030137
[100] 0.3299713

And the 100th value is 0.3299713.

Let me know if you have any doubt.

Can you find the mistake in my custom evaluation metric? XGBOOST R

1 Answers