Looping a function in R

Viewed 73

I have written a cross validation/grid search style code in R that tries to find an optimal threshold value for a given value of mtry (using the random forest algorithm). I have posted my code below using the Sonar data from the library mlbench However, there seems to be some problems with this code.

library(caret)
library(mlbench)
library(randomForest)

res <- matrix(0, nrow = 10, ncol = 6)
colnames(res) <- c("mtry","Threshhold","Accuracy", "PositivePred", "NegativePred", "F-value")
out <- matrix(0, nrow = 17, ncol = 6)
colnames(out) <- c("mtry","Threshhold","Avg.Accuracy", "Avg.PosPred", "Avg.NegPred", "Avg.F_Value")
rep <- matrix(0, nrow = 10, ncol = 6)
colnames(out) <- c("mtry","Threshhold","Avg_Accuracy", "Avg_PosPred", "Avg_NegPred", "Avg_F_Value")

data(Sonar)
N=Sonar

### creating 10 folds 


folds <- cut(seq(1,nrow(N)),breaks=10,labels=FALSE)
for (mtry in 5:14)  {
  K=mtry-4
for(thresh in seq(1,9,0.5)) {
   J = 2*thresh-1
  dataset<-N[sample(nrow(N)),]              ####  mix up the dataset N
   for(I in 1:10){
    #Segement your data by fold using the which() function 
    testIndexes <- which(folds==I,arr.ind=TRUE)
    N_test <- dataset[testIndexes, ]              ### select each fold for test
    N_train <- dataset[-testIndexes, ]            ### select rest for training 
    rf = randomForest(Class~., data = N_train, mtry=mtry, ntree=500)
    pred = predict(rf, N_test, type="prob")
    label = as.factor(ifelse(pred[,2]>=thresh,"M","R"))
    confusion = confusionMatrix(N_test$Class, label)
    res[I,1]=mtry
    res[I,2]=thresh   
    res[I,3]=confusion$overall[1]
    res[I,4]=confusion$byClass[3]
    res[I,5]=confusion$byClass[4]
    res[I,6]=confusion$byClass[7]
   } 
  print(res)
  out[J,1] = mtry
  out[J,2] = thresh
  out[J,3] = mean(res[,2])
  out[J,4] = mean(res[,3])
  out[J,5] = mean(res[,4])
  out[J,6] = mean(res[,5])

 }
 print(out)
  rep[K,1] = mtry
  rep[K,2] = thresh
  rep[K,3] = mean(out[,2])
  rep[K,4] = mean(out[,3])
  rep[K,5] = mean(out[,4])
  rep[K,6] = mean(out[,5])
}
print(rep)

Earlier, I wrote a similar code with the "iris" dataset, and I did not seem to have any problems:

library(caret)
library(randomForest)

data(iris)
N <- iris
N$Species = ifelse(N$Species == "setosa", "a", "b") 

N$Species = as.factor(N$Species) 

res <- matrix(0, nrow = 10, ncol = 5)
colnames(res) <- c("Threshhold","Accuracy", "PositivePred", "NegativePred", "F-value")
out <- matrix(0, nrow = 9, ncol = 5)
colnames(out) <- c("Threshhold","Avg.Accuracy", "Avg.PosPred", "Avg.NegPred", "Avg.F_Value")


### creating 10 folds 

folds <- cut(seq(1,nrow(N)),breaks=10,labels=FALSE)

for(J in 1:9) {
 thresh = J/10
 dataset<-N[sample(nrow(N)),]              ####  mix up the dataset N
 for(I in 1:10){
    #Segement your data by fold using the which() function 
    testIndexes <- which(folds==I,arr.ind=TRUE)
    N_test <- dataset[testIndexes, ]              ### select each fold for test
    N_train <- dataset[-testIndexes, ]            ### select rest for training 
    rf = randomForest(Species~., data = N_train, mtry=3, ntree=10)
    pred = predict(rf, N_test, type="prob")
    label = as.factor(ifelse(pred[,1]>=thresh,"a","b"))
    confusion = confusionMatrix(N_test$Species, label)
    res[I,1]=thresh   
    res[I,2]=confusion$overall[1]
    res[I,3]=confusion$byClass[3]
    res[I,4]=confusion$byClass[4]
    res[I,5]=confusion$byClass[7]
 } 
print(res)
out[J,1] = thresh
out[J,2] = mean(res[,2])
out[J,3] = mean(res[,3])
out[J,4] = mean(res[,4])
out[J,5] = mean(res[,5])

}
print(out)

Could someone please assist me in debugging the first code? Thanks

1 Answers

You need to close parenthesis ) in your for loop.

Replace this

for(thresh in seq(1,9,0.5) {

with

for(thresh in seq(1,9,0.5)) {

Update:

Also, it appears that your thresh is always above 1 giving a single value R in the label, as it is never above thresh.

label = as.factor(ifelse(pred[,2]>=thresh,"M","R"))

and that creates a problem in the next statement

confusion = confusionMatrix(N_test$Class, label)

I tested with 0.5, and I get no error.

label = as.factor(ifelse(pred[,2]>=0.5,"M","R"))

If you can define a better thresh - to stay between 0 and 1, you should be fine.

Related