I'm using this SLURM script to initialize the job:
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=11
#SBATCH --time=08:00:00
and an Rscript along these lines:
library
data
model_function <- lots of code
plan(multisession, workers=11)
sim_function <- function(sim_run) {
future_lapply(future.seed = TRUE, vals, function(vv){
res <- model_function()
names(res) <- c(...long list of names...)
dat <- reshape2::melt(res) %>% filter(L1 != "time") %>% mutate(
time = Var1,
Var1 = L1,
L1 = NULL) %>%
mutate(age_group = cut(Var2, custom_age_groups)) %>%
group_by(Var1, age_group, time) %>%
summarise(value = sum(value)) %>%
mutate(year = ceiling(time / 365),
vv=vv)
filename <- paste0(...,vv,".RDS")
filename_Rt <- paste0(...,vv,".RDS")
saveRDS(dat,filename)
saveRDS(just_Rt,filename_Rt)
})
filename <- paste0(...,"_sim_run_",sim_run,".RDS")
consolidate_files(filename)
}
lapply(1:20, function(sim_run) sim_function(sim_run))
I can't seem to get this thing to run parallel.
I've tried lots of different plans:
plan(list(tweak(cluster, workers=1),tweak(multisession, workers=11)))
plan(list(tweak(sequential),tweak(multisession)))
plan(list(tweak(sequential),tweak(multicore)))
I've tried using the future_lapply instead:
future_lapply(1:20, function(sim_run) sim_function(sim_run))
All of these still seem to run sequential despite spinning up parallel sessions, is there some topology I'm not seeing?
The code works fine on my Windows desktop so maybe I'm not appropriately using SLURM?