I'm trying to use multi processing/threading to speed up one-hot-encoding while working with UNet in Julia. If I run this on julia -t 9 -p 4 so 9 threads and 5 procs, It errors on NaN and thus seems like an out of sync calculation? This is the code where I calculate weights and then one hot labels
using Distributed
labels=[1,2]
pmap(1:target_image_size) do i
for j = 1:target_image_size
k = labels_matrix[i, j, 1]
weights[findfirst(k .== labels)] += 1
end
end
weights = weights / maximum(weights)
...
Threads.@threads for i = 1:target_image_size
for j = 1:target_image_size
k = labels_matrix[i, j, 1]
label_idx = findfirst(labels .== k)
onehot_target[i, j, label_idx] = weights[label_idx]
end
end
so if an target_image_size is 256x256 and it has [1 2 2;2 2 1]
one hot_target should be [1 0 0;0 0 1] and [0 1 1;1 1 0]
I start Julia as julia -t 9 -p 4
So the error is following :
Error During Test at model.jl:34
Got exception outside of a @test
InexactError: Int64(NaN)
Stacktrace:
[1] (::Base.var"#837#839")(x::Task)
@ Base ./asyncmap.jl:177
[2] foreach(f::Base.var"#837#839", itr::Vector{Any})
@ Base ./abstractarray.jl:2141
[3] maptwice(wrapped_f::Function, chnl::Channel{Any}, worker_tasks::Vector{Any}, c::Vector{Int64})
@ Base ./asyncmap.jl:177
[4] wrap_n_exec_twice
@ ./asyncmap.jl:153 [inlined]
[5] #async_usemap#822
@ ./asyncmap.jl:103 [inlined]
[6] #asyncmap#821
@ ./asyncmap.jl:81 [inlined]
[7] asyncmap
@ ./asyncmap.jl:81 [inlined]
[8] (::UNet.var"#10#13"{Vector{CloudLocation}, Vector{Int8}, Int64})()
@ UNet ~/data.jl:73
[9] in_scratch_dir(func::UNet.var"#10#13"{Vector{CloudLocation}, Vector{Int8}, Int64}, temp::String)
@ Jaws ~LocalDiskHelpers.jl:106
[10] in_scratch_dir (repeats 2 times)
@ ~LocalDiskHelpers.jl:96 [inlined]
[11] load_target_files
@ ~data.jl:64 [inlined]
[12] load_image_and_target_files
@ ~data.jl:15 [inlined]
[13] train(train_dataset::ImageDataset, test_dataset::ImageDataset, model::Unet, opt::RMSProp, loss::UNet.var"#loss#62"{typeof(Flux.Losses.logitcrossentropy), Unet}; batch_size::Int64, test_batch_size::Int64, num_epochs::Int64, learning_rate_drop_rate::Float64, learning_rate_step::Int64, device::Function, save_intermediate_models::Bool, save_model::Bool, out_dir::String, model_file_name::String, save_pred_image::Bool, pred_image_postfix::String, pred_image_ext::String)
@ UNet ~train.jl:112
[14] train(train_dataset::ImageDataset, test_dataset::ImageDataset, num_channels::Int64, labels::Vector{Int8}; loss_function::typeof(Flux.Losses.logitcrossentropy), learning_rate::Float64, momentum::Float64, device::Function, kwargs::Base.Iterators.Pairs{Symbol, Any, NTuple{7, Symbol}, NamedTuple{(:num_epochs, :batch_size, :test_batch_size, :save_intermediate_models, :save_model, :out_dir, :model_file_name), Tuple{Int64, Int64, Int64, Bool, Bool, String, String}}})
@ UNet ~train.jl:40
[15] macro expansion
@ ~model.jl:36 [inlined]
[16] macro expansion
@ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
[17] macro expansion
@ ~model.jl:36 [inlined]
[18] macro expansion
@ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
[19] top-level scope
@ ~model.jl:24
[20] include(fname::String)
@ Base.MainInclude ./client.jl:444
[21] top-level scope
@ ~runtests.jl:47
[22] include(fname::String)
@ Base.MainInclude ./client.jl:444
[23] top-level scope
@ REPL[1]:1
[24] eval
@ ./boot.jl:360 [inlined]
[25] eval_user_input(ast::Any, backend::REPL.REPLBackend)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:139
[26] repl_backend_loop(backend::REPL.REPLBackend)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:200
[27] start_repl_backend(backend::REPL.REPLBackend, consumer::Any)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:185
[28] run_repl(repl::REPL.AbstractREPL, consumer::Any; backend_on_current_task::Bool)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:317
[29] run_repl(repl::REPL.AbstractREPL, consumer::Any)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:305
[30] (::Base.var"#874#876"{Bool, Bool, Bool})(REPL::Module)
@ Base ./client.jl:387
[31] #invokelatest#2
@ ./essentials.jl:708 [inlined]
[32] invokelatest
@ ./essentials.jl:706 [inlined]
[33] run_main_repl(interactive::Bool, quiet::Bool, banner::Bool, history_file::Bool, color_set::Bool)
@ Base ./client.jl:372
[34] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:302
[35] _start()
@ Base ./client.jl:485
ā Warning: Opening file with JLD2.MmapIO failed, falling back to IOStream
ā @ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:233
Testing: Error During Test at model.jl:52
Got exception outside of a @test
SystemError: opening file "/tmp/jl_SjrXkR/UNet_demo.jld2": No such file or directory
Stacktrace:
[1] systemerror(p::String, errno::Int32; extrainfo::Nothing)
@ Base ./error.jl:168
[2] #systemerror#62
@ ./error.jl:167 [inlined]
[3] systemerror
@ ./error.jl:167 [inlined]
[4] open(fname::String; lock::Bool, read::Bool, write::Bool, create::Bool, truncate::Bool, append::Bool)
@ Base ./iostream.jl:293
[5] openfile
@ ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:223 [inlined]
[6] openfile(T::Type, fname::String, wr::Bool, create::Bool, truncate::Bool, fallback::Type)
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:234
[7] jldopen(fname::String, wr::Bool, create::Bool, truncate::Bool, iotype::Type{JLD2.MmapIO}; fallback::Type{IOStream}, compress::Bool, mmaparrays::Bool, typemap::Dict{String, Any})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:290
[8] jldopen(fname::String, wr::Bool, create::Bool, truncate::Bool, iotype::Type{JLD2.MmapIO})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:253
[9] jldopen(fname::String, mode::String; iotype::Type, kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:356
[10] jldopen (repeats 2 times)
@ ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:351 [inlined]
[11] jldopen(f::Function, args::String; kws::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:2
[12] jldopen
@ ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:2 [inlined]
[13] macro expansion
@ ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:145 [inlined]
[14] load_unet(filename::String)
@ UNet ~model_IO.jl:33
[15] macro expansion
@ ~model.jl:54 [inlined]
[16] macro expansion
@ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
[17] macro expansion
@ ~model.jl:54 [inlined]
[18] macro expansion
@ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
[19] top-level scope
@ ~model.jl:24
[20] include(fname::String)
@ Base.MainInclude ./client.jl:444
[21] top-level scope
@ ~/runtests.jl:47
[22] include(fname::String)
@ Base.MainInclude ./client.jl:444
[23] top-level scope
@ REPL[1]:1
[24] eval
@ ./boot.jl:360 [inlined]
[25] eval_user_input(ast::Any, backend::REPL.REPLBackend)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:139
[26] repl_backend_loop(backend::REPL.REPLBackend)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:200
[27] start_repl_backend(backend::REPL.REPLBackend, consumer::Any)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:185
[28] run_repl(repl::REPL.AbstractREPL, consumer::Any; backend_on_current_task::Bool)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:317
[29] run_repl(repl::REPL.AbstractREPL, consumer::Any)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:305
[30] (::Base.var"#874#876"{Bool, Bool, Bool})(REPL::Module)
@ Base ./client.jl:387
[31] #invokelatest#2
@ ./essentials.jl:708 [inlined]
[32] invokelatest
@ ./essentials.jl:706 [inlined]
[33] run_main_repl(interactive::Bool, quiet::Bool, banner::Bool, history_file::Bool, color_set::Bool)
@ Base ./client.jl:372
[34] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:302
[35] _start()
@ Base ./client.jl:485
caused by: SystemError: opening file "/tmp/jl_SjrXkR/UNet_demo.jld2": No such file or directory
Stacktrace:
[1] systemerror(p::String, errno::Int32; extrainfo::Nothing)
@ Base ./error.jl:168
[2] #systemerror#62
@ ./error.jl:167 [inlined]
[3] systemerror
@ ./error.jl:167 [inlined]
[4] open(fname::String; lock::Bool, read::Bool, write::Bool, create::Bool, truncate::Bool, append::Bool)
@ Base ./iostream.jl:293
[5] JLD2.MmapIO(fname::String, write::Bool, create::Bool, truncate::Bool)
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/mmapio.jl:105
[6] openfile
@ ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:226 [inlined]
[7] openfile(T::Type, fname::String, wr::Bool, create::Bool, truncate::Bool, fallback::Type)
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:231
[8] jldopen(fname::String, wr::Bool, create::Bool, truncate::Bool, iotype::Type{JLD2.MmapIO}; fallback::Type{IOStream}, compress::Bool, mmaparrays::Bool, typemap::Dict{String, Any})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:290
[9] jldopen(fname::String, wr::Bool, create::Bool, truncate::Bool, iotype::Type{JLD2.MmapIO})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:253
[10] jldopen(fname::String, mode::String; iotype::Type, kwargs::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:356
[11] jldopen (repeats 2 times)
@ ~/.julia/packages/JLD2/k9Gt0/src/JLD2.jl:351 [inlined]
[12] jldopen(f::Function, args::String; kws::Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}})
@ JLD2 ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:2
[13] jldopen
@ ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:2 [inlined]
[14] macro expansion
@ ~/.julia/packages/JLD2/k9Gt0/src/loadsave.jl:145 [inlined]
[15] load_unet(filename::String)
@ UNet ~model_IO.jl:33
[16] macro expansion
@ ~model.jl:54 [inlined]
[17] macro expansion
@ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
[18] macro expansion
@ ~model.jl:54 [inlined]
[19] macro expansion
@ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
[20] top-level scope
@ ~model.jl:24
[21] include(fname::String)
@ Base.MainInclude ./client.jl:444
[22] top-level scope
@ ~runtests.jl:47
[23] include(fname::String)
@ Base.MainInclude ./client.jl:444
[24] top-level scope
@ REPL[1]:1
[25] eval
@ ./boot.jl:360 [inlined]
[26] eval_user_input(ast::Any, backend::REPL.REPLBackend)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:139
[27] repl_backend_loop(backend::REPL.REPLBackend)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:200
[28] start_repl_backend(backend::REPL.REPLBackend, consumer::Any)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:185
[29] run_repl(repl::REPL.AbstractREPL, consumer::Any; backend_on_current_task::Bool)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:317
[30] run_repl(repl::REPL.AbstractREPL, consumer::Any)
@ REPL /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/REPL/src/REPL.jl:305
[31] (::Base.var"#874#876"{Bool, Bool, Bool})(REPL::Module)
@ Base ./client.jl:387
[32] #invokelatest#2
@ ./essentials.jl:708 [inlined]
[33] invokelatest
@ ./essentials.jl:706 [inlined]
[34] run_main_repl(interactive::Bool, quiet::Bool, banner::Bool, history_file::Bool, color_set::Bool)
@ Base ./client.jl:372
[35] exec_options(opts::Base.JLOptions)
@ Base ./client.jl:302
[36] _start()
@ Base ./client.jl:485