I'm running GPU on a UNet with Flux as
Flux.train!(loss, Flux.params(model), train_data_loader, opt)
but I'm getting an error at NNlib for DataType error. So 'w' is a CudaArray which is why it becomes a CuPtr at line 55. However, I'm not sure why i'm getting this error.
and the rest of the error is
This output is 1typeof(w)CuArray{Float32, 5, CUDA.Mem.DeviceBuffer}typeof(w_ptr)CuPtr{Float32}ERROR:
LoadError: TaskFailedException
nested task error: TaskFailedException
Stacktrace:
[1] wait
@ ./task.jl:322 [inlined]
[2] threading_run(func::Function)
@ Base.Threads ./threadingconstructs.jl:34
[3] macro expansion
@ ./threadingconstructs.jl:93 [inlined]
[4] conv_im2col!(y::SubArray{Float32, 5, Array{Float32, 5}, Tuple{Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}},
Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64},
Base.Slice{Base.OneTo{Int64}}}, false}, x::SubArray{Float32, 5,
Array{Float32, 5}, Tuple{Base.Slice{Base.OneTo{Int64}},
Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}},
UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false},
w::CuArray{Float32, 5, CUDA.Mem.DeviceBuffer}, cdims::DenseConvDims{3,
3, 3, 6, 3}; col::Array{Float32, 3}, alpha::Float32, beta::Float32)
@ NNlib ~/.julia/packages/NNlib/0QnJJ/src/impl/conv_im2col.jl:47
[5] conv_im2col!
@ ~/.julia/packages/NNlib/0QnJJ/src/impl/conv_im2col.jl:28 [inlined]
[6] (::NNlib.var"#262#266"{Base.Iterators.Pairs{Union{}, Union{}, Tuple{}, NamedTuple{(), Tuple{}}}, DenseConvDims{3, 3, 3, 6,
3}, SubArray{Float32, 5, Array{Float32, 5},
Tuple{Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}},
Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64},
Base.Slice{Base.OneTo{Int64}}}, false}, CuArray{Float32, 5,
CUDA.Mem.DeviceBuffer}, SubArray{Float32, 5, Array{Float32, 5},
Tuple{Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}},
Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64},
Base.Slice{Base.OneTo{Int64}}}, false}})()
@ NNlib ./threadingconstructs.jl:169
nested task error: MethodError: no method matching gemm!(::Val{false}, ::Val{false}, ::Int64, ::Int64, ::Int64,
::Float32, ::Ptr{Float32}, ::CuPtr{Float32}, ::Float32,
::Ptr{Float32})
Closest candidates are:
gemm!(::Val, ::Val, ::Int64, ::Int64, ::Int64, ::Float32, ::Ptr{Float32}, ::Ptr{Float32}, ::Float32, ::Ptr{Float32})
at /home/xyz/.julia/packages/NNlib/0QnJJ/src/gemm.jl:29
gemm!(::Val, ::Val, ::Int64, ::Int64, ::Int64, ::Float64, ::Ptr{Float64}, ::Ptr{Float64}, ::Float64, ::Ptr{Float64})
at /home/xyz/.julia/packages/NNlib/0QnJJ/src/gemm.jl:29
gemm!(::Val, ::Val, ::Int64, ::Int64, ::Int64, ::ComplexF64, ::Ptr{ComplexF64}, ::Ptr{ComplexF64}, ::ComplexF64,
::Ptr{ComplexF64}) at
/home/xyz/.julia/packages/NNlib/0QnJJ/src/gemm.jl:29
...
Stacktrace:
[1] macro expansion
@ ~/.julia/packages/NNlib/0QnJJ/src/impl/conv_im2col.jl:58 [inlined]
[2] (::NNlib.var"#909#threadsfor_fun#504"{Array{Float32, 3}, Float32, Float32, SubArray{Float32, 5, Array{Float32, 5},
Tuple{Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}},
Base.Slice{Base.OneTo{Int64}}, UnitRange{Int64},
Base.Slice{Base.OneTo{Int64}}}, false}, SubArray{Float32, 5,
Array{Float32, 5}, Tuple{Base.Slice{Base.OneTo{Int64}},
Base.Slice{Base.OneTo{Int64}}, Base.Slice{Base.OneTo{Int64}},
UnitRange{Int64}, Base.Slice{Base.OneTo{Int64}}}, false},
CuArray{Float32, 5, CUDA.Mem.DeviceBuffer}, DenseConvDims{3, 3, 3, 6,
3}, Int64, Int64, Int64, UnitRange{Int64}})(onethread::Bool)
...
@ Zygote ~/.julia/packages/Zygote/PD12J/src/compiler/interface.jl:96
[32] macro expansion
@ ~/.julia/dev/Flux/src/optimise/train.jl:129 [inlined]
[33] macro expansion
@ ~/.julia/packages/ProgressLogging/6KXlp/src/ProgressLogging.jl:328
[inlined]
[34] train!(loss::Function, ps::Zygote.Params{Zygote.Buffer{Any, Vector{Any}}}, data::MLUtils.DataLoader{Tuple{Array{Float32, 4},
Array{Int32, 4}}, Random._GLOBAL_RNG}, opt::RMSProp;
cb::Flux.Optimise.var"#38#41")
@ Flux.Optimise ~/.julia/dev/Flux/src/optimise/train.jl:127
[35] train!(loss::Function, ps::Zygote.Params{Zygote.Buffer{Any, Vector{Any}}}, data::MLUtils.DataLoader{Tuple{Array{Float32, 4},
Array{Int32, 4}}, Random._GLOBAL_RNG}, opt::RMSProp)
@ Flux.Optimise ~/.julia/dev/Flux/src/optimise/train.jl:124