Does anyone know how to add processors for Pyjulia when calling Julia function in Python? I need to use Python to call a Julia function using multiple cores and return values to Python.
The Python code, testJulia_addproc.py:
import julia
j = julia.Julia()
print("Core number = {}".format(j.nprocs()))
j.include("test_addproc.jl")
a = j.eval("test_addproc()")
print a
The Julia code test_addproc.jl:
function test_addproc()
println("line 1")
np = nprocs()
println("line 2")
if np < 12
println("line 3")
addprocs(12-np)
println("line 4")
end
println("line 5")
return(1+1)
end
==
$ python testJulia_addproc.py
I got this error.
WARNING: redefining constant JULIA_HOME
Core number = 1
line 1
line 2
line 3
ERROR (unhandled task failure): could not spawn setenv(`/anaconda/lib/python2.7/site-packages/julia-0.1.1-py2.7.egg/julia/../fake-julia/julia -Ccore2 -J/Applications/Julia-0.5.app/Contents/Resources/julia/lib/julia/sys.dylib --compile=yes --depwarn=yes --bind-to 127.0.0.1 --worker pjVTa5WBhFbDB7vs`; dir="/Users/chlin/Dropbox/Projects/StackOverflow"): no such file or directory (ENOENT)
in _jl_spawn(::String, ::Array{String,1}, ::Ptr{Void}, ::Base.Process, ::Base.DevNullStream, ::Base.PipeEndpoint, ::Base.TTY) at ./process.jl:321
in #424 at ./process.jl:478 [inlined]
in setup_stdio(::Base.##424#425{Cmd,Ptr{Void},Base.Process}, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}) at ./process.jl:466
in #spawn#423(::Nullable{Base.ProcessChain}, ::Function, ::Cmd, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./process.jl:477
in (::Base.#kw##spawn)(::Array{Any,1}, ::Base.#spawn, ::Cmd, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./<missing>:0
in #spawn#420(::Nullable{Base.ProcessChain}, ::Function, ::Base.CmdRedirect, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./process.jl:359
in (::Base.#kw##spawn)(::Array{Any,1}, ::Base.#spawn, ::Base.CmdRedirect, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./<missing>:0
in open(::Base.CmdRedirect, ::String, ::Base.DevNullStream) at ./process.jl:539
in launch(::Base.LocalManager, ::Dict{Any,Any}, ::Array{WorkerConfig,1}, ::Condition) at ./managers.jl:318
in (::Base.##666#670{Base.LocalManager,Dict{Any,Any},Array{WorkerConfig,1},Condition})() at ./event.jl:68
could not spawn setenv(`/anaconda/lib/python2.7/site-packages/julia-0.1.1-py2.7.egg/julia/../fake-julia/julia -Ccore2 -J/Applications/Julia-0.5.app/Contents/Resources/julia/lib/julia/sys.dylib --compile=yes --depwarn=yes --bind-to 127.0.0.1 --worker pjVTa5WBhFbDB7vs`; dir="/Users/chlin/Dropbox/Projects/StackOverflow"): no such file or directory (ENOENT)Traceback (most recent call last):
File "testJulia_addproc.py", line 5, in <module>
j.eval("test_addproc()")
File "//anaconda/lib/python2.7/site-packages/julia-0.1.1-py2.7.egg/julia/core.py", line 436, in eval
ans = self._call(src)
File "//anaconda/lib/python2.7/site-packages/julia-0.1.1-py2.7.egg/julia/core.py", line 399, in _call
self.check_exception(src)
File "//anaconda/lib/python2.7/site-packages/julia-0.1.1-py2.7.egg/julia/core.py", line 419, in check_exception
.format(exception_type, src))
julia.core.JuliaError: Exception 'UVError' occurred while calling julia code:
test_addproc()
Based on this post, I found the number of processors in pyjulia needs to match the number in Julia (@Isaiah: nprocs() in pure Julia must match j.nprocs()).
I tried to add procs in pyjulia.
import julia
j = julia.Julia()
j.addprocs(2)
Got errors:
ERROR (unhandled task failure): could not spawn setenv(`/anaconda/lib/python2.7/site-packages/julia-0.1.1-py2.7.egg/julia/../fake-julia/julia -Ccore2 -J/Applications/Julia-0.5.app/Contents/Resources/julia/lib/julia/sys.dylib --compile=yes --depwarn=yes --bind-to 127.0.0.1 --worker 8WpA7O0EArLzXh25`; dir="/Applications/Julia-0.5.app/Contents/Resources/julia/bin"): no such file or directory (ENOENT)
in _jl_spawn(::String, ::Array{String,1}, ::Ptr{Void}, ::Base.Process, ::Base.DevNullStream, ::Base.PipeEndpoint, ::Base.TTY) at ./process.jl:321
in #424 at ./process.jl:478 [inlined]
in setup_stdio(::Base.##424#425{Cmd,Ptr{Void},Base.Process}, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}) at ./process.jl:466
in #spawn#423(::Nullable{Base.ProcessChain}, ::Function, ::Cmd, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./process.jl:477
in (::Base.#kw##spawn)(::Array{Any,1}, ::Base.#spawn, ::Cmd, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./<missing>:0
in #spawn#420(::Nullable{Base.ProcessChain}, ::Function, ::Base.CmdRedirect, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./process.jl:359
in (::Base.#kw##spawn)(::Array{Any,1}, ::Base.#spawn, ::Base.CmdRedirect, ::Tuple{Base.DevNullStream,Pipe,Base.TTY}, ::Bool, ::Bool) at ./<missing>:0
in open(::Base.CmdRedirect, ::String, ::Base.DevNullStream) at ./process.jl:539
in launch(::Base.LocalManager, ::Dict{Any,Any}, ::Array{WorkerConfig,1}, ::Condition) at ./managers.jl:318
in (::Base.##666#670{Base.LocalManager,Dict{Any,Any},Array{WorkerConfig,1},Condition})() at ./event.jl:68
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-3-bfa063d91dec> in <module>()
----> 1 j.addprocs(2)
RuntimeError: Julia exception: could not spawn setenv(`/anaconda/lib/python2.7/site-packages/julia-0.1.1-py2.7.egg/julia/../fake-julia/julia -Ccore2 -J/Applications/Julia-0.5.app/Contents/Resources/julia/lib/julia/sys.dylib --compile=yes --depwarn=yes --bind-to 127.0.0.1 --worker 8WpA7O0EArLzXh25`; dir="/Applications/Julia-0.5.app/Contents/Resources/julia/bin"): no such file or directory (ENOENT)
Based on this post, I found there is a way around to add processors in pyjulia, but didn't return values of the Julia function.
# One way to add procs in pyjulia
import ctypes
JLPATH=b"/Applications/Julia-0.5.app/Contents/Resources/julia"
jl = ctypes.PyDLL(JLPATH+b"/lib/libjulia.dylib", ctypes.RTLD_GLOBAL)
jl.jl_init(JLPATH+b"/bin/")
jl.jl_eval_string(" include(test_addproc.jl) ")
a = jl.jl_eval_string(" test_addproc()) ")
print a
# 0
# It should be 2
My MacPro version is OS X Yosemite 10.10.5
My Python version is Python 2.7.12 :: Anaconda 4.2.0 (x86_64).
My Julia version is
Julia Version 0.5.2
Commit f4c6c9d4bb (2017-05-06 16:34 UTC)
Platform Info:
OS: macOS (x86_64-apple-darwin13.4.0)
CPU: Intel(R) Xeon(R) CPU E5-1650 v2 @ 3.50GHz
WORD_SIZE: 64
BLAS: libopenblas (USE64BITINT DYNAMIC_ARCH NO_AFFINITY Sandybridge)
LAPACK: libopenblas64_
LIBM: libopenlibm
LLVM: libLLVM-3.7.1 (ORCJIT, ivybridge)
Any suggestion or thought are welcome. Thank you.