I know this is a long shot because it will be very difficult to do a minimum viable example of the error due to the large amount of prop code being used in a deep hierarchy. But the failure to serialize looks like it occurs for a Dask class, 'Could not serialize object of type SubgraphCallable.'.
The same code worked with a combination of Dask 2.16.0 and Distributed 2.20.0. We want to update to the latest due to a bug in these versions that cause occasional exceptions.
Here is the exception:
distributed.protocol.core - CRITICAL - Failed to Serialize
Traceback (most recent call last):
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/core.py", line 70, in dumps
frames[0] = msgpack.dumps(msg, default=_encode_default, use_bin_type=True)
File "/home/ec2-user/.local/lib/python3.7/site-packages/msgpack/__init__.py", line 35, in packb
return Packer(**kwargs).pack(o)
File "msgpack/_packer.pyx", line 286, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 292, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 289, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 258, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 258, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 279, in msgpack._cmsgpack.Packer._pack
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/core.py", line 56, in _encode_default
obj, serializers=serializers, on_error=on_error, context=context
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 422, in serialize_and_split
header, frames = serialize(x, serializers, on_error, context)
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 256, in serialize
iterate_collection=True,
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 301, in serialize
for obj in x
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 301, in <listcomp>
for obj in x
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 348, in serialize
raise TypeError(msg, str(x)[:10000])
TypeError: ('Could not serialize object of type SubgraphCallable.', 'subgraph_callable')
distributed.comm.utils - ERROR - ('Could not serialize object of type SubgraphCallable.', 'subgraph_callable')
Traceback (most recent call last):
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/comm/utils.py", line 32, in _to_frames
msg, serializers=serializers, on_error=on_error, context=context
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/core.py", line 70, in dumps
frames[0] = msgpack.dumps(msg, default=_encode_default, use_bin_type=True)
File "/home/ec2-user/.local/lib/python3.7/site-packages/msgpack/__init__.py", line 35, in packb
return Packer(**kwargs).pack(o)
File "msgpack/_packer.pyx", line 286, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 292, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 289, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 258, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 258, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 279, in msgpack._cmsgpack.Packer._pack
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/core.py", line 56, in _encode_default
obj, serializers=serializers, on_error=on_error, context=context
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 422, in serialize_and_split
header, frames = serialize(x, serializers, on_error, context)
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 256, in serialize
iterate_collection=True,
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 301, in serialize
for obj in x
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 301, in <listcomp>
for obj in x
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 348, in serialize
raise TypeError(msg, str(x)[:10000])
TypeError: ('Could not serialize object of type SubgraphCallable.', 'subgraph_callable')
distributed.batched - ERROR - Error in batched write
Traceback (most recent call last):
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/batched.py", line 94, in _background_send
payload, serializers=self.serializers, on_error="raise"
File "/home/ec2-user/.local/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
value = future.result()
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/comm/tcp.py", line 243, in write
**self.handshake_options,
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/comm/utils.py", line 51, in to_frames
return _to_frames()
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/comm/utils.py", line 32, in _to_frames
msg, serializers=serializers, on_error=on_error, context=context
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/core.py", line 70, in dumps
frames[0] = msgpack.dumps(msg, default=_encode_default, use_bin_type=True)
File "/home/ec2-user/.local/lib/python3.7/site-packages/msgpack/__init__.py", line 35, in packb
return Packer(**kwargs).pack(o)
File "msgpack/_packer.pyx", line 286, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 292, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 289, in msgpack._cmsgpack.Packer.pack
File "msgpack/_packer.pyx", line 258, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 258, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 225, in msgpack._cmsgpack.Packer._pack
File "msgpack/_packer.pyx", line 279, in msgpack._cmsgpack.Packer._pack
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/core.py", line 56, in _encode_default
obj, serializers=serializers, on_error=on_error, context=context
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 422, in serialize_and_split
header, frames = serialize(x, serializers, on_error, context)
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 256, in serialize
iterate_collection=True,
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 301, in serialize
for obj in x
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 301, in <listcomp>
for obj in x
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/protocol/serialize.py", line 348, in serialize
raise TypeError(msg, str(x)[:10000])
TypeError: ('Could not serialize object of type SubgraphCallable.', 'subgraph_callable')
Traceback (most recent call last):
File "/home/ec2-user/etl/app/supplier/testextract.py", line 432, in calc_output
res = res.compute()
File "/home/ec2-user/.local/lib/python3.7/site-packages/dask/base.py", line 285, in compute
(result,) = compute(self, traverse=False, **kwargs)
File "/home/ec2-user/.local/lib/python3.7/site-packages/dask/base.py", line 567, in compute
results = schedule(dsk, keys, **kwargs)
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/client.py", line 2666, in get
results = self.gather(packed, asynchronous=asynchronous, direct=direct)
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/client.py", line 1981, in gather
asynchronous=asynchronous,
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/client.py", line 844, in sync
self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/utils.py", line 353, in sync
raise exc.with_traceback(tb)
File "/home/ec2-user/.local/lib/python3.7/site-packages/distributed/utils.py", line 336, in f
result[0] = yield future
File "/home/ec2-user/.local/lib/python3.7/site-packages/tornado/gen.py", line 735, in run
value = future.result()
concurrent.futures._base.CancelledError
If you have insights on how we might debug the cause please let me know.