The files are in a remote sftp and i search for them every morning with a automated task. The main issue is that is not failing always and is not the same file which always fails, making this error very random. Even when downloading the files manually with filezilla and uploading them again as a test, it fails. We are using paramiko's python and pysftp to connect the server.
I am posting a trace of the exception:
/opt/project/.local/lib/python3.8/site-packages/pysftp/__init__.py:61: UserWarning: Failed to load HostKeys from /opt/project/.ssh/known_hosts. You will need to explicitly load HostKeys (cnopts.hostkeys.load(filename)) or disableHostKey checking (cnopts.hostkeys = None).
warnings.warn(wmsg, UserWarning)
2022-09-14 16:57:59 [development.downloader] INFO: Succesfully connected with sftp.
2022-09-14 16:58:05 [development.downloader] INFO: Succesfully connected with sftp.
2022-09-14 16:59:55 [development.downloader] INFO: File downloaded_file_20220914_2462.csv downloaded successfully.
2022-09-14 16:59:58 [development.downloader] INFO: File downloaded_file_20220914_2316.csv downloaded successfully.
2022-09-14 17:00:02 [development.downloader] INFO: File downloaded_file_20220914_2353.csv downloaded successfully.
2022-09-14 17:00:08 [development.downloader] INFO: File downloaded_file_20220914_2632.csv downloaded successfully.
2022-09-14 17:00:12 [development.downloader] INFO: File downloaded_file_20220914_2357.csv downloaded successfully.
Traceback (most recent call last):
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 852, in _read_response
t, data = self._read_packet()
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp.py", line 201, in _read_packet
x = self._read_all(4)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp.py", line 188, in _read_all
raise EOFError()
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/opt/project/.local/lib/python3.8/site-packages/pysftp/__init__.py", line 509, in cd
yield
File "/opt/project/project/developmentdownloader.py", line 87, in get
sftp.get(filename, destination)
File "/opt/project/.local/lib/python3.8/site-packages/pysftp/__init__.py", line 249, in get
self._sftp.get(remotepath, localpath, callback=callback)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 811, in get
size = self.getfo(remotepath, fl, callback, prefetch)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 786, in getfo
return self._transfer_with_callback(
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 678, in _transfer_with_callback
data = reader.read(32768)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/file.py", line 219, in read
new_data = self._read(read_size)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_file.py", line 182, in _read
data = self._read_prefetch(size)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_file.py", line 162, in _read_prefetch
self.sftp._read_response()
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 854, in _read_response
raise SSHException("Server connection dropped: {}".format(e))
paramiko.ssh_exception.SSHException: Server connection dropped:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "manage.py", line 32, in <module>
module.run()
File "/opt/project/project/idevelopmentfiles_csv/data_collector.py", line 118, in run
integration.run()
File "/opt/project/project/developmentintegrator.py", line 438, in run
self.start_downloading()
File "/usr/local/lib/python3.8/contextlib.py", line 75, in inner
return func(*args, **kwds)
File "/opt/project/project/developmentintegrator.py", line 302, in start_downloading
self._download_multiple_branch_files()
File "/opt/project/project/developmentintegrator.py", line 197, in _download_multiple_branch_files
downloaded_file = downloader.get(file, destination)
File "/opt/project/project/developmentdownloader.py", line 89, in get
return destination
File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/opt/project/.local/lib/python3.8/site-packages/pysftp/__init__.py", line 511, in cd
self.cwd(original_path)
File "/opt/project/.local/lib/python3.8/site-packages/pysftp/__init__.py", line 524, in chdir
self._sftp.chdir(remotepath)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 659, in chdir
if not stat.S_ISDIR(self.stat(path).st_mode):
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 493, in stat
t, msg = self._request(CMD_STAT, path)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 821, in _request
num = self._async_request(type(None), t, *arg)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp_client.py", line 846, in _async_request
self._send_packet(t, msg)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp.py", line 198, in _send_packet
self._write_all(out)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/sftp.py", line 162, in _write_all
n = self.sock.send(out)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/channel.py", line 801, in send
return self._send(s, m)
File "/opt/project/.local/lib/python3.8/site-packages/paramiko/channel.py", line 1198, in _send
raise socket.error("Socket is closed")
OSError: Socket is closed
After some minutes it fails with the error shown above. But as mentioned before, the error is not always in the same file nor the same moment.
I really appreciate any help on your side