I am currently trying to create a lambda layer so that I can import csv files as pandas dataframes within a lambda function. I tried using the AWS-managed layer (AWSDataWrangler layer) but this does not work either.
I've created 'python' file containing:
I then zip this file and upload it as a lambda layer. I get this error when I try to pd.read_csv() from my s3 bucket:
"errorMessage": "Install s3fs to access S3",
"errorType": "ImportError",
"stackTrace": [
" File \"/var/task/ManifestValidationLambda/manifest_validation.py\", line 19, in handler\n df = pd.read_csv('s3://' + s3_bucket_name +'/' + s3_key, header = 0)\n",
" File \"/opt/python/pandas/util/_decorators.py\", line 311, in wrapper\n return func(*args, **kwargs)\n",
" File \"/opt/python/pandas/io/parsers/readers.py\", line 586, in read_csv\n return _read(filepath_or_buffer, kwds)\n",
" File \"/opt/python/pandas/io/parsers/readers.py\", line 482, in _read\n parser = TextFileReader(filepath_or_buffer, **kwds)\n",
" File \"/opt/python/pandas/io/parsers/readers.py\", line 811, in __init__\n self._engine = self._make_engine(self.engine)\n",
" File \"/opt/python/pandas/io/parsers/readers.py\", line 1040, in _make_engine\n return mapping[engine](self.f, **self.options) # type: ignore[call-arg]\n",
" File \"/opt/python/pandas/io/parsers/c_parser_wrapper.py\", line 51, in __init__\n self._open_handles(src, kwds)\n",
" File \"/opt/python/pandas/io/parsers/base_parser.py\", line 222, in _open_handles\n self.handles = get_handle(\n",
" File \"/opt/python/pandas/io/common.py\", line 609, in get_handle\n ioargs = _get_filepath_or_buffer(\n",
" File \"/opt/python/pandas/io/common.py\", line 358, in _get_filepath_or_buffer\n file_obj = fsspec.open(\n",
" File \"/opt/python/fsspec/core.py\", line 452, in open\n return open_files(\n",
" File \"/opt/python/fsspec/core.py\", line 286, in open_files\n fs, fs_token, paths = get_fs_token_paths(\n",
" File \"/opt/python/fsspec/core.py\", line 622, in get_fs_token_paths\n cls = get_filesystem_class(protocol)\n",
" File \"/opt/python/fsspec/registry.py\", line 221, in get_filesystem_class\n raise ImportError(bit[\"err\"]) from e\n"
]
}
Here is my python code:
s3_bucket_name = 'bucketname'
s3_key = 'csv file name'
s3_client = boto3.client('s3')
# obj = s3_client.get_object(Bucket=s3_bucket_name, Key=s3_key)
# #print(obj['Body'])
df = pd.read_csv('s3://' + s3_bucket_name +'/' + s3_key, header = 0)
print(df.head(2))
You can see that I have the s3fs package within the zip directory. How should I resolve this?