I am trying to spilt a txt file located in s3 and load the split files back to s3 location. I need to use boto3 to do this job as I am working on aws services. Is there a way I can fetch the data, spilt it, and load it back to the s3 using python script. I will be using the script in aws glue job.
import boto3
from boto3.s3.transfer import TransferConfig
import os
import threading
import sys
bucket_name = 'my-bucket'
s3_resource = boto3.resource('s3')
# multipart_chunksize : Each part size is of 1 GB
config = TransferConfig(multipart_threshold=1024 * 1024,
max_concurrency=10,
multipart_chunksize=1024 * 1024,
use_threads=True)
def multipart_download_boto3():
file_path = 'output/multi.txt'
key = 'Outbound/mx.TXT'
gen=s3_resource.Object(bucket_name,
key).download_file(file_path,
Config=config,
)
if __name__ == '__main__':
multipart_download_boto3()