I have a method that copies files from a source to an archive bucket. Below is the s3 bucket structure
s3://ingestion-bucket/manifests/initial/text.csv
s3://ingestion-bucket/manifests/initial/text_1.csv
s3://ingestion-bucket/manifests/ongoing/text_2.csv
s3://ingestion-bucket/manifests/ongoing/text_3.csv
I would like to move all files from the ingestion bucket to the archive bucket except the most recent file that landed in the ingestion bucket. Below is my code and I haven't been able to make it work
def file_archive(file_name, source_bucket, target_bucket):
"""Moves files from one s3 bucket to another.
file_name: Name of the file or table we want to archive
source_bucket: Source bucket containing the files
target_bucket: Destination bucket the files will be archived in
"""
s3_resource = boto3.resource('s3')
objects = list(s3_resource.Bucket(source_bucket).objects.filter(Prefix=f'{file_name}/'))
objects.sort(key=lambda o: o.last_modified)
latest_file = objects[0].key
print("Latest file is", latest_file)
bucket = s3_resource.Bucket(target_bucket)
source = f"s3://{source_bucket}/{file_name}/"
dest = f"s3://{target_bucket}/{file_name}/"
try:
run_command_string = 'aws s3 mv {source} {dest} --recursive --exclude "s3://{source_bucket}/{latest_file}"'.format(source=source, dest=dest, source_bucket=source_bucket, latest_file=latest_file)
print("run_command_string:", run_command_string)
proc = subprocess.Popen(run_command_string, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
out, err = proc.communicate()
returncode = proc.returncode
if returncode == 0:
log.info("Archived file {source} to {dest}".format(source=source,dest=dest))
else:
log.error("Unable to move file {source} to {dest}".format(source=source,dest=dest))
log.error(err)
raise Exception()
return True
except Exception as e:
log.error("Error in Archiving")
log.error("Error message: " + str(e))
raise Exception()
I even printed out "run_command_string" to try to tweak it and run it on cloud shell to make it work but was unsuccessful with the below variations
aws s3 cp s3://ingestion-bucket/manifests/ s3://archive-bucket/manifests/ --recursive --exclude "s3://ingestion-bucket/manifests/ongoing/text_3.csv"
aws s3 cp s3://ingestion-bucket/manifests/ s3://archive-bucket/manifests/ --dryrun --recursive --include "*" --exclude "ingestion-bucket/manifests/ongoing/text_3.csv"
aws s3 cp s3://ingestion-bucket/manifests/ s3://archive-bucket/manifests/ --dryrun --recursive --include * --exclude "manifests/ongoing/text_3.csv"