Multipart upload to Amazon Glacier: Content-Range incompatible with Content-Length

Question

I'm trying to upload a file around 1gb in size to Amazon Glacier. Somewhat arbitrarily, I've decided to break it into 32mb parts and upload them in serial.

import math
import boto3
from botocore.utils import calculate_tree_hash

client = boto3.client('glacier')
vault_name = 'my-vault'
size = 1073745600 # in bytes
size_mb = size / (2**20) # Convert to megabytes for readability
local_file = 'filename'

multi_up = client.initiate_multipart_upload(vaultName=vault_name,
                                        archiveDescription=local_file,
                                        partSize=str(2**25)) # 32 mb in bytes
parts = math.floor(size_mb / 32)
with open("/Users/alexchase/Desktop/{}".format(local_file), 'rb') as upload:
    for p in range(parts):
        # Calculate lower and upper bounds for the byte ranges. The last range
        # is bigger than the ones that come before.
        lower = (p * (2**25))
        upper = (((p + 1) * (2**25)) - 1) if (p + 1 < parts) else (size)
        up_part = client.upload_multipart_part(vaultName=vault_name,
                                           uploadId=multi_up['uploadId'],
                                           range='bytes {}-{}/*'.format(lower, upper),
                                           body=upload)
checksum = calculate_tree_hash(upload)
complete_up = client.complete_multipart_upload(archiveSize=str(size),
                                               checksum=checksum,
                                               uploadId=multi_up['uploadId'],
                                               vaultName=vault_name)

This generates an error about the first byte range.

---------------------------------------------------------------------------
InvalidParameterValueException            Traceback (most recent call last)
 in ()
     93                         uploadId=multi_up['uploadId'],
     94                         range='bytes {}-{}/*'.format(lower, upper),
---> 95                         body=upload)
     96                     upload_info.append(up_part)
     97                 checksum = calculate_tree_hash(upload)

~/anaconda/lib/python3.5/site-packages/botocore/client.py in _api_call(self, *args, **kwargs)
    251                     "%s() only accepts keyword arguments." % py_operation_name)
    252             # The "self" in this scope is referring to the BaseClient.
--> 253             return self._make_api_call(operation_name, kwargs)
    254
    255         _api_call.__name__ = str(py_operation_name)

~/anaconda/lib/python3.5/site-packages/botocore/client.py in _make_api_call(self, operation_name, api_params)
    555             error_code = parsed_response.get("Error", {}).get("Code")
    556             error_class = self.exceptions.from_code(error_code)
--> 557             raise error_class(parsed_response, operation_name)
    558         else:
    559             return parsed_response

InvalidParameterValueException: An error occurred (InvalidParameterValueException) when calling the UploadMultipartPart operation:
Content-Range: bytes 0-33554431/* is incompatible with Content-Length: 1073745600

Can anyone see what I'm doing wrong?

user_2085469 · Answer

Since the follow up answer from Alex claims it "works", I'm posting another version that worked for me under Python 3.5 and Ubuntu 16.04. I also added some environment variables from our production end to end solution.

The original post gave me an error so I tweaked it and provided some clean up. Hope this helps someone needing this Glacier functionality. Using a Shell script with awscli commands was not as clean.

import math
import boto3
import os
from botocore.utils import calculate_tree_hash

vault_name = os.getenv('GLACIER_VAULT_NAME')
file_name = os.getenv('GLACIER_UPLOAD_FILE')

if vault_name is None:
    print('GLACIER_VAULT_NAME environment variable is required. Exiting.')
    exit(1)
if file_name is None:
    print('GLACIER_UPLOAD_FILE environment variable is required. Exiting.')
    exit(2)

chunk_size = 2 ** 25
client = boto3.client('glacier')

client.create_vault(vaultName=vault_name)

upload_obj = client.initiate_multipart_upload(vaultName=vault_name,
                                              archiveDescription=file_name,
                                              partSize=str(chunk_size))
file_size = os.path.getsize(file_name)
parts = math.ceil(file_size / chunk_size)

with open(file_name, 'rb') as upload:
    for p in range(parts):
        lower = p * chunk_size
        upper = lower + chunk_size - 1

        if upper > file_size:
            upper = (file_size - lower) + lower - 1

        file_part = upload.read(chunk_size)

        up_part = client.upload_multipart_part(vaultName=vault_name,
                                               uploadId=upload_obj['uploadId'],
                                               range='bytes {}-{}/{}'.format(lower,
                                                                             upper,
                                                                             file_size),
                                               body=file_part)

# this needs a new file handler because calculate_tree_hash() processes 
# the handler in a similar way to the loop above
checksum = calculate_tree_hash(open(file_name, 'rb'))
complete_up = client.complete_multipart_upload(vaultName=vault_name,
                                               uploadId=upload_obj['uploadId'],
                                               archiveSize=str(file_size),
                                               checksum=checksum)

print(complete_up)

Multipart upload to Amazon Glacier: Content-Range incompatible with Content-Length

3 Answers