C# Azure upload file error The specified blob or block content is invalid

I'm uploading files to azure storage.

public class AzureBlob : ICloudBlob
{
    private string _fileName;
    public string FileName
    {
        get => _fileName;
        set
        {
            _fileName = value;
            _cloudBlockBlob = CloudBlobContainer.GetBlockBlobReference(value);
        }
    }
    public CloudBlobContainer CloudBlobContainer { get; set; }
    private CloudBlockBlob _cloudBlockBlob;

    public async Task UploadChunksFromPathAsync(string path, long fileLength)
    {
        const int blockSize = 256 * 1024;
        var bytesToUpload = fileLength;
        long bytesUploaded = 0;
        long startPosition = 0;

        var blockIds = new List<string>();
        var index = 0;

        do
        {
            var bytesToRead = Math.Min(blockSize, bytesToUpload);
            var blobContents = new byte[bytesToRead];

            using (var fs = new FileStream(path, FileMode.Open))
            {
                fs.Position = startPosition;
                fs.Read(blobContents, 0, (int) bytesToRead);
            }

            var blockId = Convert.ToBase64String(Encoding.UTF8.GetBytes(index.ToString("d6")));

            blockIds.Add(blockId);
            await _cloudBlockBlob.PutBlockAsync(blockId, new MemoryStream(blobContents), null);

            bytesUploaded += bytesToRead;
            bytesToUpload -= bytesToRead;
            startPosition += bytesToRead;
            index++;
        } while (bytesToUpload > 0);

        await _cloudBlockBlob.PutBlockListAsync(blockIds);   
    }
}

This works fine for one file upload, multiple file uploads calling this method one after the other throws a 400 error on _cloudBlockBlob.PutBlockListAsync with the azure error being

The specified blob or block content is invalid.

If I remove the await keyword on _cloudBlockBlob.PutBlockListAsync it works completely fine.

The blockIds are all of the same length. What am I doing wrong?

Edit

Calling code in controller:

[HttpPost]
public async Task<IActionResult> Upload([FromBody] UploadViewModel model)
{
    var audioBlob = _cloudStorage.GetBlob(CloudStorageType.Audio, model.AudioName);

    await audioBlob.UploadChunksFromPathAsync(model.AudioPath, model.FileLength);

    return Ok();
}

Storage:

public enum CloudStorageType
{
    Audio,
    Image,
}

public class AzureStorage : ICloudStorage
{
    public IDictionary<CloudStorageType, ICloudBlob> CloudBlobs { get; set; }

    public AzureStorage(IConfiguration configuration)
    {
        var storageAccount = CloudStorageAccount.Parse(configuration["ConnectionStrings:StorageConnectionString"]);
        var blobClient = storageAccount.CreateCloudBlobClient();

        CloudBlobs = new Dictionary<CloudStorageType, ICloudBlob>();

        foreach (CloudStorageType cloudStorageType in Enum.GetValues(typeof(CloudStorageType)))
        {
            CloudBlobs[cloudStorageType] = new AzureBlob(cloudStorageType.ToString().ToLower(), blobClient);
        }
    }

    public ICloudBlob GetBlob(CloudStorageType cloudStorageType, string fileName)
    {
        CloudBlobs[cloudStorageType].FileName = fileName;

        return CloudBlobs[cloudStorageType];
    }
}

Startup.cs

var azureStorage = new AzureStorage(_configuration);

// Add application services.
services.AddSingleton(_configuration);
services.AddSingleton<ICloudStorage>(azureStorage);