Windows Azure - Cleaning Up The WADLogsTable

Here's my slightly different version of @Chriseyre2000's solution, using asynchronous operations and PartitionKey querying. It's designed to run continuously within a Worker Role in my case. This one may be a bit easier on memory if you have a lot of entries to clean up.

static class LogHelper
{
    /// <summary>
    /// Periodically run a cleanup task for log data, asynchronously
    /// </summary>
    public static async void TruncateDiagnosticsAsync()
    {
        while ( true )
        {
            try
            {
                // Retrieve storage account from connection-string
                CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
                    CloudConfigurationManager.GetSetting( "CloudStorageConnectionString" ) );

                CloudTableClient tableClient = storageAccount.CreateCloudTableClient();

                CloudTable cloudTable = tableClient.GetTableReference( "WADLogsTable" );

                // keep a weeks worth of logs
                DateTime keepThreshold = DateTime.UtcNow.AddDays( -7 );

                // do this until we run out of items
                while ( true )
                {
                    TableQuery query = new TableQuery();
                    query.FilterString = string.Format( "PartitionKey lt '0{0}'", keepThreshold.Ticks );
                    var items = cloudTable.ExecuteQuery( query ).Take( 1000 );

                    if ( items.Count() == 0 )
                        break;

                    Dictionary<string, TableBatchOperation> batches = new Dictionary<string, TableBatchOperation>();
                    foreach ( var entity in items )
                    {
                        TableOperation tableOperation = TableOperation.Delete( entity );

                        // need a new batch?
                        if ( !batches.ContainsKey( entity.PartitionKey ) )
                            batches.Add( entity.PartitionKey, new TableBatchOperation() );

                        // can have only 100 per batch
                        if ( batches[entity.PartitionKey].Count < 100)
                            batches[entity.PartitionKey].Add( tableOperation );
                    }

                    // execute!
                    foreach ( var batch in batches.Values )
                        await cloudTable.ExecuteBatchAsync( batch );

                    Trace.TraceInformation( "WADLogsTable truncated: " + query.FilterString );
                }
            }
            catch ( Exception ex )
            {
                Trace.TraceError( "Truncate WADLogsTable exception {0}", ex.Message );
            }

            // run this once per day
            await Task.Delay( TimeSpan.FromDays( 1 ) );
        }
    }
}

To start the process, just call this from the OnStart method in your worker role.

// start the periodic cleanup
LogHelper.TruncateDiagnosticsAsync();

Slightly updated Chriseyre2000's code:

• using ExecuteQuerySegmented instead of ExecuteQuery

• observing TableBatchOperation limit of 100 operations

• purging all Azure tables

  public static void TruncateAllAzureTables(CloudStorageAccount storageAccount, DateTime keepThreshold)
  {
     TruncateAzureTable(storageAccount, "WADLogsTable", keepThreshold);
     TruncateAzureTable(storageAccount, "WADCrashDump", keepThreshold);
     TruncateAzureTable(storageAccount, "WADDiagnosticInfrastructureLogsTable", keepThreshold);
     TruncateAzureTable(storageAccount, "WADPerformanceCountersTable", keepThreshold);
     TruncateAzureTable(storageAccount, "WADWindowsEventLogsTable", keepThreshold);
  }
  
  public static void TruncateAzureTable(CloudStorageAccount storageAccount, string aTableName, DateTime keepThreshold)
  {
     const int maxOperationsInBatch = 100;
     var tableClient = storageAccount.CreateCloudTableClient();
  
     var cloudTable = tableClient.GetTableReference(aTableName);
  
     var query = new TableQuery { FilterString = $"Timestamp lt datetime'{keepThreshold:yyyy-MM-ddTHH:mm:ss}'" };
     TableContinuationToken continuationToken = null;
     do
     {
        var queryResult = cloudTable.ExecuteQuerySegmented(query, continuationToken);
        continuationToken = queryResult.ContinuationToken;
  
        var items = queryResult.ToList();
        var batches = new Dictionary<string, List<TableBatchOperation>>();
        foreach (var entity in items)
        {
           var tableOperation = TableOperation.Delete(entity);
  
           if (!batches.TryGetValue(entity.PartitionKey, out var batchOperationList))
           {
              batchOperationList = new List<TableBatchOperation>();
              batches.Add(entity.PartitionKey, batchOperationList);
           }
  
           var batchOperation = batchOperationList.FirstOrDefault(bo => bo.Count < maxOperationsInBatch);
           if (batchOperation == null)
           {
              batchOperation = new TableBatchOperation();
              batchOperationList.Add(batchOperation);
           }
           batchOperation.Add(tableOperation);
        }
  
        foreach (var batch in batches.Values.SelectMany(l => l))
        {
           cloudTable.ExecuteBatch(batch);
        }
     } while (continuationToken != null);
  }