I am working on a C# .NET core 3.1 application that needs to insert 300 - 500 million rows avro file data into a GBQ table. My idea is to batch insert the data using .Net Task to insert data asynchronously that doesn't block the main thread and when all tasks are finished, log the success or fail message. I did a sample code, if I use Task.Run(), it will break the batchId and lose some data. However, if using RunSynchronously works fine, but it will block the main thread and take some time, which is still acceptable. Just wondering if what's wrong with my code and is Task.Run() a good idea for my case. Thanks a lot! Here is my code: https://dotnetfiddle.net/CPKsMv Just in case, it doesn't work well, pasted here again:
using System;
using System.Collections;
using System.Threading.Tasks;
public class Program
{
public static void Main()
{
ArrayList forecasts = new ArrayList();
for(var k = 0; k < 100; k++){
forecasts.Add(k);
}
int size = 6;
var taskNum = (int) Math.Ceiling(forecasts.Count / (double) size);
Console.WriteLine("task number:" + taskNum);
Console.WriteLine("item number:" + forecasts.Count);
Task[] tasks = new Task[taskNum];
var i = 0;
for(i = 0; i < taskNum; i++) {
int start = i * size;
if (forecasts.Count - start < size) {
size = forecasts.Count - start;
}
// Method 1: This works well, but need take some time to finish
//tasks[i] = new Task(() => {
//var batchedforecastRows = forecasts.GetRange(start, size);
// GbqTable.InsertRowsAsync(batchedforecastRows);
//Console.WriteLine("batchID:" + (i + 1) + "["+string.Join( ",", batchedforecastRows.ToArray())+"]");
//});
// tasks[i].RunSynchronously();
// Method 2: will lose data: (94, 95) and batchId is messed
// Sample Print below:
// batchID:18 Inserted:[90,91,92,93]
// batchID:18 Inserted:[96,97,98,99]
tasks[i] = Task.Run(() => {
var batchedforecastRows = forecasts.GetRange(start, size);
// GbqTable.InsertRowsAsync(batchedforecastRows);
Console.WriteLine("batchID:" + (i + 1) + " Inserted:["+string.Join( ",", batchedforecastRows.ToArray())+"]");
});
}
}
}