i have this piece of code:
public static async Task<(IEnumerable<BsonDocument>? docs, int status, bool isSuccess)> loadAgencesWithExtraInfo()
{
(IEnumerable<BsonDocument> docs, int status, bool isSuccess) = await Helper.loadAgencesAsync("\"code\":", "\"_id\":").ConfigureAwait(false);
if(!isSuccess)
return (null, 998, false);
// i need to do **var docArray = docs.ToArray();** but i dont want
Helper.cancelTask = false;
var tasks = new List<Task<(BsonDocument? doc, int status, bool isSuccess)>>();
var part = 100;
var loop = 0;
var total = new List<(BsonDocument doc, int status, bool isSuccess)>();
do
{
// here error during the second loop because docs is empty
tasks.AddRange(docs.Skip(loop++ * part).Take(part).Select(async b => {
if (!cancelTask)
return await Helper.loadAgenceAsync(b.AsDocument, "\"codeAurore\":", "\"_id\":");
else
return (null, 999, false);
}));
var result = await Task.WhenAll(tasks.ToArray()).ConfigureAwait(false);
if (result.Count() == 0)
break;
total.AddRange(result!);
tasks.Clear();
} while (!Helper.cancelTask);
if (Helper.cancelTask)
return (null, 999, false);
else
return (total.Select(x => x.doc!), 200, true);
}
i dunno if this info is important but in debug mode: docs is System.Linq.Enumerable.SelectEnumerableIterator, the first part gets important number of records (>10000) in IEnumerable variable docs,
so in second part i need to add something to each record, so to limit number of async i have built a loop with do..while and the idea is to use Skip and Take to read right part of docs..
Here is coming the problem, i cant do that, because iteration of IEnumerable is done completely during the first loop despite Skip and Take and during the second loop, docs is empty. So the solution was to put docs in array for example and in this case all is ok. But its too expensive in memory.
How i could iterate 100 records per 100 records for example? i see the fact to use enumerator... but i dont know the way...
I should want to iterate 100 (100 async launches) , trap result and so again until the end...
Sorry for my english, hope you understant my problem.
i show here how docs is built:
private static async Task<(IEnumerable<BsonDocument>? docs, int status, bool isSuccess)> loadAgencesAsync(string? pattern, string? replace)
{
var (docs, status, isSuccess) = await Helper.API_GetDocument(Helper.agencesUrl, pattern, replace);
return (isSuccess ? docs : null, status, isSuccess);
}
public static async Task<(IEnumerable<BsonDocument>? docs, int status, bool isSuccess)> API_GetDocument(string url, string? pattern = null, string? replace = null)
{
//Log.Information("[GetDocument] url: {url}", url);
string json = "";
int status = 0;
bool isSuccess = false;
try
{
(json, status, isSuccess) = await API_GetJson(url, pattern, replace);
}
catch (Exception ex)
{
string progname = "ee";// NameOfCallingClass();
Log.Error("[GetDocument] url: {url}, {prog} -> {success} -> {status} -> exception ", url, progname, isSuccess, status);
Log.Error("[GetDocument] {ex}", ex.Message);
return (null, status, false);
}
if (!isSuccess)
{
string progname = "ee";// NameOfCallingClass();
Log.Error("[GetDocument] url: {url}, {prog} -> {success} -> {status} -> json", url, progname, isSuccess, status);
Log.Error("[GetDocument] {json}", json);
return (null, status, false);
}
if (json[0] == '[')
{
return (JsonSerializer.DeserializeArray(json).Select(x => x.AsDocument), status, isSuccess);
}
else
{
return (new List<BsonDocument> { JsonSerializer.Deserialize(json).AsDocument }, status, isSuccess);
}
}
public static async Task<(string content, int status, bool isSuccess)> API_GetJson(string url, string? pattern = null, string? replace = null)
{
string json;
int responseHttpStatusCode;
bool isSuccess;
var proxy = config[$"proxy"];
var httpClientHandler = new HttpClientHandler
{
Proxy = new WebProxy(proxy, true),
UseProxy = config.GetSection($"{url.GetSectionUrl()}:useProxy").Get<bool>()
};
using (var client = new HttpClient(httpClientHandler))
{
client.DefaultRequestHeaders.Clear();
url.GetHeadersParam().ForEach(x => client.DefaultRequestHeaders.Add(x.Item1, x.Item2));
using (HttpResponseMessage response = await client.GetAsync(url).ConfigureAwait(false))
{
responseHttpStatusCode = (Int32)response.StatusCode;
json = await response.Content.ReadAsStringAsync();
isSuccess = response.IsSuccessStatusCode;
}
}
return pattern != null ? (Regex.Replace(json, pattern, replace), responseHttpStatusCode, isSuccess) :
(json, responseHttpStatusCode, isSuccess);
}