I have created windows service in .net core.
There are 50 use cases that I want to run in parallel. each use case have different no. of ML models. I am loading all the models into memory (it take around 7.4 GB). then creating 50 tasks for 50 use cases.
As each use case has around 40-50 ml models (prediction models) which would be running sequentially. data for prediction is in CSV. so for 50 threads there will be 50 CSV.
What I observed that memory is getting increased over the period of time from 7.4 GB to 10-14 GB and when GC collects 1st time it reduced to 7.4 GB however over the period of time memory consumption goes more than 14 GB if memory available and GC call not reducing it back to 7.4 GB.
If I comment prediction code the memory usage is constant but when I uncomment code memory getting increased gradually. every 1 hour I am disposing all threads and calling GC explicitly and creating thread again. still memory is not getting released. below is the method for prediction. I set ref to null jsut in case it holds objects.
internal string LoadModelPredict(ref Guid modelGuid, ref string modelName, ref List<ModelTransformer> modelTransformers, ref string csvDataFile, ref TextLoader.Column[] textLoadercolumns, ref DateTime requestDateTime, ref BlockingCollection<PredictionResultCollection> outputQueue, ref CancellationToken stoppingToken)
{
List<float> listScore = null;
List<string> listTime = null;
List<float> listActualValues = null;
List<PredictionResult> predictionDataList = null;
MLContext mlContext = null;
IDataView dataView = null;
IDataView predictions = null;
ITransformer predictionPipeline = null;
string latestPredictedTime = "";
try
{
Logger.log.InfoFormat("[Request at {0} Model Name: {1}] IDataView load for input data Started", requestDateTime, modelName);
mlContext = new MLContext();
dataView = mlContext.Data.LoadFromTextFile(csvDataFile, textLoadercolumns, ',', hasHeader: true, allowQuoting: true);
mlContext = null;
Logger.log.InfoFormat("[Request at {0} Model Name: {1}] IDataView load for input data completed", requestDateTime, modelName);
predictionDataList = new List<PredictionResult>();
foreach (var item in modelTransformers)
{
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] model assigne start", requestDateTime, item.ModelConfigId);
predictionPipeline = item.transformer;
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] model assigne completed", requestDateTime, item.ModelConfigId);
predictions = predictionPipeline.Transform(dataView);
predictionPipeline = null;
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] prediction value extraction", requestDateTime, item.ModelConfigId);
listScore = predictions.GetColumn<float>("Score").ToList();
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] score extraction completed", requestDateTime, item.ModelConfigId);
listTime = predictions.GetColumn<string>(textLoadercolumns[0].Name).ToList();
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] Time extraction completed", requestDateTime, item.ModelConfigId);
listActualValues = predictions.GetColumn<float>(item.TextLoaderOutputColumn).ToList();
predictions = null;
Logger.log.InfoFormat("[Request at {0} Model Id: {1}] actual value extraction completed", requestDateTime, item.ModelConfigId);
string PredictedTagName = item.PredictedTagName; //ConfigParameters.PredictionTagPrefix + "_" + item.TargetVariable.Replace("-", "_");
predictionDataList.AddRange(
listTime.Zip(listScore, (time, score) => new PredictionResult
{
TagName = PredictedTagName,
Timestamp = Convert.ToDateTime(time), //.ToString("dd-MM-yyyy HH:mm:ss.000"),
Confidence = ConfigParameters.TagValueDefaultConfidence,
Value = score.ToString(),
ValueType = ValueType.SCORE.ToString()
}).ToList());
predictionDataList.AddRange(
listTime.Zip(listActualValues, (time, actualVal) => new PredictionResult
{
TagName = item.TargetVariable,
Timestamp = Convert.ToDateTime(time), //.ToString("dd-MM-yyyy HH:mm:ss.000"),
Confidence = ConfigParameters.TagValueDefaultConfidence,
Value = actualVal.ToString(),
ValueType = ValueType.ACTUAL.ToString()
}).ToList());
listScore.Clear();
listTime.Clear();
listActualValues.Clear();
listActualValues = null;
listScore = null;
listTime = null;
}
dataView = null;
//File.WriteAllLines(outputFilePath, predictionDataList.Select(x => x.TagName + "," + x.Value + "," + x.Confidence + "," + x.Timestamp.ToString("dd-MM-yyyy HH:mm:ss.000")));
if (predictionDataList.Count > 0)
{
DateTime maxTime = predictionDataList.Select(x => x.Timestamp).Max();
if (latestPredictedTime == "")
{
latestPredictedTime = maxTime.ToString();
Logger.log.InfoFormat("[Request at {0} Model Name: {1}] Adding Data to Collection.", requestDateTime, modelName);
outputQueue.Add(
new PredictionResultCollection
{
ModelGuid = modelGuid,
ModelName = modelName,
LatestPredictionTime = Convert.ToDateTime(latestPredictedTime),
RequestTime = requestDateTime,
FileOutput = predictionDataList.Select(x => x.TagName + "," + x.Value + "," + x.Confidence + "," + x.Timestamp.ToString("dd-MM-yyyy HH:mm:ss.000")).ToList()
}, stoppingToken);
Logger.log.InfoFormat("[Request at {0} Model Name: {1}] Added Data to Collection.", requestDateTime, modelName);
}
}
}
finally
{
predictionPipeline = null;
if (predictionDataList.Count > 0)
{
predictionDataList.Clear();
}
predictionDataList = null;
listActualValues = null;
listScore = null;
listTime = null;
dataView = null;
mlContext = null;
predictions = null;
}
return latestPredictedTime;
}