I'm fairly new to LINQ to SQL, so trying to get some advice regarding whether this performance is really an issue and can be improved, or is actually common. See code below, in C#/LINQ, I'm selecting all ~800,000 records from a SQL DB table, and wanted to group the ~800,000 records by user. There are 675 users and that will be 675 groups. The ultimate goal is to turn each group into a custom object DM_userSync. Some properties of DM_userSync are directly columns in DB, and can be easily assigned, for example "UserID". These actions take very little time, about ~1.3ms each. However, some properties of DM_userSync are result from processing the group, for example "SyncCount", it's the count of the group.. And these ones are quite slow - ~0.5s each and almost 6 min for the entire 675 groups.
I tried a couple approaches (GroupingMethod1 and GroupingMethod2) but essentially they are both quite slow. Is this pretty normal, Or are there better ways to achieve what I'm trying to do? Thanks a lot!.
My code (For those who'd like to help - It's probably hard to test code with specific DB links? Hope the comments help explaining my intent):
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace test_RevitDBAnalysis
{
class Program
{
static DataClassesRevitDataContext dc_RVT = new DataClassesRevitDataContext();
static void Main(string[] args)
{
GroupingMethod1();
GroupingMethod2();
Console.Read();
}
static void GroupingMethod1()
{
//~800,000 records from database
var records = from record in dc_RVT.Revit_Sync_Logs
select record;
//just for lookup
var users_Detail = from record in dc_RVT.Revit_Users
select record;
//800,000 records are grouped into 675 groups
var recordsGroupedByUser = from r in records
group r by r.userId into g
select g.Key;
//need to turn 675 groups into custom object "DM_userSync" and add to this list
List<DM_userSync> list = new List<DM_userSync>();
int userCount = 0;
foreach (var o in recordsGroupedByUser)
{
DM_userSync dm = new DM_userSync();
var recordsOfThisUser = from r in records
where r.userId == o.Value
select r;
var thisUser = (from user in users_Detail
where user.uniqueId == dm.UserID
select user).FirstOrDefault();
//~1.3ms
dm.UserID = o.Value;
//~1.3ms
if (thisUser != null)
dm.Name = thisUser.networkUserName.Trim() ?? "unknown";
//~370ms - SLOW!!
dm.SyncCount = recordsOfThisUser.Count();
//~400ms - SLOW!!
dm.ActiveDays = GetDistinctDays_1(recordsOfThisUser);
list.Add(dm);
Console.WriteLine($"{userCount} {dm.UserID}");
userCount++;
}
}
static int GetDistinctDays_1(IEnumerable<Revit_Sync_Log> g)
{
int days = (from r in g
select ((DateTime)r.startSyncTimeStamp).Date).Distinct().Count();
return days;
}
//EVEN SLOWER
static void GroupingMethod2()
{
//~800,000 records from database
var records = from record in dc_RVT.Revit_Sync_Logs
select record;
//just for lookup
var users_Detail = from record in dc_RVT.Revit_Users
select record;
//800,000 records are grouped into 675 groups
var recordsGroupedByUser = from r in records
group r by r.userId into g
select new
{
userId = g.Key,
syncs = g
};
//need to turn 675 groups into custom object "DM_userSync" and add to this list
List<DM_userSync> list = new List<DM_userSync>();
int userCount = 0;
foreach (var o in recordsGroupedByUser)
{
DM_userSync dm = new DM_userSync();
var recordsOfThisUser = from r in records
where r.userId == o.userId
select r;
var thisUser = (from user in users_Detail
where user.uniqueId == dm.UserID
select user).FirstOrDefault();
//~1.3ms
dm.UserID = (Guid)o.userId;
//~1.3ms
if (thisUser != null)
dm.Name = thisUser.networkUserName.Trim() ?? "unknown";
//~370ms
dm.SyncCount = recordsOfThisUser.Count();
//~400ms
dm.ActiveDays = GetDistinctDays_1(recordsOfThisUser);
list.Add(dm);
Console.WriteLine($"{userCount} {dm.UserID}");
userCount++;
}
}
}
}