Remove duplicate records on textfile

Alternatively, you can rearrange your file via linq.

string path = @"C:\sampleText.txt";
string[] allText = File.ReadAllLines(path, Encoding.UTF8).Where(c => c != null && c != "").Select(item => item + Environment.NewLine).Distinct().ToArray();

File.WriteAllLines(path, allText);

INPUT :

122222 08-0943-03
123333 08-0943-03
122222 08-0943-03
123333 08-0943-03
122222 08-0943-03
123333 08-0943-03
122222 08-0943-03
123333 08-0943-03
122222 08-0943-03
123333 08-0943-03

OUTPUT :

122222 08-0943-03

123333 08-0943-03

You could use LINQ to grouping data like this

using System.Linq;
.....

foreach (var row1 in dt.Rows.OfType<DataRow>().GroupBy(r => new { Component = r["Component"], Material = r["Material"] }).Select(r => r.Key))
{
  if (row["Material"].ToString().ToLower().Contains(row1.Material.ToString().ToLower()))
  {
    //// if match, then write the result
    string filename1 = "loggo2.txt";
    filename1 = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, filename1);
    using (StreamWriter w = File.AppendText(filename1))
    {
        var line = String.Format("{0} {1}", row1.Component, row1.Material);
        w.WriteLine(line);
    }
  }
}

you can read all line in file into list of string and add string from DB finally write distinct to file like this:

var strList = new List<string>();
string filename1 = "loggo2.txt";
    filename1 = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, filename1);

using (StreamReader reader = new StreamReader(filename1))
{
    while(!reader.EndOfStream)
    {
        strList.Add(reader.ReadLine());
    }
}

foreach (DataRow row1 in dt.Rows)
{
  if (row["Material"].ToString().ToLower().Contains(row1["Material"].ToString().ToLower()))
  {
    //// if match, then write the result
    strList.Add(String.Format("{0} {1}", row1["Component"], row1["Material"]));
  }
}

using (StreamWriter w = new StreamWriter(filename1, false))
{
    foreach(var line in strList.Distinct().ToList())
    {
        w.WriteLine(line);
    }
}

Let's extract a method for distinct values:

   private IEnumerable<DataRow> MyDistinctRows() {
     HashSet<string> materials = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

     foreach (DataRow row in dt.Rows) 
       if (materials.Add(row["Material"].ToString()))  
         yield return row;
   }

Now, we can save these distinct to the file

   File.AppendAllLines(filename1, MyDistinctRows()
     .Select(row => $"{row["Component"]} {row["Material"]}"));

Edit: Same idea, if you want to get rid of duplicates within a file:

   private IEnumerable<DataRow> MyDistinctRows() {
     HashSet<string> materials = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

     foreach (string line File.ReadLines(@"loggo2.txt")) 
       if (materials.Add(line.Split(' ')[0])  
         yield return line;
   }

Then again save:

   // ToList() - materialize if you want to save into the same (loggo2.txt) file  
   File.AppendAllLines(filename1, MyDistinctRows()
     .Select(row => $"{row["Component"]} {row["Material"]}")
     .ToList());

I Hope This Can Help.

You Can Do That By Using The Distinct Method Of Linq.

public static void Distinct_data()
    {
        string[] my= File.ReadAllLines(@"C:\Users\HP\source\repos\Display_turn_off\Display_turn_off\my_.txt");
        var gh= my.Distinct();
        foreach(string hj in gh)
        {
            Console.WriteLine(hj);
        }
    }