SSIS split flat file(TXT) to multiple based on the number of records it has

asdaswsad 0
Viewed 32

i have the following c# script on ssis project in visual studio that generates files based on a column value of a sql table and i would like to add a part that also splits the files when the record count reaches 200..Example of an extraction would be nameoffile_columnvalue_datetime_1.txt for the first 200,nameoffile_columnvalue_datetime_2.txt for the next 200 etc... thank you in advance!

 public void Main()
        {
            // TODO: Add your code here
            string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
            string datetime = datetime_1.Replace("/", String.Empty);

            try
            {

                //Declare Variables
                string FileNamePart = Dts.Variables["User::FlatFileNamePart"].Value.ToString();
                string DestinationFolder = Dts.Variables["User::DestinationFolder"].Value.ToString();
                string TableName = Dts.Variables["User::TableName"].Value.ToString();
                string FileDelimiter = Dts.Variables["User::FileDelimiter"].Value.ToString();
                string FileExtension = Dts.Variables["User::FileExtension"].Value.ToString();
                string ColumnNameForGrouping = Dts.Variables["User::ColumnNameForGrouping"].Value.ToString();
                string SubFolder = Dts.Variables["User::SubFolder"].Value.ToString();
                Int32 RecordCntPerFile = (Int32)Dts.Variables["User::RecordsPerFile"].Value;
                string RecordCntPerFileDecimal = RecordCntPerFile + ".0";

                //USE ADO.NET Connection from SSIS Package to get data from table
                SqlConnection myADONETConnection = new SqlConnection();
                myADONETConnection = (SqlConnection)(Dts.Connections["AR_GSLO_OLTP"].AcquireConnection(Dts.Transaction) as SqlConnection);

                //Read distinct Group Values for each flat file
                string query = "Select distinct " + ColumnNameForGrouping + " from " + TableName;

                //MessageBox.Show(query.ToString());
                SqlCommand cmd = new SqlCommand(query, myADONETConnection);
                //myADONETConnection.Open();
                DataTable dt = new DataTable();
                dt.Load(cmd.ExecuteReader());
                myADONETConnection.Close();

                //Loop through values for ColumnNameForGroup

                foreach (DataRow dt_row in dt.Rows)
                {
                    string ColumnValue = "";
                    object[] array = dt_row.ItemArray;
                    ColumnValue = array[0].ToString();


                    //Load Data into DataTable from SQL ServerTable
                    string queryString =
                     "SELECT * from " + TableName + " Where " + ColumnNameForGrouping + "='" + ColumnValue + "'";
                    SqlDataAdapter adapter = new SqlDataAdapter(queryString, myADONETConnection);
                    DataSet ds = new DataSet();
                    adapter.Fill(ds);

                    foreach (DataTable d_table in ds.Tables)
                    {
                        string FileFullPath = SubFolder + "\\" + FileNamePart + "_" + ColumnValue + "_" + datetime + FileExtension;

                        StreamWriter sw = null;
                        sw = new StreamWriter(FileFullPath, false);

                        // Write the Header Row to File an thelw na exei kai header ta kanw uncomment apo 152 mexri 160 grammi
                        int ColumnCount = d_table.Columns.Count;
                        // for (int ic = 0; ic < ColumnCount; ic++)
                        //{
                        //    sw.Write(d_table.Columns[ic]);
                        //    if (ic < ColumnCount - 1)
                        //    {
                        //        sw.Write(FileDelimiter);
                        //    }
                        //}
                        // sw.Write(sw.NewLine);

                        // Write All Rows to the File
                        foreach (DataRow dr in d_table.Rows)
                        {
                            for (int ir = 0; ir < ColumnCount; ir++)
                            {
                                if (!Convert.IsDBNull(dr[ir]))
                                {
                                    sw.Write(dr[ir].ToString());
                                }
                                if (ir < ColumnCount - 1)
                                {
                                    sw.Write(FileDelimiter);
                                }
                            }
                            sw.Write(sw.NewLine);
                        }
                        sw.Close();
                    }

                }

            }

            catch (Exception exception)
            {

                // Create Log File for Errors
                using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() + "\\" +
                    "ErrorLog_" + datetime + ".log"))
                {
                    sw.WriteLine(exception.ToString());
                    Dts.TaskResult = (int)ScriptResults.Failure;


                }
            }

        }
ssis visual-studio integration c#
2 Answers

So assuming/understanding you have a script task which is the source of your rows?

You can add a Rowcount Task to capture the amount of rows in a variable for later validation.

after that, you might need a new script task where you can do the validation of how many records there is. something like a case statement/if statement or a type of loop for each 200.. odd records to change a value/filename.

Using that loop, you can change the filename/filenamevariable.

So you would need your ConnectionManager File Destination to have an expression with this variable filename. Thus as soon as it hits 200.. records, the script should change the filename and the output should create a new file.

I am not sure if it will like the in flight name change. but think this might be a way to point your thoughts.

Inus C0

i did it first i used a foreach loop container to take the name of the txts as input variables then inside the loop i created a script with the following c# code

 public void Main()
    {
        // TODO: Add your code here
        string datetime_1 = DateTime.Now.ToString("dd/MM/yyyy");
        string datetime = datetime_1.Replace("/", String.Empty);
        StreamWriter writer = null;

        try
        {

            //Declare Variables
            string filename= Dts.Variables["User::FileName"].Value.ToString();
            //MessageBox.Show(filename);

            using (StreamReader inputfile = new System.IO.StreamReader(filename))
                {
                    int count = 0;
                    int filecount = 0;
                    string line;
                    while ((line = inputfile.ReadLine()) != null)
                    {

                        if (writer == null || count > 199)
                        {
                            count = 0;
                            if (writer != null)
                            {
                                writer.Close();
                                writer = null;
                            }
                            ++filecount;
                            writer = new System.IO.StreamWriter(filename.Replace(".txt", "_") + filecount.ToString() + ".txt", true);
                        }
                        writer.WriteLine(line);

                        ++count;
                    }

                }
            
        }

        catch (Exception exception)
        {

            // Create Log File for Errors
            using (StreamWriter sw = File.CreateText(Dts.Variables["User::LogFolder"].Value.ToString() + "\\" +
                "ErrorLog_" + datetime + ".log"))
            {
                sw.WriteLine(exception.ToString());
                Dts.TaskResult = (int)ScriptResults.Failure;


            }
        }
        finally
        {
            if (writer != null)
                writer.Close();
        }

        Dts.TaskResult = (int)ScriptResults.Success;

    }

and that was it!

asdaswsad0
Related