how to create csv file from a sequentially created data file with c language

Viewed 59

I have a question about :how to create csv file from a sequentially created data file with C language.

With a C program I make several printf of values. The output of the program is redirected to a file by : ./myprog >> file.txt

So the file is like :

0.8952
0.89647
0.3658
!!!
0.258633
0.233655
0.25475
!!!
0.5895
0.54785
0.695555
!!!

etc.

The different dimensions are separeted by "!!!"

The result I would like is :

0.8952;0.258633;0.5895
0.89647;0.233655;0.54785
0.3658;0.25475;0.695555

I tried with a two dimentions array to do so but as i have about 100 000 lines between evevy "!!!" I have a segmentation fault ex. double myTab[100000] [100000].

If you have an idea, thanks a lot. Best regards

2 Answers

Don't try to buffer everything. Just remember where each segment starts, and use fseek() wisely.

Here I use a fixed array presuming up to 10 segments. You may have to either increase this, or possibly make it dynamic and "growable".

(This is just a "rough cut", but may lead you to a solution.)

EDIT: Clarification: The upper loop indexes the beginning of each 'section', storing the offset to the first data entry of each section. Using the combination of ftell() and fseek(), the lower loop drives a single input FILE to act like many simultaneous FILE buffers reading single lines from the same file but at different locations. (Analogous to sequentially "popping top item" off multiple stacks until a stack is empty. All stacks presumed to be equally full at the outset.)

size_t offsets[ 10 ] = { 0 };
int nStored = 0;
char buf[ 128 ]; // be generous; don't scrimp.

// first pass, just remember position of 1st number
while( fgets( buf, sizeof buf, infp ) )
    if( strncmp( buf, "!!!", 3 ) )
        offsets[ nStored++ ] = ftell( infp );

// Now, make 100,000 passes until exhaust (equal sized) sections.
while( true ) {
    for( int i = 0; i < nStored; i++ ) {
        fseek( infp, offsets[ i ], SEEK_SET );
        fgets( buf, sizeof buf, infp ) )

        // Been 'chewing through' data lines so far.
        if( strncmp( buf, "!!!", 3 ) ) // section boundary?
            return; // finished with all equal sized data rows.

        *strpbrk( buf, "\n" ) = ','; // replace NL with ','
        fprintf( outfp, "%s", buf );
        offsets[ i ] = ftell( infp ); // update for next pass
    }
    fprintf( outfp, "\n" ); // Yeah, trailing comma and null field. Life, eh?
    
}

If the "sections" between "!!!" markers are NOT the same size, then do not "update" the offset value for short sections... When small sections are exhausted, output a "," to the output file to indicate "empty column". Will need a flag to indicate "no new data found during this sweep; all sections have been exhausted" and that is the clue that the job is done.

This works as far as I have tested it.
With the posted input file, I got a matching cvs file.
It is slow. It uses quite a lot of disk space. Tested with 100000 x 10000 data set and created an 8 GB binary file and at almostleast a 12 GB cvs file.
This assumes there are always the same number of double values between the !!! and that all those value will be correctly parsed by sscanf

#define _FILE_OFFSET_BITS 64

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

int main ( void) {
    char *filename = "afile.txt";
    char *binname = "adouble.bin";
    char *outname = "adouble.cvs";
    char in[100] = "";
    double dbl = 0.0;
    size_t lines = 0;
    size_t maxlines = 0;
    size_t epl = 0; // elements per line
    size_t maxepl = 0;
    size_t count = 0;
    FILE *pf = NULL;
    FILE *pfbin = NULL;
    FILE *pfout = NULL;

    if ( NULL == ( pf = fopen ( filename, "r"))) {
        perror ( filename);
        return 1;
    }

    epl = 0;
    lines = 0;
    count = 0;

    // read the file to get lines and elements per line
    while ( fgets ( in, sizeof in, pf)) {
        if ( ! strcmp ( in, "!!!\n")) {
            ++epl;
            if ( count > lines) {
                lines = count;
            }
            count = 0;
        }
        else {
            ++count;
        }
    }

    printf ( "first read complete\n");
    printf ( "lines %zu\n", lines);
    printf ( "elements per line %zu\n", epl);
    maxlines = lines;
    maxepl = epl;

    rewind ( pf); // to read the file a second time

    lines = 0;
    epl = 0;

    if ( NULL == ( pfbin = fopen ( binname, "wb"))) { // open bin file for writing
        perror ( binname);
        return 2;
    }

    while ( fgets ( in, sizeof in, pf)) {
        if ( ! strcmp ( in, "!!!\n")) {
            ++epl;
            lines = 0;
        }
        else {
            sscanf ( in, "%lf", &dbl);
            off_t pos = ( lines * maxepl + epl) * sizeof dbl; // offset into bin file
            fseeko ( pfbin, pos, SEEK_SET); // seek to offset
            fwrite ( &dbl, 1, sizeof dbl, pfbin);
            ++lines;
        }
    }

    fclose ( pf);
    fclose ( pfbin);
    printf ( "second read and create binary file complete\n");

    if ( NULL != ( pfbin = fopen ( binname, "rb"))) { // open bin file for reading
        if ( NULL != ( pfout = fopen ( outname, "w"))) { // open out file for writing
            for ( int ln = 0; ln < maxlines; ++ln) {
                for ( int ele = 0; ele < maxepl; ++ele) {
                    fread ( &dbl, 1, sizeof dbl, pfbin); // read from bin file
                    if ( ele) {
                        fprintf ( pfout, ";");
                    }
                    fprintf ( pfout, "%f", dbl);
                }
                fprintf ( pfout, "\n");
            }
            fclose ( pfout);
        }
        else {
            perror ( outname);
            return 4;
        }
        fclose ( pfbin);
    }
    else {
        perror ( binname);
        return 4;
    }

    return 0;
}
Related