The straightforward option is to do the postprocessing and line counting in one pass (counting the number of lines written to a temporary file), then creating an another temporary file but this time in the same directory as the target file, writing the header to the latter temporary file, copying the contents of the first temporary file to the latter temporary file, removing the first temporary file, and finally renaming the latter temporary file as the final file.
This would be very simple to do using POSIX C getline() (which is included in standard C libraries in Linux, BSDs, and Mac OS), but it is not available on Windows. We'd need to first write a wrapper function, that on non-Windows systems uses POSIX getline() to read unlimited-length lines; and uses fgetc() or some other function to implement same/similar function Windows.
Another approach would be to read the file using fread() into a dynamically allocated buffer, and call a callback/filtering function for each line read. This approach has the benefit of being able to support any newline encoding (NUL, LF, CR, LF CR, or CR LF) and minimizing buffer copies, but the downside of being unfamiliar approach to many programmers.
Consider the following example implementation of for_each_line():
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
typedef enum {
NEWLINE_INVALID = -1, /* Invalid NEWLINE_ enum value */
NEWLINE_NONE = 0, /* No newline at end of line; ended at end of stream */
NEWLINE_CR = 1, /* "\r" */
NEWLINE_LF = 2, /* "\n" */
NEWLINE_CRLF = 3, /* "\r\n" */
NEWLINE_LFCR = 4, /* "\n\r" */
NEWLINE_NUL = 5 /* "", or '\0', */
} newline_type;
int for_each_line(FILE *source,
int (*filter)(char *line, size_t len, void *context),
void *context)
{
char *data = NULL; /* source data buffer */
size_t size = 0; /* size of source data buffer in chars */
size_t head = 0; /* start of next line to pass to filter */
size_t tail = 0; /* next char to read into buffer */
int newline = NEWLINE_INVALID;
/* Neither source nor filter can be NULL. */
if (!source || !filter)
return -1; /* -1: Invalid parameters */
/* source stream should not be in error state. */
if (ferror(source))
return -2; /* -2: Error reading source. */
while (1) {
size_t next = head;
size_t n;
int retval;
/* Check if we have a complete line to supply to filter. */
if (tail > head) {
/* To be able to safely use strcspn(), we need a terminating nul at end of buffer. */
data[tail] = '\0';
/* Find the first occurrence of any newline character. */
n = strcspn(data + head, "\r\n");
if (n < tail - head) {
/* Found, and it wasn't the nul we added. */
if (data[head + n] == '\r') {
/* CR or CR LF */
if (data[head + n + 1] == '\n') {
newline = NEWLINE_CRLF;
next = head + n + 2;
} else {
newline = NEWLINE_CR;
next = head + n + 1;
}
} else
if (data[head + n] == '\n') {
/* LF or LF CR */
if (data[head + n + 1] == '\r') {
newline = NEWLINE_LFCR;
next = head + n + 2;
} else {
newline = NEWLINE_LF;
next = head + n + 1;
}
} else {
/* Must have been NUL */
newline = NEWLINE_NUL;
next = head + n + 1;
}
/* Note: A two-character newline might have been split at the end of a buffer,
so at this point, 'newline' is *tentative*, not exact. */
if (newline != NEWLINE_INVALID) {
/* We replace the newline with end-of-string NUL mark. */
data[head + n] = '\0';
retval = filter(data + head, n, context);
if (retval) {
free(data);
return retval;
}
head = next;
/* Check for next line, before trying to refill the buffer. */
continue;
}
}
}
/* If the buffer is at least half full, move the data to beginning of buffer. */
if (head >= tail) {
/* Buffer is completely empty. */
head = 0;
tail = 0;
} else
if (head > 0 && tail >= size / 2) {
/* Buffer is at least half full, and not aligned at start of buffer. Move. */
tail -= head;
memmove(data, data + head, tail);
head = 0;
}
/* Need to grow the buffer? */
if (tail + 2 >= size) {
/* TODO: Improve on this linear growth policy. */
const size_t new_size = (tail | 4095) + 4097 - 32;
char *new_data;
new_data = realloc(data, new_size);
if (!new_data) {
free(data);
return -3; /* Not enough memory. */
}
data = new_data;
size = new_size;
}
/* Read some data into the buffer. */
n = fread(data + tail, 1, size - tail - 1, source);
if (n > 0) {
tail += n;
/* Consume partial newline, since now we know we can. */
if (newline == NEWLINE_CR && data[head] == '\n') {
newline = NEWLINE_CRLF;
head++;
} else
if (newline == NEWLINE_LF && data[head] == '\r') {
newline = NEWLINE_LFCR;
head++;
}
} else
if (ferror(source)) {
free(data);
return -2; /* Error reading source. */
} else {
/* End of input. */
if (tail > head) {
data[tail] = '\0';
retval = filter(data + head, tail - head, context);
free(data);
return retval;
} else {
free(data); /* Note: free(NULL) is safe. */
return 0;
}
}
}
/* Execution never reaches here. */
}
struct context {
FILE *out;
const char *newline;
unsigned long long lines;
};
static int do_copy_count_lines(char *line, size_t len, void *ctxptr)
{
struct context *const ctx = ctxptr;
(void)len; /* Silence warning about unused parameter 'len'. Does nothing. */
fputs(line, ctx->out);
fputs(ctx->newline, ctx->out);
ctx->lines++;
return ferror(ctx->out);
}
int copy_count_lines(FILE *source, FILE *target, unsigned long long *linecount)
{
struct context ctx;
int retval;
if (!source || !target)
return -1; /* Invalid parameters */
ctx.out = target;
ctx.newline = "\n";
ctx.lines = 0;
retval = for_each_line(source, do_copy_count_lines, &ctx);
if (linecount)
*linecount = ctx.lines;
return retval;
}
int main(void)
{
unsigned long long count = 0;
int err;
err = copy_count_lines(stdin, stdout, &count);
if (err) {
fflush(stdout);
fprintf(stderr, "Failed: for_each_line() returned %d.\n", err);
return EXIT_FAILURE;
}
fflush(stdout);
fprintf(stderr, "Copied %llu lines from standard input to standard output.\n", count);
return EXIT_SUCCESS;
}
The above example program simply reads from standard input, writing the lines (replacing any newline convention with the default newline convention for standard output), and the number of lines copied to standard error.
Because this uses fread(), it is very fast, but not suitable for interactive input (from terminals or terminal emulators), because it reads the input in large-ish blocks.
A variant that uses fgetc() is easy to implement, well suited for interactive input, but because of the large number of function calls, is somewhat slower.
Here is an example variant for use with wide character strings. (Do note that Windows' wide character support may be broken; the following is strictly C standard compliant code.)
#include <stdlib.h>
#include <locale.h>
#include <stdio.h>
#include <wchar.h>
#include <errno.h>
/* Only EDOM, EILSEQ, and ERANGE are guaranteed to be known,
so errno constants may need to be mapped. */
/* getline(): End of stream */
#ifndef WGETLINE_EOF
#define WGETLINE_EOF 0
#endif
/* getline(): Invalid parameters */
#ifndef WGETLINE_EINVAL
#ifdef EINVAL
#define WGETLINE_EINVAL EINVAL
#else
#define WGETLINE_EINVAL EILSEQ
#endif
#endif
/* getline(): Not enough memory */
#ifndef WGETLINE_ENOMEM
#ifdef ENOMEM
#define WGETLINE_ENOMEM ENOMEM
#else
#define WGETLINE_ENOMEM ERANGE
#endif
#endif
/* getline(): Read error */
#ifndef WGETLINE_EIO
#ifdef EIO
#define WGETLINE_EIO EIO
#else
#define WGETLINE_EIO EDOM
#endif
#endif
size_t wgetline(wchar_t **lineptr, size_t *sizeptr, FILE *handle)
{
if (!lineptr || !sizeptr || !handle) {
errno = WGETLINE_EINVAL;
return 0;
} else
if (ferror(handle)) {
errno = WGETLINE_EIO;
return 0;
} else
if (feof(handle)) {
errno = WGETLINE_EOF;
return 0;
}
wchar_t *line = *lineptr;
size_t size = *sizeptr;
size_t used = 0;
wint_t wc;
if (!size)
line = NULL;
while (1) {
/* Make sure there are is room for at least three wide chars in the buffer. */
if (used + 3 > size) {
/* TODO: Better buffer growth policy? size is in wchar_t's. */
size = (used | 1023) + 1025 - 16;
line = realloc(line, size * sizeof line[0]); /* realloc(NULL, ..) is safe. */
if (!line) {
errno = WGETLINE_ENOMEM;
return 0;
}
*lineptr = line;
*sizeptr = size;
}
wc = getwc(handle);
if (wc == WEOF) {
line[used] = L'\0';
if (!used)
errno = WGETLINE_EOF;
return used;
} else {
line[used++] = wc;
if (wc == L'\n') {
line[used] = L'\0';
return used;
}
}
}
/* Never reached. */
}
int main(void)
{
wchar_t *line = NULL;
size_t size = 0;
size_t len;
unsigned long long lines = 0uLL, wchars = 0uLL;
setlocale(LC_ALL, "");
if (fwide(stdin, 1) != 1)
fprintf(stderr, "Warning: Your C library or locale does not support wide character standard input.\n");
if (fwide(stdout, 1) != 1)
fprintf(stderr, "Warning: Your C library or locale does not support wide character standard output.\n");
while (1) {
len = wgetline(&line, &size, stdin);
if (!len)
break;
lines++;
wchars += len;
fputws(line, stdout);
}
if (ferror(stdin) || !feof(stdin)) {
fprintf(stderr, "Error reading from standard input.\n");
return EXIT_FAILURE;
}
fflush(stdout);
if (ferror(stdout)) {
fprintf(stderr, "Error writing to standard output.\n");
return EXIT_FAILURE;
}
fprintf(stderr, "Copied %llu lines (%llu wide characters).\n", lines, wchars);
return EXIT_SUCCESS;
}
To give you an idea on how performant these are, time wc -l /usr/share/dict/american-english takes 5ms real time on my system; time ./ex1 < /usr/share/dict/american-english > /dev/null takes 12ms; and time ./ex2 < /usr/share/dict/american-english > /dev/null takes 85ms. All three agree it has 102,305 lines. It has 971,304 wide characters, using UTF-8 character set (with file size 971,578 bytes).
