I am implementing an encryption algorithm that I have designed in C. The algorithm is able to work in parallel using multiple threads. Without going into detail about the algorithm, the basic flow of the program is that the file to be encrypted is mmaped one gigabyte at a time with void *fileBytes = mmap(NULL, loadSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);. Then 8 threads are created and the 1 gb chunk is split up into 4096 byte blocks. These blocks are dispersed to the 8 threads and are all independent of each other. They are dispersed by passing an array of pointers to the start of each block to the threads. Then, the transformations are made to the blocks and when all the threads are finished, the memory is munmaped and the next gigabyte (or less if there is less than a gb of the file remaining) is prepared to be sent to the threads for encryption.
The code I am using to create the threads is:
for(t = 0; t < threadCount; t++) {
pthread_create(&threads[t], NULL, function, &assignments[t]);
}
for(t = 0; t < threadCount; t++) {
pthread_join(threads[t], NULL);
}
I am noticing that for files under 2 gb, this works extremely fast (1 second approx. for 1 gb) and approximately 250-300% cpu is being used on my 4 core intel machine running linux. However, for files over 2 gb the time greatly increases and the cpu usage decreases to approx. 30-40%. For example, an 8 gb file takes approximately 1m 24s. Does anyone have any idea why this may be happening? I realize that this is a very brief overview of my program so please let me know what other information I can provide.
Thank you in advance
EDIT:
I have tested a simple file modification where the first byte of each page is set to 0. 1 gb takes approximately 0.2 seconds but 8 gb takes approximately 30 seconds. Bellow is the code and fileSimpleEdit is the function that starts it. simpleMod changes the first byte and is to act in place of the encryption.
void *simpleMod(void *data) {
//get assignment info
//assignments are a struct that hold an array of jobs (which are pointers) and the job count
assignment *assignmentPtr = data;
assignment currentAssignment = *assignmentPtr;
job *jobList = currentAssignment.jobs;
unsigned long long jobCount = currentAssignment.jobCount;
//setup tracking vars
job currentJob;
union block *chunk;
//BYTE_COUNT is 4096
/*
union block {
uint_fast64_t longs[LONG_COUNT];
char bytes[BYTE_COUNT];
} block;
*/
//go through jobs one by one and set first byte to 0
for(unsigned long long j = 0; j < jobCount; j++) {
currentJob = jobList[j];
chunk = currentJob.start;
chunk->bytes[0] = 0;
}//end for each job
return NULL;
}
int runFunction(unsigned long long loadSize, unsigned long long offset, int fd,
assignment assignments[], pthread_t threads[], void *(*function)(void *)) {
//get pointer to location in file
void *fileBytes = mmap(NULL, loadSize, PROT_READ | PROT_WRITE, MAP_SHARED, fd, offset);
if(fileBytes == MAP_FAILED) { //check for error
printf("Could not open file to be encrypted\n");
return 1;
}
// malloc the jobs
unsigned long long chunkCount = loadSize / BYTE_COUNT;
unsigned long long chunksPerThread = chunkCount / threadCount;
unsigned long long remainingChunks = chunkCount - chunksPerThread * threadCount;
unsigned long long addPerAssignemnt = (remainingChunks + (threadCount - 1)) / threadCount;
for(int i = 0; i < threadCount; i++) {
assignments[i].jobCount = chunksPerThread;
if(remainingChunks == 0) {
assignments[i].jobCount += remainingChunks;
remainingChunks = 0;
} else if(remainingChunks >= addPerAssignemnt) {
assignments[i].jobCount += addPerAssignemnt;
remainingChunks -= addPerAssignemnt;
}
assignments[i].jobs = malloc(sizeof(job) * assignments[i].jobCount);
}
//then assignments need to be filled with jobs
unsigned int t = 0;
unsigned long long index = 0; //the job index within assignment;
//assign chunks as jobs to assignments
for(unsigned long long c = 0; c < chunkCount; c++) { //cycle through dimensions
//setup new job
job *newJob = &assignments[t].jobs[index];
newJob->start = PTR_PLUS_BYTES(fileBytes, c * BYTE_COUNT);
//update assignment
if(t == threadCount - 1) {
t = 0;
index++;
} else {
t++;
}
}
//setup threads for fucntion
for(t = 0; t < threadCount; t++) {
pthread_create(&threads[t], NULL, function, &assignments[t]);
}
for(t = 0; t < threadCount; t++) {
pthread_join(threads[t], NULL);
}
//free the jobs
for(t = 0; t < threadCount; t++) {
free(assignments[t].jobs);
}
//unmap mem
munmap(fileBytes, loadSize);
}//end runFunction
int disperseFunction(void *(*function)(void *), pthread_t threads[],
assignment assignments[], int fd, unsigned long long fileSize) {
struct timeval start, end;
//only load up to certain amount of file at time
for(unsigned long long lc = 0; lc < fileSize / maxLoad; lc++) {
runFunction(maxLoad, maxLoad * lc, fd, assignments, threads, function);
}
//run remainder
unsigned long long lastSize = fileSize % maxLoad;
runFunction(lastSize, fileSize - lastSize, fd, assignments, threads, function);
return 0;
}
int fileSimpleEdit(char *targetFile) {
//get file size
int fd = open(targetFile, O_RDWR);
struct stat statBuf;
fstat(fd, &statBuf);
unsigned long long originalSize = statBuf.st_size;
threadCount = 8;
pthread_t threads[threadCount];
//pad the file
unsigned short paddingSize = BYTE_COUNT - (originalSize % BYTE_COUNT);
FILE *unpaddedFile = fopen(targetFile, "a");
if(unpaddedFile == NULL) {
printf("Could not open file to be encrypted\n");
return 1;
}
//create padding
uint_fast8_t padding[paddingSize];
memset(&padding, 1, paddingSize);
//write padding
fwrite(&padding, 1, paddingSize, unpaddedFile);
fclose(unpaddedFile);
close(fd);
//open file after its padded
fd = open(targetFile, O_RDWR);
fstat(fd, &statBuf);
unsigned long long fileSize = statBuf.st_size;
//next setup assignments array
assignment assignments[threadCount];
disperseFunction(simpleMod, threads, assignments, fd, fileSize);
return 0;
} //end fileEncrypt