<commit_before>/*
 * =====================================================================================
 *
 *       Filename:  compare.c
 *
 *    Description:  Example code t
 *
 *        Version:  1.0
 *        Created:  09/16/2015 09:23:33 AM CDT
 *       Revision:  none
 *       Compiler:  gcc
 *
 *         Author:  <NAME> (<EMAIL>),
 *        Company:  Argonne National Laboratory
 *
 * =====================================================================================
 */


#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <math.h>
#include <time.h>
#include "lz.h"


float reverseFloat( const float inFloat )
{
    float retVal;
    char *floatToConvert = ( char* ) & inFloat;
    char *returnFloat = ( char* ) & retVal;
    returnFloat[0] = floatToConvert[3];
    returnFloat[1] = floatToConvert[2];
    returnFloat[2] = floatToConvert[1];
    returnFloat[3] = floatToConvert[0];
    return retVal;
}

double reverseDouble( const float inDouble )
{
    double retVal;
    char *doubleToConvert = ( char* ) & inDouble;
    char *returnDouble = ( char* ) & retVal;
    returnDouble[0] = doubleToConvert[7];
    returnDouble[1] = doubleToConvert[6];
    returnDouble[2] = doubleToConvert[5];
    returnDouble[3] = doubleToConvert[4];
    returnDouble[4] = doubleToConvert[3];
    returnDouble[5] = doubleToConvert[2];
    returnDouble[6] = doubleToConvert[1];
    returnDouble[7] = doubleToConvert[0];
    return retVal;
}

int cleanFile(char *pSrc1Fn, double min, double max, int prec, int swap)
{
    double mi = max, ma = min;
    char pSrc2Fn[64];
    ulong size1;

    FILE *pFile1 = fopen(pSrc1Fn, "rb");
    if (pFile1 == NULL)
    {
        printf("Failed to open input file.");
        return EXIT_FAILURE;
    }
    fseek(pFile1, 0, SEEK_END);
    size1 = ftell(pFile1)/prec;
    fseek(pFile1, 0, SEEK_SET);

    sprintf(pSrc2Fn, "%s.cln", pSrc1Fn);
    FILE *pFile2 = fopen(pSrc2Fn, "w");
    if (pFile2 == NULL)
    {
        printf("Failed to open input file.");
        return EXIT_FAILURE;
    }

    while (size1 > 0)
    {
        if (prec == 4)
        {
            float buf;
            fread(&buf, sizeof(float), 1, pFile1);
            if (swap) buf = reverseFloat(buf);
            if (buf >= (float)min && buf <= (float)max)
            {
                if (buf > ma) ma = buf;
                if (buf < mi) mi = buf;
                fwrite(&buf, sizeof(float), 1, pFile2);
                //printf("%f \n", buf);
            }
        } else {
            double buf;
            fread(&buf, sizeof(double), 1, pFile1);
            //if (swap) buf = reverseDouble(buf);
            if (buf >= min && buf <= max)
            {
                fwrite(&buf, sizeof(double), 1, pFile2);
                //printf("%f \n", buf);
            }
        }
        size1 = size1 - 1;
    }
    printf("All values between %f and %f\n", mi, ma);
    fclose(pFile1);<commit_msg>Cleaning for doubles, get min and max<commit_after>/*
 * =====================================================================================
 *
 *       Filename:  compare.c
 *
 *    Description:  Example code t
 *
 *        Version:  1.0
 *        Created:  09/16/2015 09:23:33 AM CDT
 *       Revision:  none
 *       Compiler:  gcc
 *
 *         Author:  <NAME> (<EMAIL>),
 *        Company:  Argonne National Laboratory
 *
 * =====================================================================================
 */


#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#include <math.h>
#include <time.h>
#include "lz.h"


float reverseFloat( const float inFloat )
{
    float retVal;
    char *floatToConvert = ( char* ) & inFloat;
    char *returnFloat = ( char* ) & retVal;
    returnFloat[0] = floatToConvert[3];
    returnFloat[1] = floatToConvert[2];
    returnFloat[2] = floatToConvert[1];
    returnFloat[3] = floatToConvert[0];
    return retVal;
}

double reverseDouble( const float inDouble )
{
    double retVal;
    char *doubleToConvert = ( char* ) & inDouble;
    char *returnDouble = ( char* ) & retVal;
    returnDouble[0] = doubleToConvert[7];
    returnDouble[1] = doubleToConvert[6];
    returnDouble[2] = doubleToConvert[5];
    returnDouble[3] = doubleToConvert[4];
    returnDouble[4] = doubleToConvert[3];
    returnDouble[5] = doubleToConvert[2];
    returnDouble[6] = doubleToConvert[1];
    returnDouble[7] = doubleToConvert[0];
    return retVal;
}

int cleanFile(char *pSrc1Fn, double min, double max, int prec, int swap)
{
    double mi = max, ma = min;
    char pSrc2Fn[64];
    ulong size1;

    FILE *pFile1 = fopen(pSrc1Fn, "rb");
    if (pFile1 == NULL)
    {
        printf("Failed to open input file.");
        return EXIT_FAILURE;
    }
    fseek(pFile1, 0, SEEK_END);
    size1 = ftell(pFile1)/prec;
    fseek(pFile1, 0, SEEK_SET);

    sprintf(pSrc2Fn, "%s.cln", pSrc1Fn);
    FILE *pFile2 = fopen(pSrc2Fn, "w");
    if (pFile2 == NULL)
    {
        printf("Failed to open input file.");
        return EXIT_FAILURE;
    }

    while (size1 > 0)
    {
        if (prec == 4)
        {
            float buf;
            fread(&buf, sizeof(float), 1, pFile1);
            if (swap) buf = reverseFloat(buf);
            if (buf >= (float)min && buf <= (float)max)
            {
                if (buf > ma) ma = buf;
                if (buf < mi) mi = buf;
                fwrite(&buf, sizeof(float), 1, pFile2);
                //printf("%f \n", buf);
            }
        } else {
            double buf;
            fread(&buf, sizeof(double), 1, pFile1);
            if (swap) buf = reverseDouble(buf);
            if (buf >= min && buf <= max)
            {
                if (buf > ma) ma = buf;
                if (buf < mi) mi = buf;
                fwrite(&buf, sizeof(double), 1, pFile2);
                //printf("%f \n", buf);
            }
        }
        size1 = size1 - 1;
    }
    printf("All values between %f and %f\n", mi, ma);
    fclose(pFile1);
{'fraction_non_alphanumeric': 0.19197178367484044, 'fraction_numerical': 0.024857238831037957, 'mean_word_length': 2.224147265836492, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2284367', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1919, 'n_words': 469}