<commit_before>#include <stdio.h>
#include <string.h>

#include "plof.h"
#include "psl.h"

#define BUFSTEP 1024

int main(int argc, char **argv)
{
    FILE *pslf;
    unsigned char *psl;
    size_t len, rd, i;
    size_t slen, stype, stl;
    struct PlofObject *context;
    struct PlofReturn ret;

    GC_INIT();

    if (argc != 2) {
        fprintf(stderr, "Use: psli <file>\n");
        return 1;
    }

    /* open the file */
    pslf = fopen(argv[1], "r");
    if (pslf == NULL) {
        perror(argv[1]);
        return 1;
    }

    /* preallocate the buffer */
    psl = GC_MALLOC(BUFSTEP);
    len = 0;

    /* now start reading */
    while (rd = fread(psl + len, 1, BUFSTEP, pslf)) {
        len += rd;
        if (rd < BUFSTEP) break;

        /* allocate more */
        psl = GC_REALLOC(psl, len + BUFSTEP);
    }

    /* FIXME: bounds checking */

    /* Go section-by-section */
    for (i = 8; i < len;) {
        /* section length */
        i += pslBignumToInt(psl + i, &slen);

        /* section type */
        stl = pslBignumToInt(psl + i, &stype);

        /* if it's program data, we found it */
        if (stype == 0) {
            i += stl;
            break;

        } else {
            /* skip this section */
            i += slen;
        }
    }<commit_msg>cplofr/psli.c: Read/write binary (stupid DOS :) )<commit_after>#include <stdio.h>
#include <string.h>

#include "plof.h"
#include "psl.h"

#define BUFSTEP 1024

int main(int argc, char **argv)
{
    FILE *pslf;
    unsigned char *psl;
    size_t len, rd, i;
    size_t slen, stype, stl;
    struct PlofObject *context;
    struct PlofReturn ret;

    GC_INIT();

    if (argc != 2) {
        fprintf(stderr, "Use: psli <file>\n");
        return 1;
    }

    /* open the file */
    pslf = fopen(argv[1], "rb");
    if (pslf == NULL) {
        perror(argv[1]);
        return 1;
    }

    /* preallocate the buffer */
    psl = GC_MALLOC(BUFSTEP);
    len = 0;

    /* now start reading */
    while (rd = fread(psl + len, 1, BUFSTEP, pslf)) {
        len += rd;
        if (rd < BUFSTEP) break;

        /* allocate more */
        psl = GC_REALLOC(psl, len + BUFSTEP);
    }
    fclose(pslf);

    /* FIXME: bounds checking */

    /* Go section-by-section */
    for (i = 8; i < len;) {
        /* section length */
        i += pslBignumToInt(psl + i, &slen);

        /* section type */
        stl = pslBignumToInt(psl + i, &stype);

        /* if it's program data, we found it */
        if (stype == 0) {
            i += stl;
            break;

        } else {
            /* skip this section */
            i += slen;
        }
    }
{'fraction_non_alphanumeric': 0.15020809685962921, 'fraction_numerical': 0.009837306091562617, 'mean_word_length': 1.6680121089808275, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7077728', 'n_tokens_mistral': 1023, 'n_tokens_neox': 962, 'n_words': 260}