<commit_before>#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <math.h>

#define PACK_T uint64_t
#define PACK_SIZE (sizeof(PACK_T) * 8)

unsigned long long pyramid(unsigned long long p)
{
    return (p*(p*p - 1))/6;
}

PACK_T* ext_shift(PACK_T ch, uint8_t pos)
{
    static PACK_T retval[2];
    retval[0] = ch << pos;
    if (pos==0)
        retval[1] = 0;
    else 
        retval[1] = ch >> (PACK_SIZE-pos);
    return retval;
}

void mark(PACK_T *set, unsigned long long pos)
{
    uint8_t bitsize = PACK_SIZE;
    uint8_t setbit = pos % bitsize;
    unsigned long long int offset = pos / bitsize;
    set[offset] |= 1ULL<<setbit;
    return;
}

PACK_T test(const PACK_T *set, unsigned long long pos)
{
    uint8_t bitsize = PACK_SIZE;
    uint8_t testbit = pos % bitsize;
    unsigned long int offset = pos / bitsize;
    return set[offset] & 1ULL<<testbit;
}

void shift_and_mark(PACK_T *source, PACK_T *target, unsigned long long length, unsigned long long pos)
{
    uint8_t bitsize = PACK_SIZE;
    uint8_t bitoff = pos % bitsize;
    unsigned long long offset = pos / bitsize;
    PACK_T *shifted;
    unsigned long long i;
    
    for (i = 0; i < length; i++) {
        if(i + offset >= length) break;
        shifted = ext_shift(source[i], bitoff);
        target[i + offset] |= shifted[0];
        if(i + offset + 1 >= length) break;
        target[i + offset + 1] |= shifted[1];
    }
    return;
}

unsigned long long count_set(PACK_T *set, unsigned long long length)
{
    unsigned long long int i,j;
    j = 0;
    for(i = 0; i < length; i++) {
        if(test(set,i))
            j++;
        if (i%1000000000ULL == 0)
            printf("N1 %lld : %lld\n",i,j);
    }
    return j;
}

void pr(const PACK_T *set, unsigned long long length)
{
    unsigned long long i;
    for(i=0;i<length*PACK_SIZE;i++) {
        if(test(set, i)) printf("X, %lld\n", i);
    }
}

void save(char *fname, PACK_T *set, unsigned long long length)
{
    FILE *ptr_file;
    unsigned long long i;
    ptr_file = fopen(fname, "wb");
    if (!ptr_file)
        printf("Unable to open file!");
    else
        fwrite(set, PACK_SIZE, length, ptr_file);
    fclose(ptr_file);
    return;
}

/*
    Overall strategy is to first mark the integers which are pyramidal
    then the integers which are the sum of two pyramidals. For sums
    of pyramidals above 3 the summations will be performed on batches
    using bit shifts in order to realize a speedup roughly proportional
    to the size of the largest machine integer. For example with a machine
    integer of size 64-bits the speedup should be roughly 64 times that of
    using integer multiplication.
*/
int main(int argc, char *argv[])
{
    unsigned long long numints,length, maxp;
    /* By default assume we want 1,000,000 digits checked*/
    if (argc != 2) {
        /*We add one since indexing is based on the cardinal value of numbers, in other
        words bit 0 in our packing always remains empty since the first pyramidal number is one.
        Therefor the cardinality for the set of 1000000 numbers is 1000001.*/ 
        numints = 1000000+1;
        length = numints / PACK_SIZE;
        if(numints % PACK_SIZE != 0)
            length++;
        maxp = (long long) floor(6.0*numints)+1ULL;
    }
    else if ((argc == 2) && (atoll(argv[1]) != 0)) {
        /*We add one since indexing is based on the cardinal value of numbers, in other
        words bit 0 in our packing always remains empty since the first pyramidal number is one.
        Therefor the cardinality for argv[1] numbers is argv[1]+1*/ 
        numints = atoll(argv[1]) + 1;
        length = numints / PACK_SIZE;
        if (numints % PACK_SIZE != 0)
            length++;
        maxp = (long long) floor(6.0*numints)+1ULL;  
    }
    else
        printf("Usage is %s length: Where length is the number of integers to check.\n", argv[0]);
    
    /* numberset is a large bit field representing the set of numbers representable
    as the sum of fewer than 5 pyramidal numbers
    */
    PACK_T *numberset = calloc(length, sizeof(PACK_T));
    PACK_T *tmpset = calloc(length, sizeof(PACK_T));
    if (numberset == NULL | tmpset == NULL) {
        printf("Failed to allocate memory. \n");<commit_msg>Added struct for packed array, small changes for strict C99 compliance<commit_after>#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <math.h>

#define PACK_T uint64_t
#define PACK_SIZE (sizeof(PACK_T) * 8)

struct bitpack {
    PACK_T *set; /* Pointer to structure containing packed bit representation
                  * of a potentially huge number of integers. For efficiency
                  * PACK_T should be the largest representable integer size on
                  * the system for which efficient standar arithmetic operations
                  * are defined.
                  */
    unsigned long long length; /* length of array pointed to by set. Alternately
                                * number of PACK_T's contained in set.
                                */
    unsigned long long maxnum; /* Number of integers packed into bits*/
};

unsigned long long pyramid(unsigned long long p)
{
    return (p*(p*p - 1))/6;
}

PACK_T* ext_shift(PACK_T ch, uint8_t pos)
{
    static PACK_T retval[2];
    retval[0] = ch << pos;
    if (pos==0)
        retval[1] = 0;
    else 
        retval[1] = ch >> (PACK_SIZE-pos);
    return retval;
}

void mark(PACK_T *set, unsigned long long pos)
{
    uint8_t setbit = pos % PACK_SIZE;
    unsigned long long int offset = pos / PACK_SIZE;
    set[offset] |= 1ULL<<setbit;
    return;
}

PACK_T test(const PACK_T *set, unsigned long long pos)
{
    uint8_t testbit = pos % PACK_SIZE;
    unsigned long int offset = pos / PACK_SIZE;
    return set[offset] & 1ULL<<testbit;
}

void shift_and_mark(PACK_T *source, PACK_T *target, unsigned long long length, unsigned long long pos)
{
    uint8_t bitoff = pos % PACK_SIZE;
    unsigned long long offset = pos / PACK_SIZE;
    PACK_T *shifted;
    unsigned long long i;
    
    for (i = 0; i < length; i++) {
        if(i + offset >= length) break;
        shifted = ext_shift(source[i], bitoff);
        target[i + offset] |= shifted[0];
        if(i + offset + 1 >= length) break;
        target[i + offset + 1] |= shifted[1];
    }
    return;
}

unsigned long long count_set(PACK_T *set, unsigned long long length)
{
    unsigned long long int i,j;
    j = 0;
    for(i = 0; i < length; i++) {
        if(test(set,i))
            j++;
        if (i%1000000000ULL == 0)
            printf("N1 %lld : %lld\n",i,j);
    }
    return j;
}

void pr(const PACK_T *set, unsigned long long length)
{
    unsigned long long i;
    for(i=0;i<length*PACK_SIZE;i++) {
        if(test(set, i)) printf("X, %lld\n", i);
    }
}

/*void save(char *fname, PACK_T *set, unsigned long long length)
{
    FILE *ptr_file;
    unsigned long long i;
    ptr_file = fopen(fname, "wb");
    if (!ptr_file)
        printf("Unable to open file!");
    else
        fwrite(set, PACK_SIZE, length, ptr_file);
    fclose(ptr_file);
    return;
}*/

/*
    Overall strategy is to first mark the integers which are pyramidal
    then the integers which are the sum of two pyramidals. For sums
    of pyramidals above 3 the summations will be performed on batches
    using bit shifts in order to realize a speedup roughly proportional
    to the size of the largest machine integer. For example with a machine
    integer of size 64-bits the speedup should be roughly 64 times that of
    using integer multiplication.
*/
int main(int argc, char *argv[])
{
    unsigned long long numints,length, maxp;
    /* By default assume we want 1,000,000 digits checked*/
    if (argc != 2) {
        /*We add one since indexing is based on the cardinal value of numbers, in other
        words bit 0 in our packing always remains empty since the first pyramidal number is one.
        Therefor the cardinality for the set of 1000000 numbers is 1000001.*/ 
        numints = 1000000+1;
        length = numints / PACK_SIZE;
        if(numints % PACK_SIZE != 0)
            length++;
        maxp = (long long) floor(6.0*numints)+1ULL;
    }
    else if ((argc == 2) && (atoll(argv[1]) != 0)) {
        /*We add one since indexing is based on the cardinal value of numbers, in other
        words bit 0 in our packing always remains empty since the first pyramidal number is one.
        Therefor the cardinality for argv[1] numbers is argv[1]+1*/ 
        numints = atoll(argv[1]) + 1;
        length = numints / PACK_SIZE;
        if (numints % PACK_SIZE != 0)
            length++;
        maxp = (long long) floor(6.0*numints)+1ULL;  
    }
    else
        printf("Usage is %s length: Where length is the number of integers to check.\n", argv[0]);
    
    /* numberset is a large bit field representing the set of numbers representable
    as the sum of fewer than 5 pyramidal numbers
    */
    PACK_T *numberset = calloc(length, sizeof(PACK_T));
    PACK_T *tmpset = calloc(length, sizeof(PACK_T));
    if ((numberset == NULL) | (tmpset == NULL)) {
        printf("Failed to allocate memory. \n");
{'fraction_non_alphanumeric': 0.10439141331589845, 'fraction_numerical': 0.020594965675057208, 'mean_word_length': 2.5725963409887114, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2673281', 'n_tokens_mistral': 3098, 'n_tokens_neox': 2892, 'n_words': 1063}