<commit_before>#include <assert.h>
#include <limits.h>
#include <math.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#include "log.h"
#include "sparse.h"
#include "dSFMT-src-2.2.1/dSFMT.h"
#include "channel_algorithms.h"

int
sample(dv_t *cp, dsfmt_t *rng) {
    float x= dsfmt_genrand_close_open(rng);
    int s= 0, e= cp->length;

    while(s+1 < e) {
        int m= (s+e)/2;

        if(x < cp->entries[m]) e= m;
        else                   s= m;
    }

    return s;
}

csc_mat_t *
sampled_matrix(dv_t *lcp, dv_t *rcp, int rows, int samples, dsfmt_t *rng) {
    bsc_hist_t *H;
    csc_mat_t *M;
    int r;

    H= bsc_hist_new();
    for(r= 0; r < rows; r++) {
        int i;

        for(i= 0; i < samples; i++) {
            int c;
            if(r < rows/2)
                c= sample(lcp, rng);
            else
                c= sample(rcp, rng);
            bsc_hist_count(H, c, r, 1);
        }
    }
    M= bsc_normalise(H);
    bsc_hist_destroy(H);
    return M;
}

dv_t *
average_cols(csc_mat_t *M) {
    dv_t *v;
    int c;
    float Ps= 0.0;

    v= dv_new(M->ncol);
    if(!v) {
        perror("dv_new");
        exit(EXIT_FAILURE);
    }
    dv_zero(v);

    /* Sum along every column. */
    for(c= 0; c < M->ncol; c++) {
        int64_t i;

        for(i= M->ci[c]; i < M->ci[c+1]; i++) {
            if(M->entries[i] == 0) continue;
            v->entries[c]+= M->entries[i];
        }
    }

    /* Rescale to ensure total prob = 1 */
    for(c= 0; c < v->length; c++) Ps+= v->entries[c];
    for(c= 0; c < v->length; c++) v->entries[c]/= Ps;

    return v;
}

dv_t *
accumulate_prob(dv_t *p) {
    dv_t *cp;
    float Pc= 0.0;
    int i;

    cp= dv_new(p->length);
    if(!cp) {
        perror("dv_new");
        exit(EXIT_FAILURE);
    }

    for(i= 0; i < p->length; i++) {
        Pc+= p->entries[i];
        cp->entries[i]= Pc;
    }

    return cp;
}

pthread_mutex_t output_lock= PTHREAD_MUTEX_INITIALIZER;

void
noisy_matrices(float cap, dv_t *lcp, dv_t *rcp, float epsilon, int n,
        int rows, int samples, dsfmt_t *rng) {
    int i;

    for(i= 0; i < n; i++) {
        csc_mat_t *M= sampled_matrix(lcp, rcp, rows, samples, rng);
        float I, e;
        csc_check(M, 1);
        I= blahut_arimoto(M, epsilon, &e);
        csc_mat_destroy(M);
        pthread_mutex_lock(&output_lock);
        printf("%.12e %.12e %.12e\n", cap, I, e);
        fflush(stdout);
        pthread_mutex_unlock(&output_lock);
    }
}

struct job {
    float cap;
    dv_t *lcp;
    dv_t *rcp;
    float epsilon;
    int n;
    int rows;
    int samples;
    dsfmt_t rng;
    pthread_t thread;
};

static void *
worker(void *arg) {
    struct job *j= (struct job *)arg;
    noisy_matrices(j->cap, j->lcp, j->rcp, j->epsilon, j->n, j->rows,
            j->samples, &j->rng);
    return (void *)0;<commit_msg>* Disabled sanity check<commit_after>#include <assert.h>
#include <limits.h>
#include <math.h>
#include <pthread.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#include "log.h"
#include "sparse.h"
#include "dSFMT-src-2.2.1/dSFMT.h"
#include "channel_algorithms.h"

int
sample(dv_t *cp, dsfmt_t *rng) {
    float x= dsfmt_genrand_close_open(rng);
    int s= 0, e= cp->length;

    while(s+1 < e) {
        int m= (s+e)/2;

        if(x < cp->entries[m]) e= m;
        else                   s= m;
    }

    return s;
}

csc_mat_t *
sampled_matrix(dv_t *lcp, dv_t *rcp, int rows, int samples, dsfmt_t *rng) {
    bsc_hist_t *H;
    csc_mat_t *M;
    int r;

    H= bsc_hist_new();
    for(r= 0; r < rows; r++) {
        int i;

        for(i= 0; i < samples; i++) {
            int c;
            if(r < rows/2)
                c= sample(lcp, rng);
            else
                c= sample(rcp, rng);
            bsc_hist_count(H, c, r, 1);
        }
    }
    M= bsc_normalise(H);
    bsc_hist_destroy(H);
    return M;
}

dv_t *
average_cols(csc_mat_t *M) {
    dv_t *v;
    int c;
    float Ps= 0.0;

    v= dv_new(M->ncol);
    if(!v) {
        perror("dv_new");
        exit(EXIT_FAILURE);
    }
    dv_zero(v);

    /* Sum along every column. */
    for(c= 0; c < M->ncol; c++) {
        int64_t i;

        for(i= M->ci[c]; i < M->ci[c+1]; i++) {
            if(M->entries[i] == 0) continue;
            v->entries[c]+= M->entries[i];
        }
    }

    /* Rescale to ensure total prob = 1 */
    for(c= 0; c < v->length; c++) Ps+= v->entries[c];
    for(c= 0; c < v->length; c++) v->entries[c]/= Ps;

    return v;
}

dv_t *
accumulate_prob(dv_t *p) {
    dv_t *cp;
    float Pc= 0.0;
    int i;

    cp= dv_new(p->length);
    if(!cp) {
        perror("dv_new");
        exit(EXIT_FAILURE);
    }

    for(i= 0; i < p->length; i++) {
        Pc+= p->entries[i];
        cp->entries[i]= Pc;
    }

    return cp;
}

pthread_mutex_t output_lock= PTHREAD_MUTEX_INITIALIZER;

void
noisy_matrices(float cap, dv_t *lcp, dv_t *rcp, float epsilon, int n,
        int rows, int samples, dsfmt_t *rng) {
    int i;

    for(i= 0; i < n; i++) {
        csc_mat_t *M= sampled_matrix(lcp, rcp, rows, samples, rng);
        float I, e;
        //csc_check(M, 1);
        I= blahut_arimoto(M, epsilon, &e);
        csc_mat_destroy(M);
        pthread_mutex_lock(&output_lock);
        printf("%.12e %.12e %.12e\n", cap, I, e);
        fflush(stdout);
        pthread_mutex_unlock(&output_lock);
    }
}

struct job {
    float cap;
    dv_t *lcp;
    dv_t *rcp;
    float epsilon;
    int n;
    int rows;
    int samples;
    dsfmt_t rng;
    pthread_t thread;
};

static void *
worker(void *arg) {
    struct job *j= (struct job *)arg;
    noisy_matrices(j->cap, j->lcp, j->rcp, j->epsilon, j->n, j->rows,
            j->samples, &j->rng);
    return (void *)0;
{'fraction_non_alphanumeric': 0.1703920220082531, 'fraction_numerical': 0.011004126547455296, 'mean_word_length': 2.0328467153284673, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5166349', 'n_tokens_mistral': 2543, 'n_tokens_neox': 2456, 'n_words': 568}