<commit_before># cython: wraparound=False
# cython: boundscheck=False
# cython: cdivision=True

import numpy as np
cimport cython
cimport cython.parallel as cp
from cython.view cimport array

from libc.stdio cimport printf
from libc.stdlib cimport srand, rand, RAND_MAX, abort, calloc, free



cdef double **calloc_2d(int m, int n) nogil:

    cdef double **a
    cdef int i

    a = <double **> calloc(m, sizeof(double*))
    if a==NULL:
        abort()

    for i in range(m):
        a[i] = <double *> calloc(n, sizeof(double))
        if a[i]==NULL:
            abort()

    return a


cdef void free_2d(double **a, int m) nogil:
    cdef int i
    for i in range(m):
        free(a[i])
    free(a)


cdef int seed_random(unsigned long seed):
    srand(seed)
    return 0


cdef double random() nogil:
    return <double>rand() / <double>RAND_MAX


cdef int init_word_top(int K, 
                       int V,
                       double [:,:] word_top,
                       double [:] beta):
    cdef int w, k
    for k in range(K):
        for w in range(V):
            word_top[w,k] = beta[w]
    return 0


cdef int init_doc_top(int K, 
                      int N,
                      double [:,:] doc_top,
                      double [:] alpha):
    cdef int w, k
    for k in range(K):
        for d in range(N):
            doc_top[d,k] = alpha[k]
    return 0


cdef int sample(int w,
                int i,
                int K,
                double *inv_top_sums,
                double **word_top,
                double **doc_top,
                double [:] dist,
                double [:] cdist) nogil:
    cdef int k
    cdef double r

    for k in range(K):
        dist[k] = inv_top_sums[k] * word_top[w][k] * doc_top[i][k]
        if k==0:
            cdist[0] = dist[0]
        else:
            cdist[k] = cdist[k-1] + dist[k]
    r = random() * cdist[K - 1]
    for k in range(K):
       if r <= cdist[k]:
           return k
    abort()
    return 0


cdef int init_cgs(int K,
                  int V,
                  int N,
                  int W,
                  int [:] indices,
                  int [:] corpus,
                  int [:] Z,
                  double [:,:] word_top,
                  double [:,:] doc_top,
                  double [:] inv_top_sums,
                  int [:] Z_cached,
                  double [:,:] word_top_cached,
                  double [:,:] doc_top_cached,
                  double [:] inv_top_sums_cached,
                  double [:] dist,
                  double [:] cdist,
                  double [:] log_probs,
                  int n_threads,
                  int iteration) nogil:

    cdef int i, j, D, start, stop, w, k, ID, u, v
    cdef double s
    cdef int *Z_local
    cdef double **word_top_local
    cdef double *inv_top_sums_local
    cdef double **doc_top_local

    with nogil, cp.parallel(num_threads=n_threads):

        Z_local = <int *>calloc(W, sizeof(int))
        if Z_local==NULL:
            abort()

        word_top_local = calloc_2d(V, K)

        inv_top_sums_local = <double *>calloc(K, sizeof(double))
        if inv_top_sums_local==NULL:
            abort()

        doc_top_local = calloc_2d(N, K)


        for u in range(W):
            Z_local[u] = Z[u]
        for u in range(V):
            for v in range(K):
                word_top_local[u][v] = word_top[u, v]
        for u in range(K):
            inv_top_sums_local[u] = inv_top_sums[u]
        for u in range(N):
            for v in range(K):
                doc_top_local[u][v] = doc_top[u, v]
                
        
        for i in cp.prange(N):

            ID = cp.threadid()
            printf('Document %i, thread %i\n', i, ID)

            if i==0:
                start = 0
                stop = indices[0]
            else:
                start = indices[i-1]<commit_msg>debugging code<commit_after># cython: wraparound=False
# cython: boundscheck=False
# cython: cdivision=True

import numpy as np
cimport cython
cimport openmp
cimport cython.parallel as cp

from libc.stdio cimport printf
from libc.stdlib cimport srand, rand, RAND_MAX, abort, calloc, free



cdef double **calloc_2d(int m, int n) nogil:

    cdef double **a
    cdef int i

    a = <double **> calloc(m, sizeof(double*))
    if a==NULL:
        abort()

    for i in range(m):
        a[i] = <double *> calloc(n, sizeof(double))
        if a[i]==NULL:
            abort()

    return a


cdef void free_2d(double **a, int m) nogil:
    cdef int i
    for i in range(m):
        free(a[i])
    free(a)


cdef int seed_random(unsigned long seed):
    srand(seed)
    return 0


cdef double random() nogil:
    return <double>rand() / <double>RAND_MAX


cdef int init_word_top(int K, 
                       int V,
                       double [:,:] word_top,
                       double [:] beta):
    cdef int w, k
    for k in range(K):
        for w in range(V):
            word_top[w,k] = beta[w]
    return 0


cdef int init_doc_top(int K, 
                      int N,
                      double [:,:] doc_top,
                      double [:] alpha):
    cdef int w, k
    for k in range(K):
        for d in range(N):
            doc_top[d,k] = alpha[k]
    return 0


cdef int sample(int w,
                int i,
                int K,
                double *inv_top_sums,
                double **word_top,
                double **doc_top,
                double [:] dist,
                double [:] cdist) nogil:
    cdef int k
    cdef double r

    for k in range(K):
        dist[k] = inv_top_sums[k] * word_top[w][k] * doc_top[i][k]
        if k==0:
            cdist[0] = dist[0]
        else:
            cdist[k] = cdist[k-1] + dist[k]
    r = random() * cdist[K - 1]
    for k in range(K):
       if r <= cdist[k]:
           return k
    abort()
    return 0


cdef int init_cgs(int K,
                  int V,
                  int N,
                  int W,
                  int [:] indices,
                  int [:] corpus,
                  int [:] Z,
                  double [:,:] word_top,
                  double [:,:] doc_top,
                  double [:] inv_top_sums,
                  int [:] Z_cached,
                  double [:,:] word_top_cached,
                  double [:,:] doc_top_cached,
                  double [:] inv_top_sums_cached,
                  double [:] dist,
                  double [:] cdist,
                  double [:] log_probs,
                  int n_threads,
                  int iteration) nogil:

    cdef int i, j, D, start, stop, w, k, ID, u, v, nt
    cdef double s
    cdef int *Z_local
    cdef double **word_top_local
    cdef double *inv_top_sums_local
    cdef double **doc_top_local

    with nogil, cp.parallel(num_threads=n_threads):

        Z_local = <int *>calloc(W, sizeof(int))
        if Z_local==NULL:
            abort()

        word_top_local = calloc_2d(V, K)

        inv_top_sums_local = <double *>calloc(K, sizeof(double))
        if inv_top_sums_local==NULL:
            abort()

        doc_top_local = calloc_2d(N, K)


        for u in range(W):
            Z_local[u] = Z[u]
        for u in range(V):
            for v in range(K):
                word_top_local[u][v] = word_top[u, v]
        for u in range(K):
            inv_top_sums_local[u] = inv_top_sums[u]
        for u in range(N):
            for v in range(K):
                doc_top_local[u][v] = doc_top[u, v]
                
        
        for i in cp.prange(N):

            nt = openmp.omp_get_num_threads()
            
            ID = cp.threadid()
            printf('Document %i, thread %i of %i\n', i, ID, nt)

            if i==0:
                start = 0
                stop = indices[0]
            else:
                start = indices[i-1]
{'fraction_non_alphanumeric': 0.10914681014604151, 'fraction_numerical': 0.004355623879067384, 'mean_word_length': 1.3962553713934929, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7621358', 'n_tokens_mistral': 2731, 'n_tokens_neox': 2555, 'n_words': 710}