<commit_before>#include "data.table.h"
#include <Rdefines.h>
#include <time.h>

// TODO: implement 'lookup' for 'gaps' and 'overlaps' arguments
SEXP lookup(SEXP ux, SEXP xlen, SEXP indices, SEXP gaps, SEXP overlaps, SEXP multArg, SEXP typeArg, SEXP verbose) {
    
    SEXP vv, tt, lookup, type_lookup;
    R_len_t i,j,k,*idx,*len1,*len2,xrows=INTEGER(xlen)[0],uxrows=LENGTH(VECTOR_ELT(ux, 0)),uxcols=LENGTH(ux);
    int *from = (int *)INTEGER(VECTOR_ELT(indices, 0));
    int *to   = (int *)INTEGER(VECTOR_ELT(indices, 1));
    clock_t pass1, pass2, pass3, start;
    enum {ALL, FIRST, LAST} mult = ALL;
    enum {ANY, WITHIN, START, END, EQUAL} type = ANY;
    
    if (!strcmp(CHAR(STRING_ELT(multArg, 0)), "all"))  mult = ALL;
    else if (!strcmp(CHAR(STRING_ELT(multArg, 0)), "first")) mult = FIRST;
    else if (!strcmp(CHAR(STRING_ELT(multArg, 0)), "last")) mult = LAST;
    else error("Internal error: invalid value for 'mult'; this should have been caught before. Please report to datatable-help");

    if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "any"))  type = ANY;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "within")) type = WITHIN;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "start")) type = START;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "end")) type = END;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "equal")) type = EQUAL;
    else error("Internal error: invalid value for 'type'; this should have been caught before. Please report to datatable-help");
    
    // For reference: uxcols-1 = type_count, uxcols-2 = count, uxcols-3 = type_lookup, uxcols-4 = lookup
    // first pass: calculate lengths first
    start = clock();
    len1 = (int *)INTEGER(VECTOR_ELT(ux, uxcols-2));
    len2 = (int *)INTEGER(VECTOR_ELT(ux, uxcols-1));
    switch (mult) {
        case FIRST: 
        for (i=0; i<xrows; i++) {
            for (j=from[i]; j<=to[i]; j++) {
                len1[j-1]++;
            }
        }
        if (type != WITHIN) {
            for (i=0; i<uxrows-1; i++)                      // TODO: this allocation can be avoided if we take care of FIRST/LAST accordingly in 'overlaps'
                if (len1[i]) len2[i] = 1;
        }
        break;
        
        case LAST :
        switch (type) {
            case ANY:
            for (i=0; i<xrows; i++) {
                for (j=from[i]; j<=to[i]; j++) {
                    len1[j-1]++;
                    if (from[i]==j && !len2[j-1]) len2[j-1]++;
                }
            }
            break;
            case START: case END: case EQUAL: case WITHIN:
            for (i=0; i<xrows; i++) {
                for (j=from[i]; j<=to[i]; j++) {
                    len1[j-1]++;
                }
            }
            if (type != WITHIN) {
                for (i=0; i<uxrows-1; i++)              // TODO: this allocation can be avoided if we take care of FIRST/LAST accordingly in 'overlaps'
                    if (len1[i]) len2[i] = 1;                    
            }
            break;
        }
        break;
        
        case ALL : 
            switch (type) {
                case START: case END:
                for (i=0; i<xrows; i++) {
                    for (j=from[i]; j<=to[i]; j++) {
                        len1[j-1]++; len2[j-1]++;       // alternatively, we could simply do with len2=len1 ?
                    }
                }
                break;
                case EQUAL:
                for (i=0; i<xrows; i++) {
                    len1[from[i]-1]++; len1[to[i]-1]++;
                    len2[from[i]-1]++; len2[to[i]-1]++;
                }
                break;
                case ANY :
                for (i=0; i<xrows; i++) {
                    k = from[i];
                    for (j=from[i]; j<=to[i]; j++) {
                        len1[j-1]++;
                        if (k==j) len2[j-1]++;
                    }
                }
                break;
                case WITHIN :
                for (i=0; i<xrows; i++) {
                    for (j=from[i]; j<=to[i]; j++) {
                        len1[j-1]++;
                    }
                }
                break;
            }
        break;
    }
    pass1 = clock() - start;
    if (LOGICAL(verbose)[0])
        Rprintf("First pass on calculating lengths in lookup ... done in %8.3f seconds\n", 1.0*(pass1)/CLOCKS_PER_SEC);
    // second pass: allocate vectors
    start = clock();
    lookup = VECTOR_ELT(ux, uxcols-4);
    type_lookup = VECTOR_ELT(ux, uxcols-3);
    for (i=0; i<uxrows; i++) {
        vv = allocVector(INTSXP, len1[i]);
        SET_VECTOR_ELT(lookup, i, vv);
        if (type != WITHIN) {
            vv = allocVector(INTSXP, len2[i]);
            SET_VECTOR_ELT(type_lookup, i, vv);
        }
    }
    pass2 = clock() - start;
    if (LOGICAL(verbose)[0])
        Rprintf("Second pass on allocation in lookup ... done in %8.3f seconds\n", 1.0*(pass2)/CLOCKS_PER_SEC);
    // generate lookup
    start = clock();
    idx = Calloc(uxrows, R_len_t); // resets bits, =0
    switch (type) {
        case ANY: case START: case END: case WITHIN:
        for (i=0; i<xrows; i++) {
            for (j=from[i]; j<=to[i]; j++) {
                vv = VECTOR_ELT(lookup, j-1);  // cache misses - memory efficiency? but 'lookups' are tiny - takes 0.036s on A.thaliana GFF for entire process)
                INTEGER(vv)[idx[j-1]++] = i+1;
            }
        }
        break;
        case EQUAL:
        for (i=0; i<xrows; i++) {
            INTEGER(VECTOR_ELT(lookup, from[i]-1))[idx[from[i]-1]++] = i+1;
            INTEGER(VECTOR_ELT(lookup, to[i]-1))[idx[to[i]-1]++] = i+1;
        }
        break;
    }
    Free(idx);
    // generate type_lookup
    if (type != WITHIN) {
        switch (mult) {
            case FIRST :
            for (i=0; i<uxrows; i++) {
                if (!len1[i]) continue;
                vv = VECTOR_ELT(lookup, i);
                tt = VECTOR_ELT(type_lookup, i);
                INTEGER(tt)[0] = INTEGER(vv)[0];
            }
            break;

            case LAST :
            for (i=0; i<uxrows; i++) {
                if (!len1[i]) continue;
                vv = VECTOR_ELT(lookup, i);
                tt = VECTOR_ELT(type_lookup, i);
                INTEGER(tt)[0] = INTEGER(vv)[len1[i]-1];
            }
        
            case ALL :
            switch (type) {
                case START: case END: case EQUAL:
                for (i=0; i<uxrows; i++)
                    SET_VECTOR_ELT(type_lookup, i, VECTOR_ELT(lookup, i));
                break;
            
                case ANY :
                for (i=0; i<uxrows-1; i++) {
                    vv = VECTOR_ELT(lookup, i);
                    tt = VECTOR_ELT(type_lookup, i);
                    k=0;
                    for (j=len1[i]-len2[i]; j<len1[i]; j++)
                        INTEGER(tt)[k++] = INTEGER(vv)[j];
                }
                break;
            
                case WITHIN :
                // for (i=0; i<uxrows-1; i++) {
                //     vv = VECTOR_ELT(lookup, i);
                //     tt = VECTOR_ELT(type_lookup, i);
                //     for (j=0; j<len2[i]; j++)
                //         INTEGER(tt)[j] = INTEGER(vv)[j];
                // }
                break;
            }
        break;
        }
    }
    pass3 = clock() - start;
    if (LOGICAL(verbose)[0])
        Rprintf("Final step in generating lookup ... done in %8.3f seconds\n", 1.0*(pass3)/CLOCKS_PER_SEC);    
    return(R_NilValue);
}

SEXP overlaps(SEXP ux, SEXP imatches, SEXP multArg, SEXP typeArg, SEXP nomatchArg, SEXP verbose) {

    R_len_t i,j,k,m,uxcols=LENGTH(ux),rows=length(VECTOR_ELT(imatches,0));
    int nomatch = INTEGER(nomatchArg)[0], totlen=0, len, thislen, wlen=0;
    int *from   = (int *)INTEGER(VECTOR_ELT(imatches, 0));
    int *to     = (int *)INTEGER(VECTOR_ELT(imatches, 1));
    int *len1   = (int *)INTEGER(VECTOR_ELT(ux, uxcols-2));<commit_msg>Avoid segfault when uxrows=1 (even if it will/should never be 1).<commit_after>#include "data.table.h"
#include <Rdefines.h>
#include <time.h>

// TODO: implement 'lookup' for 'gaps' and 'overlaps' arguments
SEXP lookup(SEXP ux, SEXP xlen, SEXP indices, SEXP gaps, SEXP overlaps, SEXP multArg, SEXP typeArg, SEXP verbose) {
    
    SEXP vv, tt, lookup, type_lookup;
    R_len_t i,j,k,*idx,*len1,*len2,xrows=INTEGER(xlen)[0],uxrows=LENGTH(VECTOR_ELT(ux, 0)),uxcols=LENGTH(ux);
    int *from = (int *)INTEGER(VECTOR_ELT(indices, 0));
    int *to   = (int *)INTEGER(VECTOR_ELT(indices, 1));
    clock_t pass1, pass2, pass3, start;
    enum {ALL, FIRST, LAST} mult = ALL;
    enum {ANY, WITHIN, START, END, EQUAL} type = ANY;
    
    if (!strcmp(CHAR(STRING_ELT(multArg, 0)), "all"))  mult = ALL;
    else if (!strcmp(CHAR(STRING_ELT(multArg, 0)), "first")) mult = FIRST;
    else if (!strcmp(CHAR(STRING_ELT(multArg, 0)), "last")) mult = LAST;
    else error("Internal error: invalid value for 'mult'; this should have been caught before. Please report to datatable-help");

    if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "any"))  type = ANY;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "within")) type = WITHIN;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "start")) type = START;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "end")) type = END;
    else if (!strcmp(CHAR(STRING_ELT(typeArg, 0)), "equal")) type = EQUAL;
    else error("Internal error: invalid value for 'type'; this should have been caught before. Please report to datatable-help");
    
    // For reference: uxcols-1 = type_count, uxcols-2 = count, uxcols-3 = type_lookup, uxcols-4 = lookup
    // first pass: calculate lengths first
    start = clock();
    len1 = (int *)INTEGER(VECTOR_ELT(ux, uxcols-2));
    len2 = (int *)INTEGER(VECTOR_ELT(ux, uxcols-1));
    switch (mult) {
        case FIRST: 
        for (i=0; i<xrows; i++) {
            for (j=from[i]; j<=to[i]; j++) {
                len1[j-1]++;
            }
        }
        if (type != WITHIN) {
            for (i=0; i<uxrows; i++)                      // TODO: this allocation can be avoided if we take care of FIRST/LAST accordingly in 'overlaps'
                if (len1[i]) len2[i] = 1;
        }
        break;
        
        case LAST :
        switch (type) {
            case ANY:
            for (i=0; i<xrows; i++) {
                for (j=from[i]; j<=to[i]; j++) {
                    len1[j-1]++;
                    if (from[i]==j && !len2[j-1]) len2[j-1]++;
                }
            }
            break;
            case START: case END: case EQUAL: case WITHIN:
            for (i=0; i<xrows; i++) {
                for (j=from[i]; j<=to[i]; j++) {
                    len1[j-1]++;
                }
            }
            if (type != WITHIN) {
                for (i=0; i<uxrows; i++)              // TODO: this allocation can be avoided if we take care of FIRST/LAST accordingly in 'overlaps'
                    if (len1[i]) len2[i] = 1;                    
            }
            break;
        }
        break;
        
        case ALL : 
            switch (type) {
                case START: case END:
                for (i=0; i<xrows; i++) {
                    for (j=from[i]; j<=to[i]; j++) {
                        len1[j-1]++; len2[j-1]++;       // alternatively, we could simply do with len2=len1 ?
                    }
                }
                break;
                case EQUAL:
                for (i=0; i<xrows; i++) {
                    len1[from[i]-1]++; len1[to[i]-1]++;
                    len2[from[i]-1]++; len2[to[i]-1]++;
                }
                break;
                case ANY :
                for (i=0; i<xrows; i++) {
                    k = from[i];
                    for (j=from[i]; j<=to[i]; j++) {
                        len1[j-1]++;
                        if (k==j) len2[j-1]++;
                    }
                }
                break;
                case WITHIN :
                for (i=0; i<xrows; i++) {
                    for (j=from[i]; j<=to[i]; j++) {
                        len1[j-1]++;
                    }
                }
                break;
            }
        break;
    }
    pass1 = clock() - start;
    if (LOGICAL(verbose)[0])
        Rprintf("First pass on calculating lengths in lookup ... done in %8.3f seconds\n", 1.0*(pass1)/CLOCKS_PER_SEC);
    // second pass: allocate vectors
    start = clock();
    lookup = VECTOR_ELT(ux, uxcols-4);
    type_lookup = VECTOR_ELT(ux, uxcols-3);
    for (i=0; i<uxrows; i++) {
        vv = allocVector(INTSXP, len1[i]);
        SET_VECTOR_ELT(lookup, i, vv);
        if (type != WITHIN) {
            vv = allocVector(INTSXP, len2[i]);
            SET_VECTOR_ELT(type_lookup, i, vv);
        }
    }
    pass2 = clock() - start;
    if (LOGICAL(verbose)[0])
        Rprintf("Second pass on allocation in lookup ... done in %8.3f seconds\n", 1.0*(pass2)/CLOCKS_PER_SEC);
    // generate lookup
    start = clock();
    idx = Calloc(uxrows, R_len_t); // resets bits, =0
    switch (type) {
        case ANY: case START: case END: case WITHIN:
        for (i=0; i<xrows; i++) {
            for (j=from[i]; j<=to[i]; j++) {
                vv = VECTOR_ELT(lookup, j-1);  // cache misses - memory efficiency? but 'lookups' are tiny - takes 0.036s on A.thaliana GFF for entire process)
                INTEGER(vv)[idx[j-1]++] = i+1;
            }
        }
        break;
        case EQUAL:
        for (i=0; i<xrows; i++) {
            INTEGER(VECTOR_ELT(lookup, from[i]-1))[idx[from[i]-1]++] = i+1;
            INTEGER(VECTOR_ELT(lookup, to[i]-1))[idx[to[i]-1]++] = i+1;
        }
        break;
    }
    Free(idx);
    // generate type_lookup
    if (type != WITHIN) {
        switch (mult) {
            case FIRST :
            for (i=0; i<uxrows; i++) {
                if (!len1[i]) continue;
                vv = VECTOR_ELT(lookup, i);
                tt = VECTOR_ELT(type_lookup, i);
                INTEGER(tt)[0] = INTEGER(vv)[0];
            }
            break;

            case LAST :
            for (i=0; i<uxrows; i++) {
                if (!len1[i]) continue;
                vv = VECTOR_ELT(lookup, i);
                tt = VECTOR_ELT(type_lookup, i);
                INTEGER(tt)[0] = INTEGER(vv)[len1[i]-1];
            }
        
            case ALL :
            switch (type) {
                case START: case END: case EQUAL:
                for (i=0; i<uxrows; i++)
                    SET_VECTOR_ELT(type_lookup, i, VECTOR_ELT(lookup, i));
                break;
            
                case ANY :
                for (i=0; i<uxrows; i++) {
                    vv = VECTOR_ELT(lookup, i);
                    tt = VECTOR_ELT(type_lookup, i);
                    k=0;
                    for (j=len1[i]-len2[i]; j<len1[i]; j++)
                        INTEGER(tt)[k++] = INTEGER(vv)[j];
                }
                break;
            
                case WITHIN :
                // for (i=0; i<uxrows; i++) {
                //     vv = VECTOR_ELT(lookup, i);
                //     tt = VECTOR_ELT(type_lookup, i);
                //     for (j=0; j<len2[i]; j++)
                //         INTEGER(tt)[j] = INTEGER(vv)[j];
                // }
                break;
            }
        break;
        }
    }
    pass3 = clock() - start;
    if (LOGICAL(verbose)[0])
        Rprintf("Final step in generating lookup ... done in %8.3f seconds\n", 1.0*(pass3)/CLOCKS_PER_SEC);    
    return(R_NilValue);
}

SEXP overlaps(SEXP ux, SEXP imatches, SEXP multArg, SEXP typeArg, SEXP nomatchArg, SEXP verbose) {

    R_len_t i,j,k,m,uxcols=LENGTH(ux),rows=length(VECTOR_ELT(imatches,0));
    int nomatch = INTEGER(nomatchArg)[0], totlen=0, len, thislen, wlen=0;
    int *from   = (int *)INTEGER(VECTOR_ELT(imatches, 0));
    int *to     = (int *)INTEGER(VECTOR_ELT(imatches, 1));
    int *len1   = (int *)INTEGER(VECTOR_ELT(ux, uxcols-2));
{'fraction_non_alphanumeric': 0.1535615751191372, 'fraction_numerical': 0.017682468021068473, 'mean_word_length': 1.5408634698104189, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1294154', 'n_tokens_mistral': 5685, 'n_tokens_neox': 5376, 'n_words': 1279}