<commit_before>
// Copied from do_radixsort in src/main/sort.c @ rev 51389
// Turns out not to be a radix sort, but a counting sort
// See http://r.789695.n4.nabble.com/method-radix-in-sort-list-isn-t-actually-a-radix-sort-tp3309470p3309470.html
// A counting sort does seem to be ideal for factors.
// Modified as follows :
// + To work with character, not integer
// + Requires sort(unique(x)) to be passed in as a hashed
//   environment e.g. stored as attribute of the column.
// + No need to scan for min and max. These are known in
//   advance to be 1 and length(hash)
// + No need for offset, removed.
// + For each character, lookup hash to get integer
// + Always increasing so removed the decreasing option
// + NA always first, so removed that option
// + It's not really a sort, but an order that's returned
// + Name change from do_radixsort to countingcharacterorder
// + Changed to a direct .Call() signature.
// + 100,000 range restriction removed.
// + replaced counts alloc (and =0 loop) with a single calloc

#include <R.h>
#define USE_RINTERNALS
#include <Rinternals.h>
#include <Rdefines.h>
#ifdef BUILD_DLL
#define EXPORT __declspec(dllexport)
EXPORT SEXP countingcharacterorder();
#endif

SEXP countingcharacterorder(SEXP x, SEXP hash)
{
    SEXP ans;
    unsigned int i, n, h, tmp, *counts;
    if (TYPEOF(x) != CHARSXP) error("x is not character");
    n = LENGTH(x);
    h = LENGTH(hash);
    PROTECT(ans = allocVector(INTSXP, n));
    counts = calloc(h+1,sizeof(unsigned int));
    for(i = 0; i < n; i++) {
        tmp = INTEGER(x)[i];  // TO DO: look up hash
        counts[(tmp==NA_INTEGER) ? 0 : tmp]++;
    }
    for(i = 1; i <= h; i++) counts[i] += counts[i-1];
    for(i = n-1; i >= 0; i--) {
        tmp = INTEGER(x)[i];  // TO DO: look up hash
	INTEGER(ans)[--counts[(tmp==NA_INTEGER) ? 0 : tmp]] = i+1;
    }
    UNPROTECT(1);
    free(counts);
    return ans;
}

<commit_msg>Counting sort for character now working. Not exposed to users yet.<commit_after>
// Copied from do_radixsort in src/main/sort.c @ rev 51389
// Turns out not to be a radix sort, but a counting sort
// See http://r.789695.n4.nabble.com/method-radix-in-sort-list-isn-t-actually-a-radix-sort-tp3309470p3309470.html
// A counting sort does seem to be ideal for factors.
// Modified as follows :
// + To work with character, not integer
// + Requires sort(unique(x)) to be passed in as ordered
//   character vector e.g. stored as attribute of the column.
// + No need to scan for min and max. These are known in
//   advance to be 1 and length(charlevels)
// + No need for range offset, removed.
// + Always increasing so removed the decreasing option
// + NA always first, so removed that option
// + It's not really a sort, but an order that's returned so
//   name change from do_radixsort to countingcharacterorder
// + Changed to a direct .Call() signature.
// + 100,000 range restriction removed.
// + Replaced counts alloc (and =0 loop) with a single calloc

#include <R.h>
#define USE_RINTERNALS
#include <Rinternals.h>
#include <Rdefines.h>
#ifdef BUILD_DLL
#define EXPORT __declspec(dllexport)
EXPORT SEXP countingcharacterorder();
#endif

SEXP countingcharacterorder(SEXP x, SEXP charlevels)
{
    SEXP ans, tmp;
    R_len_t i, n, h, *counts;
    if (TYPEOF(x) != STRSXP) error("x is not character");
    if (TYPEOF(charlevels) != STRSXP) error("charlevels is not character");
    n = LENGTH(x);
    h = LENGTH(charlevels);
    PROTECT(ans = allocVector(INTSXP, n));
    counts = calloc(h+1,sizeof(R_len_t));
    for(i=0; i<h; i++) TRUELENGTH(STRING_ELT(charlevels,i)) = i+1;
    // Using truelength as spare storage. Code and runtime inspection revealed
    // it unused and uninitiated in the global CHARSXP cache. Attrib is already
    // used internally, and would be slower to fetch anyway.
    // If r-core object, or R changes in future, we could store and restore 
    // truelength at low cost.
    for(i = 0; i < n; i++) {
        tmp = STRING_ELT(x,i);
        counts[(tmp==NA_STRING) ? 0 : TRUELENGTH(tmp)]++;
    }
    for(i = 1; i <= h; i++) counts[i] += counts[i-1];
    for(i = n-1; i >= 0; i--) {
        tmp = STRING_ELT(x,i);
        INTEGER(ans)[--counts[(tmp==NA_STRING) ? 0 : TRUELENGTH(tmp)]] = i+1;
    }
    UNPROTECT(1);
    free(counts);
    return ans;
}


{'fraction_non_alphanumeric': 0.13246572158958866, 'fraction_numerical': 0.020915640250987685, 'mean_word_length': 3.1705426356589146, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5804396', 'n_tokens_mistral': 1560, 'n_tokens_neox': 1474, 'n_words': 510}