<commit_before>/*
 ============================================================================
 Name        : hstr_history.h
 Author      : <EMAIL>
 Copyright   : Apache 2.0
 Description : Loading and processing of BASH history
 ============================================================================
*/

#include <stdio.h>
#include <limits.h>
#include <readline/history.h>
#include "include/hstr_history.h"
#include "include/hstr_regexp.h"

#define NDEBUG
#include <assert.h>

typedef struct {
    char *item;
    unsigned rank;
} RankedHistoryItem;

static HistoryItems *prioritizedHistory;
static bool dirty;

static const char *commandBlacklist[] = {
        "ls", "pwd", "cd", "cd ..", "hh", "mc",
        "ls ", "pwd ", "cd ", "cd .. ", "hh ", "mc "
};

#ifdef DEBUG_RADIX
#define DEBUG_RADIXSORT() radixsort_stat(&rs, false); exit(0)
#else
#define DEBUG_RADIXSORT()
#endif

unsigned history_ranking_function(unsigned rank, int newOccurenceOrder, size_t length) {
    long metrics=rank+(log(newOccurenceOrder)*10.0)+length;
    // alternative metrics:
    //   rank+newOccurenceOrder/10+length
    assert(metrics<UINT_MAX);
    return metrics;
}

char *get_history_file_name()
{
    char *historyFile=getenv(ENV_VAR_HISTFILE);
    if(!historyFile || strlen(historyFile)==0) {
        char *home = getenv(ENV_VAR_HOME);
        historyFile = malloc(strlen(home) + 1 + strlen(FILE_DEFAULT_HISTORY) + 1);
        strcat(strcat(strcpy(historyFile, home), "/"), FILE_DEFAULT_HISTORY);
    }
    return historyFile;
}

void dump_prioritized_history(HistoryItems *ph)
{
    printf("\n\nPrioritized history:");
    int i;
    for(i=0; i<ph->count; i++) {
        if(ph->items[i]!=NULL) {
            printf("\n%s",ph->items[i]); fflush(stdout);
        } else {
            printf("\n %d NULL",i); fflush(stdout);
        }
    }
    printf("\n"); fflush(stdout);
}

int get_item_offset()
{
    if(isZshParentShell()) {
        // In zsh history file, the format of item is
        // [:][blank][unix_timestamp][:][0][;][cmd]
        // Such as:
        // : 1420549651:0;ls /tmp/b
        // And the limit of unix timestamp 9999999999 is 2289/11/21,
        // so we could skip first 15 chars in every zsh history item to get the cmd.
        return ZSH_HISTORY_ITEM_OFFSET;
    } else {
        return BASH_HISTORY_ITEM_OFFSET;
    }
}

HistoryItems *get_prioritized_history()
{
    using_history();

    char *historyFile = get_history_file_name();
    if(read_history(historyFile)!=0) {
        fprintf(stderr, "\nUnable to read history file from '%s'!\n",historyFile);
        exit(EXIT_FAILURE);
    }
    HISTORY_STATE *historyState=history_get_history_state();

    int itemOffset = get_item_offset();

    if(historyState->length > 0) {
        HashSet rankmap;
        hashset_init(&rankmap);

        HashSet blacklist;
        int i;
        hashset_init(&blacklist);
        int length=sizeof(commandBlacklist)/sizeof(commandBlacklist[0]);
        for(i=0; i<length; i++) {
            hashset_add(&blacklist, commandBlacklist[i]);
        }

        RadixSorter rs;
        unsigned radixMaxKeyEstimate=historyState->size*1000;
        radixsort_init(&rs, (radixMaxKeyEstimate<100000?100000:radixMaxKeyEstimate));
        rs.optFloorAndInsertBigKeys=true;

        regex_t regexp;
        // HISTTIMEFORMAT defined > ^#1234567890$
        const char *histtimeformatTimestamp
          = "^#[[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]][[:digit:]]$";
        regexp_compile(&regexp, histtimeformatTimestamp);

        RankedHistoryItem *r;
        RadixItem *radixItem;
        HIST_ENTRY **historyList=history_list();
        char **rawHistory=malloc(sizeof(char*) * historyState->length);
        int rawOffset=historyState->length-1;
        char *line;
        for(i=0; i<historyState->length; i++, rawOffset--) {
            if(!regexp_match(&regexp, historyList[i]->line)) {
                continue;
            }
            if(historyList[i]->line && strlen(historyList[i]->line)>itemOffset) {
                line=historyList[i]->line+itemOffset;
            } else {
                line=historyList[i]->line;
            }
            rawHistory[rawOffset]=line;
            if(hashset_contains(&blacklist, line)) {
                continue;
            }
            if((r=hashset_get(&rankmap, line))==NULL) {
                r=malloc(sizeof(RankedHistoryItem));
                r->rank=history_ranking_function(0, i, strlen(line));<commit_msg>Making histtimeformat regexp concise.<commit_after>/*
 ============================================================================
 Name        : hstr_history.h
 Author      : <EMAIL>
 Copyright   : Apache 2.0
 Description : Loading and processing of BASH history
 ============================================================================
*/

#include <stdio.h>
#include <limits.h>
#include <readline/history.h>
#include "include/hstr_history.h"
#include "include/hstr_regexp.h"

#define NDEBUG
#include <assert.h>

typedef struct {
    char *item;
    unsigned rank;
} RankedHistoryItem;

static HistoryItems *prioritizedHistory;
static bool dirty;

static const char *commandBlacklist[] = {
        "ls", "pwd", "cd", "cd ..", "hh", "mc",
        "ls ", "pwd ", "cd ", "cd .. ", "hh ", "mc "
};

#ifdef DEBUG_RADIX
#define DEBUG_RADIXSORT() radixsort_stat(&rs, false); exit(0)
#else
#define DEBUG_RADIXSORT()
#endif

unsigned history_ranking_function(unsigned rank, int newOccurenceOrder, size_t length) {
    long metrics=rank+(log(newOccurenceOrder)*10.0)+length;
    // alternative metrics:
    //   rank+newOccurenceOrder/10+length
    assert(metrics<UINT_MAX);
    return metrics;
}

char *get_history_file_name()
{
    char *historyFile=getenv(ENV_VAR_HISTFILE);
    if(!historyFile || strlen(historyFile)==0) {
        char *home = getenv(ENV_VAR_HOME);
        historyFile = malloc(strlen(home) + 1 + strlen(FILE_DEFAULT_HISTORY) + 1);
        strcat(strcat(strcpy(historyFile, home), "/"), FILE_DEFAULT_HISTORY);
    }
    return historyFile;
}

void dump_prioritized_history(HistoryItems *ph)
{
    printf("\n\nPrioritized history:");
    int i;
    for(i=0; i<ph->count; i++) {
        if(ph->items[i]!=NULL) {
            printf("\n%s",ph->items[i]); fflush(stdout);
        } else {
            printf("\n %d NULL",i); fflush(stdout);
        }
    }
    printf("\n"); fflush(stdout);
}

int get_item_offset()
{
    if(isZshParentShell()) {
        // In zsh history file, the format of item is
        // [:][blank][unix_timestamp][:][0][;][cmd]
        // Such as:
        // : 1420549651:0;ls /tmp/b
        // And the limit of unix timestamp 9999999999 is 2289/11/21,
        // so we could skip first 15 chars in every zsh history item to get the cmd.
        return ZSH_HISTORY_ITEM_OFFSET;
    } else {
        return BASH_HISTORY_ITEM_OFFSET;
    }
}

HistoryItems *get_prioritized_history()
{
    using_history();

    char *historyFile = get_history_file_name();
    if(read_history(historyFile)!=0) {
        fprintf(stderr, "\nUnable to read history file from '%s'!\n",historyFile);
        exit(EXIT_FAILURE);
    }
    HISTORY_STATE *historyState=history_get_history_state();

    int itemOffset = get_item_offset();

    if(historyState->length > 0) {
        HashSet rankmap;
        hashset_init(&rankmap);

        HashSet blacklist;
        int i;
        hashset_init(&blacklist);
        int length=sizeof(commandBlacklist)/sizeof(commandBlacklist[0]);
        for(i=0; i<length; i++) {
            hashset_add(&blacklist, commandBlacklist[i]);
        }

        RadixSorter rs;
        unsigned radixMaxKeyEstimate=historyState->size*1000;
        radixsort_init(&rs, (radixMaxKeyEstimate<100000?100000:radixMaxKeyEstimate));
        rs.optFloorAndInsertBigKeys=true;

        regex_t regexp;
        // HISTTIMEFORMAT defined > ^#1234567890$
        const char *histtimeformatTimestamp = "^#[[:digit:]]\\{10,\\}$";
        regexp_compile(&regexp, histtimeformatTimestamp);

        RankedHistoryItem *r;
        RadixItem *radixItem;
        HIST_ENTRY **historyList=history_list();
        char **rawHistory=malloc(sizeof(char*) * historyState->length);
        int rawOffset=historyState->length-1;
        char *line;
        for(i=0; i<historyState->length; i++, rawOffset--) {
            if(!regexp_match(&regexp, historyList[i]->line)) {
                continue;
            }
            if(historyList[i]->line && strlen(historyList[i]->line)>itemOffset) {
                line=historyList[i]->line+itemOffset;
            } else {
                line=historyList[i]->line;
            }
            rawHistory[rawOffset]=line;
            if(hashset_contains(&blacklist, line)) {
                continue;
            }
            if((r=hashset_get(&rankmap, line))==NULL) {
                r=malloc(sizeof(RankedHistoryItem));
                r->rank=history_ranking_function(0, i, strlen(line));
{'fraction_non_alphanumeric': 0.17078601647739924, 'fraction_numerical': 0.01736806947227789, 'mean_word_length': 2.973020787262273, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1732614', 'n_tokens_mistral': 2984, 'n_tokens_neox': 2795, 'n_words': 524}