<commit_before>#include <fudge.h>
#include <data/lifo.h>

#define STATE_NONE    0
#define STATE_COMMAND 1
#define STATE_STDIN   2
#define STATE_STDOUT  3
#define STATE_DATA    4

static struct lifo_stack input;

static char mapUS[256] =
{
       0,   27,  '1',  '2',  '3',  '4',  '5',  '6',  '7',  '8',  '9',  '0',  '-',  '+', '\b', '\t',
     'q',  'w',  'e',  'r',  't',  'y',  'u',  'i',  'o',  'p',  '[',  ']', '\n',    0,  'a',  's',
     'd',  'f',  'g',  'h',  'j',  'k',  'l',  ';', '\'',  '`',    0, '\\',  'z',  'x',  'c',  'v',
     'b',  'n',  'm',  ',',  '.',  '/',    0,    0,    0,  ' ',    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,   27,  '!',  '@',  '#',  '$',  '%',  '^',  '&',  '*',  '(',  ')',  '_',  '=', '\b', '\t',
     'Q',  'W',  'E',  'R',  'T',  'Y',  'U',  'I',  'O',  'P',  '{',  '}', '\n',    0,  'A',  'S',
     'D',  'F',  'G',  'H',  'J',  'K',  'L',  ':', '\"',  '~',    0,  '|',  'Z',  'X',  'C',  'V',
     'B',  'N',  'M',  '<',  '>',  '?',    0,    0,    0,  ' ',    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0
};

enum symbol
{

    SYM_NULL        = 0,
    SYM_NUM         = 1,
    SYM_ALPHA       = 2,
    SYM_ALPHANUM    = 3,
    SYM_SPACE       = 4,
    SYM_LT          = 8,
    SYM_GT          = 16,
    SYM_MINUS       = 32,
    SYM_SEMICOLON   = 64,
    SYM_DOT         = 128,
    SYM_SLASH       = 256,
    SYM_WALL        = 512,
    SYM_NEWLINE     = 1024

};

static char *data;
static enum symbol sym;
static unsigned int pos;
static unsigned int len;

static void error(const char msg[])
{

}

static void next()
{

    if (pos >= len)
    {

        sym = SYM_NEWLINE;

        return;

    }

    switch (data[pos])
    {

        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':

            sym = SYM_NUM;

            break;

        case 'a':
        case 'b':
        case 'c':
        case 'd':
        case 'e':
        case 'f':
        case 'g':
        case 'h':
        case 'i':
        case 'j':
        case 'k':
        case 'l':
        case 'm':
        case 'n':
        case 'o':
        case 'p':
        case 'q':
        case 'r':
        case 's':
        case 't':
        case 'u':
        case 'v':
        case 'w':
        case 'x':
        case 'y':
        case 'z':

            sym = SYM_ALPHA;

            break;

        case ' ':
        case '\t':

            sym = SYM_SPACE;

            break;

        case '<':

            sym = SYM_LT;

            break;

        case '>':

            sym = SYM_GT;

            break;

        case '-':

            sym = SYM_MINUS;

            break;

        case ';':

            sym = SYM_SEMICOLON;

            break;

        case '.':

            sym = SYM_DOT;

            break;

        case '/':

            sym = SYM_SLASH;

            break;

        case '|':

            sym = SYM_WALL;

            break;

        case '\n':

            sym = SYM_NEWLINE;

            break;

        default:

            error("unknown symbol");

    }

    pos++;

}

static int accept(enum symbol s)
{

    if (sym & s)
    {

        next();<commit_msg>more cleanups<commit_after>#include <fudge.h>
#include <data/lifo.h>

#define STATE_NONE    0
#define STATE_COMMAND 1
#define STATE_STDIN   2
#define STATE_STDOUT  3
#define STATE_DATA    4

static struct lifo_stack input;

static char mapUS[256] =
{
       0,   27,  '1',  '2',  '3',  '4',  '5',  '6',  '7',  '8',  '9',  '0',  '-',  '+', '\b', '\t',
     'q',  'w',  'e',  'r',  't',  'y',  'u',  'i',  'o',  'p',  '[',  ']', '\n',    0,  'a',  's',
     'd',  'f',  'g',  'h',  'j',  'k',  'l',  ';', '\'',  '`',    0, '\\',  'z',  'x',  'c',  'v',
     'b',  'n',  'm',  ',',  '.',  '/',    0,    0,    0,  ' ',    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,   27,  '!',  '@',  '#',  '$',  '%',  '^',  '&',  '*',  '(',  ')',  '_',  '=', '\b', '\t',
     'Q',  'W',  'E',  'R',  'T',  'Y',  'U',  'I',  'O',  'P',  '{',  '}', '\n',    0,  'A',  'S',
     'D',  'F',  'G',  'H',  'J',  'K',  'L',  ':', '\"',  '~',    0,  '|',  'Z',  'X',  'C',  'V',
     'B',  'N',  'M',  '<',  '>',  '?',    0,    0,    0,  ' ',    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,
       0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0
};

enum symbol
{

    SYM_NULL        = 0,
    SYM_NUM         = 1,
    SYM_ALPHA       = 2,
    SYM_ALPHANUM    = 3,
    SYM_SPACE       = 4,
    SYM_LT          = 8,
    SYM_GT          = 16,
    SYM_MINUS       = 32,
    SYM_SEMICOLON   = 64,
    SYM_DOT         = 128,
    SYM_SLASH       = 256,
    SYM_WALL        = 512,
    SYM_NEWLINE     = 1024

};

static char *data;
static enum symbol sym;
static unsigned int pos;
static unsigned int len;

static void next()
{

    if (pos >= len)
    {

        sym = SYM_NEWLINE;

        return;

    }

    switch (data[pos])
    {

        case '0':
        case '1':
        case '2':
        case '3':
        case '4':
        case '5':
        case '6':
        case '7':
        case '8':
        case '9':

            sym = SYM_NUM;

            break;

        case 'a':
        case 'b':
        case 'c':
        case 'd':
        case 'e':
        case 'f':
        case 'g':
        case 'h':
        case 'i':
        case 'j':
        case 'k':
        case 'l':
        case 'm':
        case 'n':
        case 'o':
        case 'p':
        case 'q':
        case 'r':
        case 's':
        case 't':
        case 'u':
        case 'v':
        case 'w':
        case 'x':
        case 'y':
        case 'z':

            sym = SYM_ALPHA;

            break;

        case ' ':
        case '\t':

            sym = SYM_SPACE;

            break;

        case '<':

            sym = SYM_LT;

            break;

        case '>':

            sym = SYM_GT;

            break;

        case '-':

            sym = SYM_MINUS;

            break;

        case ';':

            sym = SYM_SEMICOLON;

            break;

        case '.':

            sym = SYM_DOT;

            break;

        case '/':

            sym = SYM_SLASH;

            break;

        case '|':

            sym = SYM_WALL;

            break;

        case '\n':

            sym = SYM_NEWLINE;

            break;

        default:

            sym = SYM_NULL;

            break;

    }

    pos++;

}

static int accept(enum symbol s)
{

    if (sym & s)
    {

        next();
{'fraction_non_alphanumeric': 0.19397575303087114, 'fraction_numerical': 0.05224346956630421, 'mean_word_length': 0.9281927710843374, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '895848', 'n_tokens_mistral': 3487, 'n_tokens_neox': 3309, 'n_words': 870}