<commit_before>const { readFileSync } = require('fs');
const {
  lazy,
  string,
  regex,
  alt,
  seq,
  eof
} = require('parsimmon');

const newline = regex(/\n/);
const notNewlines = regex(/[^\n]*/);
const comment = string('#').then(notNewlines).skip(newline.or(eof));
const whitespace = regex(/\s+/);
const ignore = alt(whitespace, comment).many();
const lexme = p => p.skip(ignore);

const lparen = lexme(string('('));
const rparen = lexme(string(')'));
const lbracket = lexme(string('['));
const rbracket = lexme(string(']'));
const blockterminator = lexme(string(';'));
const path = lexme(regex(/[a-zA-Z0-9._~!$&'()*+,;=:@%/-]+/)).desc('a file-path');
const str = lexme(regex(/"((?:\\.|.)*?)"/, 1)).desc('a quoted string');
const word = lexme(regex(/[a-zA-Z0-9.<>+=\\*%/-]+/)).desc('an Ait word');
const load = lexme(string('@load'));
const colon = lexme(string(':'));

const expressions = lazy('top-level expressions', function() {
  return alt(
    stringLiteral,
    definition,
    wordLiteral,
    quotation,
    tuple,
    loadDirective
  );
});

const stringLiteral = str.map(s => ({type: 'string', body: s}));
const wordLiteral = word.map(w => ({type: 'word', body: w}));
const quotation = lbracket.then(expressions.many()).skip(rbracket).map(q => ({type: 'quotation', body: q}));
const tuple = lparen.then(expressions.many()).skip(rparen).map(t => ({type: 'tuple', body: t}));
const definition = seq(word.skip(colon), expressions.many(), blockterminator)
  .map(d => ({type: 'definition', body: {keyword: d[0], body: d[1]}}));
const loadDirective = load.then(path).skip(blockterminator).map(d => ({type: 'load', body: d}));

const final = ignore.then(expressions.many());<commit_msg>Add underscore to word regex<commit_after>const { readFileSync } = require('fs');
const {
  lazy,
  string,
  regex,
  alt,
  seq,
  eof
} = require('parsimmon');

const newline = regex(/\n/);
const notNewlines = regex(/[^\n]*/);
const comment = string('#').then(notNewlines).skip(newline.or(eof));
const whitespace = regex(/\s+/);
const ignore = alt(whitespace, comment).many();
const lexme = p => p.skip(ignore);

const lparen = lexme(string('('));
const rparen = lexme(string(')'));
const lbracket = lexme(string('['));
const rbracket = lexme(string(']'));
const blockterminator = lexme(string(';'));
const path = lexme(regex(/[a-zA-Z0-9._~!$&'()*+,;=:@%/-]+/)).desc('a file-path');
const str = lexme(regex(/"((?:\\.|.)*?)"/, 1)).desc('a quoted string');
const word = lexme(regex(/[a-zA-Z0-9.<>+=\\*%/_-]+/)).desc('an Ait word');
const load = lexme(string('@load'));
const colon = lexme(string(':'));

const expressions = lazy('top-level expressions', function() {
  return alt(
    stringLiteral,
    definition,
    wordLiteral,
    quotation,
    tuple,
    loadDirective
  );
});

const stringLiteral = str.map(s => ({type: 'string', body: s}));
const wordLiteral = word.map(w => ({type: 'word', body: w}));
const quotation = lbracket.then(expressions.many()).skip(rbracket).map(q => ({type: 'quotation', body: q}));
const tuple = lparen.then(expressions.many()).skip(rparen).map(t => ({type: 'tuple', body: t}));
const definition = seq(word.skip(colon), expressions.many(), blockterminator)
  .map(d => ({type: 'definition', body: {keyword: d[0], body: d[1]}}));
const loadDirective = load.then(path).skip(blockterminator).map(d => ({type: 'load', body: d}));

const final = ignore.then(expressions.many());
{'fraction_non_alphanumeric': 0.2390158172231986, 'fraction_numerical': 0.004100761570005858, 'mean_word_length': 5.087344028520499, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '875539', 'n_tokens_mistral': 1246, 'n_tokens_neox': 1208, 'n_words': 265}