<commit_before>import urbitOb from 'urbit-ob';

const URL_REGEX = new RegExp(String(/^((\w+:\/\/)[-a-zA-Z0-9:@;?&=\/%\+\.\*!'\(\),\$_\{\}\^~\[\]`#|]+)/.source));

const isUrl = (string) => {
  try {
    return URL_REGEX.test(string);
  } catch (e) {
    return false;
  }
}

const tokenizeMessage = (text) => {
  let messages = [];
  let message = [];
  let isInCodeBlock = false;
  let endOfCodeBlock = false;
  text.split(/\r?\n/).forEach((line, index) => {
    if (index !== 0) {
      message.push('\n');
    }
    // A line of backticks enters and exits a codeblock
    if (line.startsWith('```')) {
      // But we need to check if we've ended a codeblock
      endOfCodeBlock = isInCodeBlock;
      isInCodeBlock = (!isInCodeBlock);
    } else {
      endOfCodeBlock = false;
    }

    if (isInCodeBlock || endOfCodeBlock) {
      message.push(line);<commit_msg>landscape: ignore punctuation tokenizing URLs<commit_after>import urbitOb from 'urbit-ob';

const URL_REGEX = new RegExp(String(/^((\w+:\/\/)[-a-zA-Z0-9:@;?&=\/%\+\.\*!'\(\),\$_\{\}\^~\[\]`#|]+\w)/.source));

const isUrl = (string) => {
  try {
    return URL_REGEX.test(string);
  } catch (e) {
    return false;
  }
}

const tokenizeMessage = (text) => {
  let messages = [];
  let message = [];
  let isInCodeBlock = false;
  let endOfCodeBlock = false;
  text.split(/\r?\n/).forEach((line, index) => {
    if (index !== 0) {
      message.push('\n');
    }
    // A line of backticks enters and exits a codeblock
    if (line.startsWith('```')) {
      // But we need to check if we've ended a codeblock
      endOfCodeBlock = isInCodeBlock;
      isInCodeBlock = (!isInCodeBlock);
    } else {
      endOfCodeBlock = false;
    }

    if (isInCodeBlock || endOfCodeBlock) {
      message.push(line);
{'fraction_non_alphanumeric': 0.1995490417136415, 'fraction_numerical': 0.0033821871476888386, 'mean_word_length': 2.713389121338912, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2004009', 'n_tokens_mistral': 662, 'n_tokens_neox': 648, 'n_words': 141}