<commit_before>function lineInfo(params) {
  // Localize line info
  var input = params.line;
  var cursor = params.cursor;

  // Break down the line
  var partialLeftLine = input.substr(input, cursor);
  var partialRightLine = input.substr(cursor);

  // Collect words info
  var wordDelimiter = lineInfo.wordDelimiter;
  var words = input.match(wordDelimiter) || [];
  var partialLeftWords = partialLeftLine.match(wordDelimiter) || [];
  var partialRightWords = partialRightLine.match(wordDelimiter) || [];
  var wordsIndex = partialLeftWords.length - 1;

  // Collect word info
  var partialLeftWord = partialLeftWords[partialLeftWords.length - 1] || '';
  var partialRightWord = partialRightWords[0] || '';

  // Return info
  return {
    line: {
      value: input,
      index: cursor,
      partialLeft: partialLeftLine,
      partialRight: partialRightLine,
    },
    words: {
      value: words,
      index: wordsIndex,
      partialLeft: partialLeftWords,
      partialRight: partialRightWords
    },
    word: {
      value: words[wordsIndex] || '',
      index: partialLeftWord.length,
      partialLeft: partialLeftWord,
      partialRight: partialRightWord
    }
  };
}
lineInfo.wordDelimiter = /\w+/g;

module.exports = lineInfo;<commit_msg>Added code to fix trailing whitespace ignored regression<commit_after>function lineInfo(params) {
  // Localize line info
  var input = params.line;
  var cursor = params.cursor;

  // Break down the line
  var partialLeftLine = input.substr(input, cursor);
  var partialRightLine = input.substr(cursor);

  // Collect words info
  var wordDelimiter = lineInfo.wordDelimiter;
  var words = input.split(wordDelimiter);
  var partialLeftWords = partialLeftLine.split(wordDelimiter);
  var partialRightWords = cursor === input.length ? [] : partialRightLine.split(wordDelimiter);
  var wordsIndex = partialLeftWords.length - 1;

  // Collect word info
  var partialLeftWord = partialLeftWords[partialLeftWords.length - 1] || '';
  var partialRightWord = partialRightWords[0] || '';

  // Return info
  return {
    line: {
      value: input,
      index: cursor,
      partialLeft: partialLeftLine,
      partialRight: partialRightLine,
    },
    words: {
      value: words,
      index: wordsIndex,
      partialLeft: partialLeftWords,
      partialRight: partialRightWords
    },
    word: {
      value: words[wordsIndex] || '',
      index: partialLeftWord.length,
      partialLeft: partialLeftWord,
      partialRight: partialRightWord
    }
  };
}
lineInfo.wordDelimiter = /\s+/g;

module.exports = lineInfo;
{'fraction_non_alphanumeric': 0.10644910644910645, 'fraction_numerical': 0.002331002331002331, 'mean_word_length': 3.3570219966159054, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '867426', 'n_tokens_mistral': 768, 'n_tokens_neox': 745, 'n_words': 178}