<commit_before>/*
 * Word Count content script
 */

/* global chrome:false, $:false, _:false, pluralize:false */
/* jshint bitwise:false */


/*
 * Constants
 */
var ESCAPE_KEY = 27;
var SELECTION_CHANGE_EVENT = 'selectionChange';
var COUNTED_NOUN = 'word';


/*
 * Functions for counting characters and words in selections and strings
 */
var textCount = {

  /*
   * Return the number of visible words in a string
   */
  wordCountInString: function (s) {
    if (!s || !s.length) {
      return 0;
    }

    // Consider a groups of newline whitespace to be one space each
    s = s.replace(/[\n\r]+/g, ' ');

    // Strip out control characters
    s = s.replace(/[\x00-\x1f]/g, '');

    // Remove leading and trailing whitespace, to be safe
    s = s.trim();

    // Split string on all whitespace into words
    var words = s.split(/\s+/);
<commit_msg>Count only words containing an alphanumeric char<commit_after>/*
 * Word Count content script
 */

/* global chrome:false, $:false, _:false, pluralize:false */
/* jshint bitwise:false */


/*
 * Constants
 */
var ESCAPE_KEY = 27;
var SELECTION_CHANGE_EVENT = 'selectionChange';
var COUNTED_NOUN = 'word';


/*
 * Functions for counting characters and words in selections and strings
 */
var textCount = {

  /*
   * Return the number of visible, alphanumeric words in a string
   */
  wordCountInString: function (s) {
    if (!s || !s.length) {
      return 0;
    }

    // Consider a groups of newline whitespace to be one space each
    s = s.replace(/[\n\r]+/g, ' ');

    // Strip out control characters
    s = s.replace(/[\x00-\x1f]/g, '');

    // Remove leading and trailing whitespace, to be safe
    s = s.trim();

    // Split string on all whitespace into words
    var words = s.split(/\s+/);

    // Keep only words containing at least on alphanumeric character
    words = _.filter(words, function (word) {
      return /[a-zA-Z0-9]+/.test(word);
    });

{'fraction_non_alphanumeric': 0.13619886069394097, 'fraction_numerical': 0.0072501294665976174, 'mean_word_length': 2.694072657743786, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '567857', 'n_tokens_mistral': 649, 'n_tokens_neox': 611, 'n_words': 202}