<commit_before>// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

const {assert} = chai;

import {FilterParser, BalancedJSONTokenizer, isMinified} from '/front_end/text_utils/TextUtils.js';<commit_msg>[fix-it] Add more unittests for TextUtils.js<commit_after>// Copyright 2019 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

const {assert} = chai;

import {FilterParser, BalancedJSONTokenizer, isMinified, default as Utils} from '/front_end/text_utils/TextUtils.js';

describe('Utils Object', () => {
  describe('isStopChar', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isStopChar('\0'), false, 'null was a stop char');
      assert.equal(Utils.isStopChar(' '), false, 'space was a stop char');
      assert.equal(Utils.isStopChar('!'), true, '! was not a stop char');
      assert.equal(Utils.isStopChar('/'), true, '/ was not a stop char');
      assert.equal(Utils.isStopChar('0'), false, '0 was a stop char');
      assert.equal(Utils.isStopChar('9'), false, '9 was a stop char');
      assert.equal(Utils.isStopChar('@'), true, '@ was not a stop char');
      assert.equal(Utils.isStopChar('A'), false, 'A was a stop char');
      assert.equal(Utils.isStopChar('B'), false, 'B was a stop char');
      assert.equal(Utils.isStopChar('Z'), false, 'Z was a stop char');
      assert.equal(Utils.isStopChar('['), true, '[ was not a stop char');
      assert.equal(Utils.isStopChar('_'), false, '_ was a stop char');
      assert.equal(Utils.isStopChar('`'), true, '` was not a stop char');
      assert.equal(Utils.isStopChar('a'), false, 'a was a stop char');
      assert.equal(Utils.isStopChar('b'), false, 'b was a stop char');
      assert.equal(Utils.isStopChar('z'), false, 'z was a stop char');
      assert.equal(Utils.isStopChar('{'), true, '{ was not a stop char');
    });
  });
  describe('isWordChar', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isWordChar(' '), false, 'space was a word char');
      assert.equal(Utils.isWordChar('\t'), false, 'tab was a word char');
      assert.equal(Utils.isWordChar('a'), true, 'a was not a word char');
      assert.equal(Utils.isWordChar('A'), true, 'A was not a word char');
      assert.equal(Utils.isWordChar('_'), true, '_ was not a word char');
    });
  });
  describe('isSpaceChar', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isSpaceChar(' '), true, 'space was not a space char');
      assert.equal(Utils.isSpaceChar('\t'), true, 'tab was not a space char');
      assert.equal(Utils.isSpaceChar('\f'), true, 'formfeed was not a space char');
      assert.equal(Utils.isSpaceChar('\r'), true, 'return was not a space char');
      assert.equal(Utils.isSpaceChar('\v'), true, 'vertical tab was not a space char');
      assert.equal(Utils.isSpaceChar('\xA0'), true, 'non-breaking space was not a space char');
      assert.equal(Utils.isSpaceChar('\0'), false, 'null was a space char');
      assert.equal(Utils.isSpaceChar('a'), false, 'a was a space char');
      assert.equal(Utils.isSpaceChar('A'), false, 'A was a space char');
    });
  });
  describe('isWord', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isWord(''), true, 'empty string was not a word');
      assert.equal(Utils.isWord('_'), true, '_ string was not a word');
      assert.equal(Utils.isWord('a'), true, 'a string was not a word');
      assert.equal(Utils.isWord('abc'), true, 'abc string was not a word');
      assert.equal(Utils.isWord('a{'), false, 'a{ string was a word');
      assert.equal(Utils.isWord('a`'), false, 'a` string was a word');
      assert.equal(Utils.isWord(' '), false, 'space string was a word');
    });
  });
  describe('isOpeningBraceChar', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isOpeningBraceChar('{'), true, '{ was not an opening brace');
      assert.equal(Utils.isOpeningBraceChar('('), true, '( was not an opening brace');
      assert.equal(Utils.isOpeningBraceChar('['), false, '[ was an opening brace');
      assert.equal(Utils.isOpeningBraceChar('<'), false, '< was an opening brace');
      assert.equal(Utils.isOpeningBraceChar('}'), false, '} was an opening brace');
      assert.equal(Utils.isOpeningBraceChar(')'), false, ') was an opening brace');
    });
  });
  describe('isClosingBraceChar', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isClosingBraceChar('}'), true, '} was not a closing brace');
      assert.equal(Utils.isClosingBraceChar(')'), true, ') was not a closing brace');
      assert.equal(Utils.isClosingBraceChar(']'), false, '] was a closing brace');
      assert.equal(Utils.isClosingBraceChar('>'), false, '> was a closing brace');
      assert.equal(Utils.isClosingBraceChar('{'), false, '{} was a closing brace');
      assert.equal(Utils.isClosingBraceChar('('), false, '() was a closing brace');
    });
  });
  describe('isBraceChar', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isBraceChar('{'), true, '{ was not a brace');
      assert.equal(Utils.isBraceChar('('), true, '( was not a brace');
      assert.equal(Utils.isBraceChar('}'), true, '} was not a brace');
      assert.equal(Utils.isBraceChar(')'), true, ') was not a brace');
      assert.equal(Utils.isBraceChar('['), false, '[ was a brace');
      assert.equal(Utils.isBraceChar('<'), false, '< was a brace');
      assert.equal(Utils.isBraceChar(']'), false, '] was a brace');
      assert.equal(Utils.isBraceChar('>'), false, '> was a brace');
    });
  });
  describe('textToWords', () => {
    it('returns the correct result for various inputs', () => {
      const isWordChar = Utils.isWordChar;
      const words = [];
      const callback = (word: string) => {
        words.push(word);
      };
      Utils.textToWords('', isWordChar, callback);
      assert.equal(words.length, 0, 'words was not empty');
      Utils.textToWords(' a', isWordChar, callback);
      assert.equal(words.length, 1, 'words had wrong length');
      assert.equal(words[0], 'a');
      Utils.textToWords(' a _', isWordChar, callback);
      assert.equal(words.length, 3, 'words had wrong length');
      assert.equal(words[1], 'a');
      assert.equal(words[2], '_');
    });
  });
  describe('lineIndent', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.lineIndent(''), '', 'indent was not empty');
      assert.equal(Utils.lineIndent('\tabc'), '\t', 'indent should have one tab');
      assert.equal(Utils.lineIndent(' \t abc'), ' \t ', 'indent was wrong');
    });
  });
  describe('isUpperCase', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isUpperCase('a'), false, 'a was upper case');
      assert.equal(Utils.isUpperCase('A'), true, 'A was not upper case');
      assert.equal(Utils.isUpperCase('_'), true, '_ was not upper case');
      assert.equal(Utils.isUpperCase('!'), true, '! was not upper case');
      assert.equal(Utils.isUpperCase('@'), true, '@ was not upper case');
    });
  });
  describe('isLowerCase', () => {
    it('returns the correct result for various inputs', () => {
      assert.equal(Utils.isLowerCase('a'), true, 'a was lower case');
      assert.equal(Utils.isLowerCase('A'), false, 'A was not lower case');
      assert.equal(Utils.isLowerCase('_'), true, '_ was not lower case');
      assert.equal(Utils.isLowerCase('!'), true, '! was not lower case');
      assert.equal(Utils.isLowerCase('@'), true, '@ was not lower case');
    });
  });
  describe('splitStringByRegexes', () => {
    it('returns the correct result for a single regex', () => {
      let result = Utils.splitStringByRegexes('', [/a/]);
      assert.equal(result.length, 0, 'length was wrong');

      result = Utils.splitStringByRegexes('a', [/a/]);
      assert.equal(result.length, 1, 'length was wrong');
      assert.equal(result[0].value, 'a', 'value was wrong');
      assert.equal(result[0].position, 0, 'position was wrong');
      assert.equal(result[0].regexIndex, 0, 'regex index was wrong');
      assert.deepEqual(result[0].captureGroups, [], 'capture groups was not empty');

      result = Utils.splitStringByRegexes('ba b', [/a/]);
      assert.equal(result.length, 3, 'length was wrong');
      assert.equal(result[0].value, 'b', 'value was wrong');
      assert.equal(result[0].position, 0, 'position was wrong');
      assert.equal(result[0].regexIndex, -1, 'regex index was wrong');
      assert.deepEqual(result[0].captureGroups, [], 'capture groups was not empty');
      assert.equal(result[1].value, 'a', 'value was wrong');
      assert.equal(result[1].position, 1, 'position was wrong');
      assert.equal(result[1].regexIndex, 0, 'regex index was wrong');
      assert.deepEqual(result[1].captureGroups, [], 'capture groups was not empty');
      assert.equal(result[2].value, ' b', 'value was wrong');
      assert.equal(result[2].position, 2, 'position was wrong');
      assert.equal(result[2].regexIndex, -1, 'regex index was wrong');
      assert.deepEqual(result[2].captureGroups, [], 'capture groups was not empty');
    });
    it('returns the correct result for a multiple regexs', () => {
      let result = Utils.splitStringByRegexes('', [/a/, /b/]);
      assert.equal(result.length, 0, 'length was wrong');

      result = Utils.splitStringByRegexes('a', [/a/, /b/]);
      assert.equal(result.length, 1, 'length was wrong');
      assert.equal(result[0].value, 'a', 'value was wrong');
      assert.equal(result[0].position, 0, 'position was wrong');
      assert.equal(result[0].regexIndex, 0, 'regex index was wrong');
      assert.deepEqual(result[0].captureGroups, [], 'capture groups was not empty');

      result = Utils.splitStringByRegexes('ba b', [/a/, /b/]);
      assert.equal(result.length, 4, 'length was wrong');
      assert.equal(result[0].value, 'b', 'value was wrong');
      assert.equal(result[0].position, 0, 'position was wrong');
      assert.equal(result[0].regexIndex, 1, 'regex index was wrong');
      assert.deepEqual(result[0].captureGroups, [], 'capture groups was not empty');
      assert.equal(result[1].value, 'a', 'value was wrong');
      assert.equal(result[1].position, 1, 'position was wrong');
      assert.equal(result[1].regexIndex, 0, 'regex index was wrong');
      assert.deepEqual(result[1].captureGroups, [], 'capture groups was not empty');
      assert.equal(result[2].value, ' ', 'value was wrong');
      assert.equal(result[2].position, 2, 'position was wrong');
      assert.equal(result[2].regexIndex, -1, 'regex index was wrong');
      assert.deepEqual(result[2].captureGroups, [], 'capture groups was not empty');
      assert.equal(result[3].value, 'b', 'value was wrong');
      assert.equal(result[3].position, 3, 'position was wrong');
      assert.equal(result[3].regexIndex, 1, 'regex index was wrong');
      assert.deepEqual(result[3].captureGroups, [], 'capture groups was not empty');
    });
    it('returns the correct result for global regexs', () => {
      let result = Utils.splitStringByRegexes('', [/a/g, /b/g]);
      assert.equal(result.length, 0, 'length was wrong');

      result = Utils.splitStringByRegexes('a', [/a/g, /b/g]);
      assert.equal(result.length, 1, 'length was wrong');
      assert.equal(result[0].value, 'a', 'value was wrong');
      assert.equal(result[0].position, 0, 'position was wrong');
      assert.equal(result[0].regexIndex, 0, 'regex index was wrong');
      assert.deepEqual(result[0].captureGroups, [], 'capture groups was not empty');
    });
  });
});
{'fraction_non_alphanumeric': 0.18023061497326204, 'fraction_numerical': 0.007436497326203209, 'mean_word_length': 3.695566888976069, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1555464', 'n_tokens_mistral': 3683, 'n_tokens_neox': 3642, 'n_words': 1011}