<commit_before>/**
 * @license
 * Copyright (c) 2016 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt
 */

import { matcher } from './common';
import { Token, boundaryTokenTypes } from './token';

const currentToken = Symbol('currentToken');
const cursorToken = Symbol('cursorToken');
const findNextToken = Symbol('findNextToken');

/**
 * Class that implements tokenization of significant lexical features of the
 * CSS syntax.
 */
class Tokenizer {
  /**
   * Create a Tokenizer instance.
   * @param {string} cssText The raw CSS string to be tokenized.
   */
  constructor(cssText) {
    this.cssText = cssText;
    this[cursorToken] = new Token(Token.type.none, 0, 0);
    this[currentToken] = null;
  }

  get offset() {
    return this[cursorToken].end;
  }

  /**
   * The current token that will be returned by a call to `advance`. This
   * reference is useful for "peeking" at the next token ahead in the sequence.
   * If the entire CSS text has been tokenized, the `currentToken` will be null.
   * @type {Token}
   */<commit_msg>Document class fields for Tokenizer.<commit_after>/**
 * @license
 * Copyright (c) 2016 The Polymer Project Authors. All rights reserved.
 * This code may only be used under the BSD style license found at http://polymer.github.io/LICENSE.txt
 * The complete set of authors may be found at http://polymer.github.io/AUTHORS.txt
 * The complete set of contributors may be found at http://polymer.github.io/CONTRIBUTORS.txt
 * Code distributed by Google as part of the polymer project is also
 * subject to an additional IP rights grant found at http://polymer.github.io/PATENTS.txt
 */

import { matcher } from './common';
import { Token, boundaryTokenTypes } from './token';

const currentToken = Symbol('currentToken');
const cursorToken = Symbol('cursorToken');
const findNextToken = Symbol('findNextToken');

/**
 * Class that implements tokenization of significant lexical features of the
 * CSS syntax.
 */
class Tokenizer {
  /**
   * Create a Tokenizer instance.
   * @param {string} cssText The raw CSS string to be tokenized.
   *
   */
  constructor(cssText) {
    this.cssText = cssText;
    /**
     * Tracks the position of the tokenizer in the source string.
     * Also the default head of the Token linked list.
     * @type {!Token}
     * @private
     */
    this[cursorToken] = new Token(Token.type.none, 0, 0);
    /**
     * Holds a reference to a Token that is "next" in the source string, often
     * due to having been peeked at.
     * @type {?Token}
     * @readonly
     */
    this[currentToken] = null;
  }

  get offset() {
    return this[cursorToken].end;
  }

  /**
   * The current token that will be returned by a call to `advance`. This
   * reference is useful for "peeking" at the next token ahead in the sequence.
   * If the entire CSS text has been tokenized, the `currentToken` will be null.
   * @type {Token}
   */
{'fraction_non_alphanumeric': 0.10233393177737882, 'fraction_numerical': 0.003590664272890485, 'mean_word_length': 3.3191214470284236, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4249774', 'n_tokens_mistral': 962, 'n_tokens_neox': 923, 'n_words': 387}