<commit_before>/**
 * Prism: Lightweight, robust, elegant syntax highlighting
 * MIT license http://www.opensource.org/licenses/mit-license.php/
 * @author <NAME> http://lea.verou.me
 */
define(function() {
  self = (typeof window !== 'undefined')
    ? window   // if in browser
    : (
      (typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope)
      ? self // if in worker
      : {}   // if in node js
    );

  var Prism = (function(){

  // Private helper vars
  var lang = /\blang(?:uage)?-(?!\*)(\w+)\b/i;

  var _ = self.Prism = {
    util: {
      encode: function (tokens, options) {
        if (tokens instanceof Token) {
          return new Token(tokens.type, _.util.encode(tokens.content, options), tokens.alias);
        } else if (_.util.type(tokens) === 'Array') {
          return tokens.map(function(token) {
            return _.util.encode(token, options);
          });
        } else {
          if (options && options.encoding && options.encoding.skipTokens) {
            if (options.encoding.skipTokens.indexOf(tokens) > -1) {
              return tokens
            }
          }<commit_msg>Use a regexp to match a token<commit_after>/**
 * Prism: Lightweight, robust, elegant syntax highlighting
 * MIT license http://www.opensource.org/licenses/mit-license.php/
 * @author <NAME> http://lea.verou.me
 */
define(function() {
  self = (typeof window !== 'undefined')
    ? window   // if in browser
    : (
      (typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope)
      ? self // if in worker
      : {}   // if in node js
    );

  var Prism = (function(){

  // Private helper vars
  var lang = /\blang(?:uage)?-(?!\*)(\w+)\b/i;

  var _ = self.Prism = {
    util: {
      encode: function (tokens, options) {
        if (tokens instanceof Token) {
          return new Token(tokens.type, _.util.encode(tokens.content, options), tokens.alias);
        } else if (_.util.type(tokens) === 'Array') {
          return tokens.map(function(token) {
            return _.util.encode(token, options);
          });
        } else {
          if (options && options.encoding && options.encoding.skipTokens) {
            match = (options.encoding.skipTokens).some(function(token) {
              return new RegExp(token, 'gi').test(tokens);
            });

            if (match) {
              return tokens;
            }
          }
{'fraction_non_alphanumeric': 0.1515527950310559, 'fraction_numerical': 0.00041407867494824016, 'mean_word_length': 2.1581699346405228, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1591195', 'n_tokens_mistral': 705, 'n_tokens_neox': 690, 'n_words': 173}