<commit_before>import marked from "marked";
/* eslint-disable no-unused-vars */
import React from "react";
/* eslint-enable no-unused-vars */

const markdownRenderer = {
  rendererReady: false,
  prepareMarkdownRenderer() {
    const renderer = new marked.Renderer();
    renderer.link = function() {
      const out = marked.Renderer.prototype.link.apply(this, arguments);

      return out.replace(/^<a/, '<a target="_blank"');
    };
    marked.setOptions({ renderer });
    this.rendererReady = true;
  }
};

const StringUtil = {
  arrayToJoinedString(array = [], separator = ", ") {
    if (Array.isArray(array)) {
      return array.join(separator);
    }

    return "";
  },

  getSearchTokens(searchString) {
    if (!searchString) {
      return [];
    }

    return (
      String(searchString)
        .toLowerCase()
        // split on non-word characters and slash
        .split(/[^\w/]/)
        .filter(Boolean)
    );
  },

  filterByString(objects, getter, searchString) {
    const searchItems = this.getSearchTokens(searchString);

    if (typeof getter === "function") {
      return objects.filter(obj => {
        return searchItems.some(item => {
          return this.getSearchTokens(getter(obj)).some(token => {
            return token.startsWith(item);
          });
        });
      });
    }

    return objects.filter(obj => {
      return searchItems.some(item => {
        return this.getSearchTokens(obj[getter]).some(token => {
          return token.startsWith(item);
        });
      });
    });
  },

  escapeForRegExp(str) {
    return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
  },

  isUrl(str) {
    return !!str && /^https?:\/\/.+/.test(str);<commit_msg>refactor(StringUtil): Dedupe filterByString method<commit_after>import marked from "marked";
/* eslint-disable no-unused-vars */
import React from "react";
/* eslint-enable no-unused-vars */

const markdownRenderer = {
  rendererReady: false,
  prepareMarkdownRenderer() {
    const renderer = new marked.Renderer();
    renderer.link = function() {
      const out = marked.Renderer.prototype.link.apply(this, arguments);

      return out.replace(/^<a/, '<a target="_blank"');
    };
    marked.setOptions({ renderer });
    this.rendererReady = true;
  }
};

const StringUtil = {
  arrayToJoinedString(array = [], separator = ", ") {
    if (Array.isArray(array)) {
      return array.join(separator);
    }

    return "";
  },

  getSearchTokens(searchString) {
    if (!searchString) {
      return [];
    }

    return (
      String(searchString)
        .toLowerCase()
        // split on non-word characters and slash
        .split(/[^\w/]/)
        .filter(Boolean)
    );
  },

  filterByString(objects, property, searchString) {
    const searchItems = this.getSearchTokens(searchString);

    const getter =
      typeof property === "function" ? property : obj => obj[property];

    return objects.filter(obj => {
      return searchItems.some(item => {
        return this.getSearchTokens(getter(obj)).some(token => {
          return token.startsWith(item);
        });
      });
    });
  },

  escapeForRegExp(str) {
    return str.replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&");
  },

  isUrl(str) {
    return !!str && /^https?:\/\/.+/.test(str);
{'fraction_non_alphanumeric': 0.17008259406546344, 'fraction_numerical': 0.0, 'mean_word_length': 2.5934065934065935, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5752710', 'n_tokens_mistral': 1019, 'n_tokens_neox': 1012, 'n_words': 182}