<commit_before>import React from 'react';
import _ from 'lodash';
import { stemmer as enStemmer } from 'porter-stemmer';
import ruStemmer from './ru-stemmer';


const enLetters = 'a-z';
const ruLetters = '\u0400-\u04ff';

const partRE = /"(.+?)"|(\S+)/g;
const enWordRE = new RegExp(`^[${enLetters}]+$`, 'i');
const ruWordRE = new RegExp(`^[${ruLetters}]+$`, 'i');

const hlClass = 'search-highlight';

export function parseQuery(query) {
  const terms = [];
  let m;
  while ((m = partRE.exec(query)) !== null) {
    if (m[2] && (m[2].indexOf(':') !== -1 || m[2].length === 1)) {
      continue;
    }
    if (m[1] !== undefined) {
      terms.push(new RegExp(`(^|[^${enLetters}${ruLetters}])(${_.escapeRegExp(m[1])})(?:$|[^${enLetters}${ruLetters}])`));
    } else if (enWordRE.test(m[2])) {
      terms.push(new RegExp(`(^|[^${enLetters}])(${_.escapeRegExp(stem(m[2], enStemmer))}[${enLetters}]*)`, 'i'));
    } else if (ruWordRE.test(m[2])) {
      terms.push(new RegExp(`(^|[^${ruLetters}])(${_.escapeRegExp(stem(m[2], ruStemmer))}[${ruLetters}]*)`, 'i'));
    } else {
      terms.push(new RegExp(`(^|[^${enLetters}${ruLetters}])(${_.escapeRegExp(m[2])})(?:$|[^${enLetters}${ruLetters}])`));
    }
  }
  return terms;
}

export function highlightString(text, terms) {
  if (terms.length === 0 || text === '') {
    return [text];
  }

  const result = [];
  while (text !== '') {
    let match = '', minPos = 0;
    terms.forEach((re) => {
      const m = re.exec(text);
      if (m && (match === '' || m.index < minPos)) {
        minPos = m.index + (m[1] || '').length;
        match = m[2];  // eslint-disable-line prefer-destructuring
      }
    });

    if (match !== '') {
      result.push(text.substring(0, minPos));
      result.push(<span className={hlClass}>{match}</span>);
      text = text.substring(minPos + match.length);
    } else {
      result.push(text);
      break;
    }
  }
  return result;
}

function stem(word, stemmer) {
  const stemmed = stemmer(word);
  // Find common prefix of word and stemmed
  let commonLen = 0;
  for (let i = 0; i < Math.min(word.length, stemmed.length);i++) {
    if (word.charAt(i) !== stemmed.charAt(i)) {
      break;
    }
    commonLen++;
  }
  return word.slice(0, commonLen);
}<commit_msg>Add keys to the highlighted text fragments to make React happy<commit_after>import React from 'react';
import _ from 'lodash';
import { stemmer as enStemmer } from 'porter-stemmer';
import ruStemmer from './ru-stemmer';


const enLetters = 'a-z';
const ruLetters = '\u0400-\u04ff';

const partRE = /"(.+?)"|(\S+)/g;
const enWordRE = new RegExp(`^[${enLetters}]+$`, 'i');
const ruWordRE = new RegExp(`^[${ruLetters}]+$`, 'i');

const hlClass = 'search-highlight';

export function parseQuery(query) {
  const terms = [];
  let m;
  while ((m = partRE.exec(query)) !== null) {
    if (m[2] && (m[2].indexOf(':') !== -1 || m[2].length === 1)) {
      continue;
    }
    if (m[1] !== undefined) {
      terms.push(new RegExp(`(^|[^${enLetters}${ruLetters}])(${_.escapeRegExp(m[1])})(?:$|[^${enLetters}${ruLetters}])`));
    } else if (enWordRE.test(m[2])) {
      terms.push(new RegExp(`(^|[^${enLetters}])(${_.escapeRegExp(stem(m[2], enStemmer))}[${enLetters}]*)`, 'i'));
    } else if (ruWordRE.test(m[2])) {
      terms.push(new RegExp(`(^|[^${ruLetters}])(${_.escapeRegExp(stem(m[2], ruStemmer))}[${ruLetters}]*)`, 'i'));
    } else {
      terms.push(new RegExp(`(^|[^${enLetters}${ruLetters}])(${_.escapeRegExp(m[2])})(?:$|[^${enLetters}${ruLetters}])`));
    }
  }
  return terms;
}

export function highlightString(text, terms) {
  if (terms.length === 0 || text === '') {
    return [text];
  }

  const result = [];
  while (text !== '') {
    let match = '', minPos = 0;
    terms.forEach((re) => {
      const m = re.exec(text);
      if (m && (match === '' || m.index < minPos)) {
        minPos = m.index + (m[1] || '').length;
        match = m[2];  // eslint-disable-line prefer-destructuring
      }
    });

    if (match !== '') {
      result.push(text.substring(0, minPos));
      result.push(<span key={`hl-${result.length}`} className={hlClass}>{match}</span>);
      text = text.substring(minPos + match.length);
    } else {
      result.push(text);
      break;
    }
  }
  return result;
}

function stem(word, stemmer) {
  const stemmed = stemmer(word);
  // Find common prefix of word and stemmed
  let commonLen = 0;
  for (let i = 0; i < Math.min(word.length, stemmed.length);i++) {
    if (word.charAt(i) !== stemmed.charAt(i)) {
      break;
    }
    commonLen++;
  }
  return word.slice(0, commonLen);
}
{'fraction_non_alphanumeric': 0.23557587633355107, 'fraction_numerical': 0.011321576311778794, 'mean_word_length': 3.4429400386847195, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4205324', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1709, 'n_words': 333}