<commit_before>import http from './http';

const BAD_STRINGS = [
  '(i\m ',
  '(jimmy ',
  '(kelly ',
  '(sarah ',
  'clue crew',
  'heard here',
  'seen here'
];
const isQuestionValid = (bankSize, seenQuestions, question) => {
  const lowercasedClueText = question.question.toLowerCase();
  let valid;
  try {
    valid = question.value &&
      question.value >= 600 &&
      question.value <= 1600 &&
      question.value !== 1000 &&
      question.answer.length &&
      question.question.length &&
      question.invalid_count === null &&
      lowercasedClueText.length <= 140 &&
      question.answer.match(/\w/g).length <= bankSize &&
      !BAD_STRINGS.some(s => lowercasedClueText.includes(s)) &&
      !seenQuestions.includes(question.id);
  } catch (e) {
    valid = false;
  }
  return valid;
};

const stripHtmlTags = html => {
  const node = document.createElement('div');
  node.innerHTML = html;
  return node.textContent || node.innerText || '';
};

const DOUBLED_PRICES_AIRDATE = '2001-11-26T12:00:00.000Z';
const preprocessQuestion = question => {
  if (question.airdate < DOUBLED_PRICES_AIRDATE) {
    question.value *= 2;
  }
  const processedQuestion = Object.assign({}, question, {
    answer: stripHtmlTags(question.answer)
      .split('/')[0]
      .replace(/\\/g, '')
      .replace(/^an? /, '')
      .replace(/\(.+\) ?/g, '')
      .trim(),
    question: stripHtmlTags(question.question)
      .replace(/\( +/g, '(')
      .replace(/ +\)/g, ')')
      .replace(/\\/g, '')
      .replace(/, ?([^\d])/g, ', $1')
      .replace(/: ?/g, ': ')
      .replace(/ *-- */g, '\u2014')
      .replace(/ ?& ?/g, ' and ')
      .trim()
  });
  return processedQuestion;
};

const LETTERS = `${'AEHINORST'.repeat(3)}${'BCDFGKLMPUVWY'.repeat(2)}JXQZ`;
const DIGITS = '0123456789';
const getRandomChar = pool => {
  return pool[Math.floor(Math.random() * pool.length)];
};

// Fisher-Yates
const shuffleString = string => {
  const a = string.split('');
  const n = a.length;
  let j;
  for (let i = n - 1; i; --i) {
    j = Math.floor(Math.random() * (i + 1));
    [a[i], a[j]] = [a[j], a[i]];
  }
  return a.join('');<commit_msg>Improve punctuation correction<commit_after>import http from './http';

const BAD_STRINGS = [
  '(i\m ',
  '(jimmy ',
  '(kelly ',
  '(sarah ',
  'clue crew',
  'heard here',
  'seen here'
];
const isQuestionValid = (bankSize, seenQuestions, question) => {
  const lowercasedClueText = question.question.toLowerCase();
  let valid;
  try {
    valid = question.value &&
      question.value >= 600 &&
      question.value <= 1600 &&
      question.value !== 1000 &&
      question.answer.length &&
      question.question.length &&
      question.invalid_count === null &&
      lowercasedClueText.length <= 140 &&
      question.answer.match(/\w/g).length <= bankSize &&
      !BAD_STRINGS.some(s => lowercasedClueText.includes(s)) &&
      !seenQuestions.includes(question.id);
  } catch (e) {
    valid = false;
  }
  return valid;
};

const normalizeString = html => {
  const node = document.createElement('div');
  node.innerHTML = html;
  const text = node.textContent || node.innerText || '';

  return text && text
    .replace(/\\/g, '')
    .replace(/'/g, '\u2019')
    .replace(/"(.+?)"/g, '\u201c$1\u201d')
    .replace(/ *\/ */g, ' / ')
    .replace(/ *\( */g, ' (')
    .replace(/ *\) */g, ') ')
    .replace(/ *, */g, ', ')
    .replace(/ *: */g, ': ')
    .replace(/ *; */g, '; ')
    .replace(/ *-- */g, '\u2014')
    .replace(/ *& */g, ' and ')
    .replace(/\s+/g, ' ')
    .trim();
};

const DOUBLED_PRICES_AIRDATE = '2001-11-26T12:00:00.000Z';
const preprocessQuestion = question => {
  if (question.airdate < DOUBLED_PRICES_AIRDATE) {
    question.value *= 2;
  }
  const processedQuestion = Object.assign({}, question, {
    answer: normalizeString(question.answer)
      .split('/')[0]
      .replace(/\(.+\)/g, '')
      .replace(/^an? /, '')
      .replace(/\s+/g, ' ')
      .trim(),
    question: normalizeString(question.question)
      .replace(/\s+/g, ' ')
      .trim()
  });
  const category = processedQuestion.category;
  if (category) {
    category.title = normalizeString(category.title);
  }
  return processedQuestion;
};

const LETTERS = `${'AEHINORST'.repeat(3)}${'BCDFGKLMPUVWY'.repeat(2)}JXQZ`;
const DIGITS = '0123456789';
const getRandomChar = pool => {
  return pool[Math.floor(Math.random() * pool.length)];
};

// Fisher-Yates
const shuffleString = string => {
  const a = string.split('');
  const n = a.length;
  let j;
  for (let i = n - 1; i; --i) {
    j = Math.floor(Math.random() * (i + 1));
    [a[i], a[j]] = [a[j], a[i]];
  }
  return a.join('');
{'fraction_non_alphanumeric': 0.20877230703074606, 'fraction_numerical': 0.0245108578800258, 'mean_word_length': 2.763754045307443, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '471671', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1651, 'n_words': 317}