<commit_before>/*
 * spelling.ts
 *
 * Copyright (C) 2020 by RStudio, PBC
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

import { MarkType, Schema, Node as ProsemirrorNode } from 'prosemirror-model';
import { EditorState } from 'prosemirror-state';

import { EditorWordSource, EditorWordRange, EditorUISpelling, kCharClassNonWord } from '../../api/spelling';
import { PandocMark, getMarkRange } from '../../api/mark';

export const beginDocPos = () => 1;
export const endDocPos = (doc: ProsemirrorNode) => doc.nodeSize - 2;

export function getWords(
  state: EditorState,
  start: number,
  end: number,
  spelling: EditorUISpelling,
  excluded: MarkType[],
): EditorWordSource {
  // provide defaults
  if (start === -1) {
    start = beginDocPos();
  }
  if (end === -1) {
    end = endDocPos(state.doc);
  }

  // enlarge range to begin/end
  const beginPos = findBeginWord(state, start, spelling.classifyCharacter);
  const endPos = findEndWord(state, end, spelling.classifyCharacter);

  const words: EditorWordRange[] = [];
  let currentPos = beginPos;
  while (currentPos <= endPos) {
    // advance until we find a word
    currentPos = advanceToWord(state, currentPos, spelling.classifyCharacter);
    if (currentPos >= endPos) {
      break;
    }

    // find end of word
    const endWordPos = findEndWord(state, currentPos, spelling.classifyCharacter);
    if (endWordPos === currentPos) {
      break;
    }

    // add word if it doesn't have an excluded type
    if (!excludeWord(state.doc, currentPos, endWordPos, excluded)) {<commit_msg>termination protection for spelling word extractor<commit_after>/*
 * spelling.ts
 *
 * Copyright (C) 2020 by RStudio, PBC
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

import { MarkType, Schema, Node as ProsemirrorNode } from 'prosemirror-model';
import { EditorState } from 'prosemirror-state';

import { EditorWordSource, EditorWordRange, EditorUISpelling, kCharClassNonWord } from '../../api/spelling';
import { PandocMark, getMarkRange } from '../../api/mark';

export const beginDocPos = () => 1;
export const endDocPos = (doc: ProsemirrorNode) => doc.nodeSize - 2;

export function getWords(
  state: EditorState,
  start: number,
  end: number,
  spelling: EditorUISpelling,
  excluded: MarkType[],
): EditorWordSource {
  // provide defaults
  if (start === -1) {
    start = beginDocPos();
  }
  if (end === -1) {
    end = endDocPos(state.doc);
  }

  // enlarge range to begin/end
  const beginPos = findBeginWord(state, start, spelling.classifyCharacter);
  const endPos = findEndWord(state, end, spelling.classifyCharacter);

  // create some termination protection for this loop (some reports of 
  // freezing beahvior late in the v1.4 cycle and we have already fixed
  // one instance of this loop not terminating). if e.g. advanceToWord()
  // kept returning currentPos then we wouldn't terminate
  let currentIteration = 0;

  const words: EditorWordRange[] = [];
  let currentPos = beginPos;
  while (currentPos <= endPos) {

    // termination protection
    if (currentIteration++ > (endPos - beginPos)) {
      break;
    }

    // advance until we find a word
    currentPos = advanceToWord(state, currentPos, spelling.classifyCharacter);
    if (currentPos >= endPos) {
      break;
    }

    // find end of word
    const endWordPos = findEndWord(state, currentPos, spelling.classifyCharacter);
    if (endWordPos === currentPos) {
      break;
    }

    // add word if it doesn't have an excluded type
    if (!excludeWord(state.doc, currentPos, endWordPos, excluded)) {
{'fraction_non_alphanumeric': 0.10463045414069456, 'fraction_numerical': 0.005565449688334817, 'mean_word_length': 3.6607883817427385, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1087390', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1297, 'n_words': 460}