<commit_before>import * as React from 'react'
import * as CodeMirror from 'codemirror'
import {
  IAutocompletionProvider,
  UserAutocompletionProvider,
} from '../autocompletion'
import { Doc, Position } from 'codemirror'

interface IAuthorInputProps {
  /**
   * An optional class name for the wrapper element around the
   * author input component
   */
  readonly className?: string
  // tslint:disable-next-line:react-unused-props-and-state
  readonly autocompletionProviders: ReadonlyArray<IAutocompletionProvider<any>>
}

interface IAuthorInputState {}

function prevPosition(doc: CodeMirror.Doc, pos: CodeMirror.Position) {
  return doc.posFromIndex(doc.indexFromPos(pos) - 1)
}

function nextPosition(doc: CodeMirror.Doc, pos: CodeMirror.Position) {
  return doc.posFromIndex(doc.indexFromPos(pos) + 1)
}

// mark ranges are inclusive, this checks exclusive
function posIsInsideMarkedText(doc: CodeMirror.Doc, pos: CodeMirror.Position) {
  const marks = (doc.findMarksAt(pos) as any) as ActualTextMarker[]
  const ix = doc.indexFromPos(pos)

  return marks.some(mark => {
    const pos = mark.find()
    const from = doc.indexFromPos(pos.from)
    const to = doc.indexFromPos(pos.from)

    return ix > from && ix < to
  })
}

function isMarkOrWhitespace(doc: Doc, pos: Position) {
  const line = doc.getLine(pos.line)
  if (/\s/.test(line.charAt(pos.ch))) {
    return true
  }

  return posIsInsideMarkedText(doc, pos)
}

function scanWhile(
  doc: Doc,
  start: Position,
  predicate: (doc: Doc, pos: Position) => boolean,
  iter: (doc: Doc, pos: Position) => Position
) {
  let pos = start
  let next = iter(doc, start)

  for (; predicate(doc, next); next = iter(doc, next)) {
    pos = next
  }

  return pos<commit_msg>Safeguard against infinite loops due to faulty iter functions or reaching start/end<commit_after>import * as React from 'react'
import * as CodeMirror from 'codemirror'
import {
  IAutocompletionProvider,
  UserAutocompletionProvider,
} from '../autocompletion'
import { Doc, Position } from 'codemirror'

interface IAuthorInputProps {
  /**
   * An optional class name for the wrapper element around the
   * author input component
   */
  readonly className?: string
  // tslint:disable-next-line:react-unused-props-and-state
  readonly autocompletionProviders: ReadonlyArray<IAutocompletionProvider<any>>
}

interface IAuthorInputState {}

function prevPosition(doc: CodeMirror.Doc, pos: CodeMirror.Position) {
  return doc.posFromIndex(doc.indexFromPos(pos) - 1)
}

function nextPosition(doc: CodeMirror.Doc, pos: CodeMirror.Position) {
  return doc.posFromIndex(doc.indexFromPos(pos) + 1)
}

// mark ranges are inclusive, this checks exclusive
function posIsInsideMarkedText(doc: CodeMirror.Doc, pos: CodeMirror.Position) {
  const marks = (doc.findMarksAt(pos) as any) as ActualTextMarker[]
  const ix = doc.indexFromPos(pos)

  return marks.some(mark => {
    const pos = mark.find()
    const from = doc.indexFromPos(pos.from)
    const to = doc.indexFromPos(pos.from)

    return ix > from && ix < to
  })
}

function isMarkOrWhitespace(doc: Doc, pos: Position) {
  const line = doc.getLine(pos.line)
  if (/\s/.test(line.charAt(pos.ch))) {
    return true
  }

  return posIsInsideMarkedText(doc, pos)
}

function posEquals(x: Position, y: Position) {
  return x.line === y.line && x.ch === y.ch
}

function scanWhile(
  doc: Doc,
  start: Position,
  predicate: (doc: Doc, pos: Position) => boolean,
  iter: (doc: Doc, pos: Position) => Position
) {
  let pos = start

  for (
    let next = iter(doc, start);
    predicate(doc, next) && !posEquals(pos, next);
    next = iter(doc, next)
  ) {
    pos = next
  }

  return pos
{'fraction_non_alphanumeric': 0.11819172113289761, 'fraction_numerical': 0.0010893246187363835, 'mean_word_length': 4.108484005563282, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '947634', 'n_tokens_mistral': 1230, 'n_tokens_neox': 1188, 'n_words': 318}