<commit_before>import * as vscode from 'vscode';
import { Position } from './../motion/position';
import { TextEditor } from './../textEditor';
import { Configuration } from '../../src/configuration/configuration';

export enum SearchDirection {
  Forward = 1,
  Backward = -1
}

/**
 * State involved with beginning a search (/).
 */
export class SearchState {
  private static readonly MAX_SEARCH_RANGES = 1000;
  private static specialCharactersRegex: RegExp = /[\-\[\]{}()*+?.,\\\^$|#\s]/g;

  private _matchRanges: vscode.Range[] = [];

  /**
   * Every range in the document that matches the search string.
   */
  public get matchRanges(): vscode.Range[] {
    return this._matchRanges;
  }

  private _searchCursorStartPosition: Position;
  public get searchCursorStartPosition(): Position {
    return this._searchCursorStartPosition;
  }

  private _cachedDocumentVersion: number;
  private _cachedDocumentName: String;
  private _searchDirection: SearchDirection = SearchDirection.Forward;
  private isRegex: boolean;

  private _searchString = "";
  public get searchString(): string {
    return this._searchString;
  }

  public set searchString(search: string){
    if (this._searchString !== search) {
      this._searchString = search;
      this._recalculateSearchRanges({ forceRecalc: true });
    }
  }

  private _recalculateSearchRanges({ forceRecalc }: { forceRecalc?: boolean } = {}): void {
    const search = this.searchString;
    const selection = vscode.window.activeTextEditor!.selection;
    const lineToStartAt = Math.min(selection.start.line, selection.end.line);

    if (search === "") { return; }

    // checking if the tab that is worked on has changed, or the file version has changed
    const shouldRecalculate = (this._cachedDocumentName !== TextEditor.getDocumentName()) ||
      (this._cachedDocumentVersion !== TextEditor.getDocumentVersion()) || forceRecalc;
    if (shouldRecalculate) {
      // Calculate and store all matching ranges
      this._cachedDocumentVersion = TextEditor.getDocumentVersion();
      this._cachedDocumentName = TextEditor.getDocumentName();
      this._matchRanges = [];

      /*
       * Decide whether the search is case sensitive.
       * If ignorecase is false, the search is case sensitive.
       * If ignorecase is true, the search should be case insensitive.
       * If both ignorecase and smartcase are true, the search is case sensitive only when the search string contains UpperCase character.
       */
      let ignorecase = Configuration.ignorecase;

      if (ignorecase && Configuration.smartcase && /[A-Z]/.test(search)) {
        ignorecase = false;
      }

      let searchRE = search;
      if (!this.isRegex) {
        searchRE = search.replace(SearchState.specialCharactersRegex, "\\$&");
      }

      const regexFlags = ignorecase ? 'gi' : 'g';

      let regex: RegExp;
      try {
        regex = new RegExp(searchRE, regexFlags);
      } catch (err) {
        // Couldn't compile the regexp, try again with special characters escaped
        searchRE = search.replace(SearchState.specialCharactersRegex, "\\$&");
        regex = new RegExp(searchRE, regexFlags);
      }

      let lineIdx = lineToStartAt;

      let numNewLines = (searchRE.match(/\\n/g) || []).length;
      // Start at the current line and wrap the document if we hit the end.
      outer:
      do {
        let rangeStart = new Position(lineIdx, 0);
        let rangeEnd = rangeStart.getLineEnd();
        let lineLengths = [];
        for (var i = 0; i < numNewLines; i++) {
          lineLengths.push(rangeEnd.getLineEndIncludingEOL().character);
          rangeEnd = rangeEnd.getDown(1).getLineEndIncludingEOL();
        }
        const line = TextEditor.getText(new vscode.Range(rangeStart, rangeEnd));
        let result = regex.exec(line);

        while (result) {
          if (this._matchRanges.length >= SearchState.MAX_SEARCH_RANGES) {
            break outer;
          }
          let resultToPosition = function(resultIdx: number, lineIdx: number, lineLengths: Array<number>, result: RegExpExecArray){
            let cur = resultIdx;
            for (var idx = 0; idx < lineLengths.length; idx++) {
              if (cur <= lineLengths[idx]) {
                return new Position(idx + lineIdx, cur);
              }
              cur -= lineLengths[idx];
            }
            return new Position(lineLengths.length + lineIdx, cur);
          };

          this.matchRanges.push(new vscode.Range(
            resultToPosition(result.index, lineIdx, lineLengths, result),
            resultToPosition(result.index + result[0].length, lineIdx, lineLengths, result)
          ));

          if (result.index === regex.lastIndex) {
            regex.lastIndex++;
          }

          result = regex.exec(line);
        }

        lineIdx === TextEditor.getLineCount() - 1 ? lineIdx = 0 : lineIdx++;
      } while (lineIdx !== lineToStartAt);

      this._matchRanges.sort((x, y) =>
        x.start.line < y.start.line ||
        x.start.line === y.start.line && x.start.character < y.start.character ? -1 : 1);
    }
  }

  /**
   * The position of the next search, or undefined if there is no match.
   *
   * Pass in -1 as direction to reverse the direction we search.
   */<commit_msg>rewrote searching algorithm<commit_after>import * as vscode from 'vscode';
import { Position } from './../motion/position';
import { TextEditor } from './../textEditor';
import { Configuration } from '../../src/configuration/configuration';

export enum SearchDirection {
  Forward = 1,
  Backward = -1
}

/**
 * State involved with beginning a search (/).
 */
export class SearchState {
  private static readonly MAX_SEARCH_RANGES = 1000;
  private static specialCharactersRegex: RegExp = /[\-\[\]{}()*+?.,\\\^$|#\s]/g;

  private _matchRanges: vscode.Range[] = [];

  /**
   * Every range in the document that matches the search string.
   */
  public get matchRanges(): vscode.Range[] {
    return this._matchRanges;
  }

  private _searchCursorStartPosition: Position;
  public get searchCursorStartPosition(): Position {
    return this._searchCursorStartPosition;
  }

  private _cachedDocumentVersion: number;
  private _cachedDocumentName: String;
  private _searchDirection: SearchDirection = SearchDirection.Forward;
  private isRegex: boolean;

  private _searchString = "";
  public get searchString(): string {
    return this._searchString;
  }

  public set searchString(search: string){
    if (this._searchString !== search) {
      this._searchString = search;
      this._recalculateSearchRanges({ forceRecalc: true });
    }
  }

  private _recalculateSearchRanges({ forceRecalc }: { forceRecalc?: boolean } = {}): void {
    const search = this.searchString;
    if (search === "") { return; }

    // checking if the tab that is worked on has changed, or the file version has changed
    const shouldRecalculate = (this._cachedDocumentName !== TextEditor.getDocumentName()) ||
      (this._cachedDocumentVersion !== TextEditor.getDocumentVersion()) || forceRecalc;

    if (shouldRecalculate) {
      // Calculate and store all matching ranges
      this._cachedDocumentVersion = TextEditor.getDocumentVersion();
      this._cachedDocumentName = TextEditor.getDocumentName();
      this._matchRanges = [];

      /*
       * Decide whether the search is case sensitive.
       * If ignorecase is false, the search is case sensitive.
       * If ignorecase is true, the search should be case insensitive.
       * If both ignorecase and smartcase are true, the search is case sensitive only when the search string contains UpperCase character.
       */
      let ignorecase = Configuration.ignorecase;

      if (ignorecase && Configuration.smartcase && /[A-Z]/.test(search)) {
        ignorecase = false;
      }

      let searchRE = search;
      if (!this.isRegex) {
        searchRE = search.replace(SearchState.specialCharactersRegex, "\\$&");
      }

      const regexFlags = ignorecase ? 'gi' : 'g';

      let regex: RegExp;
      try {
        regex = new RegExp(searchRE, regexFlags);
      } catch (err) {
        // Couldn't compile the regexp, try again with special characters escaped
        searchRE = search.replace(SearchState.specialCharactersRegex, "\\$&");
        regex = new RegExp(searchRE, regexFlags);
      }

      let text = TextEditor.getText(new vscode.Range(new Position(0 , 0),
          new Position(TextEditor.getLineCount() - 1, TextEditor.getLineMaxColumn(TextEditor.getLineCount() - 1))));
      // Start at the current line and wrap the document if we hit the end.
      const lineLengths = text.split("\n").map(x => x.length + 1);
      let sumLineLengths = [];
      let curLength = 0;
      for (const length of lineLengths){
        sumLineLengths.push(curLength);
        curLength += length;
      }
      let binSearch = function(val: number, l: number, r: number, arr: Array<number>): Position {
        const mid = Math.floor((l + r) / 2);
        if (l === r - 1) {
          return new Position(l, val - arr[mid]);
        }
        if (arr[mid] >= val) {
          return binSearch(val, l, mid, arr);
        } else {
          return binSearch(val, mid, r, arr);
        }
      };
      let startPos = sumLineLengths[this.searchCursorStartPosition.line] + this.searchCursorStartPosition.character;
      regex.lastIndex = startPos;
      let result = regex.exec(text);
      let wrappedOver = false;

      outer:
      while (result && !(wrappedOver && result!.index > startPos)) {
        if (this._matchRanges.length >= SearchState.MAX_SEARCH_RANGES) {
          break outer;
        }

        this.matchRanges.push(new vscode.Range(
          binSearch(result.index, 0, sumLineLengths.length, sumLineLengths),
          binSearch(result.index + result[0].length, 0, sumLineLengths.length, sumLineLengths)
        ));

        if (result.index === regex.lastIndex) {
          regex.lastIndex++;
        }
        result = regex.exec(text);
        if (!result && !wrappedOver) {
          regex.lastIndex = 0;
          wrappedOver = true;
          result = regex.exec(text);
        }
      }

      this._matchRanges.sort((x, y) =>
        (x.start.line < y.start.line) ||
        (x.start.line === y.start.line && x.start.character < y.start.character) ? -1 : 1);
    }
  }

  /**
   * The position of the next search, or undefined if there is no match.
   *
   * Pass in -1 as direction to reverse the direction we search.
   */
{'fraction_non_alphanumeric': 0.11829698066482522, 'fraction_numerical': 0.003524145156681589, 'mean_word_length': 2.6726128016789086, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6056550', 'n_tokens_mistral': 3025, 'n_tokens_neox': 2925, 'n_words': 817}