<commit_before>declare module "tree-sitter" {
  export type Point = {
    row: number;
    column: number;
  };

  export type Range = {
    start: Point;
    end: Point;
  };

  export type Edit = {
    startIndex: number;
    lengthRemoved: number;
    lengthAdded: number;
    startPosition: Point;
    extentRemoved: Point;
    extentAdded: Point;
  };

  export type Logger = (
    message: string,
    params: {[param: string]: string},
    type: "parse" | "lex"
  ) => void;

  export interface Input {
    seek(index: number): void;
    read(): any;
  }

  export interface SyntaxNode {
    isNamed: boolean;
    type: string;
    startPosition: Point;
    endPosition: Point;
    children: Array<SyntaxNode>;
    namedChildren: Array<SyntaxNode>;
    startIndex: number;
    endIndex: number;
    parent: SyntaxNode | null;
    firstChild: SyntaxNode | null;
    lastChild: SyntaxNode | null;
    firstNamedChild: SyntaxNode | null;
    lastNamedChild: SyntaxNode | null;
    nextSibling: SyntaxNode | null;
    nextNamedSibling: SyntaxNode | null;
    previousSibling: SyntaxNode | null;
    previousNamedSibling: SyntaxNode | null;

    isValid(): boolean;
    hasError(): boolean;
    hasChanges(): boolean;
    toString(): string;
    descendantForIndex(index: number): SyntaxNode;
    descendantForIndex(startIndex: number, endIndex: number): SyntaxNode;
    namedDescendantForIndex(index: number): SyntaxNode;
    namedDescendantForIndex(startIndex: number, endIndex: number): SyntaxNode;
    descendantForPosition(position: Point): SyntaxNode;
    descendantForPosition(startPosition: Point, endPosition: Point): SyntaxNode;
    namedDescendantForPosition(position: Point): SyntaxNode;
    namedDescendantForPosition(startPosition: Point, endPosition: Point): SyntaxNode;
  }

  export interface TreeCursor {
    nodeType: string;
    nodeIsNamed: boolean;
    startPosition: Point;
    endPosition: Point;
    startIndex: number;
    endIndex: number;

    gotoParent(): boolean;
    gotoFirstChild(): boolean;
    gotoNextSibling(): boolean;
  }

  export interface Tree {
    readonly rootNode: SyntaxNode;

    edit(delta: Edit): Document;
    walk(): TreeCursor;
    getChangedRanges(other: Tree): Range[];
  }

  export class Parser {

  export = Parser;
    parse(input: string | Input, previousTree?: Tree): Tree;
    getLanguage(): any;
    setLanguage(language: any);
    getLogger(): Logger;
    setLogger(logFunc: Logger): void;
  }
}
<commit_msg>Remove Parser export assignment as this breaks all other exports<commit_after>declare module "tree-sitter" {
  export type Point = {
    row: number;
    column: number;
  };

  export type Range = {
    start: Point;
    end: Point;
  };

  export type Edit = {
    startIndex: number;
    lengthRemoved: number;
    lengthAdded: number;
    startPosition: Point;
    extentRemoved: Point;
    extentAdded: Point;
  };

  export type Logger = (
    message: string,
    params: {[param: string]: string},
    type: "parse" | "lex"
  ) => void;

  export interface Input {
    seek(index: number): void;
    read(): any;
  }

  export interface SyntaxNode {
    isNamed: boolean;
    type: string;
    startPosition: Point;
    endPosition: Point;
    children: Array<SyntaxNode>;
    namedChildren: Array<SyntaxNode>;
    startIndex: number;
    endIndex: number;
    parent: SyntaxNode | null;
    firstChild: SyntaxNode | null;
    lastChild: SyntaxNode | null;
    firstNamedChild: SyntaxNode | null;
    lastNamedChild: SyntaxNode | null;
    nextSibling: SyntaxNode | null;
    nextNamedSibling: SyntaxNode | null;
    previousSibling: SyntaxNode | null;
    previousNamedSibling: SyntaxNode | null;

    isValid(): boolean;
    hasError(): boolean;
    hasChanges(): boolean;
    toString(): string;
    descendantForIndex(index: number): SyntaxNode;
    descendantForIndex(startIndex: number, endIndex: number): SyntaxNode;
    namedDescendantForIndex(index: number): SyntaxNode;
    namedDescendantForIndex(startIndex: number, endIndex: number): SyntaxNode;
    descendantForPosition(position: Point): SyntaxNode;
    descendantForPosition(startPosition: Point, endPosition: Point): SyntaxNode;
    namedDescendantForPosition(position: Point): SyntaxNode;
    namedDescendantForPosition(startPosition: Point, endPosition: Point): SyntaxNode;
  }

  export interface TreeCursor {
    nodeType: string;
    nodeIsNamed: boolean;
    startPosition: Point;
    endPosition: Point;
    startIndex: number;
    endIndex: number;

    gotoParent(): boolean;
    gotoFirstChild(): boolean;
    gotoNextSibling(): boolean;
  }

  export interface Tree {
    readonly rootNode: SyntaxNode;

    edit(delta: Edit): Document;
    walk(): TreeCursor;
    getChangedRanges(other: Tree): Range[];
  }

  export class Parser {
    parse(input: string | Input, previousTree?: Tree): Tree;
    getLanguage(): any;
    setLanguage(language: any);
    getLogger(): Logger;
    setLogger(logFunc: Logger): void;
  }
}

{'fraction_non_alphanumeric': 0.10479823328648866, 'fraction_numerical': 0.0, 'mean_word_length': 3.6823308270676693, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3900400', 'n_tokens_mistral': 1493, 'n_tokens_neox': 1483, 'n_words': 395}