<commit_before>import {Renderer, Element as Elem, RenderHandler} from "microReact";
import {Parser as MDParser} from "commonmark";
import * as CodeMirror from "codemirror";
import {debounce, uuid, unpad, Range, Position, isRange, compareRanges, comparePositions, samePosition, whollyEnclosed, adjustToWordBoundary} from "./util";

import {Span, SpanMarker, isSpanMarker, isEditorControlled, spanTypes, compareSpans, SpanChange, isSpanChange, HeadingSpan, DocumentCommentSpan} from "./ide/spans";
import * as Spans from "./ide/spans";

import {activeElements} from "./renderer";
import {send, sendEvent, indexes} from "./client";

//---------------------------------------------------------
// Navigator
//---------------------------------------------------------
/* - [x] Document Pseudo-FS
 * - [x] Table of Contents
 * - [x] Separate detail levels to control indentation / info overload
 * - [x] 2nd priority on width
 * - [x] Collapsible
 * - [x] Elision (in ToC)
 * - [x] Elision (in editor)
 */

interface TreeNode {
  id?: string,
  name: string,
  type: string,
  children?: string[],
  open?: boolean,
  span?: Span,

  hidden?: boolean,
  elisionSpan?: Span,
  level?: number
}
interface TreeMap {[id:string]: TreeNode|undefined}

class Navigator {
  labels = {
    folder: "Workspace",
    document: "Table of Contents"
  };
  open: boolean = true;

  constructor(public ide:IDE, public rootId = "root", public nodes:TreeMap = {root: {type: "folder", name: "/", children: []}}, public currentId:string = rootId) {}

  currentType():string {
    let node = this.nodes[this.currentId];
    return node && node.type || "folder";
  }

  walk(rootId:string, callback:(nodeId:string, parentId?:string) => void, parentId?:string) {
    let node = this.nodes[rootId];
    if(!node) return;
    callback(rootId, parentId);

    if(node.children) {
      for(let childId of node.children) {
        this.walk(childId, callback, rootId);
      }
    }
  }


  loadWorkspace(id:string, name:string, files:{[filename:string]: string}, parentId = this.rootId) {
    let root:TreeNode = this.nodes[id] = {id, name, type: "folder", open: true};

    let parent = root;
    for(let curId in files) {
      let node:TreeNode = {id: curId, name: curId, type: "document"};
      this.nodes[curId] = node;
      if(!parent.children) parent.children = [curId];
      else parent.children.push(curId);
    }

    if(id !== this.rootId) {
      parent = this.nodes[parentId];
      if(!parent) throw new Error(`Unable to load document into non-existent folder ${parentId}`);
      if(!parent.children) parent.children = [];
      if(parent.children.indexOf(id) === -1) {
        parent.children.push(id);
      }
    }
  }

  loadDocument(id:string, name:string) {
    let editor = this.ide.editor;
    let doc = editor.cm.getDoc();
    let headings = editor.getAllSpans("heading") as HeadingSpan[];
    headings.sort(compareSpans);

    let root:TreeNode = this.nodes[id];
    if(!root) throw new Error("Cannot load non-existent document.");
    root.open = true;
    root.children = undefined;

    let stack:TreeNode[] = [root];
    for(let heading of headings) {
      let curId = heading.id;
      let loc = heading.find();
      if(!loc) continue;

      while((stack.length > 1) && heading.source.level <= stack[stack.length - 1].level) stack.pop();
      let parent = stack[stack.length - 1];
      if(!parent.children) parent.children = [curId];
      else parent.children.push(curId);

      let old = this.nodes[curId];
      let node:TreeNode = {id: curId, name: doc.getLine(loc.from.line), type: "section", level: heading.source.level, span: heading, open: old ? old.open : true, hidden: old ? old.hidden : false, elisionSpan: old ? old.elisionSpan : undefined};
      stack.push(node);
      this.nodes[curId] = node;
    }

    this.nodes[id] = root;
  }

  updateNode(span:HeadingSpan) {
    if(this.currentType() !== "document") return;

    let nodeId = span.id;
    let node = this.nodes[nodeId];

    let loc = span.find();
    if(node && !loc) {
      if(node.elisionSpan) node.elisionSpan.clear();
      this.nodes[nodeId] = undefined;

    } else if(node) {
      if(span.isDisabled() !== node.hidden) {
        node.hidden = span.isDisabled();
      }

    } else if(!node && loc) {
      let cur = loc.from;
      let parentId:string;
      let siblingId:string|undefined;
      do {
        let parentSpan = this.ide.editor.findHeadingAt(cur);
        let parentLoc = parentSpan && parentSpan.find();
        cur = parentLoc ? parentLoc.from : {line: 0, ch: 0};
        siblingId = parentId;
        parentId = parentSpan ? parentSpan.id : this.currentId;

      } while(parentId !== this.currentId && this.nodes[parentId]!.level >= span.source.level);

      let parentNode = this.nodes[parentId]!;
      if(!parentNode.children) parentNode.children = [nodeId];
      else {
        let ix = parentNode.children.length;
        if(siblingId) {
          ix = parentNode.children.indexOf(siblingId);
          ix = (ix === -1) ? parentNode.children.length : ix;
        }
        parentNode.children.splice(ix, 0, nodeId);
      }
      let doc = this.ide.editor.cm.getDoc();
      this.nodes[nodeId] = {id: nodeId, name: doc.getLine(loc.from.line), type: "section", level: span.source.level, span, open: true, hidden: span.isDisabled()};
    }
  }

  updateElision() {
    let sections:{nodeId: string, hidden: boolean, range:Range}[] = [];

    for(let nodeId in this.nodes) {
      let node = this.nodes[nodeId];
      if(!node || node.type !== "section") continue;

      let heading = node.span as HeadingSpan;
      let range = heading.getSectionRange();
      sections.push({nodeId, hidden: node.hidden, range});
    }

    if(!sections.length) {
      // Only one source can be safely eliding at any given time.
      for(let span of this.ide.editor.getAllSpans("elision")) {
        span.clear();
      }
      return;
    }

    sections.sort((a, b) => {
      let fromDir = comparePositions(a.range.from, b.range.from);
      if(fromDir) return fromDir;
      return comparePositions(a.range.to, b.range.to);
    });

    let visibleRanges:Range[] = [];
    let currentRange:Range|undefined;
    for(let section of sections) {
      if(!section.hidden) {
        if(!currentRange) currentRange = {from: section.range.from, to: section.range.to};
        else currentRange.to = section.range.to;

      } else {
        if(currentRange) {
          if(comparePositions(section.range.from, currentRange.to) < 0) {
            currentRange.to = section.range.from;
          }
          visibleRanges.push(currentRange);
          currentRange = undefined;
        }
      }
    }

    if(currentRange) {
      visibleRanges.push(currentRange);
    }

    let editor = this.ide.editor;
    let doc = editor.cm.getDoc();
    // Capture the current topmost un-elided line in the viewport. We'll use this to maintain your scroll state (to some extent) when elisions are nuked.
    // Only one source can be safely eliding at any given time.
    let topVisible:number|undefined;
    for(let span of editor.getAllSpans("elision")) {
      let loc = span.find();
      if(loc && (!topVisible || loc.to.line < topVisible)) {
        topVisible = loc.to.line;
      }
      span.clear();
    }

    if(visibleRanges.length) {
      editor.markBetween(visibleRanges, {type: "elision"});
    } else {
      editor.markSpan({line: 0, ch: 0}, {line: doc.lineCount(), ch: 0}, {type: "elision"});
    }

    if(visibleRanges.length === 1 && topVisible) {
      let firstRange = visibleRanges[0];
      if(firstRange.from.line === 0 && firstRange.to.line >= doc.lastLine()) {
        editor.scrollToPosition({line: topVisible + 1, ch: 0});
      }
    }
  }

  // Event Handlers
  togglePane = (event:MouseEvent, elem) => {
    this.open = !this.open;
    this.ide.render();
    event.stopPropagation();
  }

  navigate = (event, elem:{nodeId:string}) => {
    this.currentId = elem.nodeId || this.rootId;
    let node = this.nodes[elem.nodeId];
    if(node && node.type === "document") {
      this.ide.loadFile(elem.nodeId);
    }
    this.ide.render();
  }

  toggleBranch = (event:MouseEvent, {nodeId}) => {
    let node = this.nodes[nodeId];
    if(!node) return;
    node.open = !node.open;
    this.ide.render();
    event.stopPropagation();
  }

  gotoSpan = (event:MouseEvent, {nodeId}) => {
    let node = this.nodes[nodeId];
    if(!node) return;
    let loc = node.span.find();
    if(!loc) return;
    if(node.span.constructor === HeadingSpan) {
      let heading = node.span as HeadingSpan;
      loc = heading.getSectionRange() || loc;
    }
    this.ide.editor.cm.scrollIntoView(loc, 20);
  }

  _inheritParentElision = (nodeId: string, parentId?: string) => {
    let node = this.nodes[nodeId];
    let parent = this.nodes[parentId];
    if(!node || !parent) return;
    node.hidden = parent.hidden;
  }

  toggleElision = (event, {nodeId}) => {
    let node = this.nodes[nodeId];
    if(!node) return;
    this.ide.editor.cm.operation( () => {
      node.hidden = !node.hidden;
      this.walk(nodeId, this._inheritParentElision);
      this.updateElision();
    });

    this.ide.render();
    event.stopPropagation();
  }

  inspectorFocus = () => {
    sendEvent([{tag: ["inspector",  "focus-current"]}]);
  }

  // Elements
  workspaceItem(nodeId:string):Elem {
    let node = this.nodes[nodeId];
    if(!node) return {c: "tree-item", nodeId};

    let subtree:Elem|undefined;
    if(node.type === "folder") {
      let items:(Elem|undefined)[] = [];
      if(node.open) {
        for(let childId of node.children || []) {
          items.push(this.workspaceItem(childId));
        }
      }
      subtree = {c: "tree-items", children: items};
    }

    return {c: `tree-item ${subtree ? "branch" : "leaf"} ${node.type} ${subtree && !node.open ? "collapsed" : ""}`, nodeId, children: [
      {c: "flex-row", children: [
        {c: `label ${subtree ? "ion-ios-arrow-down" : "no-icon"}`, text: node.name, nodeId, click: subtree ? this.toggleBranch : this.navigate}, // icon should be :before
        {c: "controls", children: [
          subtree ? {c: "new-btn ion-ios-plus-empty", click: () => console.log("new folder or document")} : undefined,
          {c: "delete-btn ion-ios-close-empty", click: () => console.log("delete folder or document w/ confirmation")}
        ]}
      ]},
      subtree
    ]};
  }

  tocItem(nodeId:string):Elem {
    let node = this.nodes[nodeId];
    if(!node) return {c: "tree-item", nodeId};

    let subtree:Elem|undefined;
    if(node.children) {
      let items:(Elem|undefined)[] = [];
      if(node.open) {
        for(let childId of node.children) {
          items.push(this.tocItem(childId));
        }
      }
      subtree = {c: "tree-items", children: items};
    }

    if(node.type === "document") {
      return {c: `tree-item ${nodeId === this.rootId ? "root" : ""} ${node.type}`, nodeId, children: [
      subtree
    ]};
    }

    return {c: `tree-item ${subtree ? "branch" : "leaf"} ${nodeId === this.rootId ? "root" : ""} ${node.type}  item-level-${node.level} ${subtree && !node.open ? "collapsed" : ""} ${node.hidden ? "hidden" : ""}`, nodeId, children: [
      {c: "flex-row", children: [
        {c: `label ${subtree && !node.level ? "ion-ios-arrow-down" : "no-icon"}`, text: node.name, nodeId, click: node.span ? this.gotoSpan : undefined}, // icon should be :before
        {c: "controls", children: [
          {c: `elide-btn ${node.hidden ? "ion-android-checkbox-outline-blank" : "ion-android-checkbox-outline"}`, nodeId, click: this.toggleElision},
        ]}
      ]},
      subtree
    ]};
  }

  inspectorControls():Elem {
    return {c: "inspector-controls", children: [
      {t: "button", c: "inspector-hide", text: "Filter to selected", click: this.inspectorFocus}<commit_msg>Make filter to selected a toggle<commit_after>import {Renderer, Element as Elem, RenderHandler} from "microReact";
import {Parser as MDParser} from "commonmark";
import * as CodeMirror from "codemirror";
import {debounce, uuid, unpad, Range, Position, isRange, compareRanges, comparePositions, samePosition, whollyEnclosed, adjustToWordBoundary} from "./util";

import {Span, SpanMarker, isSpanMarker, isEditorControlled, spanTypes, compareSpans, SpanChange, isSpanChange, HeadingSpan, DocumentCommentSpan} from "./ide/spans";
import * as Spans from "./ide/spans";

import {activeElements} from "./renderer";
import {send, sendEvent, indexes} from "./client";

//---------------------------------------------------------
// Navigator
//---------------------------------------------------------
/* - [x] Document Pseudo-FS
 * - [x] Table of Contents
 * - [x] Separate detail levels to control indentation / info overload
 * - [x] 2nd priority on width
 * - [x] Collapsible
 * - [x] Elision (in ToC)
 * - [x] Elision (in editor)
 */

interface TreeNode {
  id?: string,
  name: string,
  type: string,
  children?: string[],
  open?: boolean,
  span?: Span,

  hidden?: boolean,
  elisionSpan?: Span,
  level?: number
}
interface TreeMap {[id:string]: TreeNode|undefined}

class Navigator {
  labels = {
    folder: "Workspace",
    document: "Table of Contents"
  };
  open: boolean = true;

  constructor(public ide:IDE, public rootId = "root", public nodes:TreeMap = {root: {type: "folder", name: "/", children: []}}, public currentId:string = rootId) {}

  currentType():string {
    let node = this.nodes[this.currentId];
    return node && node.type || "folder";
  }

  walk(rootId:string, callback:(nodeId:string, parentId?:string) => void, parentId?:string) {
    let node = this.nodes[rootId];
    if(!node) return;
    callback(rootId, parentId);

    if(node.children) {
      for(let childId of node.children) {
        this.walk(childId, callback, rootId);
      }
    }
  }


  loadWorkspace(id:string, name:string, files:{[filename:string]: string}, parentId = this.rootId) {
    let root:TreeNode = this.nodes[id] = {id, name, type: "folder", open: true};

    let parent = root;
    for(let curId in files) {
      let node:TreeNode = {id: curId, name: curId, type: "document"};
      this.nodes[curId] = node;
      if(!parent.children) parent.children = [curId];
      else parent.children.push(curId);
    }

    if(id !== this.rootId) {
      parent = this.nodes[parentId];
      if(!parent) throw new Error(`Unable to load document into non-existent folder ${parentId}`);
      if(!parent.children) parent.children = [];
      if(parent.children.indexOf(id) === -1) {
        parent.children.push(id);
      }
    }
  }

  loadDocument(id:string, name:string) {
    let editor = this.ide.editor;
    let doc = editor.cm.getDoc();
    let headings = editor.getAllSpans("heading") as HeadingSpan[];
    headings.sort(compareSpans);

    let root:TreeNode = this.nodes[id];
    if(!root) throw new Error("Cannot load non-existent document.");
    root.open = true;
    root.children = undefined;

    let stack:TreeNode[] = [root];
    for(let heading of headings) {
      let curId = heading.id;
      let loc = heading.find();
      if(!loc) continue;

      while((stack.length > 1) && heading.source.level <= stack[stack.length - 1].level) stack.pop();
      let parent = stack[stack.length - 1];
      if(!parent.children) parent.children = [curId];
      else parent.children.push(curId);

      let old = this.nodes[curId];
      let node:TreeNode = {id: curId, name: doc.getLine(loc.from.line), type: "section", level: heading.source.level, span: heading, open: old ? old.open : true, hidden: old ? old.hidden : false, elisionSpan: old ? old.elisionSpan : undefined};
      stack.push(node);
      this.nodes[curId] = node;
    }

    this.nodes[id] = root;
  }

  updateNode(span:HeadingSpan) {
    if(this.currentType() !== "document") return;

    let nodeId = span.id;
    let node = this.nodes[nodeId];

    let loc = span.find();
    if(node && !loc) {
      if(node.elisionSpan) node.elisionSpan.clear();
      this.nodes[nodeId] = undefined;

    } else if(node) {
      if(span.isDisabled() !== node.hidden) {
        node.hidden = span.isDisabled();
      }

    } else if(!node && loc) {
      let cur = loc.from;
      let parentId:string;
      let siblingId:string|undefined;
      do {
        let parentSpan = this.ide.editor.findHeadingAt(cur);
        let parentLoc = parentSpan && parentSpan.find();
        cur = parentLoc ? parentLoc.from : {line: 0, ch: 0};
        siblingId = parentId;
        parentId = parentSpan ? parentSpan.id : this.currentId;

      } while(parentId !== this.currentId && this.nodes[parentId]!.level >= span.source.level);

      let parentNode = this.nodes[parentId]!;
      if(!parentNode.children) parentNode.children = [nodeId];
      else {
        let ix = parentNode.children.length;
        if(siblingId) {
          ix = parentNode.children.indexOf(siblingId);
          ix = (ix === -1) ? parentNode.children.length : ix;
        }
        parentNode.children.splice(ix, 0, nodeId);
      }
      let doc = this.ide.editor.cm.getDoc();
      this.nodes[nodeId] = {id: nodeId, name: doc.getLine(loc.from.line), type: "section", level: span.source.level, span, open: true, hidden: span.isDisabled()};
    }
  }

  updateElision() {
    let sections:{nodeId: string, hidden: boolean, range:Range}[] = [];

    for(let nodeId in this.nodes) {
      let node = this.nodes[nodeId];
      if(!node || node.type !== "section") continue;

      let heading = node.span as HeadingSpan;
      let range = heading.getSectionRange();
      sections.push({nodeId, hidden: node.hidden, range});
    }

    if(!sections.length) {
      // Only one source can be safely eliding at any given time.
      for(let span of this.ide.editor.getAllSpans("elision")) {
        span.clear();
      }
      return;
    }

    sections.sort((a, b) => {
      let fromDir = comparePositions(a.range.from, b.range.from);
      if(fromDir) return fromDir;
      return comparePositions(a.range.to, b.range.to);
    });

    let visibleRanges:Range[] = [];
    let currentRange:Range|undefined;
    for(let section of sections) {
      if(!section.hidden) {
        if(!currentRange) currentRange = {from: section.range.from, to: section.range.to};
        else currentRange.to = section.range.to;

      } else {
        if(currentRange) {
          if(comparePositions(section.range.from, currentRange.to) < 0) {
            currentRange.to = section.range.from;
          }
          visibleRanges.push(currentRange);
          currentRange = undefined;
        }
      }
    }

    if(currentRange) {
      visibleRanges.push(currentRange);
    }

    let editor = this.ide.editor;
    let doc = editor.cm.getDoc();
    // Capture the current topmost un-elided line in the viewport. We'll use this to maintain your scroll state (to some extent) when elisions are nuked.
    // Only one source can be safely eliding at any given time.
    let topVisible:number|undefined;
    for(let span of editor.getAllSpans("elision")) {
      let loc = span.find();
      if(loc && (!topVisible || loc.to.line < topVisible)) {
        topVisible = loc.to.line;
      }
      span.clear();
    }

    if(visibleRanges.length) {
      editor.markBetween(visibleRanges, {type: "elision"});
    } else {
      editor.markSpan({line: 0, ch: 0}, {line: doc.lineCount(), ch: 0}, {type: "elision"});
    }

    if(visibleRanges.length === 1 && topVisible) {
      let firstRange = visibleRanges[0];
      if(firstRange.from.line === 0 && firstRange.to.line >= doc.lastLine()) {
        editor.scrollToPosition({line: topVisible + 1, ch: 0});
      }
    }
  }

  isFocused() {
    return this.ide.editor.getAllSpans("elision").length;
  }

  // Event Handlers
  togglePane = (event:MouseEvent, elem) => {
    this.open = !this.open;
    this.ide.render();
    event.stopPropagation();
  }

  navigate = (event, elem:{nodeId:string}) => {
    this.currentId = elem.nodeId || this.rootId;
    let node = this.nodes[elem.nodeId];
    if(node && node.type === "document") {
      this.ide.loadFile(elem.nodeId);
    }
    this.ide.render();
  }

  toggleBranch = (event:MouseEvent, {nodeId}) => {
    let node = this.nodes[nodeId];
    if(!node) return;
    node.open = !node.open;
    this.ide.render();
    event.stopPropagation();
  }

  gotoSpan = (event:MouseEvent, {nodeId}) => {
    let node = this.nodes[nodeId];
    if(!node) return;
    let loc = node.span.find();
    if(!loc) return;
    if(node.span.constructor === HeadingSpan) {
      let heading = node.span as HeadingSpan;
      loc = heading.getSectionRange() || loc;
    }
    this.ide.editor.cm.scrollIntoView(loc, 20);
  }

  _inheritParentElision = (nodeId: string, parentId?: string) => {
    let node = this.nodes[nodeId];
    let parent = this.nodes[parentId];
    if(!node || !parent) return;
    node.hidden = parent.hidden;
  }

  toggleElision = (event, {nodeId}) => {
    let node = this.nodes[nodeId];
    if(!node) return;
    this.ide.editor.cm.operation( () => {
      node.hidden = !node.hidden;
      this.walk(nodeId, this._inheritParentElision);
      this.updateElision();
    });

    this.ide.render();
    event.stopPropagation();
  }

  toggleInspectorFocus = () => {
    if(this.isFocused()) {
      sendEvent([{tag: ["inspector",  "unfocus-current"]}]);
      for(let nodeId in this.nodes) {
        let node = this.nodes[nodeId];
        if(!node) continue;
        if(node.hidden) node.hidden = false;
      }
      this.updateElision();
    } else {
      sendEvent([{tag: ["inspector",  "focus-current"]}]);
    }
  }

  // Elements
  workspaceItem(nodeId:string):Elem {
    let node = this.nodes[nodeId];
    if(!node) return {c: "tree-item", nodeId};

    let subtree:Elem|undefined;
    if(node.type === "folder") {
      let items:(Elem|undefined)[] = [];
      if(node.open) {
        for(let childId of node.children || []) {
          items.push(this.workspaceItem(childId));
        }
      }
      subtree = {c: "tree-items", children: items};
    }

    return {c: `tree-item ${subtree ? "branch" : "leaf"} ${node.type} ${subtree && !node.open ? "collapsed" : ""}`, nodeId, children: [
      {c: "flex-row", children: [
        {c: `label ${subtree ? "ion-ios-arrow-down" : "no-icon"}`, text: node.name, nodeId, click: subtree ? this.toggleBranch : this.navigate}, // icon should be :before
        {c: "controls", children: [
          subtree ? {c: "new-btn ion-ios-plus-empty", click: () => console.log("new folder or document")} : undefined,
          {c: "delete-btn ion-ios-close-empty", click: () => console.log("delete folder or document w/ confirmation")}
        ]}
      ]},
      subtree
    ]};
  }

  tocItem(nodeId:string):Elem {
    let node = this.nodes[nodeId];
    if(!node) return {c: "tree-item", nodeId};

    let subtree:Elem|undefined;
    if(node.children) {
      let items:(Elem|undefined)[] = [];
      if(node.open) {
        for(let childId of node.children) {
          items.push(this.tocItem(childId));
        }
      }
      subtree = {c: "tree-items", children: items};
    }

    if(node.type === "document") {
      return {c: `tree-item ${nodeId === this.rootId ? "root" : ""} ${node.type}`, nodeId, children: [
      subtree
    ]};
    }

    return {c: `tree-item ${subtree ? "branch" : "leaf"} ${nodeId === this.rootId ? "root" : ""} ${node.type}  item-level-${node.level} ${subtree && !node.open ? "collapsed" : ""} ${node.hidden ? "hidden" : ""}`, nodeId, children: [
      {c: "flex-row", children: [
        {c: `label ${subtree && !node.level ? "ion-ios-arrow-down" : "no-icon"}`, text: node.name, nodeId, click: node.span ? this.gotoSpan : undefined}, // icon should be :before
        {c: "controls", children: [
          {c: `elide-btn ${node.hidden ? "ion-android-checkbox-outline-blank" : "ion-android-checkbox-outline"}`, nodeId, click: this.toggleElision},
        ]}
      ]},
      subtree
    ]};
  }

  inspectorControls():Elem {
    return {c: "inspector-controls", children: [
      {t: "button", c: "inspector-hide", text: this.isFocused() ? "Show all" : "Filter to selected", click: this.toggleInspectorFocus}
{'fraction_non_alphanumeric': 0.16880969115640634, 'fraction_numerical': 0.0016537809567122835, 'mean_word_length': 2.7723019338739863, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4455527', 'n_tokens_mistral': 7928, 'n_tokens_neox': 7773, 'n_words': 1776}