<commit_before>/**
 * <NAME> - http://taewon.de - @hataewon
 * See LICENSE
 */

import Cocoa

/// Contiguous piece of cells of a row that has the same attributes.
fileprivate struct RowRun: CustomStringConvertible {

  let row: Int
  let range: CountableClosedRange<Int>
  let attrs: CellAttributes

  var description: String {
    return "RowRun<\(row): \(range)\n\(attrs)>"
  }
}

public class NeoVimView: NSView, NSUserInterfaceValidations {

  public struct Config {
    let useInteractiveZsh: Bool

    public init(useInteractiveZsh: Bool) {
      self.useInteractiveZsh = useInteractiveZsh
    }
  }
  
  public static let minFontSize = CGFloat(4)
  public static let maxFontSize = CGFloat(128)
  public static let defaultFont = NSFont.userFixedPitchFont(ofSize: 13)!

  public let uuid = UUID().uuidString
  public weak var delegate: NeoVimViewDelegate?

  public fileprivate(set) var mode = Mode.Normal
  
  public var usesLigatures = false {
    didSet {
      self.drawer.usesLigatures = self.usesLigatures
      self.needsDisplay = true
    }
  }
  
  public var font: NSFont {
    get {
      return self._font
    }

    set {
      guard newValue.isFixedPitch else {
        return
      }

      let size = newValue.pointSize
      guard size >= NeoVimView.minFontSize && size <= NeoVimView.maxFontSize else {
        return
      }

      self._font = newValue
      self.drawer.font = self.font
      self.cellSize = self.drawer.cellSize
      self.descent = self.drawer.descent
      self.leading = self.drawer.leading
      
      self.resizeNeoVimUiTo(size: self.bounds.size)
    }
  }

  public var cwd: URL {
    get {
      return URL(fileURLWithPath: self.agent.vimCommandOutput("silent pwd"))
    }

    set {
      let path = newValue.path
      let escapedCwd = self.agent.escapedFileName(path)
      self.agent.vimCommand("silent cd \(escapedCwd)")
    }
  }

  override public var acceptsFirstResponder: Bool {
    return true
  }

  fileprivate static let emojis: [UInt32] = [
    0x1F600...0x1F64F,
    0x1F910...0x1F918,
    0x1F980...0x1F984,
    0x1F9C0...0x1F9C0
    ].flatMap { $0 }
  
  fileprivate var _font = NeoVimView.defaultFont

  fileprivate let agent: NeoVimAgent
  fileprivate let drawer: TextDrawer
  fileprivate let fontManager = NSFontManager.shared()
  fileprivate let pasteboard = NSPasteboard.general()

  fileprivate let grid = Grid()

  fileprivate var markedText: String?

  /// We store the last marked text because Cocoa's text input system does the following:
  /// 하 -> hanja popup -> insertText(하) -> attributedSubstring...() -> setMarkedText(下) -> ...
  /// We want to return "하" in attributedSubstring...()
  fileprivate var lastMarkedText: String?
  
  fileprivate var markedPosition = Position.null
  fileprivate var keyDownDone = true

  fileprivate var lastClickedCellPosition = Position.null
  
  fileprivate var xOffset = CGFloat(0)
  fileprivate var yOffset = CGFloat(0)
  fileprivate var cellSize = CGSize.zero
  fileprivate var descent = CGFloat(0)
  fileprivate var leading = CGFloat(0)

  fileprivate let maxScrollDeltaX = 30
  fileprivate let maxScrollDeltaY = 30
  fileprivate let scrollLimiterX = CGFloat(20)
  fileprivate let scrollLimiterY = CGFloat(20)
  fileprivate var scrollGuardCounterX = 5
  fileprivate var scrollGuardCounterY = 5
  fileprivate let scrollGuardYield = 5
  
  fileprivate var isCurrentlyPinching = false
  fileprivate var pinchTargetScale = CGFloat(1)
  fileprivate var pinchImage = NSImage()

  fileprivate var currentlyResizing = false
  fileprivate var currentEmoji = "😎"
  fileprivate let emojiAttrs = [
    NSFontAttributeName: NSFont(name: "AppleColorEmoji", size: 72)!
  ]
  fileprivate let resizeTextAttrs = [
    NSFontAttributeName: NSFont.systemFont(ofSize: 18),
    NSForegroundColorAttributeName: NSColor.darkGray
  ]

  public init(frame rect: NSRect, config: Config) {
    self.drawer = TextDrawer(font: self._font, useLigatures: false)
    self.agent = NeoVimAgent(uuid: self.uuid)

    super.init(frame: CGRect.zero)

    self.wantsLayer = true
    self.cellSize = self.drawer.cellSize
    self.descent = self.drawer.descent
    self.leading = self.drawer.leading

    // We cannot set bridge in init since self is not available before super.init()...
    self.agent.bridge = self
    self.agent.useInteractiveZsh = config.useInteractiveZsh
    let noErrorDuringInitialization = self.agent.runLocalServerAndNeoVim()

    // Neovim is ready now: resize neovim to bounds.
    DispatchUtils.gui {
      if noErrorDuringInitialization == false {
        let alert = NSAlert()
        alert.alertStyle = .warning
        alert.messageText = "Error during initialization"
        alert.informativeText = "There was an error during the initialization of NeoVim. "
          + "Use :messages to view the error messages."
        alert.runModal()
      }

      self.resizeNeoVimUiTo(size: self.bounds.size)
    }
  }
  
  convenience override init(frame rect: NSRect) {
    self.init(frame: rect, config: Config(useInteractiveZsh: false))
  }

  required public init?(coder: NSCoder) {
    fatalError("init(coder:) has not been implemented")
  }

  @IBAction public func debug1(_ sender: AnyObject!) {
    NSLog("DEBUG 1 - Start")
    let buffers = self.agent.tabs().map { $0.allBuffers() }.flatMap { $0 }
    NSLog("\(Set(buffers))")
    NSLog("DEBUG 1 - End")
  }
}

// MARK: - API
extension NeoVimView {

  public func enterResizeMode() {
    self.currentlyResizing = true
    self.needsDisplay = true
  }

  public func exitResizeMode() {
    self.currentlyResizing = false
    self.needsDisplay = true
    self.resizeNeoVimUiTo(size: self.bounds.size)
  }

  /**
   - returns: nil when for exampls a quickfix panel is open.
   */
  public func currentBuffer() -> NeoVimBuffer? {
    return self.agent.buffers().filter { $0.isCurrent }.first
  }

  public func hasDirtyDocs() -> Bool {
    return self.agent.hasDirtyDocs()
  }

  public func isCurrentBufferDirty() -> Bool {
    let curBuf = self.currentBuffer()
    return curBuf?.isDirty ?? true
  }
  
  public func newTab() {
    self.exec(command: "tabe")
  }

  public func open(urls: [URL]) {
    let tabs = self.agent.tabs()
    let buffers = tabs.map { $0.allBuffers() }.flatMap { $0 }
    let currentBufferIsTransient = buffers.filter { $0.isCurrent }.first?.isTransient ?? false

    urls.enumerated().forEach { (idx, url) in
      let path = url.path
      if buffers.filter({ $0.fileName == path }).first != nil {
        for window in tabs.map({ $0.windows }).flatMap({ $0 }) {

          if window.buffer.fileName == path {
            Swift.print(window)
            self.agent.select(window)
            return
          }
        }
      }

      if idx == 0 && currentBufferIsTransient {
        self.open(url, cmd: "e")
      } else {
        self.open(url, cmd: "tabe")
      }
    }
  }
  
  public func openInNewTab(urls: [URL]) {
    urls.forEach { self.open($0, cmd: "tabe") }
  }
  
  public func openInCurrentTab(url: URL) {
    self.open(url, cmd: "e")
  }

  public func closeCurrentTab() {
    self.exec(command: "q")
  }
  
  public func saveCurrentTab() {
    self.exec(command: "w")
  }
  
  public func saveCurrentTab(url: URL) {
    let path = url.path
    let escapedFileName = self.agent.escapedFileName(path)
    self.exec(command: "w \(escapedFileName)")
  }

  public func closeCurrentTabWithoutSaving() {
    self.exec(command: "q!")
  }

  public func closeAllWindows() {
    self.exec(command: "qa")
  }
  
  public func closeAllWindowsWithoutSaving() {
    self.exec(command: "qa!")
  }

  /// Does the following
  /// - `Mode.Normal`: `:command<CR>`
  /// - else: `:<Esc>:command<CR>`
  fileprivate func exec(command cmd: String) {
    switch self.mode {
    case .Normal:
      self.agent.vimInput(":\(cmd)<CR>")
    default:
      self.agent.vimInput("<Esc>:\(cmd)<CR>")
    }
  }
  
  fileprivate func open(_ url: URL, cmd: String) {
    let path = url.path
    let escapedFileName = self.agent.escapedFileName(path)
    self.exec(command: "\(cmd) \(escapedFileName)")
  }
}

// MARK: - Resizing
extension NeoVimView {

  override public func setFrameSize(_ newSize: NSSize) {
    super.setFrameSize(newSize)

    // initial resizing is done when grid has data
    guard self.grid.hasData else {
      return
    }

    if self.inLiveResize || self.currentlyResizing {
      // TODO: Turn off live resizing for now.
      // self.resizeNeoVimUiTo(size: newSize)
      return
    }

    // There can be cases where the frame is resized not by live resizing, eg when the window is resized by window
    // management tools. Thus, we make sure that the resize call is made when this happens.
    self.resizeNeoVimUiTo(size: newSize)
  }

  override public func viewDidEndLiveResize() {
    super.viewDidEndLiveResize()
    self.resizeNeoVimUiTo(size: self.bounds.size)
  }

  fileprivate func resizeNeoVimUiTo(size: CGSize) {
    self.currentEmoji = self.randomEmoji()

//    NSLog("\(#function): \(size)")
    let discreteSize = self.discreteSize(size: size)

    if discreteSize == self.grid.size {
      return
    }

    self.xOffset = floor((size.width - self.cellSize.width * CGFloat(discreteSize.width)) / 2)
    self.yOffset = floor((size.height - self.cellSize.height * CGFloat(discreteSize.height)) / 2)

    self.agent.resize(toWidth: Int32(discreteSize.width), height: Int32(discreteSize.height))
  }
  
  fileprivate func discreteSize(size: CGSize) -> Size {
    return Size(width: Int(floor(size.width / self.cellSize.width)),
                height: Int(floor(size.height / self.cellSize.height)))
  }
}

// MARK: - Drawing
extension NeoVimView {

  override public func draw(_ dirtyUnionRect: NSRect) {
    guard self.grid.hasData else {
      return
    }

    if self.inLiveResize || self.currentlyResizing {
      NSColor.windowBackgroundColor.set()
      dirtyUnionRect.fill()
      
      let boundsSize = self.bounds.size

      let emojiSize = self.currentEmoji.size(withAttributes: self.emojiAttrs)
      let emojiX = (boundsSize.width - emojiSize.width) / 2
      let emojiY = (boundsSize.height - emojiSize.height) / 2

      let discreteSize = self.discreteSize(size: boundsSize)
      let displayStr = "\(discreteSize.width) × \(discreteSize.height)"

      let size = displayStr.size(withAttributes: self.resizeTextAttrs)
      let x = (boundsSize.width - size.width) / 2
      let y = emojiY - size.height

      self.currentEmoji.draw(at: CGPoint(x: emojiX, y: emojiY), withAttributes: self.emojiAttrs)
      displayStr.draw(at: CGPoint(x: x, y: y), withAttributes: self.resizeTextAttrs)

      return
    }

//    NSLog("\(#function): \(dirtyUnionRect)")
    let context = NSGraphicsContext.current()!.cgContext
    
    if self.isCurrentlyPinching {
      let boundsSize = self.bounds.size
      let targetSize = CGSize(width: boundsSize.width * self.pinchTargetScale,
                              height: boundsSize.height * self.pinchTargetScale)
      self.pinchImage.draw(in: CGRect(origin: self.bounds.origin, size: targetSize))
      return
    }

    context.textMatrix = CGAffineTransform.identity;
    context.setTextDrawingMode(.fill);

    let dirtyRects = self.rectsBeingDrawn()
//    NSLog("\(dirtyRects)")
<commit_msg>GH-239, GH-312 Use font smoothing<commit_after>/**
 * <NAME> - http://taewon.de - @hataewon
 * See LICENSE
 */

import Cocoa

/// Contiguous piece of cells of a row that has the same attributes.
fileprivate struct RowRun: CustomStringConvertible {

  let row: Int
  let range: CountableClosedRange<Int>
  let attrs: CellAttributes

  var description: String {
    return "RowRun<\(row): \(range)\n\(attrs)>"
  }
}

public class NeoVimView: NSView, NSUserInterfaceValidations {

  public struct Config {
    let useInteractiveZsh: Bool

    public init(useInteractiveZsh: Bool) {
      self.useInteractiveZsh = useInteractiveZsh
    }
  }
  
  public static let minFontSize = CGFloat(4)
  public static let maxFontSize = CGFloat(128)
  public static let defaultFont = NSFont.userFixedPitchFont(ofSize: 13)!

  public let uuid = UUID().uuidString
  public weak var delegate: NeoVimViewDelegate?

  public fileprivate(set) var mode = Mode.Normal
  
  public var usesLigatures = false {
    didSet {
      self.drawer.usesLigatures = self.usesLigatures
      self.needsDisplay = true
    }
  }
  
  public var font: NSFont {
    get {
      return self._font
    }

    set {
      guard newValue.isFixedPitch else {
        return
      }

      let size = newValue.pointSize
      guard size >= NeoVimView.minFontSize && size <= NeoVimView.maxFontSize else {
        return
      }

      self._font = newValue
      self.drawer.font = self.font
      self.cellSize = self.drawer.cellSize
      self.descent = self.drawer.descent
      self.leading = self.drawer.leading
      
      self.resizeNeoVimUiTo(size: self.bounds.size)
    }
  }

  public var cwd: URL {
    get {
      return URL(fileURLWithPath: self.agent.vimCommandOutput("silent pwd"))
    }

    set {
      let path = newValue.path
      let escapedCwd = self.agent.escapedFileName(path)
      self.agent.vimCommand("silent cd \(escapedCwd)")
    }
  }

  override public var acceptsFirstResponder: Bool {
    return true
  }

  fileprivate static let emojis: [UInt32] = [
    0x1F600...0x1F64F,
    0x1F910...0x1F918,
    0x1F980...0x1F984,
    0x1F9C0...0x1F9C0
    ].flatMap { $0 }
  
  fileprivate var _font = NeoVimView.defaultFont

  fileprivate let agent: NeoVimAgent
  fileprivate let drawer: TextDrawer
  fileprivate let fontManager = NSFontManager.shared()
  fileprivate let pasteboard = NSPasteboard.general()

  fileprivate let grid = Grid()

  fileprivate var markedText: String?

  /// We store the last marked text because Cocoa's text input system does the following:
  /// 하 -> hanja popup -> insertText(하) -> attributedSubstring...() -> setMarkedText(下) -> ...
  /// We want to return "하" in attributedSubstring...()
  fileprivate var lastMarkedText: String?
  
  fileprivate var markedPosition = Position.null
  fileprivate var keyDownDone = true

  fileprivate var lastClickedCellPosition = Position.null
  
  fileprivate var xOffset = CGFloat(0)
  fileprivate var yOffset = CGFloat(0)
  fileprivate var cellSize = CGSize.zero
  fileprivate var descent = CGFloat(0)
  fileprivate var leading = CGFloat(0)

  fileprivate let maxScrollDeltaX = 30
  fileprivate let maxScrollDeltaY = 30
  fileprivate let scrollLimiterX = CGFloat(20)
  fileprivate let scrollLimiterY = CGFloat(20)
  fileprivate var scrollGuardCounterX = 5
  fileprivate var scrollGuardCounterY = 5
  fileprivate let scrollGuardYield = 5
  
  fileprivate var isCurrentlyPinching = false
  fileprivate var pinchTargetScale = CGFloat(1)
  fileprivate var pinchImage = NSImage()

  fileprivate var currentlyResizing = false
  fileprivate var currentEmoji = "😎"
  fileprivate let emojiAttrs = [
    NSFontAttributeName: NSFont(name: "AppleColorEmoji", size: 72)!
  ]
  fileprivate let resizeTextAttrs = [
    NSFontAttributeName: NSFont.systemFont(ofSize: 18),
    NSForegroundColorAttributeName: NSColor.darkGray
  ]

  public init(frame rect: NSRect, config: Config) {
    self.drawer = TextDrawer(font: self._font, useLigatures: false)
    self.agent = NeoVimAgent(uuid: self.uuid)

    super.init(frame: CGRect.zero)

    self.wantsLayer = true
    self.cellSize = self.drawer.cellSize
    self.descent = self.drawer.descent
    self.leading = self.drawer.leading

    // We cannot set bridge in init since self is not available before super.init()...
    self.agent.bridge = self
    self.agent.useInteractiveZsh = config.useInteractiveZsh
    let noErrorDuringInitialization = self.agent.runLocalServerAndNeoVim()

    // Neovim is ready now: resize neovim to bounds.
    DispatchUtils.gui {
      if noErrorDuringInitialization == false {
        let alert = NSAlert()
        alert.alertStyle = .warning
        alert.messageText = "Error during initialization"
        alert.informativeText = "There was an error during the initialization of NeoVim. "
          + "Use :messages to view the error messages."
        alert.runModal()
      }

      self.resizeNeoVimUiTo(size: self.bounds.size)
    }
  }
  
  convenience override init(frame rect: NSRect) {
    self.init(frame: rect, config: Config(useInteractiveZsh: false))
  }

  required public init?(coder: NSCoder) {
    fatalError("init(coder:) has not been implemented")
  }

  @IBAction public func debug1(_ sender: AnyObject!) {
    NSLog("DEBUG 1 - Start")
    let buffers = self.agent.tabs().map { $0.allBuffers() }.flatMap { $0 }
    NSLog("\(Set(buffers))")
    NSLog("DEBUG 1 - End")
  }
}

// MARK: - API
extension NeoVimView {

  public func enterResizeMode() {
    self.currentlyResizing = true
    self.needsDisplay = true
  }

  public func exitResizeMode() {
    self.currentlyResizing = false
    self.needsDisplay = true
    self.resizeNeoVimUiTo(size: self.bounds.size)
  }

  /**
   - returns: nil when for exampls a quickfix panel is open.
   */
  public func currentBuffer() -> NeoVimBuffer? {
    return self.agent.buffers().filter { $0.isCurrent }.first
  }

  public func hasDirtyDocs() -> Bool {
    return self.agent.hasDirtyDocs()
  }

  public func isCurrentBufferDirty() -> Bool {
    let curBuf = self.currentBuffer()
    return curBuf?.isDirty ?? true
  }
  
  public func newTab() {
    self.exec(command: "tabe")
  }

  public func open(urls: [URL]) {
    let tabs = self.agent.tabs()
    let buffers = tabs.map { $0.allBuffers() }.flatMap { $0 }
    let currentBufferIsTransient = buffers.filter { $0.isCurrent }.first?.isTransient ?? false

    urls.enumerated().forEach { (idx, url) in
      let path = url.path
      if buffers.filter({ $0.fileName == path }).first != nil {
        for window in tabs.map({ $0.windows }).flatMap({ $0 }) {

          if window.buffer.fileName == path {
            Swift.print(window)
            self.agent.select(window)
            return
          }
        }
      }

      if idx == 0 && currentBufferIsTransient {
        self.open(url, cmd: "e")
      } else {
        self.open(url, cmd: "tabe")
      }
    }
  }
  
  public func openInNewTab(urls: [URL]) {
    urls.forEach { self.open($0, cmd: "tabe") }
  }
  
  public func openInCurrentTab(url: URL) {
    self.open(url, cmd: "e")
  }

  public func closeCurrentTab() {
    self.exec(command: "q")
  }
  
  public func saveCurrentTab() {
    self.exec(command: "w")
  }
  
  public func saveCurrentTab(url: URL) {
    let path = url.path
    let escapedFileName = self.agent.escapedFileName(path)
    self.exec(command: "w \(escapedFileName)")
  }

  public func closeCurrentTabWithoutSaving() {
    self.exec(command: "q!")
  }

  public func closeAllWindows() {
    self.exec(command: "qa")
  }
  
  public func closeAllWindowsWithoutSaving() {
    self.exec(command: "qa!")
  }

  /// Does the following
  /// - `Mode.Normal`: `:command<CR>`
  /// - else: `:<Esc>:command<CR>`
  fileprivate func exec(command cmd: String) {
    switch self.mode {
    case .Normal:
      self.agent.vimInput(":\(cmd)<CR>")
    default:
      self.agent.vimInput("<Esc>:\(cmd)<CR>")
    }
  }
  
  fileprivate func open(_ url: URL, cmd: String) {
    let path = url.path
    let escapedFileName = self.agent.escapedFileName(path)
    self.exec(command: "\(cmd) \(escapedFileName)")
  }
}

// MARK: - Resizing
extension NeoVimView {

  override public func setFrameSize(_ newSize: NSSize) {
    super.setFrameSize(newSize)

    // initial resizing is done when grid has data
    guard self.grid.hasData else {
      return
    }

    if self.inLiveResize || self.currentlyResizing {
      // TODO: Turn off live resizing for now.
      // self.resizeNeoVimUiTo(size: newSize)
      return
    }

    // There can be cases where the frame is resized not by live resizing, eg when the window is resized by window
    // management tools. Thus, we make sure that the resize call is made when this happens.
    self.resizeNeoVimUiTo(size: newSize)
  }

  override public func viewDidEndLiveResize() {
    super.viewDidEndLiveResize()
    self.resizeNeoVimUiTo(size: self.bounds.size)
  }

  fileprivate func resizeNeoVimUiTo(size: CGSize) {
    self.currentEmoji = self.randomEmoji()

//    NSLog("\(#function): \(size)")
    let discreteSize = self.discreteSize(size: size)

    if discreteSize == self.grid.size {
      return
    }

    self.xOffset = floor((size.width - self.cellSize.width * CGFloat(discreteSize.width)) / 2)
    self.yOffset = floor((size.height - self.cellSize.height * CGFloat(discreteSize.height)) / 2)

    self.agent.resize(toWidth: Int32(discreteSize.width), height: Int32(discreteSize.height))
  }
  
  fileprivate func discreteSize(size: CGSize) -> Size {
    return Size(width: Int(floor(size.width / self.cellSize.width)),
                height: Int(floor(size.height / self.cellSize.height)))
  }
}

// MARK: - Drawing
extension NeoVimView {

  override public func draw(_ dirtyUnionRect: NSRect) {
    guard self.grid.hasData else {
      return
    }

    if self.inLiveResize || self.currentlyResizing {
      NSColor.windowBackgroundColor.set()
      dirtyUnionRect.fill()
      
      let boundsSize = self.bounds.size

      let emojiSize = self.currentEmoji.size(withAttributes: self.emojiAttrs)
      let emojiX = (boundsSize.width - emojiSize.width) / 2
      let emojiY = (boundsSize.height - emojiSize.height) / 2

      let discreteSize = self.discreteSize(size: boundsSize)
      let displayStr = "\(discreteSize.width) × \(discreteSize.height)"

      let size = displayStr.size(withAttributes: self.resizeTextAttrs)
      let x = (boundsSize.width - size.width) / 2
      let y = emojiY - size.height

      self.currentEmoji.draw(at: CGPoint(x: emojiX, y: emojiY), withAttributes: self.emojiAttrs)
      displayStr.draw(at: CGPoint(x: x, y: y), withAttributes: self.resizeTextAttrs)

      return
    }

//    NSLog("\(#function): \(dirtyUnionRect)")
    let context = NSGraphicsContext.current()!.cgContext
    
    if self.isCurrentlyPinching {
      let boundsSize = self.bounds.size
      let targetSize = CGSize(width: boundsSize.width * self.pinchTargetScale,
                              height: boundsSize.height * self.pinchTargetScale)
      self.pinchImage.draw(in: CGRect(origin: self.bounds.origin, size: targetSize))
      return
    }

    // When both anti-aliasing and font smoothing is turned on, then the "Use LCD font smoothing when available" setting
    // is used to render texts, cf. chapter 11 from "Programming with Quartz".
    context.setShouldSmoothFonts(true);
    context.textMatrix = CGAffineTransform.identity;
    context.setTextDrawingMode(.fill);

    let dirtyRects = self.rectsBeingDrawn()
//    NSLog("\(dirtyRects)")

{'fraction_non_alphanumeric': 0.10917222027358821, 'fraction_numerical': 0.00815503332164153, 'mean_word_length': 2.9461937716262976, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4484024', 'n_tokens_mistral': 7606, 'n_tokens_neox': 7298, 'n_words': 1783}