<commit_before>Flow.Renderers = (_, _sandbox) ->
  h1: -> Flow.Heading _, 'h1'
  h2: -> Flow.Heading _, 'h2'
  h3: -> Flow.Heading _, 'h3'
  h4: -> Flow.Heading _, 'h4'
  h5: -> Flow.Heading _, 'h5'
  h6: -> Flow.Heading _, 'h6'
  md: -> Flow.Markdown _
  cs: (guid) -> Flow.Coffeescript _, guid, _sandbox
  raw: -> Flow.Raw _

Flow.Notebook = (_, _renderers) ->
  _localName = signal 'Untitled Flow'
  _remoteName = signal null

  _isEditingName = signal no
  editName = -> _isEditingName yes
  saveName = -> _isEditingName no

  _cells = signals []
  _selectedCell = null
  _selectedCellIndex = -1
  _clipboardCell = null
  _lastDeletedCell = null
  _areInputsHidden = signal no
  _areOutputsHidden = signal no
  _isSidebarHidden = signal no
  _isRunningAll = signal no
  _runningCaption = signal 'Running'
  _runningPercent = signal '0%'
  _runningCellInput = signal ''
  _status = Flow.Status _
  _sidebar = Flow.Sidebar _, _cells
  _about = Flow.About _
  _dialogs = Flow.Dialogs _

  serialize = ->
    cells = for cell in _cells()
      type: cell.type()
      input: cell.input()

    version: '1.0.0'
    cells: cells

  deserialize = (localName, remoteName, doc) ->
    _localName localName
    _remoteName remoteName

    cells = for cell in doc.cells
      createCell cell.type, cell.input
    _cells cells

    selectCell head cells

    # Execute all non-code cells (headings, markdown, etc.)
    for cell in _cells()
      cell.execute() unless cell.isCode() 

    return

  createCell = (type='cs', input='') ->
    Flow.Cell _, _renderers, type, input

  checkConsistency = ->
    selectionCount = 0
    for cell, i in _cells()
      unless cell
        error "index #{i} is empty"
      else
        if cell.isSelected()
          selectionCount++
    error "selected cell count = #{selectionCount}" if selectionCount isnt 1
    return

  selectCell = (target, scrollIntoView=yes, scrollImmediately=no) ->
    return if _selectedCell is target
    _selectedCell.isSelected no if _selectedCell
    _selectedCell = target
    #TODO also set focus so that tabs don't jump to the first cell
    _selectedCell.isSelected yes
    _selectedCellIndex = _cells.indexOf _selectedCell
    checkConsistency()
    if scrollIntoView
      defer ->
        _selectedCell.scrollIntoView scrollImmediately
    _selectedCell

  cloneCell = (cell) ->
    createCell cell.type(), cell.input()

  switchToCommandMode = ->
    _selectedCell.isActive no

  switchToEditMode = ->
    _selectedCell.isActive yes
    no

  convertCellToCode = ->
    _selectedCell.type 'cs'

  convertCellToHeading = (level) -> -> 
    _selectedCell.type "h#{level}"
    _selectedCell.execute()

  convertCellToMarkdown = ->
    _selectedCell.type 'md'
    _selectedCell.execute()

  convertCellToRaw = ->
    _selectedCell.type 'raw'
    _selectedCell.execute()

  copyCell = ->
    _clipboardCell = _selectedCell

  cutCell = ->
    copyCell()
    removeCell()

  deleteCell = ->
    _lastDeletedCell = _selectedCell
    removeCell()

  removeCell = ->
    cells = _cells()
    if cells.length > 1
      if _selectedCellIndex is cells.length - 1
        #TODO call dispose() on this cell
        removedCell = head splice _cells, _selectedCellIndex, 1
        selectCell cells[_selectedCellIndex - 1]
      else
        #TODO call dispose() on this cell
        removedCell = head splice _cells, _selectedCellIndex, 1
        selectCell cells[_selectedCellIndex]
      _.saveClip 'trash', removedCell.type(), removedCell.input() if removedCell
    return
    
  insertCell = (index, cell) ->
    splice _cells, index, 0, cell
    selectCell cell
    cell

  insertAbove = (cell) ->
    insertCell _selectedCellIndex, cell

  insertBelow = (cell) ->
    insertCell _selectedCellIndex + 1, cell

  appendCell = (cell) ->
    insertCell _cells().length, cell

  insertCellAbove = (type, input) ->
    insertAbove createCell type, input

  insertCellBelow = (type, input) ->
    insertBelow createCell type, input

  insertNewCellAbove = ->
    insertAbove createCell 'cs'

  insertNewCellBelow = ->
    insertBelow createCell 'cs'

  insertCellAboveAndRun = (type, input) ->
    cell = insertAbove createCell type, input
    cell.execute()
    cell

  insertCellBelowAndRun = (type, input) ->
    cell = insertBelow createCell type, input
    cell.execute()
    cell

  appendCellAndRun = (type, input) ->
    cell = appendCell createCell type, input 
    cell.execute()
    cell


  moveCellDown = ->
    cells = _cells()
    unless _selectedCellIndex is cells.length - 1
      splice _cells, _selectedCellIndex, 1
      _selectedCellIndex++
      splice _cells, _selectedCellIndex, 0, _selectedCell
    return

  moveCellUp = ->
    unless _selectedCellIndex is 0
      cells = _cells()
      splice _cells, _selectedCellIndex, 1
      _selectedCellIndex--
      splice _cells, _selectedCellIndex, 0, _selectedCell
    return

  mergeCellBelow = ->
    cells = _cells()
    unless _selectedCellIndex is cells.length - 1
      nextCell = cells[_selectedCellIndex + 1]
      if _selectedCell.type() is nextCell.type()
        nextCell.input _selectedCell.input() + '\n' + nextCell.input()
        removeCell()
    return

  splitCell = ->
    if _selectedCell.isActive()
      input = _selectedCell.input()
      if input.length > 1
        cursorPosition = _selectedCell.getCursorPosition()
        if 0 < cursorPosition < input.length - 1
          left = substr input, 0, cursorPosition
          right = substr input, cursorPosition
          _selectedCell.input left
          insertCell _selectedCellIndex + 1, createCell 'cs', right
          _selectedCell.isActive yes
    return

  pasteCellAbove = ->
    insertCell _selectedCellIndex, cloneCell _clipboardCell if _clipboardCell

  pasteCellBelow = ->
    insertCell _selectedCellIndex + 1, cloneCell _clipboardCell if _clipboardCell

  undoLastDelete = ->
    insertCell _selectedCellIndex + 1, _lastDeletedCell if _lastDeletedCell
    _lastDeletedCell = null

  runCell = ->
    _selectedCell.execute()
    no

  runCellAndInsertBelow = ->
    _selectedCell.execute -> insertNewCellBelow()
    no

  #TODO ipython has inconsistent behavior here. seems to be doing runCellAndInsertBelow if executed on the lowermost cell.
  runCellAndSelectBelow = ->
    _selectedCell.execute -> selectNextCell()
    no

  checkIfNameIsInUse = (name, go) ->
    _.requestObjectExists 'notebook', name, (error, exists) -> go exists

  storeNotebook = (localName, remoteName) ->
    _.requestPutObject 'notebook', localName, serialize(), (error) ->
      if error
        _.alert "Error saving notebook: #{error.message}"
      else
        _remoteName localName
        _localName localName
        if remoteName isnt localName # renamed document
          _.requestDeleteObject 'notebook', remoteName, (error) ->
            if error
              _.alert "Error deleting remote notebook [#{remoteName}]: #{error.message}"
            _.saved()
        else
          _.saved()

  saveNotebook = ->
    localName = Flow.Util.sanitizeName _localName()
    return _.alert 'Invalid notebook name.' if localName is ''

    remoteName = _remoteName()
    if remoteName # saved document
      storeNotebook localName, remoteName
    else # unsaved document
      checkIfNameIsInUse localName, (isNameInUse) ->
        if isNameInUse
          _.confirm "A notebook with that name already exists.\nDo you want to replace it with the one you're saving?", { acceptCaption: 'Replace', declineCaption: 'Cancel' }, (accept) ->
            if accept
              storeNotebook localName, remoteName
        else
          storeNotebook localName, remoteName
    return

  promptForNotebook = ->
    _.dialog Flow.FileOpenDialog, (result) ->
      if result
        { error, filename } = result
        if error
          _.growl error.message ? error
        else
          loadNotebook filename
          _.loaded()

  uploadFile = ->
    _.dialog Flow.FileUploadDialog, (result) ->
      if result
        { error } = result
        if error
          _.growl error.message ? error
        else
          _.growl 'File uploaded successfully!'
          _.insertAndExecuteCell 'cs', "setupParse source_frames: [ #{stringify result.result.destination_frame }]"

  toggleInput = ->
    _selectedCell.toggleInput()

  toggleOutput = ->
    _selectedCell.toggleOutput()

  toggleAllInputs = ->
    wereHidden = _areInputsHidden()
    _areInputsHidden not wereHidden
    #
    # If cells are generated while inputs are hidden, the input boxes
    #   do not resize to fit contents. So explicitly ask all cells 
    #   to resize themselves.
    #
    if wereHidden
      for cell in _cells()
        cell.autoResize()
    return

  toggleAllOutputs = ->
    _areOutputsHidden not _areOutputsHidden()

  toggleSidebar = ->
    _isSidebarHidden not _isSidebarHidden()

  showBrowser = ->
    _isSidebarHidden no
    _.showBrowser()

  showOutline = ->
    _isSidebarHidden no
    _.showOutline()

  showClipboard = ->
    _isSidebarHidden no
    _.showClipboard()

  selectNextCell = ->
    cells = _cells()
    unless _selectedCellIndex is cells.length - 1
      selectCell cells[_selectedCellIndex + 1]
    return no # prevent arrow keys from scrolling the page

  selectPreviousCell = ->
    unless _selectedCellIndex is 0
      cells = _cells()
      selectCell cells[_selectedCellIndex - 1]
    return no # prevent arrow keys from scrolling the page

  displayKeyboardShortcuts = ->
    $('#keyboardHelpDialog').modal()

  findBuildProperty = (caption) ->
    if Flow.BuildProperties
      if entry = (find Flow.BuildProperties, (entry) -> entry.caption is caption)
        entry.value
      else
        undefined
    else
      undefined

  displayDocumentation = ->
    gitBranch = findBuildProperty 'H2O Build git branch'
    projectVersion = findBuildProperty 'H2O Build project version' 
    buildVersion = if projectVersion then last projectVersion.split '.' else undefined

    if buildVersion and buildVersion isnt '99999'
      window.open "http://h2o-release.s3.amazonaws.com/h2o/#{gitBranch}/#{buildVersion}/docs-website/h2o-docs/index.html", '_blank'
    else
      gitHash = (findBuildProperty 'H2O Build git hash') or 'master'
      window.open "https://github.com/h2oai/h2o-dev/blob/#{gitHash}/h2o-docs/src/product/flow/README.md", '_blank'

  executeCommand = (command) -> ->
    _.insertAndExecuteCell 'cs', command

  displayAbout = ->
    $('#aboutDialog').modal()

  shutdown = ->
    _.requestShutdown (error, result) ->
      if error
        _.growl "Shutdown failed: #{error.message}", 'danger'
      else
        _.growl 'Shutdown complete!', 'warning'


  showHelp = ->
    _isSidebarHidden no
    _.showHelp()

  createNotebook = ->
    _.confirm 'This action will replace your active notebook.\nAre you sure you want to continue?', { acceptCaption: 'Create New Notebook', declineCaption: 'Cancel' }, (accept) ->
      currentTime = (new Date()).getTime()
      deserialize 'Untitled Flow', null,
        cells: [
          type: 'cs'
          input: ''
        ]

  duplicateNotebook = ->
    deserialize "Copy of #{_localName()}", null, serialize()

  openNotebook = (name, doc) ->
    deserialize name, null, doc

  loadNotebook = (name) ->
    _.requestObject 'notebook', name, (error, doc) ->
      if error
        _.alert error.message ? error
      else
        deserialize name, name, doc

  exportNotebook = ->
    if remoteName = _remoteName()
      window.open "/3/NodePersistentStorage.bin/notebook/#{remoteName}", '_blank'
    else
      _.alert "Please save this notebook before exporting."

  goToUrl = (url) -> ->
    window.open url, '_blank'

  executeAllCells = (fromBeginning, go) ->
    _isRunningAll yes

    cells = slice _cells(), 0
    cellCount = cells.length
    cellIndex = 0<commit_msg>PUBDEV-1198 Cancel File -> New operation when cancelled (D'oh)<commit_after>Flow.Renderers = (_, _sandbox) ->
  h1: -> Flow.Heading _, 'h1'
  h2: -> Flow.Heading _, 'h2'
  h3: -> Flow.Heading _, 'h3'
  h4: -> Flow.Heading _, 'h4'
  h5: -> Flow.Heading _, 'h5'
  h6: -> Flow.Heading _, 'h6'
  md: -> Flow.Markdown _
  cs: (guid) -> Flow.Coffeescript _, guid, _sandbox
  raw: -> Flow.Raw _

Flow.Notebook = (_, _renderers) ->
  _localName = signal 'Untitled Flow'
  _remoteName = signal null

  _isEditingName = signal no
  editName = -> _isEditingName yes
  saveName = -> _isEditingName no

  _cells = signals []
  _selectedCell = null
  _selectedCellIndex = -1
  _clipboardCell = null
  _lastDeletedCell = null
  _areInputsHidden = signal no
  _areOutputsHidden = signal no
  _isSidebarHidden = signal no
  _isRunningAll = signal no
  _runningCaption = signal 'Running'
  _runningPercent = signal '0%'
  _runningCellInput = signal ''
  _status = Flow.Status _
  _sidebar = Flow.Sidebar _, _cells
  _about = Flow.About _
  _dialogs = Flow.Dialogs _

  serialize = ->
    cells = for cell in _cells()
      type: cell.type()
      input: cell.input()

    version: '1.0.0'
    cells: cells

  deserialize = (localName, remoteName, doc) ->
    _localName localName
    _remoteName remoteName

    cells = for cell in doc.cells
      createCell cell.type, cell.input
    _cells cells

    selectCell head cells

    # Execute all non-code cells (headings, markdown, etc.)
    for cell in _cells()
      cell.execute() unless cell.isCode() 

    return

  createCell = (type='cs', input='') ->
    Flow.Cell _, _renderers, type, input

  checkConsistency = ->
    selectionCount = 0
    for cell, i in _cells()
      unless cell
        error "index #{i} is empty"
      else
        if cell.isSelected()
          selectionCount++
    error "selected cell count = #{selectionCount}" if selectionCount isnt 1
    return

  selectCell = (target, scrollIntoView=yes, scrollImmediately=no) ->
    return if _selectedCell is target
    _selectedCell.isSelected no if _selectedCell
    _selectedCell = target
    #TODO also set focus so that tabs don't jump to the first cell
    _selectedCell.isSelected yes
    _selectedCellIndex = _cells.indexOf _selectedCell
    checkConsistency()
    if scrollIntoView
      defer ->
        _selectedCell.scrollIntoView scrollImmediately
    _selectedCell

  cloneCell = (cell) ->
    createCell cell.type(), cell.input()

  switchToCommandMode = ->
    _selectedCell.isActive no

  switchToEditMode = ->
    _selectedCell.isActive yes
    no

  convertCellToCode = ->
    _selectedCell.type 'cs'

  convertCellToHeading = (level) -> -> 
    _selectedCell.type "h#{level}"
    _selectedCell.execute()

  convertCellToMarkdown = ->
    _selectedCell.type 'md'
    _selectedCell.execute()

  convertCellToRaw = ->
    _selectedCell.type 'raw'
    _selectedCell.execute()

  copyCell = ->
    _clipboardCell = _selectedCell

  cutCell = ->
    copyCell()
    removeCell()

  deleteCell = ->
    _lastDeletedCell = _selectedCell
    removeCell()

  removeCell = ->
    cells = _cells()
    if cells.length > 1
      if _selectedCellIndex is cells.length - 1
        #TODO call dispose() on this cell
        removedCell = head splice _cells, _selectedCellIndex, 1
        selectCell cells[_selectedCellIndex - 1]
      else
        #TODO call dispose() on this cell
        removedCell = head splice _cells, _selectedCellIndex, 1
        selectCell cells[_selectedCellIndex]
      _.saveClip 'trash', removedCell.type(), removedCell.input() if removedCell
    return
    
  insertCell = (index, cell) ->
    splice _cells, index, 0, cell
    selectCell cell
    cell

  insertAbove = (cell) ->
    insertCell _selectedCellIndex, cell

  insertBelow = (cell) ->
    insertCell _selectedCellIndex + 1, cell

  appendCell = (cell) ->
    insertCell _cells().length, cell

  insertCellAbove = (type, input) ->
    insertAbove createCell type, input

  insertCellBelow = (type, input) ->
    insertBelow createCell type, input

  insertNewCellAbove = ->
    insertAbove createCell 'cs'

  insertNewCellBelow = ->
    insertBelow createCell 'cs'

  insertCellAboveAndRun = (type, input) ->
    cell = insertAbove createCell type, input
    cell.execute()
    cell

  insertCellBelowAndRun = (type, input) ->
    cell = insertBelow createCell type, input
    cell.execute()
    cell

  appendCellAndRun = (type, input) ->
    cell = appendCell createCell type, input 
    cell.execute()
    cell


  moveCellDown = ->
    cells = _cells()
    unless _selectedCellIndex is cells.length - 1
      splice _cells, _selectedCellIndex, 1
      _selectedCellIndex++
      splice _cells, _selectedCellIndex, 0, _selectedCell
    return

  moveCellUp = ->
    unless _selectedCellIndex is 0
      cells = _cells()
      splice _cells, _selectedCellIndex, 1
      _selectedCellIndex--
      splice _cells, _selectedCellIndex, 0, _selectedCell
    return

  mergeCellBelow = ->
    cells = _cells()
    unless _selectedCellIndex is cells.length - 1
      nextCell = cells[_selectedCellIndex + 1]
      if _selectedCell.type() is nextCell.type()
        nextCell.input _selectedCell.input() + '\n' + nextCell.input()
        removeCell()
    return

  splitCell = ->
    if _selectedCell.isActive()
      input = _selectedCell.input()
      if input.length > 1
        cursorPosition = _selectedCell.getCursorPosition()
        if 0 < cursorPosition < input.length - 1
          left = substr input, 0, cursorPosition
          right = substr input, cursorPosition
          _selectedCell.input left
          insertCell _selectedCellIndex + 1, createCell 'cs', right
          _selectedCell.isActive yes
    return

  pasteCellAbove = ->
    insertCell _selectedCellIndex, cloneCell _clipboardCell if _clipboardCell

  pasteCellBelow = ->
    insertCell _selectedCellIndex + 1, cloneCell _clipboardCell if _clipboardCell

  undoLastDelete = ->
    insertCell _selectedCellIndex + 1, _lastDeletedCell if _lastDeletedCell
    _lastDeletedCell = null

  runCell = ->
    _selectedCell.execute()
    no

  runCellAndInsertBelow = ->
    _selectedCell.execute -> insertNewCellBelow()
    no

  #TODO ipython has inconsistent behavior here. seems to be doing runCellAndInsertBelow if executed on the lowermost cell.
  runCellAndSelectBelow = ->
    _selectedCell.execute -> selectNextCell()
    no

  checkIfNameIsInUse = (name, go) ->
    _.requestObjectExists 'notebook', name, (error, exists) -> go exists

  storeNotebook = (localName, remoteName) ->
    _.requestPutObject 'notebook', localName, serialize(), (error) ->
      if error
        _.alert "Error saving notebook: #{error.message}"
      else
        _remoteName localName
        _localName localName
        if remoteName isnt localName # renamed document
          _.requestDeleteObject 'notebook', remoteName, (error) ->
            if error
              _.alert "Error deleting remote notebook [#{remoteName}]: #{error.message}"
            _.saved()
        else
          _.saved()

  saveNotebook = ->
    localName = Flow.Util.sanitizeName _localName()
    return _.alert 'Invalid notebook name.' if localName is ''

    remoteName = _remoteName()
    if remoteName # saved document
      storeNotebook localName, remoteName
    else # unsaved document
      checkIfNameIsInUse localName, (isNameInUse) ->
        if isNameInUse
          _.confirm "A notebook with that name already exists.\nDo you want to replace it with the one you're saving?", { acceptCaption: 'Replace', declineCaption: 'Cancel' }, (accept) ->
            if accept
              storeNotebook localName, remoteName
        else
          storeNotebook localName, remoteName
    return

  promptForNotebook = ->
    _.dialog Flow.FileOpenDialog, (result) ->
      if result
        { error, filename } = result
        if error
          _.growl error.message ? error
        else
          loadNotebook filename
          _.loaded()

  uploadFile = ->
    _.dialog Flow.FileUploadDialog, (result) ->
      if result
        { error } = result
        if error
          _.growl error.message ? error
        else
          _.growl 'File uploaded successfully!'
          _.insertAndExecuteCell 'cs', "setupParse source_frames: [ #{stringify result.result.destination_frame }]"

  toggleInput = ->
    _selectedCell.toggleInput()

  toggleOutput = ->
    _selectedCell.toggleOutput()

  toggleAllInputs = ->
    wereHidden = _areInputsHidden()
    _areInputsHidden not wereHidden
    #
    # If cells are generated while inputs are hidden, the input boxes
    #   do not resize to fit contents. So explicitly ask all cells 
    #   to resize themselves.
    #
    if wereHidden
      for cell in _cells()
        cell.autoResize()
    return

  toggleAllOutputs = ->
    _areOutputsHidden not _areOutputsHidden()

  toggleSidebar = ->
    _isSidebarHidden not _isSidebarHidden()

  showBrowser = ->
    _isSidebarHidden no
    _.showBrowser()

  showOutline = ->
    _isSidebarHidden no
    _.showOutline()

  showClipboard = ->
    _isSidebarHidden no
    _.showClipboard()

  selectNextCell = ->
    cells = _cells()
    unless _selectedCellIndex is cells.length - 1
      selectCell cells[_selectedCellIndex + 1]
    return no # prevent arrow keys from scrolling the page

  selectPreviousCell = ->
    unless _selectedCellIndex is 0
      cells = _cells()
      selectCell cells[_selectedCellIndex - 1]
    return no # prevent arrow keys from scrolling the page

  displayKeyboardShortcuts = ->
    $('#keyboardHelpDialog').modal()

  findBuildProperty = (caption) ->
    if Flow.BuildProperties
      if entry = (find Flow.BuildProperties, (entry) -> entry.caption is caption)
        entry.value
      else
        undefined
    else
      undefined

  displayDocumentation = ->
    gitBranch = findBuildProperty 'H2O Build git branch'
    projectVersion = findBuildProperty 'H2O Build project version' 
    buildVersion = if projectVersion then last projectVersion.split '.' else undefined

    if buildVersion and buildVersion isnt '99999'
      window.open "http://h2o-release.s3.amazonaws.com/h2o/#{gitBranch}/#{buildVersion}/docs-website/h2o-docs/index.html", '_blank'
    else
      gitHash = (findBuildProperty 'H2O Build git hash') or 'master'
      window.open "https://github.com/h2oai/h2o-dev/blob/#{gitHash}/h2o-docs/src/product/flow/README.md", '_blank'

  executeCommand = (command) -> ->
    _.insertAndExecuteCell 'cs', command

  displayAbout = ->
    $('#aboutDialog').modal()

  shutdown = ->
    _.requestShutdown (error, result) ->
      if error
        _.growl "Shutdown failed: #{error.message}", 'danger'
      else
        _.growl 'Shutdown complete!', 'warning'


  showHelp = ->
    _isSidebarHidden no
    _.showHelp()

  createNotebook = ->
    _.confirm 'This action will replace your active notebook.\nAre you sure you want to continue?', { acceptCaption: 'Create New Notebook', declineCaption: 'Cancel' }, (accept) ->
      if accept
        currentTime = (new Date()).getTime()
        deserialize 'Untitled Flow', null,
          cells: [
            type: 'cs'
            input: ''
          ]

  duplicateNotebook = ->
    deserialize "Copy of #{_localName()}", null, serialize()

  openNotebook = (name, doc) ->
    deserialize name, null, doc

  loadNotebook = (name) ->
    _.requestObject 'notebook', name, (error, doc) ->
      if error
        _.alert error.message ? error
      else
        deserialize name, name, doc

  exportNotebook = ->
    if remoteName = _remoteName()
      window.open "/3/NodePersistentStorage.bin/notebook/#{remoteName}", '_blank'
    else
      _.alert "Please save this notebook before exporting."

  goToUrl = (url) -> ->
    window.open url, '_blank'

  executeAllCells = (fromBeginning, go) ->
    _isRunningAll yes

    cells = slice _cells(), 0
    cellCount = cells.length
    cellIndex = 0
{'fraction_non_alphanumeric': 0.0936120105975861, 'fraction_numerical': 0.005467008705160015, 'mean_word_length': 2.7231877250665413, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 200, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4502862', 'n_tokens_mistral': 7217, 'n_tokens_neox': 6881, 'n_words': 1938}