<commit_before>
DokGen do(
  Collected = [{},{},{},{}] mimic do(
    from = method(files, kinds, cells, specs {},
      newObj = self mimic
      newObj[0] = files
      newObj[1] = kinds
      newObj[2] = cells
      newObj[3] = specs
      newObj)

    collectedFiles = method([0])
    collectedKinds = method([1])
    collectedCells = method([2])
    collectedSpecs = method([3])
  )

  document = method(
    "Takes a list of command line arguments, parses these and then builds up the documentation about all data in the system",
    arguments,

    outputDir = "dok"

    combineWithSpecs = true
    specsPattern = "test/**/*_spec.ik"

    collected = Collected from({}, {"Ground" => Ground}, {})

    collect(Ground, collected)

    if(combineWithSpecs,
      collectSpecs(specsPattern, collected collectedSpecs, collected)
    )

    generate(outputDir, collected)
  )
)<commit_msg>Added Dokgen OptionParser<commit_after>use("iopt")

DokGen do(
  Collected = [{},{},{},{}] mimic do(
    from = method(files, kinds, cells, specs {},
      newObj = self mimic
      newObj[0] = files
      newObj[1] = kinds
      newObj[2] = cells
      newObj[3] = specs
      newObj)

    collectedFiles = method([0])
    collectedKinds = method([1])
    collectedCells = method([2])
    collectedSpecs = method([3])
  )

  OptionParser = IOpt mimic do(
    banner = "Usage: dokgen [options]"
    
    combineWithSpecs? = true
    outputDir = "dok"
    specsPattern = "test/**/*_spec.ik"

    ["-o", "--output-dir"] = method("Output directory", dir outputDir, 
      @outputDir = dir)

    ["--[no-]combine-specs"] = method("Combine with specs?", v, 
      @combineWithSpecs? = v)
    
    ["--spec"] = method("Specs pattern", glob specsPattern,
      @combineWithSpecs? = true
      @specsPattern = glob)
    
    ["-h", "--help"] = method("Display usage", @println. System exit)
    ["-h"] priority = -10
  )

  document = method(
    "Takes a list of command line arguments, parses these and then builds up the documentation about all data in the system",
    arguments,

    opt = OptionParser mimic
    opt parse(arguments)
    
    collected = Collected from({}, {"Ground" => Ground}, {})

    collect(Ground, collected)

    if(opt combineWithSpecs?,
      collectSpecs(opt specsPattern, collected collectedSpecs, collected))

    generate(opt outputDir, collected)
  )
)
{'fraction_non_alphanumeric': 0.13755274261603376, 'fraction_numerical': 0.007594936708860759, 'mean_word_length': 2.958263772954925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5184687', 'n_tokens_mistral': 771, 'n_tokens_neox': 744, 'n_words': 202}