<commit_before># -*- coding: utf-8 -*-
require 'yaml'
require 'pp'

class OrgToYaml
  attr_accessor :help_cont

  def initialize(file)
    @help_cont = {} #{ head: [File.basename(file, '.org')] }
    @head_sym = nil
    @conts = ''
    org_to_yaml(File.readlines(file))
  end

  def make_options(line)
    head, desc = line.split(':')
    desc ||= head
    { short: "-#{head[0]}", long: "--#{head}", desc: desc.to_s }
  end

  def next_cont(head)
    @help_cont[@head_sym][:cont] = @conts if @head_sym
    return if head == 'EOF'
    @conts = ''
    @head_sym = head.to_sym
    @help_cont[@head_sym] = {
      opts: make_options(head), title: head, cont: ''
    }
  end

  def org_to_yaml(lines)
    lines.each do |line|
      if m = line.match(/^\* (.+)/)
        next_cont m[1]
      else
        @conts << line
      end
    end
    next_cont 'EOF'
  end
end

if __FILE__ == $1
  helps = OrgToYaml.new(ARGV[0])
  pp helps.help_cont
end
<commit_msg>rev some<commit_after># -*- coding: utf-8 -*-
require 'yaml'
require 'pp'

class OrgToYaml
  attr_accessor :help_cont

  def initialize(file)
    @help_cont = {} #{ head: [File.basename(file, '.org')] }
    @head_sym = nil
    @conts = ''
    org_to_yaml(File.readlines(file))
  end

  def make_options(line)
    head, desc = line.split(':')
    desc ||= head.to_s
    short = "-#{head[0]}"
    { short: short, long: "--#{head}", desc: desc }
  end

  def next_cont(head)
    @help_cont[@head_sym][:cont] = @conts if @head_sym
    return if head == 'EOF'
    @conts = ''
    @head_sym = head.to_sym
    @help_cont[@head_sym] = {
      opts: make_options(head), title: head, cont: ''
    }
  end

  def org_to_yaml(lines)
    lines.each do |line|
      if m = line.match(/^\* (.+)/)
        next_cont m[1]
      else
        @conts << line
      end
    end
    next_cont 'EOF'
  end
end

if __FILE__ == $1
  helps = OrgToYaml.new(ARGV[0])
  pp helps.help_cont
end

{'fraction_non_alphanumeric': 0.16449086161879894, 'fraction_numerical': 0.005221932114882507, 'mean_word_length': 2.5679702048417132, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5593372', 'n_tokens_mistral': 758, 'n_tokens_neox': 753, 'n_words': 169}