<commit_before>#!/usr/bin/env ruby
# encoding utf-8

require './toc_tracker.rb'
require 'tempfile'

TOC_INDENT = 4

def parse_options(args)
  if ARGV.count == 0
    raise ArgumentError.new("no input file supplied")
  end

  options = {infile: ARGV[0]}

  if ARGV.count == 2
    options[:outfile] = ARGV[1]
  end

  options
end

def process_file(file_path)
  content = File.read(file_path)

  content = number_chapters(content)
  content = write_toc(content)

  content
end

def number_chapters(content)
  node_index = 0
  character_heading_regexp = /^(#+)\s*((?:\d\.?)*)\s*(.+)$/

  content = content.gsub(character_heading_regexp) do
    marker =  Regexp.last_match[1]
    # skip match[2]: this is the "old" numbering
    title = Regexp.last_match[3]

    depth = marker.length # 1-based to account for root node

    new_node = @toc_tracker.add_node(depth, title)
    title = numbered_title(new_node)
    anchor_link = "<a name=\"md-toc_#{node_index}\"></a>"
    node_index += 1

    "#{marker} #{anchor_link}#{title}"
  end

  content
end

def write_toc(content)
  nodes = @toc_tracker.get_flat_node_list[1..-1]
  toc = nodes.collect do |node|
    title = numbered_title(node)

    node_index = nodes.index(node)
    link = "#md-toc_#{node_index}"

    indentation = '&nbsp;' * (TOC_INDENT * (node.depth - 1))
    "#{indentation}[#{title}](#{link})"<commit_msg>included already generated anchors into regexp<commit_after>#!/usr/bin/env ruby
# encoding utf-8

require './toc_tracker.rb'
require 'tempfile'

TOC_INDENT = 4

def parse_options(args)
  if ARGV.count == 0
    raise ArgumentError.new("no input file supplied")
  end

  options = {infile: ARGV[0]}

  if ARGV.count == 2
    options[:outfile] = ARGV[1]
  end

  options
end

def process_file(file_path)
  content = File.read(file_path)

  content = number_chapters(content)
  content = write_toc(content)

  content
end

def number_chapters(content)
  node_index = 0
  character_heading_regexp = /
    ^(\#+)\s*                         # heading tag
    (?:<a\sname="md-toc_\d+"><\/a>)?  # already generated anchor
    ((?:\d\.?)*)\s*                   # heading number
    (.+)$                             # heading content
  /x

  content = content.gsub(character_heading_regexp) do
    marker =  Regexp.last_match[1]
    # skip match[2]: this is the "old" numbering
    title = Regexp.last_match[3]

    depth = marker.length # 1-based to account for root node

    new_node = @toc_tracker.add_node(depth, title)
    title = numbered_title(new_node)
    anchor_link = "<a name=\"md-toc_#{node_index}\"></a>"
    node_index += 1

    "#{marker} #{anchor_link}#{title}"
  end

  content
end

def write_toc(content)
  nodes = @toc_tracker.get_flat_node_list[1..-1]
  toc = nodes.collect do |node|
    title = numbered_title(node)

    node_index = nodes.index(node)
    link = "#md-toc_#{node_index}"

    indentation = '&nbsp;' * (TOC_INDENT * (node.depth - 1))
    "#{indentation}[#{title}](#{link})"
{'fraction_non_alphanumeric': 0.14536256323777402, 'fraction_numerical': 0.01011804384485666, 'mean_word_length': 3.255380200860832, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3167188', 'n_tokens_mistral': 1137, 'n_tokens_neox': 1064, 'n_words': 226}