<commit_before>require_relative 'parse'
require 'active_support/core_ext/hash/slice'

module Precedent
  # Convert a list of block-level element hashes into a hash
  # representing the structure of the document as a whole.
  #
  # This format is more useful for certain output formatters, especially
  # those that need to generate footnotes inline (e.g. LaTeX) rather
  # than at the end of output (e.g. HTML).
  def self.load(source)
    blank = { :meta => {}, :content => [], :footnotes => {} }
    injected = inject_footnotes(parse(source).reduce(blank) do |mem, e|
      case e[:type]
      when :meta
        mem.merge({ :meta => mem[:meta].merge(e[:content]) })
      when :footnote
        note_hash = { e[:marker] => e[:content] }
        mem.merge({ :footnotes => mem[:footnotes].merge(note_hash) })
      else
        mem.merge({ :content => mem[:content] + [e] })
      end
    end)
    injected.merge(content: numbered(injected[:content], 1).first)
  end

  NUMBERED_TYPES = [:indented, :flush, :raggedleft]

  def self.numbered(node, start=1)
    if node.is_a?(Array)
      mapped = node.map { |n| ret, start = numbered(n, start) ; ret }
      [mapped, start]
    elsif node.is_a?(Hash) && node[:content]<commit_msg>fixed: number ragged_left, not raggedleft<commit_after>require_relative 'parse'
require 'active_support/core_ext/hash/slice'

module Precedent
  # Convert a list of block-level element hashes into a hash
  # representing the structure of the document as a whole.
  #
  # This format is more useful for certain output formatters, especially
  # those that need to generate footnotes inline (e.g. LaTeX) rather
  # than at the end of output (e.g. HTML).
  def self.load(source)
    blank = { :meta => {}, :content => [], :footnotes => {} }
    injected = inject_footnotes(parse(source).reduce(blank) do |mem, e|
      case e[:type]
      when :meta
        mem.merge({ :meta => mem[:meta].merge(e[:content]) })
      when :footnote
        note_hash = { e[:marker] => e[:content] }
        mem.merge({ :footnotes => mem[:footnotes].merge(note_hash) })
      else
        mem.merge({ :content => mem[:content] + [e] })
      end
    end)
    injected.merge(content: numbered(injected[:content], 1).first)
  end

  NUMBERED_TYPES = [:indented, :flush, :ragged_left]

  def self.numbered(node, start=1)
    if node.is_a?(Array)
      mapped = node.map { |n| ret, start = numbered(n, start) ; ret }
      [mapped, start]
    elsif node.is_a?(Hash) && node[:content]
{'fraction_non_alphanumeric': 0.14142225793491361, 'fraction_numerical': 0.0016070711128967456, 'mean_word_length': 3.2710120068610635, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2603186', 'n_tokens_mistral': 812, 'n_tokens_neox': 799, 'n_words': 227}