<commit_before>module PROIEL
  module Converter
    # This converts to the CoNLL-X format as described on http://ilk.uvt.nl/conll/#dataformat.
    class CoNLLX
      class << self
        def process(tb, options)
          tb.sources.each do |source|
            source.divs.each do |div|
              div.sentences.each do |sentence|
                id_to_number = {}

# probably an error here with prodroptokens, see conll-u script
                
                sentence.tokens.map(&:id).each_with_index.each do |id, i|
                  id_to_number[id] = i + 1
                end

                id_to_token = {}

                sentence.tokens.each do |token|
                  id_to_token[token.id] = token
                end

                sentence.tokens.each do |token|
                  unless token.is_empty?
                    this_number = id_to_number[token.id]
                    head_number, relation = find_lexical_head_and_relation(id_to_number, id_to_token, token)
                    form = token.form.gsub(' ', '')
                    lemma = token.lemma.gsub(' ', '')
                    pos_major = token.part_of_speech_hash[:major]
                    pos_full = token.part_of_speech
                    morphology = format_morphology(token)

                    puts [this_number, form, lemma, pos_major, pos_full,
                          morphology, head_number, relation, "_", "_"].join("\t")
                  end
                end

                puts
              end
            end
          end
        end

        def format_morphology(token)
          token.morphology_hash.map do |k, v|
            # Remove inflection tag unless when set to inflecting
            if k == :inflection and v =='i'
              nil
            else
              "#{k.upcase[0..3]}#{v}"
            end
          end.compact.join('|')
        end
<commit_msg>Fixed regression in conll-x.rb after new numbering of prodrop tokens in proiel-xml<commit_after>module PROIEL
  module Converter
    # This converts to the CoNLL-X format as described on http://ilk.uvt.nl/conll/#dataformat.
    class CoNLLX
      class << self
        def process(tb, options)
          tb.sources.each do |source|
            source.divs.each do |div|
              div.sentences.each do |sentence|
                id_to_number = {}

                # Do not care about prodrop tokens
                tk = sentence.tokens.reject { |t| t.empty_token_sort == 'P' }
                
                # Renumber to make the sequence continguous after prodrop tokens where left out
                tk.map(&:id).each_with_index.each do |id, i|
                  id_to_number[id] = i + 1
                end

                id_to_token = tk.inject({}) { |h, t| h.merge({t.id => t}) }

                tk.each do |token|
                  unless token.is_empty?
                    this_number = id_to_number[token.id]
                    head_number, relation = find_lexical_head_and_relation(id_to_number, id_to_token, token)
                    form = token.form.gsub(/[[:space:]]/, '.')
                    lemma = token.lemma.gsub(/[[:space:]]/, '.')
                    pos_major = token.part_of_speech_hash[:major]
                    pos_full = token.part_of_speech
                    morphology = format_morphology(token)

                    puts [this_number, form, lemma, pos_major, pos_full,
                          morphology, head_number, relation, "_", "_"].join("\t")
                  end
                end

                puts
              end
            end
          end
        end

        def format_morphology(token)
          token.morphology_hash.map do |k, v|
            # Remove inflection tag unless when set to inflecting
            if k == :inflection and v =='i'
              nil
            else
              "#{k.upcase[0..3]}#{v}"
            end
          end.compact.join('|')
        end

{'fraction_non_alphanumeric': 0.09224531170805879, 'fraction_numerical': 0.0015205271160669033, 'mean_word_length': 1.3368857312018947, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 1}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4608976', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1088, 'n_words': 264}