<commit_before>require 'ref_parsers'

class RefParsers::RISParser
  def initialize
    @type_key = "TY"
    @types = %w(ABST ADVS ART BILL BOOK CASE CHAP COMP CONF CTLG DATA ELEC GEN HEAR ICOMM INPR JFULL JOUR MAP MGZN MPCT MUSIC NEWS PAMP PAT PCOMM RPRT SER SLIDE SOUND STAT THES UNBILl UNPB VIDEO WEB)
    @terminator_key = "ER"
    @line_regex = /^([A-Z][A-Z0-9])  -( (.*))?$/
    @key_regex_order = 1
    @value_regex_order = 3
    @regex_match_length = 4
    @key_value_separator = "  - "
    @file_type = "RIS"
  end
end

class RefParsers::EndNoteParser
  def initialize
    @type_key = "0"
    @types = ["Generic", "Artwork", "Audiovisual Material", "Bill", "Book", "Book Section", "Case", "Chart or Table", "Classical Work", "Computer Program", "Conference Paper", "Conference Proceedings", "Edited Book", "Equation", "Electronic Article", "Electronic Book", "Electronic Source", "Figure", "Film or Broadcast", "Government Document", "Hearing", "Journal Article", "Legal Rule/Regulation", "Magazine Article", "Manuscript", "Map", "Newspaper Article", "Online Database", "Online Multimedia", "Patent", "Personal Communication", "Report", "Statute", "Thesis", "Unpublished Work", "Unused 1", "Unused 2", "Unused 3"]
    @terminator_key = nil
    @line_regex = /^%([A-NP-Z0-9\?\@\!\#\$\]\&\(\)\*\+\^\>\<\[\=\~])\s+(.*)$/
    @key_regex_order = 1
    @value_regex_order = 2
    @regex_match_length = 3
    @key_value_separator = " "
    @file_type = "EndNote"
  end
end

class RefParsers::LineParser
  def open(filename)
    # Remove the initial hidden characters
    body = File.open(filename, "r:bom|utf-8").read().strip
    # Add keys to multivalued fields separated by newlines
    last_key = ""
    File.open(filename, 'w') do |new_file| 
      body.lines.each do |line|
        if !line.blank?
          m = line.match(@line_regex)
          if m && m.length == @regex_match_length
            last_key = m[@key_regex_order]
            new_line = line
          else
            new_line = last_key + @key_value_separator + line
          end
        else
          new_line = line
        end
        new_file << new_line
      end
    end
    # Parse them into entries
    entries = parse(File.read(filename, encoding: 'UTF-8'))
    # Convert entries to reference objects
    references = []
    entries.each do |entry|
      references << RefParsers::Reference.new(entry, @file_type)
    end
    references
  end
end

class RefParsers::Reference
  attr_accessor :hash, :file_type

  def initialize(hash, file_type)
    @hash = hash
    @file_type = file_type
  end

  def addresses
    if @file_type == "RIS"<commit_msg>Fixed key prefix for endnote format.<commit_after>require 'ref_parsers'

class RefParsers::RISParser
  def initialize
    @type_key = "TY"
    @types = %w(ABST ADVS ART BILL BOOK CASE CHAP COMP CONF CTLG DATA ELEC GEN HEAR ICOMM INPR JFULL JOUR MAP MGZN MPCT MUSIC NEWS PAMP PAT PCOMM RPRT SER SLIDE SOUND STAT THES UNBILl UNPB VIDEO WEB)
    @terminator_key = "ER"
    @line_regex = /^([A-Z][A-Z0-9])  -( (.*))?$/
    @key_regex_order = 1
    @value_regex_order = 3
    @regex_match_length = 4
    @key_value_separator = "  - "
    @file_type = "RIS"
    @key_prefix = ""
  end
end

class RefParsers::EndNoteParser
  def initialize
    @type_key = "0"
    @types = ["Generic", "Artwork", "Audiovisual Material", "Bill", "Book", "Book Section", "Case", "Chart or Table", "Classical Work", "Computer Program", "Conference Paper", "Conference Proceedings", "Edited Book", "Equation", "Electronic Article", "Electronic Book", "Electronic Source", "Figure", "Film or Broadcast", "Government Document", "Hearing", "Journal Article", "Legal Rule/Regulation", "Magazine Article", "Manuscript", "Map", "Newspaper Article", "Online Database", "Online Multimedia", "Patent", "Personal Communication", "Report", "Statute", "Thesis", "Unpublished Work", "Unused 1", "Unused 2", "Unused 3"]
    @terminator_key = nil
    @line_regex = /^%([A-NP-Z0-9\?\@\!\#\$\]\&\(\)\*\+\^\>\<\[\=\~])\s+(.*)$/
    @key_regex_order = 1
    @value_regex_order = 2
    @regex_match_length = 3
    @key_value_separator = " "
    @file_type = "EndNote"
    @key_prefix = "%"
  end
end

class RefParsers::LineParser
  def open(filename)
    # Remove the initial hidden characters
    body = File.open(filename, "r:bom|utf-8").read().strip
    # Add keys to multivalued fields separated by newlines
    last_key = ""
    File.open(filename, 'w') do |new_file| 
      body.lines.each do |line|
        if !line.blank?
          m = line.match(@line_regex)
          if m && m.length == @regex_match_length
            last_key = m[@key_regex_order]
            new_line = line
          else
            new_line = @key_prefix + last_key + @key_value_separator + line
          end
        else
          new_line = line
        end
        new_file << new_line
      end
    end
    # Parse them into entries
    entries = parse(File.read(filename, encoding: 'UTF-8'))
    # Convert entries to reference objects
    references = []
    entries.each do |entry|
      references << RefParsers::Reference.new(entry, @file_type)
    end
    references
  end
end

class RefParsers::Reference
  attr_accessor :hash, :file_type

  def initialize(hash, file_type)
    @hash = hash
    @file_type = file_type
  end

  def addresses
    if @file_type == "RIS"
{'fraction_non_alphanumeric': 0.13612565445026178, 'fraction_numerical': 0.005983545250560957, 'mean_word_length': 3.1177829099307157, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7526964', 'n_tokens_mistral': 1859, 'n_tokens_neox': 1772, 'n_words': 491}