<commit_before>module OntologyVersion::Files
  extend ActiveSupport::Concern
  
  included do
    # virtual attribute for upload
    attr_accessible :raw_file
    before_create :commit_raw_file, unless: :commit_oid?
  end

  def raw_file=(value)
    @raw_file = value
  end

  def commit_raw_file
    return true if ontology.parent
    raise "raw file missing" unless @raw_file
    ontology.path = @raw_file.original_filename if ontology.path.nil?
    # otherwise the file upload is broken (no implicit conversion of ActionDispatch::Http::UploadedFile into String):
    tmp_file = if @raw_file.class == ActionDispatch::Http::UploadedFile 
        @raw_file.tempfile
      else
        @raw_file
      end
    repository.save_file(tmp_file, ontology.path, "message", user)
  end

  def tmp_dir
    Rails.root.join("tmp","commits",commit_oid)
  end

  # path to the raw file
  def raw_path
    tmp_dir.join("raw",ontology.path)
  end

  # path to the raw file, checks out the raw file if is missing
  def raw_path!
    checkout_raw!
    raw_path
  end

  # path to xml file (hets output)
  def xml_path
    tmp_dir.join("xml", ontology.path)
  end

  # path to xml file (hets output) with code positions
  def code_reference_path
    tmp_dir.join("xml",
                 ontology.path.sub(ontology.file_extension, '.pp.xml'))
  end

  def xml_file?
    File.exists? xml_path
  end

  def raw_file?
    File.exists? raw_path
  end

  # checks out the raw file
  def checkout_raw!
    unless raw_file?
      FileUtils.mkdir_p raw_path.dirname
      File.open(raw_path, "w"){|f| f.write raw_data }
    end
  end

  # returns the raw data directly from the repository
  def raw_data
    repository.read_file(ontology.path, commit_oid)[:content]
  end
  
end<commit_msg>set encoding to UTF-8 before sending file to hets<commit_after>module OntologyVersion::Files
  extend ActiveSupport::Concern
  
  included do
    # virtual attribute for upload
    attr_accessible :raw_file
    before_create :commit_raw_file, unless: :commit_oid?
  end

  def raw_file=(value)
    @raw_file = value
  end

  def commit_raw_file
    return true if ontology.parent
    raise "raw file missing" unless @raw_file
    ontology.path = @raw_file.original_filename if ontology.path.nil?
    # otherwise the file upload is broken (no implicit conversion of ActionDispatch::Http::UploadedFile into String):
    tmp_file = if @raw_file.class == ActionDispatch::Http::UploadedFile 
        @raw_file.tempfile
      else
        @raw_file
      end
    repository.save_file(tmp_file, ontology.path, "message", user)
  end

  def tmp_dir
    Rails.root.join("tmp","commits",commit_oid)
  end

  # path to the raw file
  def raw_path
    tmp_dir.join("raw",ontology.path)
  end

  # path to the raw file, checks out the raw file if is missing
  def raw_path!
    checkout_raw!
    raw_path
  end

  # path to xml file (hets output)
  def xml_path
    tmp_dir.join("xml", ontology.path)
  end

  # path to xml file (hets output) with code positions
  def code_reference_path
    tmp_dir.join("xml",
                 ontology.path.sub(ontology.file_extension, '.pp.xml'))
  end

  def xml_file?
    File.exists? xml_path
  end

  def raw_file?
    File.exists? raw_path
  end

  # checks out the raw file
  def checkout_raw!
    unless raw_file?
      FileUtils.mkdir_p raw_path.dirname
      File.open(raw_path, "w"){|f| f.write raw_data }
    end
  end

  # returns the raw data directly from the repository
  def raw_data
    repository.read_file(ontology.path, commit_oid)[:content].force_encoding("UTF-8")
  end
  
end
{'fraction_non_alphanumeric': 0.08059118795315115, 'fraction_numerical': 0.0005577244841048522, 'mean_word_length': 3.1709302325581397, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '956822', 'n_tokens_mistral': 1193, 'n_tokens_neox': 1149, 'n_words': 331}