<commit_before>class ExternalRepository

  class << self

    def repository
      Repository.where(name: Settings.external_repository_name).first_or_create
    end

    def add_to_repository(iri, message, user)
      tmp_path = download_iri(iri)
      repository.save_file_only(tmp_path, determine_path(iri, :fullpath),
                                message, user)
    end

    def determine_iri(external_iri)
      "http://#{Settings.hostname}/#{repository.path}/#{determine_path(external_iri, :fullpath)}"
    end

    def determine_path(external_iri, symbol)
      case symbol
      when :fullpath then determine_filepath(external_iri)
      when :dirpath then determine_filepath(external_iri, false)
      when :basepath then determine_basepath(external_iri, false)
      when :basename then determine_basename(external_iri, false)
      when :extension then determine_extension(external_iri)
      else nil
      end
    end

    private
    def determine_filepath(external_iri, with_file=true)
      fullpath = iri_split(external_iri)
      with_file ? fullpath : fullpath.sub(determine_basename(external_iri), '')
    end

    def determine_basepath(external_iri, with_extension=true)
      basepath = iri_split(external_iri)
      with_extension ? basepath : basepath.sub(/\.[^\.]+\z/, '')
    end

    def determine_basename(external_iri, with_extension=true)
      File.basename(determine_basepath(external_iri, with_extension))
    end

    def determine_extension(external_iri)
      determine_basepath(external_iri) =~ /(\.[^\.]+)\z/
      $1
    end

    # split iri into wget -r style
    def iri_split(iri)
      match = URI::regexp(['http','https']).match(iri)
      if match
        host = match[4]
        path = match[7]
        File.join(host, path)
      else
        iri
      end
    end

    def download_iri(external_iri)
      file_content = Net::HTTP.get(URI.parse(external_iri))
      dir = Pathname.new('/tmp/reference_ontologies/').
        join(determine_path(external_iri, :dirpath))
      begin
        dir.mkpath<commit_msg>use File.extname to determine extension.<commit_after>class ExternalRepository

  class << self

    def repository
      Repository.where(name: Settings.external_repository_name).first_or_create
    end

    def add_to_repository(iri, message, user)
      tmp_path = download_iri(iri)
      repository.save_file_only(tmp_path, determine_path(iri, :fullpath),
                                message, user)
    end

    def determine_iri(external_iri)
      "http://#{Settings.hostname}/#{repository.path}/#{determine_path(external_iri, :fullpath)}"
    end

    def determine_path(external_iri, symbol)
      case symbol
      when :fullpath then determine_filepath(external_iri)
      when :dirpath then determine_filepath(external_iri, false)
      when :basepath then determine_basepath(external_iri, false)
      when :basename then determine_basename(external_iri, false)
      when :extension then determine_extension(external_iri)
      else nil
      end
    end

    private
    def determine_filepath(external_iri, with_file=true)
      fullpath = iri_split(external_iri)
      with_file ? fullpath : fullpath.sub(determine_basename(external_iri), '')
    end

    def determine_basepath(external_iri, with_extension=true)
      basepath = iri_split(external_iri)
      with_extension ? basepath : basepath.sub(determine_extension(external_iri), '')
    end

    def determine_basename(external_iri, with_extension=true)
      File.basename(determine_basepath(external_iri, with_extension))
    end

    def determine_extension(external_iri)
      File.extname(determine_basepath(external_iri))
    end

    # split iri into wget -r style
    def iri_split(iri)
      match = URI::regexp(['http','https']).match(iri)
      if match
        host = match[4]
        path = match[7]
        File.join(host, path)
      else
        iri
      end
    end

    def download_iri(external_iri)
      file_content = Net::HTTP.get(URI.parse(external_iri))
      dir = Pathname.new('/tmp/reference_ontologies/').
        join(determine_path(external_iri, :dirpath))
      begin
        dir.mkpath
{'fraction_non_alphanumeric': 0.09281077182014907, 'fraction_numerical': 0.0012022120702091848, 'mean_word_length': 3.155844155844156, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1176265', 'n_tokens_mistral': 1267, 'n_tokens_neox': 1245, 'n_words': 243}