<commit_before>require 'fileutils'
require 'open-uri'

module PkgForge
  ##
  # Add source methods to Forge
  class Forge
    attr_writer :source

    Contract None => HashOf[Symbol => Any]
    def source
      @source ||= { type: 'git', path: 'upstream' }
    end

    private

    Contract None => nil
    def prepare_source!
      type_method = "#{source[:type]}_prepare_source"
      return send(type_method) if respond_to?(type_method, true)
      raise("Unknown source type: #{source[:type]}")
    end

    Contract None => nil
    def git_prepare_source
      run_local 'git submodule update --init --recursive'
      run_local "git clone --recursive '#{source[:path]}' #{tmpdir(:build)}"
    end

    Contract None => nil
    def tar_prepare_source
      dest_file = tmpfile(:source_tar)
      File.open(dest_file, 'wb') do |fh|
        open(source[:url], 'rb') { |request| fh.write request.read }
      end
      run "tar -xf #{dest_file} --strip-components=1"
    end

    Contract None => nil
    def empty_prepare_source
      # This source type is a no-op
    end
  end

  module DSL
    ##
    # Add source methods to Forge DSL
    class Forge
      Contract HashOf[Symbol => Any] => nil
      def source(params)
        @forge.source = params
        nil
      end
    end
  end<commit_msg>support checksumming for tar source, fixes #14<commit_after>require 'fileutils'
require 'open-uri'

module PkgForge
  ##
  # Add source methods to Forge
  class Forge
    attr_writer :source

    Contract None => HashOf[Symbol => Any]
    def source
      @source ||= { type: 'git', path: 'upstream' }
    end

    private

    Contract None => nil
    def prepare_source!
      type_method = "#{source[:type]}_prepare_source"
      return send(type_method) if respond_to?(type_method, true)
      raise("Unknown source type: #{source[:type]}")
    end

    Contract None => nil
    def git_prepare_source
      run_local 'git submodule update --init --recursive'
      run_local "git clone --recursive '#{source[:path]}' #{tmpdir(:build)}"
    end

    Contract None => nil
    def tar_prepare_source
      dest_file = tmpfile(:source_tar)
      File.open(dest_file, 'wb') do |fh|
        open(source[:url], 'rb') { |request| fh.write request.read }
        verify_file(dest_file, source[:checksum])
      end
      run "tar -xf #{dest_file} --strip-components=1"
    end

    Contract None => nil
    def empty_prepare_source
      # This source type is a no-op
    end
  end

  module DSL
    ##
    # Add source methods to Forge DSL
    class Forge
      Contract HashOf[Symbol => Any] => nil
      def source(params)
        @forge.source = params
        nil
      end
    end
  end
{'fraction_non_alphanumeric': 0.1084292610471593, 'fraction_numerical': 0.0014853323431117712, 'mean_word_length': 2.5682119205298015, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '907996', 'n_tokens_mistral': 856, 'n_tokens_neox': 829, 'n_words': 249}