<commit_before>require 'uri'
require 'zipfile'
require 'stored_csv'
require 'schema_processor'
require 'processor_helpers'
require 'stored_chunk'
require 'fog_storage'
require 'pusher'

class PackageProcessor
  include ProcessorHelpers

  def initialize(params, package_id)
    @params = params
    @package_id = package_id
  end

  def pusher_channel
    channel = "package_#{@package_id}"
    Pusher[channel]
  end

  def process
    read_files unless @params[:files_data].blank?
    fetch_uploaded_files unless @params[:file_ids].blank?
    open_files unless @params[:files].blank?
    unzip_urls unless @params[:urls].blank?

    @files.flatten! if @files

    create_package
    pusher_channel.trigger('complete', {
      url: Rails.application.routes.url_helpers.package_url(@package)
    })
  end

  def package
    Package.find(@package_id)
  end

  def create_package
    if schema_present?
      schema = SchemaProcessor.new(url: @params[:schema_url], file: @params[:schema_file], data: @params[:schema_data])
      package.create_package(@files || @params[:urls], schema.url, schema.schema)
    else
      package.create_package(@files || @params[:urls])
    end<commit_msg>Only return path<commit_after>require 'uri'
require 'zipfile'
require 'stored_csv'
require 'schema_processor'
require 'processor_helpers'
require 'stored_chunk'
require 'fog_storage'
require 'pusher'

class PackageProcessor
  include ProcessorHelpers

  def initialize(params, package_id)
    @params = params
    @package_id = package_id
  end

  def pusher_channel
    channel = "package_#{@package_id}"
    Pusher[channel]
  end

  def process
    read_files unless @params[:files_data].blank?
    fetch_uploaded_files unless @params[:file_ids].blank?
    open_files unless @params[:files].blank?
    unzip_urls unless @params[:urls].blank?

    @files.flatten! if @files

    create_package
    pusher_channel.trigger('complete', {
      url: Rails.application.routes.url_helpers.package_url(package, only_path: true)
    })
  end

  def package
    Package.find(@package_id)
  end

  def create_package
    if schema_present?
      schema = SchemaProcessor.new(url: @params[:schema_url], file: @params[:schema_file], data: @params[:schema_data])
      package.create_package(@files || @params[:urls], schema.url, schema.schema)
    else
      package.create_package(@files || @params[:urls])
    end
{'fraction_non_alphanumeric': 0.10841356215989954, 'fraction_numerical': 0.0, 'mean_word_length': 4.419501133786848, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2098529', 'n_tokens_mistral': 747, 'n_tokens_neox': 754, 'n_words': 161}