<commit_before>class PackagistCrawler < Versioneye::Crawl


  def self.logger
    ActiveSupport::BufferedLogger.new('log/packagist.log')
  end


  def self.crawl serial = false 
    start_time = Time.now
    packages = PackagistCrawler.get_first_level_list
    packages.each do |name|
      if serial == true 
        PackagistCrawler.crawle_package( name ) 
      else 
        PackagistCrawlProducer.new( name ) # Let the rabbit do the work! 
      end
    end
    duration = Time.now - start_time
    self.logger.info(" *** This crawl took #{duration} *** ")
    return nil
  end


  def self.get_first_level_list
    body = JSON.parse HTTParty.get('http://packagist.org/packages/list.json' ).response.body
    body['packageNames']
  end


  def self.crawle_package name
    self.logger.info "crawl #{name}"
    return nil if name.to_s.empty?

    resource     = "http://packagist.org/packages/#{name}.json"
    pack         = JSON.parse HTTParty.get( resource ).response.body
    package      = pack['package']
    package_name = package['name']
    versions     = package['versions']
    return nil if versions.nil? || versions.empty?

    product = PackagistCrawler.init_product package_name
    PackagistCrawler.update_product product, package
    PackagistCrawler.update_packagist_link product, package_name

    versions.each do |version|
      self.process_version version, product
    end
    ProductService.update_version_data( product )
  rescue => e
    self.logger.error "ERROR in crawle_package Message:   #{e.message}"
    self.logger.error e.backtrace.join("\n")
  end


  def self.process_version version, product
    version_number = String.new(version[0])
    version_obj = version[1]

    version_number.gsub!(/^v/, '') if version_number.to_s.match(/^v[0-9]+\..*/)
    db_version  = product.version_by_number version_number
    if db_version.nil?
      if has_tag_variants?(product, version_number, version_obj) == false 
        return nil 
      end

      PackagistCrawler.create_new_version( product, version_number, version_obj )
      return nil
    end

    if version_number.match(/\Adev\-/)
      Dependency.remove_dependencies Product::A_LANGUAGE_PHP, product.prod_key, version_number
      ComposerUtils.create_dependencies product, version_number, version_obj
      Versionarchive.remove_archives Product::A_LANGUAGE_PHP, product.prod_key, version_number
      ComposerUtils.create_archive product, version_number, version_obj
    end
  end<commit_msg>Update Packagist Crawler. Track dev-master<commit_after>class PackagistCrawler < Versioneye::Crawl


  def self.logger
    ActiveSupport::BufferedLogger.new('log/packagist.log')
  end


  def self.crawl serial = false 
    start_time = Time.now
    packages = PackagistCrawler.get_first_level_list
    packages.each do |name|
      if serial == true 
        PackagistCrawler.crawle_package( name ) 
      else 
        PackagistCrawlProducer.new( name ) # Let the rabbit do the work! 
      end
    end
    duration = Time.now - start_time
    self.logger.info(" *** This crawl took #{duration} *** ")
    return nil
  end


  def self.get_first_level_list
    body = JSON.parse HTTParty.get('http://packagist.org/packages/list.json' ).response.body
    body['packageNames']
  end


  def self.crawle_package name
    self.logger.info "crawl #{name}"
    return nil if name.to_s.empty?

    resource     = "http://packagist.org/packages/#{name}.json"
    pack         = JSON.parse HTTParty.get( resource ).response.body
    package      = pack['package']
    package_name = package['name']
    versions     = package['versions']
    return nil if versions.nil? || versions.empty?

    product = PackagistCrawler.init_product package_name
    PackagistCrawler.update_product product, package
    PackagistCrawler.update_packagist_link product, package_name

    versions.each do |version|
      self.process_version version, product
    end
    ProductService.update_version_data( product )
  rescue => e
    self.logger.error "ERROR in crawle_package Message:   #{e.message}"
    self.logger.error e.backtrace.join("\n")
  end


  def self.process_version version, product
    version_number = String.new(version[0])
    version_obj = version[1]

    version_number.gsub!(/^v/, '') if version_number.to_s.match(/^v[0-9]+\..*/)
    db_version  = product.version_by_number version_number
    if db_version.nil?
      if !version_number.eql?('dev-master') && has_tag_variants?(product, version_number, version_obj) == false 
        return nil 
      end

      PackagistCrawler.create_new_version( product, version_number, version_obj )
      return nil
    end

    if version_number.match(/\Adev\-/)
      Dependency.remove_dependencies Product::A_LANGUAGE_PHP, product.prod_key, version_number
      ComposerUtils.create_dependencies product, version_number, version_obj
      Versionarchive.remove_archives Product::A_LANGUAGE_PHP, product.prod_key, version_number
      ComposerUtils.create_archive product, version_number, version_obj
    end
  end
{'fraction_non_alphanumeric': 0.09684461202619567, 'fraction_numerical': 0.0015876165905933718, 'mean_word_length': 3.3485763589301123, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '818591', 'n_tokens_mistral': 1609, 'n_tokens_neox': 1546, 'n_words': 326}