<commit_before>class BowerProjectsCrawler < Bower 

  
  def self.process_project task, token, skipKnownVersions = true 
    check_request_limit(token)
    repo_response = Github.repo_info(task[:repo_fullname], token, true, task[:crawled_at])

    if repo_response.nil? or repo_response.is_a?(Boolean)
      logger.error "crawl_projects | Did not get repo_info for #{task[:repo_fullname]}"
      return false 
    end

    if repo_response.code == 304
      logger.debug "crawl_projects | no changes for #{task[:repo_fullname]}, since #{task[:crawled_at]}"
      return false 
    end

    if repo_response.body.to_s.empty?
      logger.error "Did not get any repo info for #{task[:repo_fullname]}. Response code: #{repo_response.code}"
      return false 
    end

    if repo_response.code != 200 && repo_response.code != 201 
      logger.error "crawl_projects | cant read information for #{task[:repo_fullname]} - response body: #{repo_info}. Response code: #{repo_response.code}"
      return false
    end

    repo_info = JSON.parse(repo_response.body, symbolize_names: true)
    repo_info[:repo_fullname] = task[:repo_fullname]
    product = add_bower_package(task, repo_info,  token, skipKnownVersions)
    if product.nil?
      logger.error "crawl_projects | cant add bower package for #{task[:repo_fullname]}."
      return false
    end

    task.update_attributes({crawled_at: DateTime.now})
    to_version_task(task, product[:prod_key]) # add new version task when everything went oK
    
    true
  end


  def self.add_bower_package(task, repo_info, token, skipKnownVersions = true)
    logger.info "#-- reading #{task[:repo_fullname]} from url: #{task[:url]} branch: #{repo_info[:default_branch]}"
    pkg_file = self.read_project_file_from_github(task, token, repo_info[:default_branch])
    if pkg_file.nil?
      logger.error "add_bower_package | Didnt get any project file for #{task[:repo_fullname]}"
      return nil
    end

    pkg_file[:repo_fullname]  = repo_info[:repo_fullname]
    pkg_file[:default_branch] = repo_info[:default_branch]
    
    pkg_file[:name] = task[:registry_name]  if task.has_attribute?(:registry_name) # if task has prod_key then use it - dont trust user's unvalidated bower.json
    pkg_file[:name] = repo_info[:name]      if pkg_file[:name].to_s.strip.empty?
    pkg_file[:name] = repo_info[:full_name] if pkg_file[:name].to_s.strip.empty?

    prod_key        = make_prod_key(task)
    product         = create_bower_package(prod_key, pkg_file, repo_info, token, skipKnownVersions)
    if product.nil?
      logger.error "add_bower_package | cant create_or_find product for #{task[:repo_fullname]}"
      return nil
    end

    product.save
    product
  rescue => e
    logger.error "add_bower_package | cant save product for #{repo_info} - #{e.message}"
    if product && product.errors
      logger.error "#{product.errors.full_messages.to_sentence}"
    end
    logger.error e.backtrace.join("\n")
    nil
  end


  # Saves product and save sub/related docs
  def self.create_bower_package(prod_key, pkg_info, repo_info, token, skipKnownVersions = true)
    prod = create_or_update_product(prod_key, pkg_info, token, repo_info[:language], skipKnownVersions)

    version = pkg_info[:version].to_s

    # Version exist already! <commit_msg>Better logging<commit_after>class BowerProjectsCrawler < Bower 

  
  def self.process_project task, token, skipKnownVersions = true 
    check_request_limit(token)
    repo_response = Github.repo_info(task[:repo_fullname], token, true, task[:crawled_at])

    if repo_response.nil? or repo_response.is_a?(Boolean)
      logger.error "ERROR: crawl_projects | Did not get repo_info for #{task[:repo_fullname]}"
      return false 
    end

    if repo_response.code == 304
      logger.debug "ERROR: crawl_projects | no changes for #{task[:repo_fullname]}, since #{task[:crawled_at]}"
      return false 
    end

    if repo_response.body.to_s.empty?
      logger.error "ERROR: Did not get any repo info for #{task[:repo_fullname]}. Response code: #{repo_response.code}"
      return false 
    end

    if repo_response.code != 200 && repo_response.code != 201 
      logger.error "ERROR: crawl_projects | cant read information for #{task[:repo_fullname]} - response body: #{repo_info}. Response code: #{repo_response.code}"
      return false
    end

    repo_info = JSON.parse(repo_response.body, symbolize_names: true)
    repo_info[:repo_fullname] = task[:repo_fullname]
    product = add_bower_package(task, repo_info,  token, skipKnownVersions)
    if product.nil?
      logger.error "ERROR: crawl_projects | cant add bower package for #{task[:repo_fullname]}."
      return false
    end

    task.update_attributes({crawled_at: DateTime.now})
    to_version_task(task, product[:prod_key]) # add new version task when everything went oK
    
    true
  end


  def self.add_bower_package(task, repo_info, token, skipKnownVersions = true)
    logger.info "#-- reading #{task[:repo_fullname]} from url: #{task[:url]} branch: #{repo_info[:default_branch]}"
    pkg_file = self.read_project_file_from_github(task, token, repo_info[:default_branch])
    if pkg_file.nil?
      logger.error "ERROR: add_bower_package | Didnt get any project file for #{task[:repo_fullname]}"
      return nil
    end

    pkg_file[:repo_fullname]  = repo_info[:repo_fullname]
    pkg_file[:default_branch] = repo_info[:default_branch]
    
    pkg_file[:name] = task[:registry_name]  if task.has_attribute?(:registry_name) # if task has prod_key then use it - dont trust user's unvalidated bower.json
    pkg_file[:name] = repo_info[:name]      if pkg_file[:name].to_s.strip.empty?
    pkg_file[:name] = repo_info[:full_name] if pkg_file[:name].to_s.strip.empty?

    prod_key        = make_prod_key(task)
    product         = create_bower_package(prod_key, pkg_file, repo_info, token, skipKnownVersions)
    if product.nil?
      logger.error "ERROR: add_bower_package | cant create_or_find product for #{task[:repo_fullname]}"
      return nil
    end

    product.save
    product
  rescue => e
    logger.error "ERROR: add_bower_package | cant save product for #{repo_info} - #{e.message}"
    if product && product.errors
      logger.error "#{product.errors.full_messages.to_sentence}"
    end
    logger.error e.backtrace.join("\n")
    nil
  end


  # Saves product and save sub/related docs
  def self.create_bower_package(prod_key, pkg_info, repo_info, token, skipKnownVersions = true)
    prod = create_or_update_product(prod_key, pkg_info, token, repo_info[:language], skipKnownVersions)

    version = pkg_info[:version].to_s

    # Version exist already! 
{'fraction_non_alphanumeric': 0.10638617580766341, 'fraction_numerical': 0.00270473328324568, 'mean_word_length': 3.923076923076923, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3628792', 'n_tokens_mistral': 2208, 'n_tokens_neox': 2096, 'n_words': 501}