<commit_before>class CocoapodsCrawler < Versioneye::Crawl

  def logger
    ActiveSupport::BufferedLogger.new('log/cocoapods.log')
  end

  def run cmd_string
    logger.info "Running: $ #{cmd_string}"
    `#{cmd_string}`
  end

  def self.crawl
    self.new.crawl
  end

  def crawl
    git_url = Settings.instance.cocoapods_spec_git
    dir = Settings.instance.cocoapods_spec

    run "git clone #{git_url} #{dir}"
    run "(cd #{dir} && git pull)"

    i = 0
    all_spec_files do |filepath|
      i += 1
      logger.info "Parse CocoaPods Spec ##{i}: #{filepath}"
      p "Parse CocoaPods Spec ##{i}: #{filepath}"
      parse_spec filepath
    end
  end

  def parse_spec filepath
    # parse every podspec file
    parser  = CocoapodsPodspecParser.new
    product = parser.parse_file filepath
    if product
      ProductService.update_version_data product, false
      product.save
    else
      logger.warn 'NO PRODUCT'
    end
  rescue => e
    logger.error e.message
  end

  # traverse directory, search for .podspec files
  def all_spec_files(&block)
    dir = Settings.instance.cocoapods_spec
    Dir.glob "#{dir}/**/*.podspec" do |filepath|
      block.call filepath
    end<commit_msg>Catch Exception in cocoapods crawler<commit_after>class CocoapodsCrawler < Versioneye::Crawl


  def logger
    ActiveSupport::BufferedLogger.new('log/cocoapods.log')
  end


  def run cmd_string
    logger.info "Running: $ #{cmd_string}"
    `#{cmd_string}`
  end


  def self.crawl
    self.new.crawl
  end


  def crawl
    git_url = Settings.instance.cocoapods_spec_git
    dir = Settings.instance.cocoapods_spec

    run "git clone #{git_url} #{dir}"
    run "(cd #{dir} && git pull)"

    i = 0
    all_spec_files do |filepath|
      i += 1
      logger.info "Parse CocoaPods Spec ##{i}: #{filepath}"
      p "Parse CocoaPods Spec ##{i}: #{filepath}"
      parse_spec filepath
    end
  rescue => e
    logger.error e.message
    logger.error e.backtrace.join('\n')
  end


  def parse_spec filepath
    # parse every podspec file
    parser  = CocoapodsPodspecParser.new
    product = parser.parse_file filepath
    if product
      ProductService.update_version_data product, false
      product.save
    else
      logger.warn 'NO PRODUCT'
    end
  rescue => e
    logger.error e.message
    logger.error e.backtrace.join('\n')
  end


  # traverse directory, search for .podspec files
  def all_spec_files(&block)
    dir = Settings.instance.cocoapods_spec
    Dir.glob "#{dir}/**/*.podspec" do |filepath|
      block.call filepath
    end
  rescue => e
    logger.error e.message
    logger.error e.backtrace.join('\n')
{'fraction_non_alphanumeric': 0.10262257696693272, 'fraction_numerical': 0.0015203344735841885, 'mean_word_length': 2.8423357664233575, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3390716', 'n_tokens_mistral': 925, 'n_tokens_neox': 880, 'n_words': 223}