<commit_before>class FeedEaterFeedWorker
  include Sidekiq::Worker

  PYTHON = './virtualenv/bin/python'
  FEEDVALIDATOR = './virtualenv/bin/feedvalidator.py'

  def perform(feed_onestop_id)
    # Download the feed
    puts "feed_onestop_id: #{feed_onestop_id}"
    feed = Feed.find_by(onestop_id: feed_onestop_id)
    logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Downloading #{feed.url}"
    updated = feed.fetch_and_check_for_updated_version
    return unless updated

    # Clear out old log files
    log_file_path = FeedEaterFeedWorker.artifact_file_path("#{feed_onestop_id}.log")
    validation_report_path = FeedEaterFeedWorker.artifact_file_path("#{feed_onestop_id}.html")
    FileUtils.rm(log_file_path) if File.exist?(log_file_path)
    FileUtils.rm(validation_report_path) if File.exist?(validation_report_path)

    # Create import record
    feed_import = FeedImport.create(feed: feed)

    # Validate and import feed
    begin
      logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Validating feed"
      run_python('./lib/feedeater/validate.py', "--feedvalidator #{FEEDVALIDATOR} --log #{log_file_path} #{feed_onestop_id}")
      logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Uploading feed"
      run_python('./lib/feedeater/post.py', "--log #{log_file_path} #{feed_onestop_id}")
      logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Creating GTFS artifact"
      run_python('./lib/feedeater/artifact.py', "--log #{log_file_path} #{feed_onestop_id}")
      if Figaro.env.upload_feed_eater_artifacts_to_s3.present? &&
         Figaro.env.upload_feed_eater_artifacts_to_s3 == 'true'
        logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Enqueuing a job to upload artifacts to S3"
        UploadFeedEaterArtifactsToS3Worker.perform_async(feed_onestop_id)
      end
    rescue Exception => e
      # NOTE: we're catching all exceptions, including Interrupt,<commit_msg>(remove debugging puts statement)<commit_after>class FeedEaterFeedWorker
  include Sidekiq::Worker

  PYTHON = './virtualenv/bin/python'
  FEEDVALIDATOR = './virtualenv/bin/feedvalidator.py'

  def perform(feed_onestop_id)
    # Download the feed
    feed = Feed.find_by(onestop_id: feed_onestop_id)
    logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Downloading #{feed.url}"
    updated = feed.fetch_and_check_for_updated_version
    return unless updated

    # Clear out old log files
    log_file_path = FeedEaterFeedWorker.artifact_file_path("#{feed_onestop_id}.log")
    validation_report_path = FeedEaterFeedWorker.artifact_file_path("#{feed_onestop_id}.html")
    FileUtils.rm(log_file_path) if File.exist?(log_file_path)
    FileUtils.rm(validation_report_path) if File.exist?(validation_report_path)

    # Create import record
    feed_import = FeedImport.create(feed: feed)

    # Validate and import feed
    begin
      logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Validating feed"
      run_python('./lib/feedeater/validate.py', "--feedvalidator #{FEEDVALIDATOR} --log #{log_file_path} #{feed_onestop_id}")
      logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Uploading feed"
      run_python('./lib/feedeater/post.py', "--log #{log_file_path} #{feed_onestop_id}")
      logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Creating GTFS artifact"
      run_python('./lib/feedeater/artifact.py', "--log #{log_file_path} #{feed_onestop_id}")
      if Figaro.env.upload_feed_eater_artifacts_to_s3.present? &&
         Figaro.env.upload_feed_eater_artifacts_to_s3 == 'true'
        logger.info "FeedEaterFeedWorker #{feed_onestop_id}: Enqueuing a job to upload artifacts to S3"
        UploadFeedEaterArtifactsToS3Worker.perform_async(feed_onestop_id)
      end
    rescue Exception => e
      # NOTE: we're catching all exceptions, including Interrupt,
{'fraction_non_alphanumeric': 0.1063997893073479, 'fraction_numerical': 0.0021069265209375824, 'mean_word_length': 4.660208643815201, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5433934', 'n_tokens_mistral': 1289, 'n_tokens_neox': 1265, 'n_words': 223}