<commit_before>class FeedEaterWorker
  include Sidekiq::Worker

  def perform(feed_onestop_ids: [])
    logger.info '0. fetching latest onestop-id-registry'
    OnestopIdClient::Registry.repo(force_update: true)

    logger.info '1. downloading feeds that have been updated'
    # system "cd #{Figaro.env.onestop_id_registry_local_path} &&
    #         python -m onestop_updater.checkfeeds
    #         #{feed_onestop_ids.join(' ')}"
    system "python -m feedeater.fetch #{feed_onestop_ids.join(' ')}"

    # 1.1. feedvalidator.py
    system "python -m feedeater.validate #{feed_onestop_ids.join(' ')}"
    
    # 2. onestop-updater: Merge & POST changesets to Datastore
    # TODO: datastore_api_base_url = Figaro.env.DATASTORE_PROTOCOL + "://" + Figaro.env.DATASTORE_HOST +
    system "pythom -m feedeater.post #{feed_onestop_ids.join(' ')}"

    # 3. onestop-updater: Write out GTFS+onestop
    system "python -m feedeater.merge #{feed_onestop_ids.join(' ')}"
  
    # 3.1: feedeater reports
  
    # 4. onestop-updater: Push to Amazon S3
    # aws sync

    # 5. build report & upload validator output to S3
  end
end<commit_msg>Reflect current feedeater program names<commit_after>class FeedEaterWorker
  include Sidekiq::Worker

  def perform(feed_onestop_ids: [])
    # TODO: datastore_api_base_url = Figaro.env.DATASTORE_PROTOCOL + "://" + Figaro.env.DATASTORE_HOST +
    logger.info '0. Fetching latest onestop-id-registry'
    OnestopIdClient::Registry.repo(force_update: true)

    logger.info '1. Downloading feeds that have been updated'
    system "python -m feedeater.fetch #{feed_onestop_ids.join(' ')}"

    logger.info '1. Validating feeds'
    system "python -m feedeater.validate #{feed_onestop_ids.join(' ')}"
    
    logger.info '3. Uploading feed to datastore'
    system "pythom -m feedeater.post #{feed_onestop_ids.join(' ')}"

    logger.info '4. Creating GTFS artifacts'
    system "python -m feedeater.artifact #{feed_onestop_ids.join(' ')}"
  
    # logger.info '5. Creating FeedEater Reports'

    # logger.info '6. Uploading to S3'
  end
end
{'fraction_non_alphanumeric': 0.11897880539499037, 'fraction_numerical': 0.009633911368015413, 'mean_word_length': 3.5448577680525166, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5809131', 'n_tokens_mistral': 766, 'n_tokens_neox': 744, 'n_words': 162}