<commit_before>require 'fileutils'
require 'open-uri'
require 'net/ftp'

module Itrp
  module Export
    module Monitor

      class Service

        def initialize
          @failed_exports = Set.new
          @missing_export_ids = Set.new
          @options = Itrp::Export::Monitor.configuration.current
          @options[:ids] = (@options[:ids] || []) + [@options[:id]].flatten.compact.map(&:to_i)
          [:root, :ids, :imap_user_name, :imap_password].each do |required_option|
            raise ::Itrp::Exception.new("Missing required configuration option #{required_option}") if option(required_option).blank?
          end
          @logger = @options[:logger]
        end

        # Retrieve an option
        def option(key)
          @options[key]
        end

        def process(mail)
          mail = Itrp::Export::Monitor::Mail.new(mail)
          return if @failed_exports.include?(mail.download_uri)

          if option(:ids).include?(mail.export_id)
            begin
              @logger.info { "Processing ITRP Export mail:\n  Subject: #{mail.original.subject}\n  Export ID: #{mail.export_id}\n  Token: #{mail.token}\n  URI: #{mail.download_uri}" }
              store_export(mail)
            rescue ::Exception => ex
              @failed_exports.add(mail.download_uri)
              @logger.error { "Processing of mail '#{mail.original.subject}' failed: #{ex.message}\n  #{ex.backtrace.join("\n  ")}" }
              handle_exception(ex, mail)
              mail.ignore # leave mail in the mailbox
            end
          else
            mail.ignore # leave mail in the mailbox
            unless @missing_export_ids.include?(mail.export_id)
              @missing_export_ids.add(mail.export_id)
              @logger.info { mail.export_id ? "Skipping mail. ITRP Export ID #{mail.export_id} not configured for monitoring" : "Skipping mail. Not an ITRP Export mail: #{mail.original.subject}" }
            end
          end
        end

        # Generate a clacks config file based on the export config
        def generate_clacks_config
          clacks_config_filename = "#{dir(:tmp)}/clacks_config.#{monitor_id}.rb"
          File.open(clacks_config_filename, 'w') do |clacks_config|
            clacks_config.write(<<EOF)
# -- DO NOT EDIT --
# Generated by the Export Monitor

pid "#{dir(:pids)}/#{monitor_id}.pid"
stdout_path "#{dir(:log)}/#{monitor_id}.log"
stderr_path "#{dir(:log)}/#{monitor_id}.log"

imap({
  address:    '#{option(:imap_address)}',
  port:       #{option(:imap_port)},
  user_name:  '#{option(:imap_user_name)}',
  password:   '#{option(:imap_password)}',
  enable_ssl: #{option(:imap_ssl)}
})

find_options({
  mailbox:           '#{option(:imap_mailbox)}',
  archivebox:        '#{option(:imap_archive)}',
  keys:              'FROM ITRP HEADER X-ITRP-ExportID ""',
  delete_after_find: true # Note that only the processed export mails will be deleted
})

on_mail do |mail|
  Itrp::Export::Monitor.process(mail)
end
EOF
          end
          clacks_config_filename
        end

        private

        def store_export(mail)
          # download export file to the downloads directory
          local_filename = download_export(mail)
          # copy the file to the :to directory
          copy_export(local_filename) if option(:to)
          # ftp the file
          ftp_export(local_filename) if option(:to_ftp)
        end

        def download_export(mail)
          local_filename = "#{dir(:downloads)}/#{mail.filename}"
          File.open(local_filename, 'w') { |local| open(mail.download_uri) { |remote| local.write(remote.read) }}
          local_filename
        end

        def copy_export(local_filename)
          FileUtils.mkpath(option(:to))
          to_filename = "#{option(:to)}/#{File.basename(local_filename)}"
          FileUtils.copy(local_filename, "#{to_filename}.in_progress")
          FileUtils.move("#{to_filename}.in_progress", to_filename)
          @logger.info { "Copied export '#{local_filename}' to '#{to_filename}'" }
        end

        def ftp_export(local_filename)
          remote_filename = File.basename(local_filename)
          Net::FTP.open(option(:to_ftp), option(:ftp_user_name), option(:ftp_password)) do |ftp|
            ftp.putbinaryfile(local_filename, "#{remote_filename}.in_progress")
            ftp.rename("#{remote_filename}.in_progress", remote_filename)
          end
          @logger.info { "FTP export '#{local_filename}' to '#{option(:to_ftp)}/#{remote_filename}'" }
        end

        def dir(subdir)
          directory = File.expand_path(subdir.to_s, option(:root))
          FileUtils.mkpath(directory)
          directory
        end

        def monitor_id
          @monitor_id ||= "export_monitor.#{option(:ids).map(&:to_s).join('.')}"<commit_msg>Downloaded zip files are corrupted on Windows.<commit_after>require 'fileutils'
require 'open-uri'
require 'net/ftp'

module Itrp
  module Export
    module Monitor

      class Service

        def initialize
          @failed_exports = Set.new
          @missing_export_ids = Set.new
          @options = Itrp::Export::Monitor.configuration.current
          @options[:ids] = (@options[:ids] || []) + [@options[:id]].flatten.compact.map(&:to_i)
          [:root, :ids, :imap_user_name, :imap_password].each do |required_option|
            raise ::Itrp::Exception.new("Missing required configuration option #{required_option}") if option(required_option).blank?
          end
          @logger = @options[:logger]
        end

        # Retrieve an option
        def option(key)
          @options[key]
        end

        def process(mail)
          mail = Itrp::Export::Monitor::Mail.new(mail)
          return if @failed_exports.include?(mail.download_uri)

          if option(:ids).include?(mail.export_id)
            begin
              @logger.info { "Processing ITRP Export mail:\n  Subject: #{mail.original.subject}\n  Export ID: #{mail.export_id}\n  Token: #{mail.token}\n  URI: #{mail.download_uri}" }
              store_export(mail)
            rescue ::Exception => ex
              @failed_exports.add(mail.download_uri)
              @logger.error { "Processing of mail '#{mail.original.subject}' failed: #{ex.message}\n  #{ex.backtrace.join("\n  ")}" }
              handle_exception(ex, mail)
              mail.ignore # leave mail in the mailbox
            end
          else
            mail.ignore # leave mail in the mailbox
            unless @missing_export_ids.include?(mail.export_id)
              @missing_export_ids.add(mail.export_id)
              @logger.info { mail.export_id ? "Skipping mail. ITRP Export ID #{mail.export_id} not configured for monitoring" : "Skipping mail. Not an ITRP Export mail: #{mail.original.subject}" }
            end
          end
        end

        # Generate a clacks config file based on the export config
        def generate_clacks_config
          clacks_config_filename = "#{dir(:tmp)}/clacks_config.#{monitor_id}.rb"
          File.open(clacks_config_filename, 'w') do |clacks_config|
            clacks_config.write(<<EOF)
# -- DO NOT EDIT --
# Generated by the Export Monitor

pid "#{dir(:pids)}/#{monitor_id}.pid"
stdout_path "#{dir(:log)}/#{monitor_id}.log"
stderr_path "#{dir(:log)}/#{monitor_id}.log"

imap({
  address:    '#{option(:imap_address)}',
  port:       #{option(:imap_port)},
  user_name:  '#{option(:imap_user_name)}',
  password:   '#{option(:imap_password)}',
  enable_ssl: #{option(:imap_ssl)}
})

find_options({
  mailbox:           '#{option(:imap_mailbox)}',
  archivebox:        '#{option(:imap_archive)}',
  keys:              'FROM ITRP HEADER X-ITRP-ExportID ""',
  delete_after_find: true # Note that only the processed export mails will be deleted
})

on_mail do |mail|
  Itrp::Export::Monitor.process(mail)
end
EOF
          end
          clacks_config_filename
        end

        private

        def store_export(mail)
          # download export file to the downloads directory
          local_filename = download_export(mail)
          # copy the file to the :to directory
          copy_export(local_filename) if option(:to)
          # ftp the file
          ftp_export(local_filename) if option(:to_ftp)
        end

        def download_export(mail)
          local_filename = "#{dir(:downloads)}/#{mail.filename}"
          File.open(local_filename, 'w') { |f| f.binmode; f.write(open(mail.download_uri).read) }
          local_filename
        end

        def copy_export(local_filename)
          FileUtils.mkpath(option(:to))
          to_filename = "#{option(:to)}/#{File.basename(local_filename)}"
          FileUtils.copy(local_filename, "#{to_filename}.in_progress")
          FileUtils.move("#{to_filename}.in_progress", to_filename)
          @logger.info { "Copied export '#{local_filename}' to '#{to_filename}'" }
        end

        def ftp_export(local_filename)
          remote_filename = File.basename(local_filename)
          Net::FTP.open(option(:to_ftp), option(:ftp_user_name), option(:ftp_password)) do |ftp|
            ftp.putbinaryfile(local_filename, "#{remote_filename}.in_progress")
            ftp.rename("#{remote_filename}.in_progress", remote_filename)
          end
          @logger.info { "FTP export '#{local_filename}' to '#{option(:to_ftp)}/#{remote_filename}'" }
        end

        def dir(subdir)
          directory = File.expand_path(subdir.to_s, option(:root))
          FileUtils.mkpath(directory)
          directory
        end

        def monitor_id
          @monitor_id ||= "export_monitor.#{option(:ids).map(&:to_s).join('.')}"
{'fraction_non_alphanumeric': 0.12923750912694273, 'fraction_numerical': 0.0, 'mean_word_length': 2.5842990654205606, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '801010', 'n_tokens_mistral': 2902, 'n_tokens_neox': 2856, 'n_words': 592}