<commit_before>import os
import boto.swf
import json
import random
import datetime
import importlib
import calendar
import time

import zipfile
import requests
import urlparse
import glob
import shutil
import re

import activity

import boto.s3
from boto.s3.connection import S3Connection

import provider.simpleDB as dblib

"""
PublishPOA activity
"""

class activity_PublishPOA(activity.activity):
    
    def __init__(self, settings, logger, conn = None, token = None, activity_task = None):
        activity.activity.__init__(self, settings, logger, conn, token, activity_task)

        self.name = "PublishPOA"
        self.version = "1"
        self.default_task_heartbeat_timeout = 30
        self.default_task_schedule_to_close_timeout = 60*30
        self.default_task_schedule_to_start_timeout = 30
        self.default_task_start_to_close_timeout= 60*15
        self.description = "Download POA files in outbox, zip, and publish."
        
        # Directory where POA library is stored
        self.poa_lib_dir_name = "elife-poa-xml-generation"
        
        # Where we specify the library to be imported
        self.elife_poa_lib = None
        
        # Import the libraries we will need
        self.import_imports()
        
        # Create output directories
        self.create_activity_directories()
        
        # Data provider where email body is saved
        self.db = dblib.SimpleDB(settings)
        
        # Bucket for outgoing files
        self.publish_bucket = settings.poa_packaging_bucket
        self.outbox_folder = "outbox/"
        self.published_folder = "published/"
        
        # Folder for crossref XML
        self.crossref_outbox_folder = "crossref/outbox/"
        
        # Folder for pubmed XML
        self.pubmed_outbox_folder = "pubmed/outbox/"
        
        # Folder for publication email
        self.publication_email_outbox_folder = "publication_email/outbox/"
        
        # Subfolders on the FTP site to deliver into
        self.ftp_subfolder_poa = "poa"
        self.ftp_subfolder_ds = "ds"
        
        # Track the success of some steps
        self.activity_status = None
        self.prepare_status = None
        self.approve_status = None
        self.ftp_status = None
        self.go_status = None
        self.outbox_status = None
        self.publish_status = None
        
        self.outbox_s3_key_names = None
        
        # Track XML files selected for crossref XML
        self.crossref_outbox_s3_key_names = None
        self.crossref_articles_not_uploaded_to_outbox = None
        
        # Track XML files selected for pubmed XML
        self.pubmed_outbox_s3_key_names = None
        self.pubmed_articles_not_uploaded_to_outbox = None
        
        # Track XML files selected for publication email
        self.publication_email_outbox_s3_key_names = None
        self.publication_email_articles_not_uploaded_to_outbox = None
        
        # More file status tracking for reporting in email
        self.malformed_ds_file_names = []
        self.empty_ds_file_names = []
        self.unmatched_ds_file_names = []
    
    def do_activity(self, data = None):
        """
        Activity, do the work
        """
        if(self.logger):
            self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4))
        
        # Download the S3 objects
        self.download_files_from_s3_outbox()
        
        # Prepare for HW
        self.prepare_status = self.prepare_for_hw()
        
        # Approve files for publishing
        self.approve_status = self.approve_for_publishing()
        
        if self.approve_status is True:
            try:
                # Publish files
                
                # Feb 26, 2016: Disable FTP delivery
                #self.ftp_files_to_endpoint(file_type = "/*_ds.zip", sub_dir = self.ftp_subfolder_ds)
                #self.ftp_files_to_endpoint(file_type = "/*[0-9].zip", sub_dir = self.ftp_subfolder_poa)
                
                self.ftp_status = True
            except:
                self.ftp_status = False
                
            if self.ftp_status is True:
                try:
                    # Add go.xml files
                    self.create_go_xml_file("pap", self.ftp_subfolder_poa)
                    self.create_go_xml_file("ds", self.ftp_subfolder_ds)
                    
                    # Feb 26, 2016: Disable FTP delivery
                    #self.ftp_go_xml_to_endpoint("pap", self.ftp_subfolder_poa)
                    #self.ftp_go_xml_to_endpoint("ds", self.ftp_subfolder_ds)
                    
                    self.go_status = True
                except:
                    self.go_status = False
            
            if self.ftp_status is True and self.go_status is True:
                # Clean up outbox
                print "Moving files from outbox folder to published folder"
                self.clean_outbox()
                self.upload_xml_to_crossref_outbox_s3()
                self.upload_xml_to_pubmed_outbox_s3()
                self.upload_xml_to_publication_email_outbox_s3()
                self.outbox_status = True
                
            # Set the activity status of this activity based on successes
            if (self.prepare_status is True and
                self.ftp_status is True and
                self.go_status is True):
                # Published!
                self.publish_status = True
            else:
                self.publish_status = False
            
        # Set the activity status of this activity based on successes
        if self.publish_status is not False:<commit_msg>Do not add XML to outboxes during PublishPOA anymore, this is done elsewhere now.<commit_after>import os
import boto.swf
import json
import random
import datetime
import importlib
import calendar
import time

import zipfile
import requests
import urlparse
import glob
import shutil
import re

import activity

import boto.s3
from boto.s3.connection import S3Connection

import provider.simpleDB as dblib

"""
PublishPOA activity
"""

class activity_PublishPOA(activity.activity):
    
    def __init__(self, settings, logger, conn = None, token = None, activity_task = None):
        activity.activity.__init__(self, settings, logger, conn, token, activity_task)

        self.name = "PublishPOA"
        self.version = "1"
        self.default_task_heartbeat_timeout = 30
        self.default_task_schedule_to_close_timeout = 60*30
        self.default_task_schedule_to_start_timeout = 30
        self.default_task_start_to_close_timeout= 60*15
        self.description = "Download POA files in outbox, zip, and publish."
        
        # Directory where POA library is stored
        self.poa_lib_dir_name = "elife-poa-xml-generation"
        
        # Where we specify the library to be imported
        self.elife_poa_lib = None
        
        # Import the libraries we will need
        self.import_imports()
        
        # Create output directories
        self.create_activity_directories()
        
        # Data provider where email body is saved
        self.db = dblib.SimpleDB(settings)
        
        # Bucket for outgoing files
        self.publish_bucket = settings.poa_packaging_bucket
        self.outbox_folder = "outbox/"
        self.published_folder = "published/"
        
        # Folder for crossref XML
        self.crossref_outbox_folder = "crossref/outbox/"
        
        # Folder for pubmed XML
        self.pubmed_outbox_folder = "pubmed/outbox/"
        
        # Folder for publication email
        self.publication_email_outbox_folder = "publication_email/outbox/"
        
        # Subfolders on the FTP site to deliver into
        self.ftp_subfolder_poa = "poa"
        self.ftp_subfolder_ds = "ds"
        
        # Track the success of some steps
        self.activity_status = None
        self.prepare_status = None
        self.approve_status = None
        self.ftp_status = None
        self.go_status = None
        self.outbox_status = None
        self.publish_status = None
        
        self.outbox_s3_key_names = None
        
        # Track XML files selected for crossref XML
        self.crossref_outbox_s3_key_names = None
        self.crossref_articles_not_uploaded_to_outbox = None
        
        # Track XML files selected for pubmed XML
        self.pubmed_outbox_s3_key_names = None
        self.pubmed_articles_not_uploaded_to_outbox = None
        
        # Track XML files selected for publication email
        self.publication_email_outbox_s3_key_names = None
        self.publication_email_articles_not_uploaded_to_outbox = None
        
        # More file status tracking for reporting in email
        self.malformed_ds_file_names = []
        self.empty_ds_file_names = []
        self.unmatched_ds_file_names = []
    
    def do_activity(self, data = None):
        """
        Activity, do the work
        """
        if(self.logger):
            self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4))
        
        # Download the S3 objects
        self.download_files_from_s3_outbox()
        
        # Prepare for HW
        self.prepare_status = self.prepare_for_hw()
        
        # Approve files for publishing
        self.approve_status = self.approve_for_publishing()
        
        if self.approve_status is True:
            try:
                # Publish files
                
                # Feb 26, 2016: Disable FTP delivery
                #self.ftp_files_to_endpoint(file_type = "/*_ds.zip", sub_dir = self.ftp_subfolder_ds)
                #self.ftp_files_to_endpoint(file_type = "/*[0-9].zip", sub_dir = self.ftp_subfolder_poa)
                
                self.ftp_status = True
            except:
                self.ftp_status = False
                
            if self.ftp_status is True:
                try:
                    # Add go.xml files
                    self.create_go_xml_file("pap", self.ftp_subfolder_poa)
                    self.create_go_xml_file("ds", self.ftp_subfolder_ds)
                    
                    # Feb 26, 2016: Disable FTP delivery
                    #self.ftp_go_xml_to_endpoint("pap", self.ftp_subfolder_poa)
                    #self.ftp_go_xml_to_endpoint("ds", self.ftp_subfolder_ds)
                    
                    self.go_status = True
                except:
                    self.go_status = False
            
            if self.ftp_status is True and self.go_status is True:
                # Clean up outbox
                print "Moving files from outbox folder to published folder"
                self.clean_outbox()
                #self.upload_xml_to_crossref_outbox_s3()
                #self.upload_xml_to_pubmed_outbox_s3()
                #self.upload_xml_to_publication_email_outbox_s3()
                self.outbox_status = True
                
            # Set the activity status of this activity based on successes
            if (self.prepare_status is True and
                self.ftp_status is True and
                self.go_status is True):
                # Published!
                self.publish_status = True
            else:
                self.publish_status = False
            
        # Set the activity status of this activity based on successes
        if self.publish_status is not False:
{'fraction_non_alphanumeric': 0.06094255486578648, 'fraction_numerical': 0.0069948413045379035, 'mean_word_length': 1.833292048550904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2951843', 'n_tokens_mistral': 3331, 'n_tokens_neox': 3169, 'n_words': 766}