<commit_before>import os
import boto.swf
import json
import random
import datetime
import importlib
import calendar
import time

import zipfile
import requests
import urlparse
import glob
import shutil
import re

import activity

from xml.etree import ElementTree
from xml.etree.ElementTree import Element, SubElement

from elifetools import parseJATS as parser
from elifetools import xmlio

import boto.s3
from boto.s3.connection import S3Connection

import provider.s3lib as s3lib

"""
PublishFinalPOA activity
"""

class activity_PublishFinalPOA(activity.activity):
    
    def __init__(self, settings, logger, conn = None, token = None, activity_task = None):
        activity.activity.__init__(self, settings, logger, conn, token, activity_task)

        self.name = "PublishFinalPOA"
        self.version = "1"
        self.default_task_heartbeat_timeout = 60 * 30
        self.default_task_schedule_to_close_timeout = 60 * 30
        self.default_task_schedule_to_start_timeout = 30
        self.default_task_start_to_close_timeout= 60 * 15
        self.description = "Download POA files from a bucket, zip each article separately, and upload to final bucket."
        
        # Local directory settings
        self.TMP_DIR = self.get_tmp_dir() + os.sep + "tmp_dir"
        self.INPUT_DIR = self.get_tmp_dir() + os.sep + "input_dir"
        self.OUTPUT_DIR = self.get_tmp_dir() + os.sep + "output_dir"
        self.JUNK_DIR = self.get_tmp_dir() + os.sep + "junk_dir"
        self.DONE_DIR = self.get_tmp_dir() + os.sep + "done_dir"
        
        # Bucket for outgoing files
        self.input_bucket = settings.poa_packaging_bucket
        self.published_folder_prefix = "published/"
        self.published_folder_name = None
        
        self.publish_bucket = settings.publishing_buckets_prefix + settings.production_bucket
        
        # Track the success of some steps
        self.activity_status = None
        self.prepare_status = None
        self.approve_status = None
        self.publish_status = None
        
        # More file status tracking for reporting in email
        self.malformed_ds_file_names = []
        self.empty_ds_file_names = []
        self.unmatched_ds_file_names = []
    
    def do_activity(self, data = None):
        """
        Activity, do the work
        """
        if(self.logger):
            self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4))
        
        # Create output directories
        self.create_activity_directories()
        
        # Set the published folder name
        # For now use todays date
        self.published_folder_name = (self.published_folder_prefix
                                      + str(datetime.datetime.utcnow().strftime('%Y%m%d'))
                                      + '/')
        
        self.prepare_status = self.check_published_folder_exists()
        
        if self.prepare_status:
            # Download the S3 objects
            self.download_files_from_s3()
        
        # Approve files for publishing
        self.approve_status = self.approve_for_publishing()
        
        if self.approve_status is True:
            
            article_filenames_map = self.profile_article_files()
            
            for doi_id,filenames in article_filenames_map.iteritems():
                
                article_xml_file_name = self.article_xml_from_filename_map(filenames)

                new_filenames = self.new_filenames(doi_id, filenames)
                
                if article_xml_file_name:
                    xml_file = self.INPUT_DIR + os.sep + article_xml_file_name
                    try:
                        self.convert_xml(doi_id, xml_file, filenames, new_filenames)
                    except ParseError:
                        # One possible error is an entirely blank XML file or a malformed xml file
                        continue
                    
                revision = self.next_revision_number(doi_id)
                zip_file_name = self.new_zip_file_name(doi_id, revision)
                if revision and zip_file_name:
                    self.zip_article_files(doi_id, filenames, new_filenames, zip_file_name)<commit_msg>Exception handling fix after testing PublishFinalPOA.<commit_after>import os
import boto.swf
import json
import random
import datetime
import importlib
import calendar
import time

import zipfile
import requests
import urlparse
import glob
import shutil
import re

import activity

from xml.etree import ElementTree
from xml.etree.ElementTree import Element, SubElement

from elifetools import parseJATS as parser
from elifetools import xmlio

import boto.s3
from boto.s3.connection import S3Connection

import provider.s3lib as s3lib

"""
PublishFinalPOA activity
"""

class activity_PublishFinalPOA(activity.activity):
    
    def __init__(self, settings, logger, conn = None, token = None, activity_task = None):
        activity.activity.__init__(self, settings, logger, conn, token, activity_task)

        self.name = "PublishFinalPOA"
        self.version = "1"
        self.default_task_heartbeat_timeout = 60 * 30
        self.default_task_schedule_to_close_timeout = 60 * 30
        self.default_task_schedule_to_start_timeout = 30
        self.default_task_start_to_close_timeout= 60 * 15
        self.description = "Download POA files from a bucket, zip each article separately, and upload to final bucket."
        
        # Local directory settings
        self.TMP_DIR = self.get_tmp_dir() + os.sep + "tmp_dir"
        self.INPUT_DIR = self.get_tmp_dir() + os.sep + "input_dir"
        self.OUTPUT_DIR = self.get_tmp_dir() + os.sep + "output_dir"
        self.JUNK_DIR = self.get_tmp_dir() + os.sep + "junk_dir"
        self.DONE_DIR = self.get_tmp_dir() + os.sep + "done_dir"
        
        # Bucket for outgoing files
        self.input_bucket = settings.poa_packaging_bucket
        self.published_folder_prefix = "published/"
        self.published_folder_name = None
        
        self.publish_bucket = settings.publishing_buckets_prefix + settings.production_bucket
        
        # Track the success of some steps
        self.activity_status = None
        self.prepare_status = None
        self.approve_status = None
        self.publish_status = None
        
        # More file status tracking for reporting in email
        self.malformed_ds_file_names = []
        self.empty_ds_file_names = []
        self.unmatched_ds_file_names = []
    
    def do_activity(self, data = None):
        """
        Activity, do the work
        """
        if(self.logger):
            self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4))
        
        # Create output directories
        self.create_activity_directories()
        
        # Set the published folder name
        # For now use todays date
        self.published_folder_name = (self.published_folder_prefix
                                      + str(datetime.datetime.utcnow().strftime('%Y%m%d'))
                                      + '/')
        
        self.prepare_status = self.check_published_folder_exists()
        
        if self.prepare_status:
            # Download the S3 objects
            self.download_files_from_s3()
        
        # Approve files for publishing
        self.approve_status = self.approve_for_publishing()
        
        if self.approve_status is True:
            
            article_filenames_map = self.profile_article_files()
            
            for doi_id,filenames in article_filenames_map.iteritems():
                
                article_xml_file_name = self.article_xml_from_filename_map(filenames)

                new_filenames = self.new_filenames(doi_id, filenames)
                
                if article_xml_file_name:
                    xml_file = self.INPUT_DIR + os.sep + article_xml_file_name
                    try:
                        self.convert_xml(doi_id, xml_file, filenames, new_filenames)
                    except Exception as e:
                        # One possible error is an entirely blank XML file or a malformed xml file
                        if self.logger:
                            self.logger.exception("Exception when converting XML for doi %s, %s" %
                                                  str(doi_id), e.message)
                    
                revision = self.next_revision_number(doi_id)
                zip_file_name = self.new_zip_file_name(doi_id, revision)
                if revision and zip_file_name:
                    self.zip_article_files(doi_id, filenames, new_filenames, zip_file_name)
{'fraction_non_alphanumeric': 0.06818181818181818, 'fraction_numerical': 0.0053068758652515, 'mean_word_length': 2.094966083541592, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3813884', 'n_tokens_mistral': 2486, 'n_tokens_neox': 2402, 'n_words': 563}