<commit_before>import os
import boto.swf
import json
import random
import datetime
import importlib

import zipfile
import requests
import urlparse
import glob
import shutil

import activity

import boto.s3
from boto.s3.connection import S3Connection

import provider.ejp as ejplib

"""
PackagePOA activity
"""

class activity_PackagePOA(activity.activity):
    
    def __init__(self, settings, logger, conn = None, token = None, activity_task = None):
        activity.activity.__init__(self, settings, logger, conn, token, activity_task)

        self.name = "PackagePOA"
        self.version = "1"
        self.default_task_heartbeat_timeout = 30
        self.default_task_schedule_to_close_timeout = 60*30
        self.default_task_schedule_to_start_timeout = 30
        self.default_task_start_to_close_timeout= 60*15
        self.description = "Process POA zip file input, repackage, and save to S3."
        
        # Directory where POA library is stored
        self.poa_lib_dir_name = "elife-poa-xml-generation"
        
        # Where we specify the library to be imported
        self.elife_poa_lib = None
        
        # Import the libraries we will need
        self.import_imports()
        
        # Create output directories
        self.create_activity_directories()
        
        # Create an EJP provider to access S3 bucket holding CSV files
        self.ejp = ejplib.EJP(settings, self.get_tmp_dir())
        
        # Bucket for outgoing files
        self.publish_bucket = settings.poa_packaging_bucket
        self.outbox_folder = "outbox/"
        
        # Some values to set later
        self.poa_zip_filename = None
        self.doi = None
    
    def do_activity(self, data = None):
        """
        Activity, do the work
        """
        if(self.logger):
            self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4))
        
        # Download the S3 object
        document = data["data"]["document"]
        
        # Download POA zip file
        self.download_poa_zip(document)
        
        # Get the DOI from the zip file
        self.get_doi_from_zip_file()
        
        # Transform zip file
        self.process_poa_zipfile()
        
        # Set the DOI and generate XML
        doi_id = self.get_doi_id_from_doi(self.doi)
        self.download_latest_csv()
        self.generate_xml(doi_id)
    
        # Copy finished files to S3 outbox
        self.copy_files_to_s3_outbox()
        
        # TODO!  Assume all worked for now
        result = True

        return result

    def get_doi_id_from_doi(self, doi):
        """
        Extract just the integer doi_id value from the DOI string
        """
        return int(doi.split(".")[-1])

    def download_poa_zip(self, document, bucket_name = None):
        """
        Given the s3 object name as document, download it from the
        POA delivery bucket and save file to disk in the EJP_INPUT_DIR
        """
        if bucket_name is None:
            # Default bucket
            bucket_name = self.settings.poa_bucket
            
        #print bucket_name
        #print document

        # Connect to S3 and bucket
        s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key)
        bucket = s3_conn.lookup(bucket_name)
        # Get the S3 object key
        s3_key = bucket.get_key(document)
        
        # Download and save to disk
        contents = s3_key.get_contents_as_string()
        filename_plus_path = self.elife_poa_lib.settings.EJP_INPUT_DIR + os.sep + document
        mode = "wb"
        f = open(filename_plus_path, mode)
        f.write(contents)
        f.close()
        
        # Save the zip file name for later use
        self.poa_zip_filename = filename_plus_path
            
    def get_doi_from_zip_file(self, filename = None):
        """
        Get the DOI from the zip file manifest.xml using the POA library
        Use the object variable as the default if not specified
        """
        if filename is None:
            filename = self.poa_zip_filename
        if filename is None:
            return None
        
        # Good, continue
        current_zipfile = zipfile.ZipFile(filename, 'r')
        doi = self.elife_poa_lib.transform.get_doi_from_zipfile(current_zipfile)
        
        self.doi = doi
    
    def process_poa_zipfile(self):
        """
        Using the POA transform-ejp-zip-to-hw-zip module
        """
        self.elife_poa_lib.transform.process_zipfile(
            zipfile_name = self.poa_zip_filename,
            output_dir   = self.elife_poa_lib.settings.STAGING_TO_HW_DIR
        )
        <commit_msg>Changes to logically check for bad data, first check for a DOI present.<commit_after>import os
import boto.swf
import json
import random
import datetime
import importlib

import zipfile
import requests
import urlparse
import glob
import shutil

import activity

import boto.s3
from boto.s3.connection import S3Connection

import provider.ejp as ejplib

"""
PackagePOA activity
"""

class activity_PackagePOA(activity.activity):
    
    def __init__(self, settings, logger, conn = None, token = None, activity_task = None):
        activity.activity.__init__(self, settings, logger, conn, token, activity_task)

        self.name = "PackagePOA"
        self.version = "1"
        self.default_task_heartbeat_timeout = 30
        self.default_task_schedule_to_close_timeout = 60*30
        self.default_task_schedule_to_start_timeout = 30
        self.default_task_start_to_close_timeout= 60*15
        self.description = "Process POA zip file input, repackage, and save to S3."
        
        # Directory where POA library is stored
        self.poa_lib_dir_name = "elife-poa-xml-generation"
        
        # Where we specify the library to be imported
        self.elife_poa_lib = None
        
        # Import the libraries we will need
        self.import_imports()
        
        # Create output directories
        self.create_activity_directories()
        
        # Create an EJP provider to access S3 bucket holding CSV files
        self.ejp = ejplib.EJP(settings, self.get_tmp_dir())
        
        # Bucket for outgoing files
        self.publish_bucket = settings.poa_packaging_bucket
        self.outbox_folder = "outbox/"
        
        # Some values to set later
        self.poa_zip_filename = None
        self.doi = None
    
    def do_activity(self, data = None):
        """
        Activity, do the work
        """
        if(self.logger):
            self.logger.info('data: %s' % json.dumps(data, sort_keys=True, indent=4))
        
        # Download the S3 object
        document = data["data"]["document"]
        
        # Download POA zip file
        self.download_poa_zip(document)
        
        # Get the DOI from the zip file
        self.get_doi_from_zip_file()
        doi_id = self.get_doi_id_from_doi(self.doi)
        
        if self.approve_for_packaging(doi_id) is False:
            # Bad. Fail the activity
            result = False
            
        else:
            # Good, continue
            
            # Transform zip file
            self.process_poa_zipfile()
            
            # Set the DOI and generate XML
            
            self.download_latest_csv()
            self.generate_xml(doi_id)
        
            # Copy finished files to S3 outbox
            self.copy_files_to_s3_outbox()
            
            # TODO!  Assume all worked for now
            result = True
            
        # Send email - TODO!!!

        # Return the activity result, True or False
        return result

    def get_doi_id_from_doi(self, doi):
        """
        Extract just the integer doi_id value from the DOI string
        """
        try:
            doi_id = int(doi.split(".")[-1])
        except:
            doi_id = None
            
        return doi_id

    def download_poa_zip(self, document, bucket_name = None):
        """
        Given the s3 object name as document, download it from the
        POA delivery bucket and save file to disk in the EJP_INPUT_DIR
        """
        if bucket_name is None:
            # Default bucket
            bucket_name = self.settings.poa_bucket
            
        #print bucket_name
        #print document

        # Connect to S3 and bucket
        s3_conn = S3Connection(self.settings.aws_access_key_id, self.settings.aws_secret_access_key)
        bucket = s3_conn.lookup(bucket_name)
        # Get the S3 object key
        s3_key = bucket.get_key(document)
        
        # Download and save to disk
        contents = s3_key.get_contents_as_string()
        filename_plus_path = self.elife_poa_lib.settings.EJP_INPUT_DIR + os.sep + document
        mode = "wb"
        f = open(filename_plus_path, mode)
        f.write(contents)
        f.close()
        
        # Save the zip file name for later use
        self.poa_zip_filename = filename_plus_path
            
    def get_doi_from_zip_file(self, filename = None):
        """
        Get the DOI from the zip file manifest.xml using the POA library
        Use the object variable as the default if not specified
        """
        if filename is None:
            filename = self.poa_zip_filename
        if filename is None:
            return None
        
        # Good, continue
        current_zipfile = zipfile.ZipFile(filename, 'r')
        doi = self.elife_poa_lib.transform.get_doi_from_zipfile(current_zipfile)
        
        self.doi = doi
    
    def approve_for_packaging(self, doi_id):
        """
        After downloading the zip file but before starting to package it,
        do all the pre-packaging steps and checks, including to have a DOI
        """
        if doi_id is None:
            return False
        return True
    
    def process_poa_zipfile(self):
        """
        Using the POA transform-ejp-zip-to-hw-zip module
        """
        self.elife_poa_lib.transform.process_zipfile(
            zipfile_name = self.poa_zip_filename,
            output_dir   = self.elife_poa_lib.settings.STAGING_TO_HW_DIR
        )
        
{'fraction_non_alphanumeric': 0.0675848499754058, 'fraction_numerical': 0.006099360550909985, 'mean_word_length': 1.9664429530201342, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '753888', 'n_tokens_mistral': 3016, 'n_tokens_neox': 2913, 'n_words': 793}