<commit_before>import json
import unittest
import os
import inspect
import time
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from baseTest import BaseTest
from testUtils import TestUtils
from dataactcore.aws.s3UrlHandler import s3UrlHandler
from dataactcore.models.jobModels import Submission, JobStatus
from dataactcore.models.errorModels import ErrorData, FileStatus
from dataactbroker.handlers.fileHandler import FileHandler

class FileTests(BaseTest):
    """ Test file submission routes """
    fileResponse = None
    CHECK_ERROR_REPORTS = False # If True, must provide submission ID below
    ERROR_REPORT_SUBMISSION_ID = 11
    JOB_ID_FILE = "jobId.json"
    SUBMISSION_ID_FILE = "submissionId"
    TABLES_CLEARED_FILE = "tablesCleared" # Holds a boolean flag in a file so it can be checked before doing table setup
    tablesCleared = False # Set to true after first time setup is run
    submissionId = None
    passed = False

    def __init__(self,methodName,interfaces):
        """ Run scripts to clear the job tables and populate with a defined test set """
        super(FileTests,self).__init__(methodName=methodName)
        self.methodName = methodName
        jobTracker = interfaces.jobDb
        self.jobTracker = jobTracker
        self.errorDatabase = interfaces.errorDb
        self.interfaces = interfaces
        try:
            self.tablesCleared = self.toBool(open(self.TABLES_CLEARED_FILE,"r").read())
        except Exception as e:
            # Could not read from file or cast as boolean
            print(str(e))
            self.tablesCleared = False

        if(not self.tablesCleared):
            # Clear job tracker
            #clearJobs()
            self.tablesCleared = True
            open(self.TABLES_CLEARED_FILE,"w").write(str(True))
            # Create submission ID

            self.submissionId = self.insertSubmission(self.jobTracker)

            # Create jobs
            jobValues = {}
            jobValues["uploadFinished"] = [1,4,1]
            jobValues["recordRunning"] = [1,3,2]
            jobValues["externalWaiting"] = [1,1,5]
            jobValues["awardFin"] = [2,2,2]
            jobValues["appropriations"] = [3,2,2]
            jobValues["procurement"] = [4,2,2]
            self.jobIdDict = {}

            for jobKey, values in jobValues.items():
                job_id = self.insertJob(
                    self.jobTracker,
                    filetype = values[0],
                    status = values[1],
                    type_id = values[2],
                    submission = self.submissionId
                )
                self.jobIdDict[jobKey] = job_id

            # Save jobIdDict to file
            open(self.JOB_ID_FILE,"w").write(json.dumps(self.jobIdDict))
            open(self.SUBMISSION_ID_FILE,"w").write(str(self.submissionId))

        else:
            # Read job ID dict from file
            self.jobIdDict = json.loads(open(self.JOB_ID_FILE,"r").read())
            self.submissionId = int(open(self.SUBMISSION_ID_FILE,"r").read())

    @staticmethod
    def toBool(stringValue):
        if(stringValue.lower() == "true"):
            return True
        elif(stringValue.lower() == "false"):
            return False
        else:
            raise ValueError("Invalid string passed to toBool function")

    def call_file_submission(self):
        # If fileResponse doesn't exist, send the request
        fileJson = '{"appropriations":"test1.csv","award_financial":"test2.csv","award":"test3.csv","procurement":"test4.csv"}'
        if(self.fileResponse == None):
            self.utils.login()
            self.fileResponse = self.utils.postRequest("/v1/submit_files/",fileJson)

    def test_file_submission(self):
        open(FileHandler.VALIDATOR_RESPONSE_FILE,"w").write(str(-1))
        self.call_file_submission()
        self.response = self.fileResponse # Used for displaying failures
        # Test that status is 200
        assert(self.fileResponse.status_code==200)
        # Test Content-Type header
        assert("Content-Type" in self.fileResponse.headers)
        assert(self.fileResponse.headers["Content-Type"]=="application/json")
        # Test message parts for urls
        assert("_test1.csv" in self.fileResponse.json()["appropriations_key"] )
        assert("_test2.csv" in self.fileResponse.json()["award_financial_key"])
        assert("_test3.csv" in self.fileResponse.json()["award_key"])
        assert("_test4.csv" in self.fileResponse.json()["procurement_key"])

        for requiredField in ["AccessKeyId","SecretAccessKey","SessionToken","SessionToken"] :
            assert(len(self.fileResponse.json()["credentials"][requiredField]) > 0)

        assert(len(self.fileResponse.json()["bucket_name"]) > 0)

        self.uploadFileByURL("/"+self.fileResponse.json()["appropriations_key"],"test1.csv")
        # Test that job ids are returned
        responseDict = self.fileResponse.json()
        idKeys = ["procurement_id", "award_id", "award_financial_id", "appropriations_id"]
        for key in idKeys:
            assert(key in responseDict)
            try:
                int(responseDict[key])
            except:
                self.fail("One of the job ids returned was not an integer")

        # Test that correct user ID is on submission
        submissionId = responseDict["submission_id"]
        user = self.interfaces.userDb.getUserByEmail("user3")
        userId = user.user_id
        submission = self.interfaces.jobDb.getSubmissionById(submissionId)
        assert(submission.user_id == userId) # Check that submission got mapped to the correct user

        # Call upload complete route for each id
        self.check_upload_complete(responseDict["appropriations_id"])
        self.passed = True

    @staticmethod
    def waitOnJob(jobTracker, jobId, status):
        currentID = jobTracker.getStatusId("running")
        targetStatus = jobTracker.getStatusId(status)
        while jobTracker.getStatus(jobId) == currentID:
            time.sleep(1)
        assert(targetStatus == jobTracker.getStatus(jobId))

    def test_check_status(self):
        """ Check that test status route returns correct JSON"""
        utils = TestUtils()
        utils.login()
        self.response = utils.postRequest("/v1/check_status/",'{"submission_id":'+str(self.submissionId)+'}')

        assert(self.response.status_code == 200)
        assert(self.response.json()[str(self.jobIdDict["uploadFinished"])]["status"]=="finished")
        assert(self.response.json()[str(self.jobIdDict["uploadFinished"])]["job_type"]=="file_upload")
        assert(self.response.json()[str(self.jobIdDict["uploadFinished"])]["file_type"]=="award")
        assert(self.response.json()[str(self.jobIdDict["recordRunning"])]["status"]=="running")
        assert(self.response.json()[str(self.jobIdDict["recordRunning"])]["job_type"]=="csv_record_validation")
        assert(self.response.json()[str(self.jobIdDict["recordRunning"])]["file_type"]=="award")
        assert(self.response.json()[str(self.jobIdDict["externalWaiting"])]["status"]=="waiting")
        assert(self.response.json()[str(self.jobIdDict["externalWaiting"])]["job_type"]=="external_validation")
        assert(self.response.json()[str(self.jobIdDict["externalWaiting"])]["file_type"]=="award")
        assert(self.response.json()[str(self.jobIdDict["appropriations"])]["status"]=="ready")
        assert(self.response.json()[str(self.jobIdDict["appropriations"])]["job_type"]=="csv_record_validation")
        assert(self.response.json()[str(self.jobIdDict["appropriations"])]["file_type"]=="appropriations")
        self.passed = True

    def check_error_route(self,jobId,submissonId) :
        jobJson = json.dumps({"upload_id":jobId})
        urlData = self.utils.postRequest("/v1/job_error_report/",jobJson)
        assert("submission_"+str(submissonId)+"_procurement_error_report" in urlData.json()["error_url"] )
        assert("?Signature" in urlData.json()["error_url"] )
        assert("&AWSAccessKeyId" in urlData.json()["error_url"])

    def check_upload_complete(self, jobId):
        jobJson = json.dumps({"upload_id":jobId})
        self.utils.login()
        finalizeResponse = self.utils.postRequest("/v1/finalize_job/",jobJson)
        print("Finalize response:")
        print(str(finalizeResponse.status_code))
        print(str(finalizeResponse.json()))<commit_msg>Removed extra debug messages in fileTests<commit_after>import json
import unittest
import os
import inspect
import time
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from baseTest import BaseTest
from testUtils import TestUtils
from dataactcore.aws.s3UrlHandler import s3UrlHandler
from dataactcore.models.jobModels import Submission, JobStatus
from dataactcore.models.errorModels import ErrorData, FileStatus
from dataactbroker.handlers.fileHandler import FileHandler

class FileTests(BaseTest):
    """ Test file submission routes """
    fileResponse = None
    CHECK_ERROR_REPORTS = False # If True, must provide submission ID below
    ERROR_REPORT_SUBMISSION_ID = 11
    JOB_ID_FILE = "jobId.json"
    SUBMISSION_ID_FILE = "submissionId"
    TABLES_CLEARED_FILE = "tablesCleared" # Holds a boolean flag in a file so it can be checked before doing table setup
    tablesCleared = False # Set to true after first time setup is run
    submissionId = None
    passed = False

    def __init__(self,methodName,interfaces):
        """ Run scripts to clear the job tables and populate with a defined test set """
        super(FileTests,self).__init__(methodName=methodName)
        self.methodName = methodName
        jobTracker = interfaces.jobDb
        self.jobTracker = jobTracker
        self.errorDatabase = interfaces.errorDb
        self.interfaces = interfaces
        try:
            self.tablesCleared = self.toBool(open(self.TABLES_CLEARED_FILE,"r").read())
        except Exception as e:
            # Could not read from file or cast as boolean
            print(str(e))
            self.tablesCleared = False

        if(not self.tablesCleared):
            # Clear job tracker
            #clearJobs()
            self.tablesCleared = True
            open(self.TABLES_CLEARED_FILE,"w").write(str(True))
            # Create submission ID

            self.submissionId = self.insertSubmission(self.jobTracker)

            # Create jobs
            jobValues = {}
            jobValues["uploadFinished"] = [1,4,1]
            jobValues["recordRunning"] = [1,3,2]
            jobValues["externalWaiting"] = [1,1,5]
            jobValues["awardFin"] = [2,2,2]
            jobValues["appropriations"] = [3,2,2]
            jobValues["procurement"] = [4,2,2]
            self.jobIdDict = {}

            for jobKey, values in jobValues.items():
                job_id = self.insertJob(
                    self.jobTracker,
                    filetype = values[0],
                    status = values[1],
                    type_id = values[2],
                    submission = self.submissionId
                )
                self.jobIdDict[jobKey] = job_id

            # Save jobIdDict to file
            open(self.JOB_ID_FILE,"w").write(json.dumps(self.jobIdDict))
            open(self.SUBMISSION_ID_FILE,"w").write(str(self.submissionId))

        else:
            # Read job ID dict from file
            self.jobIdDict = json.loads(open(self.JOB_ID_FILE,"r").read())
            self.submissionId = int(open(self.SUBMISSION_ID_FILE,"r").read())

    @staticmethod
    def toBool(stringValue):
        if(stringValue.lower() == "true"):
            return True
        elif(stringValue.lower() == "false"):
            return False
        else:
            raise ValueError("Invalid string passed to toBool function")

    def call_file_submission(self):
        # If fileResponse doesn't exist, send the request
        fileJson = '{"appropriations":"test1.csv","award_financial":"test2.csv","award":"test3.csv","procurement":"test4.csv"}'
        if(self.fileResponse == None):
            self.utils.login()
            self.fileResponse = self.utils.postRequest("/v1/submit_files/",fileJson)

    def test_file_submission(self):
        open(FileHandler.VALIDATOR_RESPONSE_FILE,"w").write(str(-1))
        self.call_file_submission()
        self.response = self.fileResponse # Used for displaying failures
        # Test that status is 200
        assert(self.fileResponse.status_code==200)
        # Test Content-Type header
        assert("Content-Type" in self.fileResponse.headers)
        assert(self.fileResponse.headers["Content-Type"]=="application/json")
        # Test message parts for urls
        assert("_test1.csv" in self.fileResponse.json()["appropriations_key"] )
        assert("_test2.csv" in self.fileResponse.json()["award_financial_key"])
        assert("_test3.csv" in self.fileResponse.json()["award_key"])
        assert("_test4.csv" in self.fileResponse.json()["procurement_key"])

        for requiredField in ["AccessKeyId","SecretAccessKey","SessionToken","SessionToken"] :
            assert(len(self.fileResponse.json()["credentials"][requiredField]) > 0)

        assert(len(self.fileResponse.json()["bucket_name"]) > 0)

        self.uploadFileByURL("/"+self.fileResponse.json()["appropriations_key"],"test1.csv")
        # Test that job ids are returned
        responseDict = self.fileResponse.json()
        idKeys = ["procurement_id", "award_id", "award_financial_id", "appropriations_id"]
        for key in idKeys:
            assert(key in responseDict)
            try:
                int(responseDict[key])
            except:
                self.fail("One of the job ids returned was not an integer")

        # Test that correct user ID is on submission
        submissionId = responseDict["submission_id"]
        user = self.interfaces.userDb.getUserByEmail("user3")
        userId = user.user_id
        submission = self.interfaces.jobDb.getSubmissionById(submissionId)
        assert(submission.user_id == userId) # Check that submission got mapped to the correct user

        # Call upload complete route for each id
        self.check_upload_complete(responseDict["appropriations_id"])
        self.passed = True

    @staticmethod
    def waitOnJob(jobTracker, jobId, status):
        currentID = jobTracker.getStatusId("running")
        targetStatus = jobTracker.getStatusId(status)
        while jobTracker.getStatus(jobId) == currentID:
            time.sleep(1)
        assert(targetStatus == jobTracker.getStatus(jobId))

    def test_check_status(self):
        """ Check that test status route returns correct JSON"""
        utils = TestUtils()
        utils.login()
        self.response = utils.postRequest("/v1/check_status/",'{"submission_id":'+str(self.submissionId)+'}')

        assert(self.response.status_code == 200)
        assert(self.response.json()[str(self.jobIdDict["uploadFinished"])]["status"]=="finished")
        assert(self.response.json()[str(self.jobIdDict["uploadFinished"])]["job_type"]=="file_upload")
        assert(self.response.json()[str(self.jobIdDict["uploadFinished"])]["file_type"]=="award")
        assert(self.response.json()[str(self.jobIdDict["recordRunning"])]["status"]=="running")
        assert(self.response.json()[str(self.jobIdDict["recordRunning"])]["job_type"]=="csv_record_validation")
        assert(self.response.json()[str(self.jobIdDict["recordRunning"])]["file_type"]=="award")
        assert(self.response.json()[str(self.jobIdDict["externalWaiting"])]["status"]=="waiting")
        assert(self.response.json()[str(self.jobIdDict["externalWaiting"])]["job_type"]=="external_validation")
        assert(self.response.json()[str(self.jobIdDict["externalWaiting"])]["file_type"]=="award")
        assert(self.response.json()[str(self.jobIdDict["appropriations"])]["status"]=="ready")
        assert(self.response.json()[str(self.jobIdDict["appropriations"])]["job_type"]=="csv_record_validation")
        assert(self.response.json()[str(self.jobIdDict["appropriations"])]["file_type"]=="appropriations")
        self.passed = True

    def check_error_route(self,jobId,submissonId) :
        jobJson = json.dumps({"upload_id":jobId})
        urlData = self.utils.postRequest("/v1/job_error_report/",jobJson)
        assert("submission_"+str(submissonId)+"_procurement_error_report" in urlData.json()["error_url"] )
        assert("?Signature" in urlData.json()["error_url"] )
        assert("&AWSAccessKeyId" in urlData.json()["error_url"])

    def check_upload_complete(self, jobId):
        jobJson = json.dumps({"upload_id":jobId})
        self.utils.login()
        finalizeResponse = self.utils.postRequest("/v1/finalize_job/",jobJson)
{'fraction_non_alphanumeric': 0.12635163390883566, 'fraction_numerical': 0.006571479777764502, 'mean_word_length': 3.0829268292682928, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3469441', 'n_tokens_mistral': 4921, 'n_tokens_neox': 4835, 'n_words': 876}