<commit_before>import unittest
import os
import inspect
import boto
from datetime import datetime
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from baseTestAPI import BaseTestAPI
from dataactcore.models.jobModels import Submission, Job
from dataactcore.models.errorModels import ErrorMetadata, File
from dataactcore.config import CONFIG_BROKER
from dataactbroker.handlers.jobHandler import JobHandler
from shutil import copy

class FileTests(BaseTestAPI):
    """Test file submission routes."""

    updateSubmissionId = None
    filesSubmitted = False
    submitFilesResponse = None

    @classmethod
    def setUpClass(cls):
        """Set up class-wide resources (test data)"""
        super(FileTests, cls).setUpClass()
        #TODO: refactor into a pytest fixture

        # get the submission test user
        submission_user = cls.userDb.getUserByEmail(
            cls.test_users['submission_email'])
        cls.submission_user_id = submission_user.user_id

        # setup submission/jobs data for test_check_status
        cls.status_check_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS", startDate = "10/2015", endDate = "06/2016", is_quarter = True)

        cls.jobIdDict = cls.setupJobsForStatusCheck(cls.interfaces,
            cls.status_check_submission_id)

        # setup submission/jobs data for test_error_report
        cls.error_report_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS")
        cls.setupJobsForReports(cls.jobTracker, cls.error_report_submission_id)

        # setup file status data for test_metrics
        cls.test_metrics_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS")
        cls.setupFileData(cls.jobTracker, cls.errorDatabase,
            cls.test_metrics_submission_id)

        cls.row_error_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS", startDate = "10/2015", endDate = "06/2016", is_quarter = True)
        cls.setupSubmissionWithError(cls.interfaces, cls.row_error_submission_id)

    def setUp(self):
        """Test set-up."""
        super(FileTests, self).setUp()
        self.login_other_user(
            self.test_users["submission_email"], self.user_password)

    def call_file_submission(self):
        """Call the broker file submission route."""
        if not self.filesSubmitted:
            if(CONFIG_BROKER["use_aws"]):
                self.filenames = {"appropriations":"test1.csv",
                    "award_financial":"test2.csv", "award":"d2_file.csv.csv",
                    "program_activity":"test4.csv", "cgac_code": "SYS",
                    "reporting_period_start_date":"01/2001",
                    "reporting_period_end_date":"01/2001", "is_quarter":True}
            else:
                # If local must use full destination path
                filePath = CONFIG_BROKER["broker_files"]
                self.filenames = {"appropriations":os.path.join(filePath,"test1.csv"),
                    "award_financial":os.path.join(filePath,"test2.csv"), "award":os.path.join(filePath,"d2_file.csv.csv"),
                    "program_activity":os.path.join(filePath,"test4.csv"), "cgac_code": "SYS",
                    "reporting_period_start_date":"01/2001",
                    "reporting_period_end_date":"01/2001", "is_quarter":True}
            self.submitFilesResponse = self.app.post_json("/v1/submit_files/", self.filenames, headers={"x-session-id":self.session_id})
            self.updateSubmissionId = self.submitFilesResponse.json["submission_id"]
        return self.submitFilesResponse

    def test_file_submission(self):
        """Test broker file submission and response."""
        response = self.call_file_submission()
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.headers.get("Content-Type"), "application/json")

        json = response.json
        self.assertIn("test1.csv", json["appropriations_key"])
        self.assertIn("test2.csv", json["award_financial_key"])
        self.assertIn("d2_file.csv", json["award_key"])
        self.assertIn("test4.csv", json["program_activity_key"])
        self.assertIn("credentials", json)

        credentials = json["credentials"]
        for requiredField in ["AccessKeyId", "SecretAccessKey",
            "SessionToken", "SessionToken"]:
            self.assertIn(requiredField, credentials)
            self.assertTrue(len(credentials[requiredField]))

        self.assertIn("bucket_name", json)
        self.assertTrue(len(json["bucket_name"]))
<commit_msg>updating files names in unit tests to match d2<commit_after>import unittest
import os
import inspect
import boto
from datetime import datetime
from boto.s3.connection import S3Connection
from boto.s3.key import Key
from baseTestAPI import BaseTestAPI
from dataactcore.models.jobModels import Submission, Job
from dataactcore.models.errorModels import ErrorMetadata, File
from dataactcore.config import CONFIG_BROKER
from dataactbroker.handlers.jobHandler import JobHandler
from shutil import copy

class FileTests(BaseTestAPI):
    """Test file submission routes."""

    updateSubmissionId = None
    filesSubmitted = False
    submitFilesResponse = None

    @classmethod
    def setUpClass(cls):
        """Set up class-wide resources (test data)"""
        super(FileTests, cls).setUpClass()
        #TODO: refactor into a pytest fixture

        # get the submission test user
        submission_user = cls.userDb.getUserByEmail(
            cls.test_users['submission_email'])
        cls.submission_user_id = submission_user.user_id

        # setup submission/jobs data for test_check_status
        cls.status_check_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS", startDate = "10/2015", endDate = "06/2016", is_quarter = True)

        cls.jobIdDict = cls.setupJobsForStatusCheck(cls.interfaces,
            cls.status_check_submission_id)

        # setup submission/jobs data for test_error_report
        cls.error_report_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS")
        cls.setupJobsForReports(cls.jobTracker, cls.error_report_submission_id)

        # setup file status data for test_metrics
        cls.test_metrics_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS")
        cls.setupFileData(cls.jobTracker, cls.errorDatabase,
            cls.test_metrics_submission_id)

        cls.row_error_submission_id = cls.insertSubmission(
            cls.jobTracker, cls.submission_user_id, cgac_code = "SYS", startDate = "10/2015", endDate = "06/2016", is_quarter = True)
        cls.setupSubmissionWithError(cls.interfaces, cls.row_error_submission_id)

    def setUp(self):
        """Test set-up."""
        super(FileTests, self).setUp()
        self.login_other_user(
            self.test_users["submission_email"], self.user_password)

    def call_file_submission(self):
        """Call the broker file submission route."""
        if not self.filesSubmitted:
            if(CONFIG_BROKER["use_aws"]):
                self.filenames = {"appropriations":"test1.csv",
                    "award_financial":"test2.csv", "award":CONFIG_BROKER["d2_file_name"],
                    "program_activity":"test4.csv", "cgac_code": "SYS",
                    "reporting_period_start_date":"01/2001",
                    "reporting_period_end_date":"01/2001", "is_quarter":True}
            else:
                # If local must use full destination path
                filePath = CONFIG_BROKER["broker_files"]
                self.filenames = {"appropriations":os.path.join(filePath,"test1.csv"),
                    "award_financial":os.path.join(filePath,"test2.csv"), "award":os.path.join(filePath,CONFIG_BROKER["d2_file_name"]),
                    "program_activity":os.path.join(filePath,"test4.csv"), "cgac_code": "SYS",
                    "reporting_period_start_date":"01/2001",
                    "reporting_period_end_date":"01/2001", "is_quarter":True}
            self.submitFilesResponse = self.app.post_json("/v1/submit_files/", self.filenames, headers={"x-session-id":self.session_id})
            self.updateSubmissionId = self.submitFilesResponse.json["submission_id"]
        return self.submitFilesResponse

    def test_file_submission(self):
        """Test broker file submission and response."""
        response = self.call_file_submission()
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.headers.get("Content-Type"), "application/json")

        json = response.json
        self.assertIn("test1.csv", json["appropriations_key"])
        self.assertIn("test2.csv", json["award_financial_key"])
        self.assertIn(CONFIG_BROKER["d2_file_name"], json["award_key"])
        self.assertIn("test4.csv", json["program_activity_key"])
        self.assertIn("credentials", json)

        credentials = json["credentials"]
        for requiredField in ["AccessKeyId", "SecretAccessKey",
            "SessionToken", "SessionToken"]:
            self.assertIn(requiredField, credentials)
            self.assertTrue(len(credentials[requiredField]))

        self.assertIn("bucket_name", json)
        self.assertTrue(len(json["bucket_name"]))

{'fraction_non_alphanumeric': 0.108497006616952, 'fraction_numerical': 0.014179182858943389, 'mean_word_length': 3.11495246326707, 'pattern_counts': {'":': 34, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7443202', 'n_tokens_mistral': 2899, 'n_tokens_neox': 2781, 'n_words': 502}