<commit_before>import argparse
import re
import logging
import os
import csv
import boto3
import datetime
import json

from dataactcore.logging import configure_logging
from dataactcore.interfaces.db import GlobalDB
from dataactcore.config import CONFIG_BROKER
from dataactvalidator.health_check import create_app

logger = logging.getLogger(__name__)

'''
This script is used to pull updated financial assistance records (from --date to present) for FSRS.
It can also run with --auto to poll the specified S3 bucket (BUCKET_NAME/BUCKET_PREFIX}) for the most
recent file that was uploaded, and use the boto3 response for --date.
'''

BUCKET_NAME = CONFIG_BROKER['data_extracts_bucket']
BUCKET_PREFIX = 'fsrs_award_extracts/'


def main():
    now = datetime.datetime.now()
    parser = argparse.ArgumentParser(description='Pull')
    parser.add_argument('--date',
                        help='Specify modified date in mm/dd/yyyy format. Overrides --auto option.',
                        nargs=1, type=str)
    parser.add_argument('--auto',
                        help='Polls S3 for the most recently uploaded FABS_for_FSRS file, '
                             + 'and uses that as the modified date.',
                        action='store_true')
    args = parser.parse_args()

    metrics_json = {
        'script_name': 'get_fsrs_updates.py',
        'start_time': str(now),
        'records_provided': 0,
        'start_date': ''
    }

    if args.auto:
        s3_resource = boto3.resource('s3', region_name='us-gov-west-1')
        extract_bucket = s3_resource.Bucket(BUCKET_NAME)
        all_fsrs_extracts = extract_bucket.objects.filter(Prefix=BUCKET_PREFIX)
        mod_date = max(all_fsrs_extracts, key=lambda k: k.last_modified).last_modified.strftime("%m/%d/%Y")

    if args.date:
        arg_date = args.date[0]
        given_date = arg_date.split('/')
        if not re.match('^\d{2}$', given_date[0]) or not re.match('^\d{2}$', given_date[1])\
                or not re.match('^\d{4}$', given_date[2]):
            logger.error("Date " + arg_date + " not in proper mm/dd/yyyy format")
            return
        mod_date = arg_date

    if not mod_date:
        logger.error("Date or auto setting is required.")
        return

    metrics_json['start_date'] = mod_date

    logger.info("Starting SQL query of financial assistance records from {} to present...".format(mod_date))
    sess = GlobalDB.db().session
    results = sess.execute("""
    WITH base_transaction AS (
    SELECT fain,
        MIN(pafa_b.action_date) as base_date,
        MIN(pafa_b.period_of_performance_star) as earliest_start,
        MAX(pafa_b.period_of_performance_curr) as latest_end,
        MAX(pafa_b.modified_at) as max_mod,
        SUM(CASE WHEN pafa_b.is_active = True
                    THEN pafa_b.federal_action_obligation
                    ELSE 0
                    END) as obligation_sum,
        CASE WHEN EXISTS (SELECT 1
                            FROM published_award_financial_assistance AS sub_pafa_b
                            WHERE is_active = True<commit_msg>Adding a plain english summary for the FSRS Extract<commit_after>import argparse
import re
import logging
import os
import csv
import boto3
import datetime
import json

from dataactcore.logging import configure_logging
from dataactcore.interfaces.db import GlobalDB
from dataactcore.config import CONFIG_BROKER
from dataactvalidator.health_check import create_app

logger = logging.getLogger(__name__)

'''
This script is used to pull updated financial assistance records (from --date to present) for FSRS.
It can also run with --auto to poll the specified S3 bucket (BUCKET_NAME/BUCKET_PREFIX}) for the most
recent file that was uploaded, and use the boto3 response for --date.
'''

BUCKET_NAME = CONFIG_BROKER['data_extracts_bucket']
BUCKET_PREFIX = 'fsrs_award_extracts/'


def main():
    now = datetime.datetime.now()
    parser = argparse.ArgumentParser(description='Pull')
    parser.add_argument('--date',
                        help='Specify modified date in mm/dd/yyyy format. Overrides --auto option.',
                        nargs=1, type=str)
    parser.add_argument('--auto',
                        help='Polls S3 for the most recently uploaded FABS_for_FSRS file, '
                             + 'and uses that as the modified date.',
                        action='store_true')
    args = parser.parse_args()

    metrics_json = {
        'script_name': 'get_fsrs_updates.py',
        'start_time': str(now),
        'records_provided': 0,
        'start_date': ''
    }

    if args.auto:
        s3_resource = boto3.resource('s3', region_name='us-gov-west-1')
        extract_bucket = s3_resource.Bucket(BUCKET_NAME)
        all_fsrs_extracts = extract_bucket.objects.filter(Prefix=BUCKET_PREFIX)
        mod_date = max(all_fsrs_extracts, key=lambda k: k.last_modified).last_modified.strftime("%m/%d/%Y")

    if args.date:
        arg_date = args.date[0]
        given_date = arg_date.split('/')
        if not re.match('^\d{2}$', given_date[0]) or not re.match('^\d{2}$', given_date[1])\
                or not re.match('^\d{4}$', given_date[2]):
            logger.error("Date " + arg_date + " not in proper mm/dd/yyyy format")
            return
        mod_date = arg_date

    if not mod_date:
        logger.error("Date or auto setting is required.")
        return

    metrics_json['start_date'] = mod_date

    logger.info("Starting SQL query of financial assistance records from {} to present...".format(mod_date))
    sess = GlobalDB.db().session
    """
    Query Summary:
    Each row is the *latest transaction of an award* with the transaction’s modified_date being within the past day and
    also includes summary data about the award associated with the transaction. 
    """
    results = sess.execute("""
    WITH base_transaction AS (
    SELECT fain,
        MIN(pafa_b.action_date) as base_date,
        MIN(pafa_b.period_of_performance_star) as earliest_start,
        MAX(pafa_b.period_of_performance_curr) as latest_end,
        MAX(pafa_b.modified_at) as max_mod,
        SUM(CASE WHEN pafa_b.is_active = True
                    THEN pafa_b.federal_action_obligation
                    ELSE 0
                    END) as obligation_sum,
        CASE WHEN EXISTS (SELECT 1
                            FROM published_award_financial_assistance AS sub_pafa_b
                            WHERE is_active = True
{'fraction_non_alphanumeric': 0.09419615145872129, 'fraction_numerical': 0.006207324643078833, 'mean_word_length': 2.6066032456631225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6526172', 'n_tokens_mistral': 1983, 'n_tokens_neox': 1899, 'n_words': 511}