<commit_before>#!/usr/bin/env python3
""" <NAME> 2019-02-11 for DOAJ - uploaded manually (todo: we should upload this in the release script) """

import boto3
import botocore
import json
from datetime import datetime, timezone, timedelta

s3 = boto3.client('s3')

# Check the doaj elasticsearch snapshot bucket has been updated today (should happen daily at 0600 via background job)
buckets = ['doaj-index-backups']

# TODO: Check the doaj-nginx logs bucket has been updated today (should happen daily at 0630 via cron logrotate)
# buckets += ['doaj-nginx-logs']


def lambda_handler(event, context):
    """ The main function executed by Lambda"""

    summary = {'success': [], 'fail': []}
    for b in buckets:

        # First check the bucket actually exists
        try:
            s3.head_bucket(Bucket=b)
        except botocore.exceptions.ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 404:
                send_alert_email(b, last_mod=None)

        # Then check the expected entry exists in the bucket's objects.
        files = list_bucket_keys(bucket_name=b)
        old_to_new = sorted(files, key=lambda f: f['LastModified'])
        newest = old_to_new[-1]

        # If the newest file is older than 1 day old, our backups are not up to date.
        if datetime.now(timezone.utc) - newest['LastModified'] > timedelta(days=1):
            # Send an alert
            send_alert_email(bucket=b, last_mod=newest['LastModified'])<commit_msg>Enable lambda alerting on the nginx logs backup<commit_after>#!/usr/bin/env python3
""" <NAME> 2020-02-07 for DOAJ - uploaded manually (todo: we should upload this in the release script) """

import boto3
import botocore
import json
from datetime import datetime, timezone, timedelta

s3 = boto3.client('s3')

# Check the doaj elasticsearch snapshot bucket has been updated today (should happen daily at 0600 via background job)
buckets = ['doaj-index-backups']

# Check the doaj-nginx logs bucket has been updated today (should happen daily at 0630 via cron logrotate)
buckets += ['doaj-nginx-logs']


def lambda_handler(event, context):
    """ The main function executed by Lambda"""

    summary = {'success': [], 'fail': []}
    for b in buckets:

        # First check the bucket actually exists
        try:
            s3.head_bucket(Bucket=b)
        except botocore.exceptions.ClientError as e:
            error_code = int(e.response['Error']['Code'])
            if error_code == 404:
                send_alert_email(b, last_mod=None)

        # Then check the expected entry exists in the bucket's objects.
        files = list_bucket_keys(bucket_name=b)
        old_to_new = sorted(files, key=lambda f: f['LastModified'])
        newest = old_to_new[-1]

        # If the newest file is older than 1 day old, our backups are not up to date.
        if datetime.now(timezone.utc) - newest['LastModified'] > timedelta(days=1):
            # Send an alert
            send_alert_email(bucket=b, last_mod=newest['LastModified'])
{'fraction_non_alphanumeric': 0.10091743119266056, 'fraction_numerical': 0.01834862385321101, 'mean_word_length': 3.1368563685636857, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5201551', 'n_tokens_mistral': 943, 'n_tokens_neox': 879, 'n_words': 307}