<commit_before>from __future__ import print_function
import boto3
import json
import decimal
from boto3.dynamodb.conditions  import Key, Attr
from botocore.exceptions import ClientError


class DynamoDB(object):
    def db_find(self, table):
        x = []
        for i in table.find():
            x.append(i)
        return x
    def db_find_one(self, table, key_val):
        val = key_val['uid']
        return ddb_find(table, val)
    def db_insert_one(self, table, doc):
        create_new(table, doc)
    def load_old_Text(self, key_val):
        x = ddb_find('entries',key_val['uid'])[0]['payload']
        y = x.replace("'",'"')
        x = json.loads(y)
        old_text = x['past_lives'][0]
        return old_text
    def find_previous_Entry(self, key_val):
        return ddb_find('entries',key_val['uid'])[0]
    def find_clean_save(self, key_val):
        clean_and_save(key_val['uid'])
    def update_DB(self, doc):
        update('entries', doc['uid'], doc['parsed']['text'], str(doc))
        e = ddb_find('entries', doc['uid'])
        return e
    def check_db(self, text):
        table = dynamodb.Table('entries')
        entries = table.scan()['Items']
        for entry in entries:
            if entry['text'] == text:
                payload = entry['payload']
                x = payload.replace("'",'"')
                entry = json.loads(x)
                if entry['parsed']['intent']['confidence'] > 0.6:
                    return entry
        return False



#Helper class to convert a DynamoDB item to json
class DecimalEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, decimal.Decimal):
            if o % 1 > 0:
                return float(o)
            else:
                return int(o)
        return super(DecimalEncoder, self).default(o)

dynamodb = boto3.resource('dynamodb', region_name='us-west-2', endpoint_url="http://localhost:8000")


def create_new(t_name, doc):
    table = dynamodb.Table(t_name)
    #if t_name == 'entries':
    response = table.put_item(
        Item={
            'uid': doc['_id'],
            'text': doc['text'],
            'date': doc['date'],
            'past_lives': doc['past_lives']
             }
        )
    x = list(doc)
    if 'payload' in x:
        update(t_name, doc['_id'],doc['text'],doc['payload'])
    print("PutItem succeeded")
    return json.dumps(response, indent=4, cls=DecimalEncoder)


def get(t_name, uid):
    table = dynamodb.Table(t_name)
    try:
        response = table.get_item(
            Key={
                'year': year,
                'title': title
            }
        )
    except ClientError as e:
        print(e.response['Error']['Message'])
    else:
        item = response['Item']
        print("GetItem succeeded:")
        x = json.dumps(item, indent=4,cls=DecimalEncoder)
        the_dict = json.loads(x)
        return the_dict


def update(t_name, uid, text, p):
    table = dynamodb.Table(t_name)
    response = table.update_item(
        Key={
            'uid': uid,
            'text': text
        },
        UpdateExpression="set payload = :p",
        ExpressionAttributeValues={
            ':p': p
        },
        ReturnValues="UPDATED_NEW"
    )

    print("UpdateItem succeeded:")
    print(json.dumps(response, indent=4, cls=DecimalEncoder))



def create_table(name):
    table = dynamodb.create_table(
        TableName=name,
        KeySchema=[
            {
                'AttributeName': 'uid',
                'KeyType': 'HASH'  #Partition key
            },
            {
                'AttributeName': 'text',
                'KeyType': 'RANGE'  #Sort key
            }
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'uid',
                'AttributeType': 'S'
            },
            {
                'AttributeName': 'text',
                'AttributeType': 'S'
            },

        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 10,
            'WriteCapacityUnits': 10
        }
    )

    print("Table status:", table.table_status)


def ddb_find(t_name, value):
    table = dynamodb.Table(t_name)
    response = table.query(
        KeyConditionExpression=Key('uid').eq(value)
    )

    return response['Items']


def delete_table(table_name):
    table = dynamodb.Table(table_name)
    table.delete()


def clean_and_save(uid):
    dirty = ddb_find('entries',uid)[0]['payload']
    y = dirty.replace("'",'"')
    dirty = json.loads(y)
    cleansed = dirty['parsed']
    try:
        cleansed['intent'] = cleansed['intent']['name']
    except:
        pass
    if cleansed['entities'] != []:
        leng = len(cleansed['entities'])
        for i in range(0,leng):
            if cleansed['entities'][i]['entity'] != 'date_time':
                try:
                    item = cleansed['entities'][i]
                    item.pop('extractor')
                    item.pop('processors')
                except:
                    pass
            else:
                return
        cleansed = {'_id': uid,
                    'text': 'runtime_training_data',
                    'date': str(datetime.datetime.now()),
                    'past_lives': [],
                    'payload': cleansed}
        create_new('runtime_training_data', cleansed)<commit_msg>Update poly_dynamo.py<commit_after>from __future__ import print_function
import boto3
import json
import decimal
from boto3.dynamodb.conditions  import Key, Attr
from botocore.exceptions import ClientError


class DynamoDB(object):
    def db_find(self, table):
        x = []
        for i in table.find():
            x.append(i)
        return x
    def db_find_one(self, table, key_val):
        val = key_val['uid']
        return ddb_find(table, val)
    def db_insert_one(self, doc):
        create_new('entries', doc)
    def load_old_Text(self, key_val):
        x = ddb_find('entries',key_val['uid'])[0]['payload']
        y = x.replace("'",'"')
        x = json.loads(y)
        old_text = x['past_lives'][0]
        return old_text
    def find_previous_Entry(self, key_val):
        return ddb_find('entries',key_val['uid'])[0]
    def find_clean_save(self, key_val):
        clean_and_save(key_val['uid'])
    def update_DB(self, doc):
        update('entries', doc['uid'], doc['parsed']['text'], str(doc))
        e = ddb_find('entries', doc['uid'])
        return e
    def check_db(self, text):
        table = dynamodb.Table('entries')
        entries = table.scan()['Items']
        for entry in entries:
            if entry['text'] == text:
                payload = entry['payload']
                x = payload.replace("'",'"')
                entry = json.loads(x)
                if entry['parsed']['intent']['confidence'] > 0.6:
                    return entry
        return False



#Helper class to convert a DynamoDB item to json
class DecimalEncoder(json.JSONEncoder):
    def default(self, o):
        if isinstance(o, decimal.Decimal):
            if o % 1 > 0:
                return float(o)
            else:
                return int(o)
        return super(DecimalEncoder, self).default(o)

dynamodb = boto3.resource('dynamodb', region_name='us-west-2', endpoint_url="http://localhost:8000")


def create_new(t_name, doc):
    table = dynamodb.Table(t_name)
    #if t_name == 'entries':
    response = table.put_item(
        Item={
            'uid': doc['_id'],
            'text': doc['text'],
            'date': doc['date'],
            'past_lives': doc['past_lives']
             }
        )
    x = list(doc)
    if 'payload' in x:
        update(t_name, doc['_id'],doc['text'],doc['payload'])
    print("PutItem succeeded")
    return json.dumps(response, indent=4, cls=DecimalEncoder)


def get(t_name, uid):
    table = dynamodb.Table(t_name)
    try:
        response = table.get_item(
            Key={
                'year': year,
                'title': title
            }
        )
    except ClientError as e:
        print(e.response['Error']['Message'])
    else:
        item = response['Item']
        print("GetItem succeeded:")
        x = json.dumps(item, indent=4,cls=DecimalEncoder)
        the_dict = json.loads(x)
        return the_dict


def update(t_name, uid, text, p):
    table = dynamodb.Table(t_name)
    response = table.update_item(
        Key={
            'uid': uid,
            'text': text
        },
        UpdateExpression="set payload = :p",
        ExpressionAttributeValues={
            ':p': p
        },
        ReturnValues="UPDATED_NEW"
    )

    print("UpdateItem succeeded:")
    print(json.dumps(response, indent=4, cls=DecimalEncoder))



def create_table(name):
    table = dynamodb.create_table(
        TableName=name,
        KeySchema=[
            {
                'AttributeName': 'uid',
                'KeyType': 'HASH'  #Partition key
            },
            {
                'AttributeName': 'text',
                'KeyType': 'RANGE'  #Sort key
            }
        ],
        AttributeDefinitions=[
            {
                'AttributeName': 'uid',
                'AttributeType': 'S'
            },
            {
                'AttributeName': 'text',
                'AttributeType': 'S'
            },

        ],
        ProvisionedThroughput={
            'ReadCapacityUnits': 10,
            'WriteCapacityUnits': 10
        }
    )

    print("Table status:", table.table_status)


def ddb_find(t_name, value):
    table = dynamodb.Table(t_name)
    response = table.query(
        KeyConditionExpression=Key('uid').eq(value)
    )

    return response['Items']


def delete_table(table_name):
    table = dynamodb.Table(table_name)
    table.delete()


def clean_and_save(uid):
    dirty = ddb_find('entries',uid)[0]['payload']
    y = dirty.replace("'",'"')
    dirty = json.loads(y)
    cleansed = dirty['parsed']
    try:
        cleansed['intent'] = cleansed['intent']['name']
    except:
        pass
    if cleansed['entities'] != []:
        leng = len(cleansed['entities'])
        for i in range(0,leng):
            if cleansed['entities'][i]['entity'] != 'date_time':
                try:
                    item = cleansed['entities'][i]
                    item.pop('extractor')
                    item.pop('processors')
                except:
                    pass
            else:
                return
        cleansed = {'_id': uid,
                    'text': 'runtime_training_data',
                    'date': str(datetime.datetime.now()),
                    'past_lives': [],
                    'payload': cleansed}
        create_new('runtime_training_data', cleansed)

{'fraction_non_alphanumeric': 0.13570759137769448, 'fraction_numerical': 0.004498594189315839, 'mean_word_length': 1.8832747905971359, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6423921', 'n_tokens_mistral': 3267, 'n_tokens_neox': 3147, 'n_words': 653}