<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2020 damian <<EMAIL>>
#
# Distributed under terms of the MIT license.

import boto3
import datetime
from boto3.dynamodb.conditions import Key, Attr

dynamodb = boto3.resource('dynamodb', region_name='eu-west-1')
table = dynamodb.Table('dev.xxwmm.storepick.items_v2')

batch_id = "16"

start_time = datetime.datetime.now()
print("Start index-based DB query...")
resp = table.query(
    IndexName = 'batchId-index',
    KeyConditionExpression=Key('batchId').eq(batch_id)
)
end_time = datetime.datetime.now()

delta = end_time - start_time
print(f"index-based query took: {delta}")
print(f"got {len(resp['Items'])} items for batch_id {batch_id}")

start_time = datetime.datetime.now()
print("Start non-index-based DB queries...")

items_list = []
NUMBER_ITEMS_PARTITION = 50
for index in range(0, NUMBER_ITEMS_PARTITION + 1):
    tmp_resp = table.query(
        KeyConditionExpression=Key('outOfStock').eq(f'outOfStock_{batch_id}_{index}'),
        ConsistentRead=True
    )
    items_list += tmp_resp['Items']

resp = {}
resp['Items'] = items_list

# THE BELOW QUERY THROWS EXCEPTION AS ONLY `eq` is allowed for KeyConditionExpression here...
#resp = table.query(
    #KeyConditionExpression=Key('outOfStock').begins_with('outOfStock_{batch_id}'),
    #ConsistentRead=True
#)
end_time = datetime.datetime.now()

delta = end_time - start_time

print(f"no-index-based query took: {delta}")
print(f"got {len(resp['Items'])} items for batch_id {batch_id}")

#print("The query returned the following items:")
#for item in resp['Items']:
    #print(item)

#config_table = dynamodb.Table('dev.xxwmm.storepick.config')

#response_array = []
#for status in ['NEW', 'PENDING', 'ITEMS_REVIEW', 'ITEMS_UPLOADED', 'IN_PROGRESS', 'CANCELLED', 'COMPLETED']:
    #table_items = config_table.query(
        #IndexName = 'status-index',
        #KeyConditionExpression = Key('status').eq(status),
    #)
    #response_array = response_array + (table_items['Items'])

#print(len(response_array))

<commit_msg>aws: dynamodb_query => implement dynamodb script and make it generic<commit_after>#! /usr/bin/env python

import fire
import boto3
import logging
from boto3.dynamodb.conditions import Key, Attr

"""
Example usage:
    python dynamodb_query.py --table_name="dev.orders" --query_key="orderDate" --query_value="10122020"
"""

def query_dynamodb(table_name, query_key, query_value):
    print(f"Querying dynamodb table: {table_name} with filter => {query_key}={query_value}")
    dynamodb = boto3.resource('dynamodb')
    table = dynamodb.Table(table_name)

    response = table.query(
        KeyConditionExpression=Key(str(query_key)).eq(str(query_value))
    )

    for item in response['Items']:
        print(str(item))

    items_number = len(response['Items'])
    print(f"number of items for query '{query_key}={query_value}': {items_number}")

if __name__ == "__main__":
    fire.Fire(query_dynamodb)

{'fraction_non_alphanumeric': 0.14720642355302777, 'fraction_numerical': 0.00936768149882904, 'mean_word_length': 4.968063872255489, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4600904', 'n_tokens_mistral': 1053, 'n_tokens_neox': 1022, 'n_words': 215}