<commit_before>import botocore
import boto3
import cbor
import copy
import functools
import time
import random
from base58 import b58encode
from boto3.dynamodb.types import Binary
from multihash import encode as multihash_encode, SHA2_256

from mediachain.datastore.data_objects import Record, MultihashReference

__AWS_CONFIG=dict()
__DB_INSTANCE=None

def set_aws_config(cfg):
    global __AWS_CONFIG
    __AWS_CONFIG = cfg

def get_aws_config():
    global __AWS_CONFIG
    return __AWS_CONFIG

def get_db():
    global __DB_INSTANCE
    if __DB_INSTANCE is None:
        __DB_INSTANCE = DynamoDatastore()

    return __DB_INSTANCE


BACKOFF_COEFF = 50.0
MAX_ATTEMPTS = 10
RETRYABLE_ERRORS = [
    "InternalServerError",
    "ProvisionedThroughputExceededException"
]


def with_retry(func, *args, **kwargs):
    operation = func.__name__
    attempts = 1
    while True:
        try:
            output = func(*args, **kwargs)
        except botocore.exceptions.ClientError as e:
            error_code = e.response['Error']['Code']
            if error_code not in RETRYABLE_ERRORS:
                raise e
            if attempts >= MAX_ATTEMPTS:
                print('Operation {} failed after {} attempts'.format(
                    operation, attempts
                ))
                raise e
        else:
            return output<commit_msg>expose aws error as mediachain.datastore.dynamo.DynamoError<commit_after>import botocore
import boto3
import cbor
import copy
import functools
import time
import random
from base58 import b58encode
from boto3.dynamodb.types import Binary
from botocore.exceptions import ClientError as DynamoError
from multihash import encode as multihash_encode, SHA2_256

from mediachain.datastore.data_objects import Record, MultihashReference

__AWS_CONFIG=dict()
__DB_INSTANCE=None

def set_aws_config(cfg):
    global __AWS_CONFIG
    __AWS_CONFIG = cfg

def get_aws_config():
    global __AWS_CONFIG
    return __AWS_CONFIG

def get_db():
    global __DB_INSTANCE
    if __DB_INSTANCE is None:
        __DB_INSTANCE = DynamoDatastore()

    return __DB_INSTANCE


BACKOFF_COEFF = 50.0
MAX_ATTEMPTS = 10
RETRYABLE_ERRORS = [
    "InternalServerError",
    "ProvisionedThroughputExceededException"
]


def with_retry(func, *args, **kwargs):
    operation = func.__name__
    attempts = 1
    while True:
        try:
            output = func(*args, **kwargs)
        except DynamoError as e:
            error_code = e.response['Error']['Code']
            if error_code not in RETRYABLE_ERRORS:
                raise e
            if attempts >= MAX_ATTEMPTS:
                print('Operation {} failed after {} attempts'.format(
                    operation, attempts
                ))
                raise e
        else:
            return output
{'fraction_non_alphanumeric': 0.06359414076455877, 'fraction_numerical': 0.011432654519471239, 'mean_word_length': 2.7583892617449663, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '902174', 'n_tokens_mistral': 872, 'n_tokens_neox': 838, 'n_words': 225}