<commit_before>import os
import time

from boto.dynamodb2.table import Table
from boto.dynamodb2.fields import HashKey, RangeKey, GlobalAllIndex

from dbhelper import *

class ShingleTableManager:
    """
    Manage shingle-docid pairs stored on AWS dynamo db database.

    Contains functions for CRUD on table, adding and querying entries etc.
    """

    def __init__(self):
        """
        Instantiates a new instance of DbItemManager class
        """

        self.tableConnString = os.environ['SHINGLETABLE_CONNECTIONSTRING'];

    def __getTable(self):
        """
        Get the shingle table.
        """

        shingleTableConnectionParams = parseConnectionString(
            self.tableConnString);

        return Table(
            shingleTableConnectionParams['name'],
            connection = getDbConnection(shingleTableConnectionParams));

    def createFreshTable(self):
        """
        Create a fresh empty shingle table.
        """

        # delete existing table if it exists
        try:
            self.__getTable().delete();
            time.sleep(5)
        except:
            pass;# do nothing. Maybe there was no existing table

        # create new table
        shingleTableConnectionParams = parseConnectionString(
            self.tableConnString);
        return Table.create(
            shingleTableConnectionParams['name'],
            schema = [
                HashKey('shingle'),
                RangeKey('docId')
            ], throughput = {
                'read': 14,
                'write': 1,
            }, global_indexes = [
                GlobalAllIndex('docIdIndex', parts = [
                    HashKey('docId'),
                    RangeKey('shingle')
                ],
                throughput = {
                    'read': 1,
                    'write': 1,
                })
            ],
            connection = getDbConnection(shingleTableConnectionParams))

    def addEntries(self, docId, shingles):
        """
        Add a entries in shingles table for shingles and docId passed.
        """

        shingles = list(set(shingles)) # remove duplicate shingles
        shingleTable = self.__getTable()
        with shingleTable.batch_write() as batch:
            for shingle in shingles:
                batch.put_item(data={
                    'docId': docId,
                    'shingle': shingle})

    def queryByShingle(self, shingle):
        """<commit_msg>Increasing the default write throughput in shingle tables<commit_after>import os
import time

from boto.dynamodb2.table import Table
from boto.dynamodb2.fields import HashKey, RangeKey, GlobalAllIndex

from dbhelper import *

class ShingleTableManager:
    """
    Manage shingle-docid pairs stored on AWS dynamo db database.

    Contains functions for CRUD on table, adding and querying entries etc.
    """

    def __init__(self):
        """
        Instantiates a new instance of DbItemManager class
        """

        self.tableConnString = os.environ['SHINGLETABLE_CONNECTIONSTRING'];

    def __getTable(self):
        """
        Get the shingle table.
        """

        shingleTableConnectionParams = parseConnectionString(
            self.tableConnString);

        return Table(
            shingleTableConnectionParams['name'],
            connection = getDbConnection(shingleTableConnectionParams));

    def createFreshTable(self):
        """
        Create a fresh empty shingle table.
        """

        # delete existing table if it exists
        try:
            self.__getTable().delete();
            time.sleep(5)
        except:
            pass;# do nothing. Maybe there was no existing table

        # create new table
        shingleTableConnectionParams = parseConnectionString(
            self.tableConnString);
        return Table.create(
            shingleTableConnectionParams['name'],
            schema = [
                HashKey('shingle'),
                RangeKey('docId')
            ], throughput = {
                'read': 14,
                'write': 20,
            }, global_indexes = [
                GlobalAllIndex('docIdIndex', parts = [
                    HashKey('docId'),
                    RangeKey('shingle')
                ],
                throughput = {
                    'read': 1,
                    'write': 20,
                })
            ],
            connection = getDbConnection(shingleTableConnectionParams))

    def addEntries(self, docId, shingles):
        """
        Add a entries in shingles table for shingles and docId passed.
        """

        shingles = list(set(shingles)) # remove duplicate shingles
        shingleTable = self.__getTable()
        with shingleTable.batch_write() as batch:
            for shingle in shingles:
                batch.put_item(data={
                    'docId': docId,
                    'shingle': shingle})

    def queryByShingle(self, shingle):
        """
{'fraction_non_alphanumeric': 0.08558740411788454, 'fraction_numerical': 0.0036334275333064193, 'mean_word_length': 1.7436323366555924, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2950876', 'n_tokens_mistral': 1292, 'n_tokens_neox': 1213, 'n_words': 323}