<commit_before># -*- coding: utf-8 -*-
"""
Potential SQLite backing for Dynamo. 
This would be useful for local development

:copyright: (c) 2012 by <NAME>.
:license: ISC, see LICENSE for more details.
"""
import json
import pprint

import sqlalchemy
from sqlalchemy import sql, Table, Column, String, LargeBinary

from dynochemy import db
from dynochemy import errors
from dynochemy import utils

DEFAULT_LIMIT = 100

def create_db(db_name, metadata, has_range_key=True):
    if has_range_key:
        dynamo_tbl = Table(db_name, metadata,
                           Column('hash_key', String, primary_key=True),
                           Column('range_key', String, primary_key=True),
                           Column('content', LargeBinary),
                        )
    else:
        dynamo_tbl = Table(db_name, metadata,
                           Column('hash_key', String, primary_key=True),
                           Column('content', LargeBinary),
                        )

    return dynamo_tbl


class CommandError(errors.Error): pass


def compare(operator, lval, rval):
    if operator == 'EQ':
        return lval == rval
    elif operator == 'GT':
        return lval > rval
    elif operator == 'LT':
        return lval < rval
    else:
        raise NotImplementedError(operator)


class SQLClient(object):
    def __init__(self, engine, name, key_spec):
        self.engine = engine
        self.key_spec = key_spec

        metadata = sqlalchemy.MetaData(bind=engine)

        try:
            self.table = Table(name, metadata, autoload=True, autoload_with=self.engine)
        except sqlalchemy.exc.NoSuchTableError:
            self.table = create_db(name, metadata, has_range_key=bool(len(key_spec) > 1))
            self.table.create(engine)

    def do_getitem(self, args):
        if len(self.key_spec) > 1:
            expr = sql.and_(self.table.c.hash_key == utils.parse_value(args['Key']['HashKeyElement']), 
                              self.table.c.range_key == utils.parse_value(args['Key']['RangeKeyElement']))
        else:
            expr = self.table.c.hash_key == utils.parse_value(args['Key']['HashKeyElement'])

        q = sql.select([self.table], expr)
        try:
            res = list(self.engine.execute(q))[0]
        except IndexError:
            out = {}
        else:
            out = {
                'Item': json.loads(res[self.table.c.content])
            }


        return out

    def _put_item(self, item_data):
        item = utils.parse_item(item_data)
        hash_key = item[self.key_spec[0]]
        if len(self.key_spec) == 2:
            range_key = item[self.key_spec[1]]
            del_q = self.table.delete().where(sql.and_(self.table.c.hash_key==hash_key, self.table.c.range_key==range_key))
            ins = self.table.insert().values(hash_key=hash_key, range_key=range_key, content=json.dumps(item_data))
        else:
            del_q = self.table.delete().where(self.table.c.hash_key==hash_key)
            ins = self.table.insert().values(hash_key=hash_key, content=json.dumps(item_data))

        try:
            self.engine.execute(ins)
        except sqlalchemy.exc.IntegrityError:
            # Try again
            self.engine.execute(del_q)
            self.engine.execute(ins)

    def do_putitem(self, args):
        item_data = args['Item']
        if 'Expected' in args:
            raise NotImplementedError

        self._put_item(item_data)

    def do_updateitem(self, args):
        if 'Expected' in args:
            raise NotImplementedError

        if args.get('ReturnValues') not in (None, 'ALL_OLD', 'ALL_NEW'):
            raise NotImplementedError

        if len(self.key_spec) > 1:
            key = (utils.parse_value(args['Key']['HashKeyElement']), 
                   utils.parse_value(args['Key']['RangeKeyElement']))
            expr = sql.and_(self.table.c.hash_key == key[0],
                              self.table.c.range_key == key[1])
        else:
            key = (utils.parse_value(args['Key']['HashKeyElement']),)
            expr = self.table.c.hash_key == key[0]

        q = sql.select([self.table], expr)
        res = list(self.engine.execute(q))
        if res:
            item = json.loads(res[0][self.table.c.content])
        else:
            item = {}
            item.update(utils.format_key(self.key_spec, key))

        real_item = utils.parse_item(item)

        # Apply our updates
        for attribute, value_update in args['AttributeUpdates'].iteritems():
            if value_update['Action'] == "ADD":
                if attribute in real_item:
                    if isinstance(real_item[attribute], (int,float)):
                        real_item[attribute] += utils.parse_value(value_update['Value'])
                    else:
                        real_item[attribute].append(utils.parse_value(value_update['Value']))
                else:
                    real_item[attribute] = utils.parse_value(value_update['Value'])

            elif value_update['Action'] == "PUT":
                real_item[attribute] = utils.parse_value(value_update['Value'])
            elif value_update['Action'] == "DELETE":
                del real_item[attribute]
            else:
                raise ValueError(value_update['Action'])
        
        # write to the db
        self._put_item(utils.format_item(real_item))

        if args.get('ReturnValues', 'NONE') == 'NONE':
            return {}
        elif args['ReturnValues'] == 'ALL_NEW':
            return {'Attributes': utils.format_item(real_item)}
        elif args['ReturnValues'] == 'ALL_OLD':
            return {'Attributes': item}

    def do_batchwriteitem(self, args):
        for _, requests in args['RequestItems'].iteritems():
            for request in requests:
                req_type = request.keys()[0]
                item = request[req_type]['Item']

                if req_type == "PutRequest":
                    self._put_item(item)
                elif req_type == "DeleteRequest":
                    raise NotImplementedError
                else:
                    raise NotImplementedError

        return {}

    def do_scan(self, args):
        q = sql.select([self.table])
        if 'Limit' in args:
            q = q.limit(args['Limit'])

        unsupported_keys = set(args.keys()) - set(['Limit', 'TableName', 'ScanFilter']) 
        if unsupported_keys:
            raise NotImplementedError(unsupported_keys)

        out = {'Items': []}
        for res in self.engine.execute(q):
            item_data = json.loads(res[self.table.c.content])
            item = utils.parse_item(item_data)
            if 'ScanFilter' in args:
                for attribute, filter_spec in args['ScanFilter'].iteritems():
                    if attribute not in item:
                        continue
                    for value_spec in filter_spec['AttributeValueList']:
                        value = utils.parse_value(value_spec)
                        if compare(filter_spec['ComparisonOperator'], item[attribute], value):
                            out['Items'].append(item_data)
                            break
            else:<commit_msg>Delete support for SQL backing<commit_after># -*- coding: utf-8 -*-
"""
Potential SQLite backing for Dynamo. 
This would be useful for local development

:copyright: (c) 2012 by <NAME>.
:license: ISC, see LICENSE for more details.
"""
import json
import pprint

import sqlalchemy
from sqlalchemy import sql, Table, Column, String, LargeBinary

from dynochemy import db
from dynochemy import errors
from dynochemy import utils

DEFAULT_LIMIT = 100

def create_db(db_name, metadata, has_range_key=True):
    if has_range_key:
        dynamo_tbl = Table(db_name, metadata,
                           Column('hash_key', String, primary_key=True),
                           Column('range_key', String, primary_key=True),
                           Column('content', LargeBinary),
                        )
    else:
        dynamo_tbl = Table(db_name, metadata,
                           Column('hash_key', String, primary_key=True),
                           Column('content', LargeBinary),
                        )

    return dynamo_tbl


class CommandError(errors.Error): pass


def compare(operator, lval, rval):
    if operator == 'EQ':
        return lval == rval
    elif operator == 'GT':
        return lval > rval
    elif operator == 'LT':
        return lval < rval
    else:
        raise NotImplementedError(operator)


class SQLClient(object):
    def __init__(self, engine, name, key_spec):
        self.engine = engine
        self.key_spec = key_spec

        metadata = sqlalchemy.MetaData(bind=engine)

        try:
            self.table = Table(name, metadata, autoload=True, autoload_with=self.engine)
        except sqlalchemy.exc.NoSuchTableError:
            self.table = create_db(name, metadata, has_range_key=bool(len(key_spec) > 1))
            self.table.create(engine)

    def do_getitem(self, args):
        if len(self.key_spec) > 1:
            expr = sql.and_(self.table.c.hash_key == utils.parse_value(args['Key']['HashKeyElement']), 
                              self.table.c.range_key == utils.parse_value(args['Key']['RangeKeyElement']))
        else:
            expr = self.table.c.hash_key == utils.parse_value(args['Key']['HashKeyElement'])

        q = sql.select([self.table], expr)
        try:
            res = list(self.engine.execute(q))[0]
        except IndexError:
            out = {}
        else:
            out = {
                'Item': json.loads(res[self.table.c.content])
            }


        return out

    def _put_item(self, item_data):
        item = utils.parse_item(item_data)
        hash_key = item[self.key_spec[0]]
        if len(self.key_spec) == 2:
            range_key = item[self.key_spec[1]]
            del_q = self.table.delete().where(sql.and_(self.table.c.hash_key==hash_key, self.table.c.range_key==range_key))
            ins = self.table.insert().values(hash_key=hash_key, range_key=range_key, content=json.dumps(item_data))
        else:
            del_q = self.table.delete().where(self.table.c.hash_key==hash_key)
            ins = self.table.insert().values(hash_key=hash_key, content=json.dumps(item_data))

        try:
            self.engine.execute(ins)
        except sqlalchemy.exc.IntegrityError:
            # Try again
            self.engine.execute(del_q)
            self.engine.execute(ins)

    def do_putitem(self, args):
        item_data = args['Item']
        if 'Expected' in args:
            raise NotImplementedError

        self._put_item(item_data)

    def do_deleteitem(self, args):
        if len(self.key_spec) > 1:
            expr = sql.and_(self.table.c.hash_key == utils.parse_value(args['Key']['HashKeyElement']), 
                              self.table.c.range_key == utils.parse_value(args['Key']['RangeKeyElement']))
        else:
            expr = self.table.c.hash_key == utils.parse_value(args['Key']['HashKeyElement'])

        item = self.do_getitem(args)

        del_q = self.table.delete().where(expr)
        res = self.engine.execute(del_q)

        out = {
            'Attributes': item['Item']
        }

        return out

    def do_updateitem(self, args):
        if 'Expected' in args:
            raise NotImplementedError

        if args.get('ReturnValues') not in (None, 'ALL_OLD', 'ALL_NEW'):
            raise NotImplementedError

        if len(self.key_spec) > 1:
            key = (utils.parse_value(args['Key']['HashKeyElement']), 
                   utils.parse_value(args['Key']['RangeKeyElement']))
            expr = sql.and_(self.table.c.hash_key == key[0],
                              self.table.c.range_key == key[1])
        else:
            key = (utils.parse_value(args['Key']['HashKeyElement']),)
            expr = self.table.c.hash_key == key[0]

        q = sql.select([self.table], expr)
        res = list(self.engine.execute(q))
        if res:
            item = json.loads(res[0][self.table.c.content])
        else:
            item = {}
            item.update(utils.format_key(self.key_spec, key))

        real_item = utils.parse_item(item)

        # Apply our updates
        for attribute, value_update in args['AttributeUpdates'].iteritems():
            if value_update['Action'] == "ADD":
                if attribute in real_item:
                    if isinstance(real_item[attribute], (int,float)):
                        real_item[attribute] += utils.parse_value(value_update['Value'])
                    else:
                        real_item[attribute].append(utils.parse_value(value_update['Value']))
                else:
                    real_item[attribute] = utils.parse_value(value_update['Value'])

            elif value_update['Action'] == "PUT":
                real_item[attribute] = utils.parse_value(value_update['Value'])
            elif value_update['Action'] == "DELETE":
                del real_item[attribute]
            else:
                raise ValueError(value_update['Action'])
        
        # write to the db
        self._put_item(utils.format_item(real_item))

        if args.get('ReturnValues', 'NONE') == 'NONE':
            return {}
        elif args['ReturnValues'] == 'ALL_NEW':
            return {'Attributes': utils.format_item(real_item)}
        elif args['ReturnValues'] == 'ALL_OLD':
            return {'Attributes': item}

    def do_batchwriteitem(self, args):
        for _, requests in args['RequestItems'].iteritems():
            for request in requests:
                req_type = request.keys()[0]
                args = request[req_type]

                if req_type == "PutRequest":
                    self._put_item(args['Item'])
                elif req_type == "DeleteRequest":
                    self.do_deleteitem(args)
                else:
                    raise NotImplementedError

        return {}

    def do_scan(self, args):
        q = sql.select([self.table])
        if 'Limit' in args:
            q = q.limit(args['Limit'])

        unsupported_keys = set(args.keys()) - set(['Limit', 'TableName', 'ScanFilter']) 
        if unsupported_keys:
            raise NotImplementedError(unsupported_keys)

        out = {'Items': []}
        for res in self.engine.execute(q):
            item_data = json.loads(res[self.table.c.content])
            item = utils.parse_item(item_data)
            if 'ScanFilter' in args:
                for attribute, filter_spec in args['ScanFilter'].iteritems():
                    if attribute not in item:
                        continue
                    for value_spec in filter_spec['AttributeValueList']:
                        value = utils.parse_value(value_spec)
                        if compare(filter_spec['ComparisonOperator'], item[attribute], value):
                            out['Items'].append(item_data)
                            break
            else:
{'fraction_non_alphanumeric': 0.1181727242414138, 'fraction_numerical': 0.0027342447482494167, 'mean_word_length': 1.9583744328269876, 'pattern_counts': {'":': 10, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1598606', 'n_tokens_mistral': 4320, 'n_tokens_neox': 4199, 'n_words': 846}