<commit_before># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2013, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
import io
import json
from uuid import UUID

import psycopg2
import psycopg2.extras
from psycopg2 import Binary
from psycopg2.extensions import STATUS_READY

from .main import BaseStorage
from ..models import Document, Resource, create_content, MEDIATYPES
from .database import SQL


psycopg2.extras.register_uuid()
psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)

JSON_FIELDS = ['authors', 'publishers', 'licensors', 'editors', 'translators']


# cribbed from http://stackoverflow.com/questions/19048017/python-extract-substitution-vars-from-format-string
def get_format_keys(s):
    d = {}
    while True:
        try:
            s % d
        except KeyError as exc:
            # exc.args[0] contains the name of the key that was not found;
            # 0 is used because it appears to work with all types of placeholders.
            d[exc.args[0]] = 0
        else:
            break
    return d.keys()


def check_args(s, kwargs):
    format_keys = get_format_keys(s)
    for k in kwargs:
        if k not in format_keys:
            raise KeyError(k)


def checked_execute(cur, s, kwargs):
    check_args(s, kwargs)
    cur.execute(s, kwargs)


class PostgresqlStorage(BaseStorage):
    """Utility for managing and interfacing with the the storage medium."""

    Error = psycopg2.Error

    def __init__(self, db_connection_string=None):
        #initialize db
        self.conn = psycopg2.connect(db_connection_string)

    def get(self, type_=Document, **kwargs):
        """Retrieve ``Document`` objects from storage."""
        for obj in self.get_all(type_=type_, **kwargs):
            return obj

    def get_all(self, type_=Document, user_id=None, permissions=None,
                **kwargs):
        """Retrieve ``Document`` objects from storage."""
        # all kwargs are expected to match attributes of the stored Document.
        # We're trusting the names of the args to match table column names, but not
        # trusting the values

        in_progress = ( self.conn.status != STATUS_READY )

        cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

        type_name = type_.__name__.lower()

        # if ID is not a well formed uuid, there's no need to even hit the db
        if 'id' in kwargs and type(kwargs['id']) != UUID:
            try:
                kwargs['id'] = UUID(kwargs['id'])
            except ValueError:
                return

        match_clauses = []
        match_values = kwargs.copy()
        for k, v in kwargs.items():
            if isinstance(v, dict):
                for json_k, json_v in v.items():
                    match_clauses.append(
                            "{field}->>'{json_key}' = %({field}_{json_key})s"
                            .format(field=k, json_key=json_k))
                    match_values['{field}_{json_key}'.format(
                        field=k, json_key=json_k)] = json_v
                match_values.pop(k)
            elif k == 'contained_in': # Array based storage , assumes singular key
                if v.startswith('not:'):
                    match_clauses.append(' NOT %(contained_in)s = ANY (contained_in) ')
                    match_values[k] = v[4:]
                else:
                    match_clauses.append(' %(contained_in)s = ANY (contained_in) ')
            else:
                if  str(v).startswith('not:'):
                    match_clauses.append(' NOT {field} = %({field})s'.format(field=k))
                    match_values[k] = str(v)[4:]
                else:
                    match_clauses.append('{field} = %({field})s'.format(field=k))

        # 1 = 1 in case where clause is empty
        where_clause = ' AND '.join(match_clauses) or '1 = 1'
        if type_name in ('document', 'binder') and user_id and permissions:
            match_values.update({
                'user_id': user_id,
                'permissions': permissions})
            checked_execute(cursor, SQL['get-document'].format(
                where_clause=where_clause), match_values)
        else:
            checked_execute(cursor, SQL['get'].format(
                tablename=type_name, where_clause=where_clause), match_values)
        res = cursor.fetchall()
        if not in_progress:
            self.conn.rollback() # Frees the connection
        if res:
            for r in res:
                if 'license' in r:
                    r['license'] = eval(r['license'])
                    rd = dict(r)
                    for field in ('user_id', 'permission', 'uuid'):
                        if field in rd:
                            rd.pop(field)
                    if rd['media_type'] == MEDIATYPES['binder']:
                        rd['tree'] = json.loads(rd.pop('content'))
                    document = create_content(**rd)
                    checked_execute(cursor, SQL['get'].format(
                        tablename='document_acl',
                        where_clause='uuid = %(uuid)s'), {'uuid': rd['id']})
                    permissions_by_users = {}
                    for acl in cursor.fetchall():
                        permissions_by_users.setdefault(acl['user_id'], [])
                        permissions_by_users[acl['user_id']].append(
                                acl['permission'])
                    for user_id, permissions in permissions_by_users.items():
                        document.acls.append((user_id,) + tuple(permissions))
                    yield document
                else:
                    rd = dict(r)
                    rd.pop('hash')
                    rd['data'] = io.BytesIO(rd['data'][:])
                    yield type_(**dict(rd))<commit_msg>Read license_acceptance from postgresql storage.<commit_after># -*- coding: utf-8 -*-
# ###
# Copyright (c) 2013, Rice University
# This software is subject to the provisions of the GNU Affero General
# Public License version 3 (AGPLv3).
# See LICENCE.txt for details.
# ###
import io
import json
from uuid import UUID

import psycopg2
import psycopg2.extras
from psycopg2 import Binary
from psycopg2.extensions import STATUS_READY

from .main import BaseStorage
from ..models import Document, Resource, create_content, MEDIATYPES
from .database import SQL


psycopg2.extras.register_uuid()
psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json)

JSON_FIELDS = ['authors', 'publishers', 'licensors', 'editors', 'translators']


# cribbed from http://stackoverflow.com/questions/19048017/python-extract-substitution-vars-from-format-string
def get_format_keys(s):
    d = {}
    while True:
        try:
            s % d
        except KeyError as exc:
            # exc.args[0] contains the name of the key that was not found;
            # 0 is used because it appears to work with all types of placeholders.
            d[exc.args[0]] = 0
        else:
            break
    return d.keys()


def check_args(s, kwargs):
    format_keys = get_format_keys(s)
    for k in kwargs:
        if k not in format_keys:
            raise KeyError(k)


def checked_execute(cur, s, kwargs):
    check_args(s, kwargs)
    cur.execute(s, kwargs)


class PostgresqlStorage(BaseStorage):
    """Utility for managing and interfacing with the the storage medium."""

    Error = psycopg2.Error

    def __init__(self, db_connection_string=None):
        #initialize db
        self.conn = psycopg2.connect(db_connection_string)

    def get(self, type_=Document, **kwargs):
        """Retrieve ``Document`` objects from storage."""
        for obj in self.get_all(type_=type_, **kwargs):
            return obj

    def get_all(self, type_=Document, user_id=None, permissions=None,
                **kwargs):
        """Retrieve ``Document`` objects from storage."""
        # all kwargs are expected to match attributes of the stored Document.
        # We're trusting the names of the args to match table column names, but not
        # trusting the values

        in_progress = ( self.conn.status != STATUS_READY )

        cursor = self.conn.cursor(cursor_factory=psycopg2.extras.DictCursor)

        type_name = type_.__name__.lower()

        # if ID is not a well formed uuid, there's no need to even hit the db
        if 'id' in kwargs and type(kwargs['id']) != UUID:
            try:
                kwargs['id'] = UUID(kwargs['id'])
            except ValueError:
                return

        match_clauses = []
        match_values = kwargs.copy()
        for k, v in kwargs.items():
            if isinstance(v, dict):
                for json_k, json_v in v.items():
                    match_clauses.append(
                            "{field}->>'{json_key}' = %({field}_{json_key})s"
                            .format(field=k, json_key=json_k))
                    match_values['{field}_{json_key}'.format(
                        field=k, json_key=json_k)] = json_v
                match_values.pop(k)
            elif k == 'contained_in': # Array based storage , assumes singular key
                if v.startswith('not:'):
                    match_clauses.append(' NOT %(contained_in)s = ANY (contained_in) ')
                    match_values[k] = v[4:]
                else:
                    match_clauses.append(' %(contained_in)s = ANY (contained_in) ')
            else:
                if  str(v).startswith('not:'):
                    match_clauses.append(' NOT {field} = %({field})s'.format(field=k))
                    match_values[k] = str(v)[4:]
                else:
                    match_clauses.append('{field} = %({field})s'.format(field=k))

        # 1 = 1 in case where clause is empty
        where_clause = ' AND '.join(match_clauses) or '1 = 1'
        if type_name in ('document', 'binder') and user_id and permissions:
            match_values.update({
                'user_id': user_id,
                'permissions': permissions})
            checked_execute(cursor, SQL['get-document'].format(
                where_clause=where_clause), match_values)
        else:
            checked_execute(cursor, SQL['get'].format(
                tablename=type_name, where_clause=where_clause), match_values)
        res = cursor.fetchall()
        if not in_progress:
            self.conn.rollback() # Frees the connection
        if res:
            for r in res:
                if 'license' in r:
                    r['license'] = eval(r['license'])
                    rd = dict(r)
                    for field in ('user_id', 'permission', 'uuid'):
                        if field in rd:
                            rd.pop(field)
                    if rd['media_type'] == MEDIATYPES['binder']:
                        rd['tree'] = json.loads(rd.pop('content'))
                    document = create_content(**rd)
                    checked_execute(cursor, SQL['get'].format(
                        tablename='document_acl',
                        where_clause='uuid = %(uuid)s'), {'uuid': rd['id']})
                    permissions_by_users = {}
                    for acl in cursor.fetchall():
                        permissions_by_users.setdefault(acl['user_id'], [])
                        permissions_by_users[acl['user_id']].append(
                                acl['permission'])
                    for user_id, permissions in permissions_by_users.items():
                        document.acls.append((user_id,) + tuple(permissions))
                    checked_execute(cursor, SQL['get'].format(
                        tablename='document_licensor_acceptance',
                        where_clause='uuid = %(uuid)s'), {'uuid': rd['id']})
                    licensor_acceptance = [
                        {'id': r['user_id'], 'has_accepted': r['has_accepted']}
                        for r in cursor.fetchall()]
                    document.licensor_acceptance = licensor_acceptance
                    yield document
                else:
                    rd = dict(r)
                    rd.pop('hash')
                    rd['data'] = io.BytesIO(rd['data'][:])
                    yield type_(**dict(rd))
{'fraction_non_alphanumeric': 0.11200585651537336, 'fraction_numerical': 0.00569383439075972, 'mean_word_length': 1.8520064950127582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1345648', 'n_tokens_mistral': 3550, 'n_tokens_neox': 3370, 'n_words': 873}