<commit_before>
# 
# Copyright 2013-2017 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""ERMREST URL abstract syntax tree (AST) for data resources.

"""

import cStringIO
import web
import tempfile

from ..api import Api
from . import path
from ....model.predicate import predicatecls
from ....model.name import Name
from .... import ermpath, exception
from webauthn2.util import urlquote

def _preprocess_attributes(epath, attributes):
    """Expand '*' wildcards in attributes into explicit projections understood by ermpath."""
    results = []
    for item in attributes:
        if type(item) is tuple:
            # make preprocessing resolution idempotent
            attribute, col, base = item
        else:
            attribute = item
            if len(attribute.nameparts) > 2:
                raise exception.BadSyntax('Column name %s, qualified by schema and table names, not allowed as attribute.' % attribute)
            elif len(attribute.nameparts) > 1 and attribute.nameparts[0] not in epath.aliases:
                raise exception.BadSyntax('Alias %s, qualifying column name %s, not bound in path.' % (attribute.nameparts[0], attribute))
            col, base = attribute.resolve_column(epath._model, epath)

        if col.is_star_column() and not hasattr(attribute, 'aggfunc'):
            # expand '*' wildcard sugar as if user referenced each column
            if hasattr(attribute, 'nbins'):
                raise exception.BadSyntax('Wildcard column %s does not support binning.' % attribute)
            if attribute.alias is not None:
                raise exception.BadSyntax('Wildcard column %s cannot be given an alias.' % attribute)
            if base == epath:
                # columns from final entity path element
                for col in epath._path[epath.current_entity_position()].table.columns_in_order():
                    results.append((Name([col.name]), col, base))
            elif base in epath.aliases:
                # columns from interior path referenced by alias
                for col in epath[base].table.columns_in_order():
                    results.append((Name([base, col.name]).set_alias('%s:%s' % (base, col.name)), col, base))
            else:
                raise NotImplementedError('Unresolvable * column violates program invariants!')
        else:
            results.append((attribute, col, base))
            
    return results

def _GET(handler, uri, dresource, vresource):
    """Perform HTTP GET of generic data resources.
    """
    content_type = handler.negotiated_content_type()
    limit = handler.negotiated_limit()

    if content_type == 'text/csv':
        results = tempfile.TemporaryFile()
    else:
        results = None
        
    def body(conn, cur):
        handler.set_http_etag( vresource.etag(cur) )
        handler.http_check_preconditions()
        dresource.add_sort(handler.sort)
        dresource.add_paging(handler.after, handler.before)
        return dresource.get(conn, cur, content_type=content_type, output_file=results, limit=limit)

    def post_commit(lines):
        handler.emit_headers()
        if lines is None:
            return
        web.header('Content-Type', content_type)
        if 'download' in handler.queryopts and handler.queryopts['download']:
            fname = handler.queryopts['download']
            fname += {
                'application/json': '.json',<commit_msg>use REPEATABLE_READ rather than SERIALIZABLE transactions for data GET requests<commit_after>
# 
# Copyright 2013-2017 University of Southern California
# 
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# 
#    http://www.apache.org/licenses/LICENSE-2.0
# 
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""ERMREST URL abstract syntax tree (AST) for data resources.

"""

import cStringIO
import web
import tempfile
import psycopg2

from ..api import Api
from . import path
from ....model.predicate import predicatecls
from ....model.name import Name
from .... import ermpath, exception
from webauthn2.util import urlquote

def _preprocess_attributes(epath, attributes):
    """Expand '*' wildcards in attributes into explicit projections understood by ermpath."""
    results = []
    for item in attributes:
        if type(item) is tuple:
            # make preprocessing resolution idempotent
            attribute, col, base = item
        else:
            attribute = item
            if len(attribute.nameparts) > 2:
                raise exception.BadSyntax('Column name %s, qualified by schema and table names, not allowed as attribute.' % attribute)
            elif len(attribute.nameparts) > 1 and attribute.nameparts[0] not in epath.aliases:
                raise exception.BadSyntax('Alias %s, qualifying column name %s, not bound in path.' % (attribute.nameparts[0], attribute))
            col, base = attribute.resolve_column(epath._model, epath)

        if col.is_star_column() and not hasattr(attribute, 'aggfunc'):
            # expand '*' wildcard sugar as if user referenced each column
            if hasattr(attribute, 'nbins'):
                raise exception.BadSyntax('Wildcard column %s does not support binning.' % attribute)
            if attribute.alias is not None:
                raise exception.BadSyntax('Wildcard column %s cannot be given an alias.' % attribute)
            if base == epath:
                # columns from final entity path element
                for col in epath._path[epath.current_entity_position()].table.columns_in_order():
                    results.append((Name([col.name]), col, base))
            elif base in epath.aliases:
                # columns from interior path referenced by alias
                for col in epath[base].table.columns_in_order():
                    results.append((Name([base, col.name]).set_alias('%s:%s' % (base, col.name)), col, base))
            else:
                raise NotImplementedError('Unresolvable * column violates program invariants!')
        else:
            results.append((attribute, col, base))
            
    return results

def _GET(handler, uri, dresource, vresource):
    """Perform HTTP GET of generic data resources.
    """
    content_type = handler.negotiated_content_type()
    limit = handler.negotiated_limit()

    if content_type == 'text/csv':
        results = tempfile.TemporaryFile()
    else:
        results = None
        
    def body(conn, cur):
        try:
            conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_REPEATABLE_READ)
            handler.set_http_etag( vresource.etag(cur) )
            handler.http_check_preconditions()
            dresource.add_sort(handler.sort)
            dresource.add_paging(handler.after, handler.before)
            return dresource.get(conn, cur, content_type=content_type, output_file=results, limit=limit)
        finally:
            try:
                conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE)
            except:
                pass

    def post_commit(lines):
        handler.emit_headers()
        if lines is None:
            return
        web.header('Content-Type', content_type)
        if 'download' in handler.queryopts and handler.queryopts['download']:
            fname = handler.queryopts['download']
            fname += {
                'application/json': '.json',
{'fraction_non_alphanumeric': 0.09034907597535935, 'fraction_numerical': 0.004469138784877401, 'mean_word_length': 2.610989969472307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7464655', 'n_tokens_mistral': 2215, 'n_tokens_neox': 2125, 'n_words': 721}