<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2012-2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from copy import copy
from sqlalchemy import MetaData, Table
from rdc.etl.error import ProhibitedOperationError
from rdc.etl.hash import Hash
from rdc.etl.io import STDIN, INSERT, UPDATE, SELECT, STDERR
from rdc.etl.transform import Transform
from rdc.etl.util import now, cached_property


class DatabaseLoad(Transform):
    """

    TODO doc this !!! test this !!!!

    """

    engine = None
    table_name = None
    fetch_columns = None
    insert_only_fields = ()
    discriminant = ('id', )
    created_at_field = 'created_at'
    updated_at_field = 'updated_at'
    allowed_operations = (INSERT, UPDATE, )

    def __init__(self, engine=None, table_name=None, fetch_columns=None, discriminant=None, created_at_field=None,
                 updated_at_field=None, insert_only_fields=None, allowed_operations=None):
        super(DatabaseLoad, self).__init__()

        self.engine = engine or self.engine
        self.table_name = table_name or self.table_name

        # xxx should take self.fetch_columns into account if provided
        self.fetch_columns = {}
        if isinstance(fetch_columns, list):
            self.add_fetch_column(*fetch_columns)
        elif isinstance(fetch_columns, dict):
            self.add_fetch_column(**fetch_columns)<commit_msg>Allows DatabaseLoad to take a tuple as well as a list for its fetch_columns argument (was only working with lists).<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2012-2014 <NAME>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from copy import copy
from sqlalchemy import MetaData, Table
from rdc.etl.error import ProhibitedOperationError
from rdc.etl.hash import Hash
from rdc.etl.io import STDIN, INSERT, UPDATE, SELECT, STDERR
from rdc.etl.transform import Transform
from rdc.etl.util import now, cached_property


class DatabaseLoad(Transform):
    """

    TODO doc this !!! test this !!!!

    """

    engine = None
    table_name = None
    fetch_columns = None
    insert_only_fields = ()
    discriminant = ('id', )
    created_at_field = 'created_at'
    updated_at_field = 'updated_at'
    allowed_operations = (INSERT, UPDATE, )

    def __init__(self, engine=None, table_name=None, fetch_columns=None, discriminant=None, created_at_field=None,
                 updated_at_field=None, insert_only_fields=None, allowed_operations=None):
        super(DatabaseLoad, self).__init__()

        self.engine = engine or self.engine
        self.table_name = table_name or self.table_name

        # XXX should take self.fetch_columns into account if provided
        self.fetch_columns = {}
        if isinstance(fetch_columns, (list, tuple, )):
            self.add_fetch_column(*fetch_columns)
        elif isinstance(fetch_columns, dict):
            self.add_fetch_column(**fetch_columns)
{'fraction_non_alphanumeric': 0.08502440277421013, 'fraction_numerical': 0.006678653994348831, 'mean_word_length': 3.6412395709177594, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5491010', 'n_tokens_mistral': 1136, 'n_tokens_neox': 1083, 'n_words': 375}