<commit_before># Copyright 2014-2015 SpendRight, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Building the scratch (intermediate) database."""
from logging import getLogger
from os import remove
from os import rename
from os.path import exists
from os.path import getmtime

from .db import create_index
from .db import create_table
from .db import insert_row
from .db import open_db
from .db import show_tables
from .norm import clean_string
from .table import TABLES

log = getLogger(__name__)


def build_scratch_db(
        scratch_db_path, input_db_paths, *, force=False):
    """Take data from the various input databases, and put it into
    a single, indexed database with correct table definitions.

    Does nothing if the scratch DB is newer than all the input
    DBs, unless *force* is true.

    Unlike the output database, every table in the scratch database
    has a scraper_id field. The names of each input database are used
    as a "namespace" which is prepended to the scraper_id value
    (if any) in the input data. For example a row with scraper_id
    "coca_cola" from a db named sr.company.sqlite would end up in
    the scratch DB with scraper_id value "sr.company.coca_cola".

    This also cleans smart quotes, excess whitespace, etc. out of the
    input data.
    """
    # TODO: might also want to apply custom corrections here
    if exists(scratch_db_path) and not force:
        mtime = getmtime(scratch_db_path)
        if all(exists(db_path) and getmtime(db_path) < mtime
               for db_path in input_db_paths):
            log.info('{} already exists and is up-to-date'.format(
                scratch_db_path))
            return

    scratch_db_tmp_path = scratch_db_path + '.tmp'
    if exists(scratch_db_tmp_path):
        remove(scratch_db_tmp_path)

    log.info('building {}...'.format(scratch_db_tmp_path))

    with open_db(scratch_db_tmp_path) as scratch_db:

        init_scratch_tables(scratch_db)

        for input_db_path in input_db_paths:
            log.info('dumping data from {} -> {}'.format(
                input_db_path, scratch_db_tmp_path))

            scraper_prefix = db_path_to_scraper_prefix(input_db_path)
            with open_db(input_db_path) as input_db:

                dump_db_to_scratch(input_db, scratch_db, scraper_prefix)

    log.info('moving {} -> {}'.format(scratch_db_tmp_path, scratch_db_path))
    rename(scratch_db_tmp_path, scratch_db_path)


def init_scratch_tables(scratch_db):
    """Add tables to the given (open) SQLite DB."""
    for table_name, table_def in sorted(TABLES.items()):
        columns = table_def['columns'].copy()
        columns['scraper_id'] = 'text'

        create_table(scratch_db, table_name, columns)

        # add "primary key" (non-unique) index
        index_cols = list(table_def.get('primary_key', ()))
        if 'scraper_id' not in index_cols:
            index_cols = ['scraper_id'] + index_cols
        create_index(scratch_db, table_name, index_cols)

        # add other indexes
        for index_cols in table_def.get('indexes', ()):
            create_index(scratch_db, table_name, index_cols)


def db_path_to_scraper_prefix(path):
    idx = path.lower().rfind('.sqlite')
    if idx == -1:
        raise ValueError
    return path[:idx]


def dump_db_to_scratch(input_db, scratch_db, scraper_prefix=''):
    input_table_names = set(show_tables(input_db))

    extra_table_names = input_table_names - set(TABLES)
    if extra_table_names:<commit_msg>"create" don't "init" scratch tables<commit_after># Copyright 2014-2015 SpendRight, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Building the scratch (intermediate) database."""
from logging import getLogger
from os import remove
from os import rename
from os.path import exists
from os.path import getmtime

from .db import create_index
from .db import create_table
from .db import insert_row
from .db import open_db
from .db import show_tables
from .norm import clean_string
from .table import TABLES

log = getLogger(__name__)


def build_scratch_db(
        scratch_db_path, input_db_paths, *, force=False):
    """Take data from the various input databases, and put it into
    a single, indexed database with correct table definitions.

    Does nothing if the scratch DB is newer than all the input
    DBs, unless *force* is true.

    Unlike the output database, every table in the scratch database
    has a scraper_id field. The names of each input database are used
    as a "namespace" which is prepended to the scraper_id value
    (if any) in the input data. For example a row with scraper_id
    "coca_cola" from a db named sr.company.sqlite would end up in
    the scratch DB with scraper_id value "sr.company.coca_cola".

    This also cleans smart quotes, excess whitespace, etc. out of the
    input data.
    """
    # TODO: might also want to apply custom corrections here
    if exists(scratch_db_path) and not force:
        mtime = getmtime(scratch_db_path)
        if all(exists(db_path) and getmtime(db_path) < mtime
               for db_path in input_db_paths):
            log.info('{} already exists and is up-to-date'.format(
                scratch_db_path))
            return

    scratch_db_tmp_path = scratch_db_path + '.tmp'
    if exists(scratch_db_tmp_path):
        remove(scratch_db_tmp_path)

    log.info('building {}...'.format(scratch_db_tmp_path))

    with open_db(scratch_db_tmp_path) as scratch_db:

        create_scratch_tables(scratch_db)

        for input_db_path in input_db_paths:
            log.info('dumping data from {} -> {}'.format(
                input_db_path, scratch_db_tmp_path))

            scraper_prefix = db_path_to_scraper_prefix(input_db_path)
            with open_db(input_db_path) as input_db:

                dump_db_to_scratch(input_db, scratch_db, scraper_prefix)

    log.info('moving {} -> {}'.format(scratch_db_tmp_path, scratch_db_path))
    rename(scratch_db_tmp_path, scratch_db_path)


def create_scratch_tables(scratch_db):
    """Add tables to the given (open) SQLite DB."""
    for table_name in sorted(TABLES):
        create_scratch_table(scratch_db, table_name)


def create_scratch_table(scratch_db, table_name):
    table_def = TABLES[table_name]

    columns = table_def['columns'].copy()
    columns['scraper_id'] = 'text'

    create_table(scratch_db, table_name, columns)

    # add "primary key" (non-unique) index
    index_cols = list(table_def.get('primary_key', ()))
    if 'scraper_id' not in index_cols:
        index_cols = ['scraper_id'] + index_cols
    create_index(scratch_db, table_name, index_cols)

    # add other indexes
    for index_cols in table_def.get('indexes', ()):
        create_index(scratch_db, table_name, index_cols)


def db_path_to_scraper_prefix(path):
    idx = path.lower().rfind('.sqlite')
    if idx == -1:
        raise ValueError
    return path[:idx]


def dump_db_to_scratch(input_db, scratch_db, scraper_prefix=''):
    input_table_names = set(show_tables(input_db))

    extra_table_names = input_table_names - set(TABLES)
    if extra_table_names:
{'fraction_non_alphanumeric': 0.08169571128414288, 'fraction_numerical': 0.0032134470399209, 'mean_word_length': 3.4855875831485585, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4283143', 'n_tokens_mistral': 2521, 'n_tokens_neox': 2453, 'n_words': 799}