<commit_before># -*- coding: utf-8 -*-

#   Copyright 2014 SpendRight, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Download and open databases."""

import logging
from os import rename
from os import remove
from os.path import exists
from os.path import getmtime

from srs.db import DEFAULT_DB_NAME
from srs.db import TABLE_TO_KEY_FIELDS
from srs.db import create_table_if_not_exists
from srs.db import download_db
from srs.db import get_db_path
from srs.db import open_db
from srs.db import open_dt
from srs.db import show_tables


SOURCE_DBS = {
    'campaigns',
    'companies',
}

CAMPAIGNS_PREFIX = 'campaigns:'

INPUT_DB_NAME = 'input'
INPUT_DB_TMP_NAME = INPUT_DB_NAME + '.tmp'

OUTPUT_DB_NAME = DEFAULT_DB_NAME + '.tmp'

URLS_DB_NAME = 'urls'

log = logging.getLogger(__name__)


def download_and_merge_dbs(force=False):
    """Download the various scraper DBs and merge into a single
    "input" DB."""
    # if input DB already exists and all its deps are up-to-date, we're done
    input_db_path = get_db_path(INPUT_DB_NAME)
    if exists(input_db_path) and not force:
        mtime = getmtime(input_db_path)
        if all(exists(db_path) and getmtime(db_path) < mtime for db_path
               in (get_db_path(db_name) for db_name in SOURCE_DBS)):
            return

    input_db_tmp_path = get_db_path(INPUT_DB_TMP_NAME)
    if exists(input_db_tmp_path):
        log.debug('Removing old version of {}'.format(input_db_tmp_path))
        remove(input_db_tmp_path)

    db = open_db(INPUT_DB_TMP_NAME)
    dt = open_dt(INPUT_DB_TMP_NAME)

    for src_db_name in sorted(SOURCE_DBS):
        download_db(src_db_name, force=force)
        src_db = open_db(src_db_name)

        for table in show_tables(src_db):
            if table not in TABLE_TO_KEY_FIELDS:
                log.warn('Unknown table `{}` in {} db, skipping'.format(
                    table, src_db_name))

            log.info('{}.{} -> {}'.format(src_db_name, table, input_db_path))

            create_table_if_not_exists(table, db=db)

            for row in src_db.execute('SELECT * FROM `{}`'.format(table)):
                # prepend db name to scraper ID
                scraper_id = '{}:{}'.format(src_db_name, row['scraper_id'])
                row = dict(row, scraper_id=scraper_id)
                dt.upsert(row, table)

    log.info('closing {} and moving to {}'.format(
        input_db_tmp_path, input_db_path))
    db.close()
    dt.close()
    rename(input_db_tmp_path, input_db_path)


def open_input_db():
    # TODO: does it actually save us anything to cache the input db?
    if not hasattr(open_input_db, '_db'):
        open_input_db._db = open_db(INPUT_DB_NAME)

    return open_input_db._db


def init_output_db():
    """Create a fresh output DB and its tables."""
    output_db_path = get_db_path(OUTPUT_DB_NAME)

    if exists(output_db_path):
        log.debug('Removing old version of {}'.format(
            output_db_path))
        remove(output_db_path)

    open_output_db()


def open_output_db():
    """Open a DB for output into a temp file."""
    if not hasattr(open_output_db, '_db'):
        open_output_db._db = open_db(OUTPUT_DB_NAME)

    return open_output_db._db


def open_output_dt():
    """Get the DumpTruck for the output DB"""
    if not hasattr(open_output_dt, '_dt'):
        open_output_db()
        open_output_dt._dt = open_dt(OUTPUT_DB_NAME)

    return open_output_dt._dt


def open_urls_db():
    """Open urls.sqlite, downloading it if necessary."""
    if not hasattr(open_urls_db, '_db'):
        download_db(URLS_DB_NAME)
        open_urls_db._db = open_db(URLS_DB_NAME)

    return open_urls_db._db


def output_row(row, table):
    """Output row to the given table. Also remove empty fields
    and remove the scraper_id field if the table name doesn't
    start with "scraper_".
    """
    row = clean_row(row)
    if 'scraper_id' in row and not table.startswith('scraper_'):
        del row['scraper_id']

    log.debug('{}: {}'.format(table, repr(row)))

    dt = open_output_dt()
    dt.upsert(row, table)


def close_output_db():
    """Move output_db into place."""
    output_db_path = get_db_path(OUTPUT_DB_NAME)
    default_db_path = get_db_path(DEFAULT_DB_NAME)<commit_msg>added primary keys to output DB<commit_after># -*- coding: utf-8 -*-

#   Copyright 2014 SpendRight, Inc.
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Download and open databases."""

import logging
from os import rename
from os import remove
from os.path import exists
from os.path import getmtime

from srs.db import DEFAULT_DB_NAME
from srs.db import TABLE_TO_KEY_FIELDS
from srs.db import create_table_if_not_exists
from srs.db import download_db
from srs.db import get_db_path
from srs.db import open_db
from srs.db import open_dt
from srs.db import show_tables


SOURCE_DBS = {
    'campaigns',
    'companies',
}

CAMPAIGNS_PREFIX = 'campaigns:'

INPUT_DB_NAME = 'input'
INPUT_DB_TMP_NAME = INPUT_DB_NAME + '.tmp'

OUTPUT_DB_NAME = DEFAULT_DB_NAME + '.tmp'

URLS_DB_NAME = 'urls'

log = logging.getLogger(__name__)


def download_and_merge_dbs(force=False):
    """Download the various scraper DBs and merge into a single
    "input" DB."""
    # if input DB already exists and all its deps are up-to-date, we're done
    input_db_path = get_db_path(INPUT_DB_NAME)
    if exists(input_db_path) and not force:
        mtime = getmtime(input_db_path)
        if all(exists(db_path) and getmtime(db_path) < mtime for db_path
               in (get_db_path(db_name) for db_name in SOURCE_DBS)):
            return

    input_db_tmp_path = get_db_path(INPUT_DB_TMP_NAME)
    if exists(input_db_tmp_path):
        log.debug('Removing old version of {}'.format(input_db_tmp_path))
        remove(input_db_tmp_path)

    db = open_db(INPUT_DB_TMP_NAME)
    dt = open_dt(INPUT_DB_TMP_NAME)

    for src_db_name in sorted(SOURCE_DBS):
        download_db(src_db_name, force=force)
        src_db = open_db(src_db_name)

        for table in show_tables(src_db):
            if table not in TABLE_TO_KEY_FIELDS:
                log.warn('Unknown table `{}` in {} db, skipping'.format(
                    table, src_db_name))

            log.info('{}.{} -> {}'.format(src_db_name, table, input_db_path))

            create_table_if_not_exists(table, db=db)

            for row in src_db.execute('SELECT * FROM `{}`'.format(table)):
                # prepend db name to scraper ID
                scraper_id = '{}:{}'.format(src_db_name, row['scraper_id'])
                row = dict(row, scraper_id=scraper_id)
                dt.upsert(row, table)

    log.info('closing {} and moving to {}'.format(
        input_db_tmp_path, input_db_path))
    db.close()
    dt.close()
    rename(input_db_tmp_path, input_db_path)


def open_input_db():
    # TODO: does it actually save us anything to cache the input db?
    if not hasattr(open_input_db, '_db'):
        open_input_db._db = open_db(INPUT_DB_NAME)

    return open_input_db._db


def init_output_db():
    """Create a fresh output DB and its tables."""
    output_db_path = get_db_path(OUTPUT_DB_NAME)

    if exists(output_db_path):
        log.debug('Removing old version of {}'.format(
            output_db_path))
        remove(output_db_path)

    open_output_db()


def open_output_db():
    """Open a DB for output into a temp file."""
    if not hasattr(open_output_db, '_db'):
        open_output_db._db = open_db(OUTPUT_DB_NAME)

    return open_output_db._db


def open_output_dt():
    """Get the DumpTruck for the output DB"""
    if not hasattr(open_output_dt, '_dt'):
        open_output_db()
        open_output_dt._dt = open_dt(OUTPUT_DB_NAME)

    return open_output_dt._dt


def open_urls_db():
    """Open urls.sqlite, downloading it if necessary."""
    if not hasattr(open_urls_db, '_db'):
        download_db(URLS_DB_NAME)
        open_urls_db._db = open_db(URLS_DB_NAME)

    return open_urls_db._db


def output_row(row, table):
    """Output row to the given table. Also remove empty fields
    and remove the scraper_id field if the table name doesn't
    start with "scraper_".
    """
    row = clean_row(row)
    if 'scraper_id' in row and not table.startswith('scraper_'):
        del row['scraper_id']

    log.debug('{}: {}'.format(table, repr(row)))

    db = open_output_db()
    create_table_if_not_exists(table, db=db)

    dt = open_output_dt()
    dt.upsert(row, table)


def close_output_db():
    """Move output_db into place."""
    output_db_path = get_db_path(OUTPUT_DB_NAME)
    default_db_path = get_db_path(DEFAULT_DB_NAME)
{'fraction_non_alphanumeric': 0.0971267957526546, 'fraction_numerical': 0.0018738288569643974, 'mean_word_length': 3.243374558303887, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5844956', 'n_tokens_mistral': 3279, 'n_tokens_neox': 3204, 'n_words': 874}