<commit_before># Copyright 2017 Verily Life Sciences Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Mock library for interacting with BigQuery, backed by SQLite.

Sample usage:

    client = mock_bq.Client(project_id)
    result = client.Query(query)
"""

# Workaround for https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2366
from __future__ import absolute_import

import datetime
import logging
import re

from google.cloud.bigquery.schema import SchemaField
from pysqlite2 import dbapi2 as sqlite3
from verily.bigquery_wrapper.bq_base import (BQ_PATH_DELIMITER,
                                             BigqueryBaseClient)

# SQLite only uses . to separate database and table, so we need a
# different delimiter. It's pretty strict about what special characters it
# will accept as a part of a table name so I'm using Z since it's an uncommon
# character. The change should be invisible to users except in debugging test
# cases and tables with the delimiters replaced are never backparsed into Bigquery
# tables so there's no danger in using a table path with a Z in it.
MOCK_DELIMITER = 'Z'

TRUE_STR = 'True'
FALSE_STR = 'False'


# Pylint thinks this is an abstract class because it throws NotImplementedErrors.
# The warning is disabled because in this case those errors indicate that those methods
# are just difficult or impossible to mock.
# pylint: disable=R0921
class Client(BigqueryBaseClient):
    """Stores pointers to a mock BigQuery project, backed by an in-memory
    sqlite database instead.

    Args:
        project_id: The id of the project to associate with the client.
        default_dataset: The default dataset to use in function calls if none is explicitly provided.
        maximum_billing_tier: Unused in this implementation.
        print_before_and_after: If set to True, this will print out queries as passed in, then
            print out the query after it's been reformatted to match appropriate SQLite conventions.
            Helpful for debugging.
    """

    def __init__(self, project_id, default_dataset=None, maximum_billing_tier=None,
                 print_before_and_after=False):
        # SQLite does not allow dashes in table paths.
        formatted_project_id = project_id.replace('-', '_')
        super(Client, self).__init__(formatted_project_id, default_dataset, maximum_billing_tier)

        # Connect to an in-memory database path.
        self.conn = sqlite3.connect(':memory:')
        self.cursor = self.conn.cursor()

        # We use dictionaries to simulate the project/dataset/table structure present in BQ.
        # Mostly these will serve as checks in tests to make sure that a project and dataset
        # are created before a table is inserted into them.
        self.project_map = {}
        self.project_map[self.project_id] = []

        self.table_map = {}
        self.table_map[default_dataset] = []

        self.print_before_and_after = print_before_and_after

    def get_delimiter(self):
        return MOCK_DELIMITER

    def _convert_to_mock_path(self, table_path):
        """
        Converts a table path from being delimited with BQ_PATH_DELIMITER to MOCK_DELIMITER.
        """
        table_project, dataset_id, table_name = self.parse_table_path(table_path,
                                                                      delimiter=BQ_PATH_DELIMITER)
        return self.path(table_name, dataset_id, table_project)

    @staticmethod
    def _bq_type_to_sqlite_type(typename):
        """ Maps BQ types to SQLite types. One limitation of this class is that there's not a 1:1
        mapping, but this will cover most use cases."""
        if typename in ['FLOAT', 'FLOAT64']:
            return 'REAL'<commit_msg>Create default dataset with create method<commit_after># Copyright 2017 Verily Life Sciences Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Mock library for interacting with BigQuery, backed by SQLite.

Sample usage:

    client = mock_bq.Client(project_id)
    result = client.Query(query)
"""

# Workaround for https://github.com/GoogleCloudPlatform/google-cloud-python/issues/2366
from __future__ import absolute_import

import datetime
import logging
import re

from google.cloud.bigquery.schema import SchemaField
from pysqlite2 import dbapi2 as sqlite3
from verily.bigquery_wrapper.bq_base import (BQ_PATH_DELIMITER,
                                             BigqueryBaseClient)

# SQLite only uses . to separate database and table, so we need a
# different delimiter. It's pretty strict about what special characters it
# will accept as a part of a table name so I'm using Z since it's an uncommon
# character. The change should be invisible to users except in debugging test
# cases and tables with the delimiters replaced are never backparsed into Bigquery
# tables so there's no danger in using a table path with a Z in it.
MOCK_DELIMITER = 'Z'

TRUE_STR = 'True'
FALSE_STR = 'False'


# Pylint thinks this is an abstract class because it throws NotImplementedErrors.
# The warning is disabled because in this case those errors indicate that those methods
# are just difficult or impossible to mock.
# pylint: disable=R0921
class Client(BigqueryBaseClient):
    """Stores pointers to a mock BigQuery project, backed by an in-memory
    sqlite database instead.

    Args:
        project_id: The id of the project to associate with the client.
        default_dataset: The default dataset to use in function calls if none is explicitly provided.
        maximum_billing_tier: Unused in this implementation.
        print_before_and_after: If set to True, this will print out queries as passed in, then
            print out the query after it's been reformatted to match appropriate SQLite conventions.
            Helpful for debugging.
    """

    def __init__(self, project_id, default_dataset=None, maximum_billing_tier=None,
                 print_before_and_after=False):
        # SQLite does not allow dashes in table paths.
        formatted_project_id = project_id.replace('-', '_')
        super(Client, self).__init__(formatted_project_id, default_dataset, maximum_billing_tier)

        # Connect to an in-memory database path.
        self.conn = sqlite3.connect(':memory:')
        self.cursor = self.conn.cursor()

        # We use dictionaries to simulate the project/dataset/table structure present in BQ.
        # Mostly these will serve as checks in tests to make sure that a project and dataset
        # are created before a table is inserted into them.
        self.project_map = {}
        self.project_map[self.project_id] = []

        self.table_map = {}
        self.create_dataset_by_name(default_dataset)

        self.print_before_and_after = print_before_and_after

    def get_delimiter(self):
        return MOCK_DELIMITER

    def _convert_to_mock_path(self, table_path):
        """
        Converts a table path from being delimited with BQ_PATH_DELIMITER to MOCK_DELIMITER.
        """
        table_project, dataset_id, table_name = self.parse_table_path(table_path,
                                                                      delimiter=BQ_PATH_DELIMITER)
        return self.path(table_name, dataset_id, table_project)

    @staticmethod
    def _bq_type_to_sqlite_type(typename):
        """ Maps BQ types to SQLite types. One limitation of this class is that there's not a 1:1
        mapping, but this will cover most use cases."""
        if typename in ['FLOAT', 'FLOAT64']:
            return 'REAL'
{'fraction_non_alphanumeric': 0.061565420560747666, 'fraction_numerical': 0.005607476635514018, 'mean_word_length': 3.1397485493230173, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '651294', 'n_tokens_mistral': 2318, 'n_tokens_neox': 2227, 'n_words': 924}