# Path: epitator/get_database_connection.py
# def get_database_connection(create_database=False):
#     databse_exists = os.path.exists(ANNOTATOR_DB_PATH)
#     if databse_exists or create_database:
#         if not databse_exists:
#             print("Creating database at:", ANNOTATOR_DB_PATH)
#         connection = sqlite3.connect(ANNOTATOR_DB_PATH)
#         cur = connection.cursor()
#         cur.execute("PRAGMA foreign_keys = ON")
#         cur.execute("""
#         CREATE TABLE IF NOT EXISTS metadata (
#             property TEXT PRIMARY KEY ASC, value TEXT
#         )""")
#         if not databse_exists:
#             # Initialize the database
#             cur.execute("""
#             CREATE TABLE entities (
#                 id TEXT PRIMARY KEY, label TEXT, type TEXT, source TEXT
#             )""")
#             cur.execute("""
#             CREATE TABLE synonyms (
#                 synonym TEXT,
#                 entity_id TEXT REFERENCES entities(id),
#                 weight INTEGER
#             )""")
#             cur.execute('''
#             CREATE INDEX synonym_index ON synonyms (synonym);
#             ''')
#             cur.execute("INSERT INTO metadata VALUES ('dbversion', '0.0.1')")
#             connection.commit()
#         db_version = next(cur.execute("""
#         SELECT value AS version FROM metadata WHERE property = 'dbversion'
#         """), None)
#         if not db_version or db_version[0] != "0.0.1":
#             raise Exception("The database at " + ANNOTATOR_DB_PATH +
#                             " has a version that is not compatible by this version of EpiTator.\n"
#                             "You will need to rerun the data import scripts.")
#         return connection
#     else:
#         raise Exception("There is no EpiTator database at: " + ANNOTATOR_DB_PATH +
#                         "\nRun `python -m epitator.importers.import_all` to create a new database"
#                         "\nor set ANNOTATOR_DB_PATH to use a database at a different location.")
# 
# Path: epitator/utils.py
# def parse_number(num, default=None):
#     try:
#         return int(num)
#     except ValueError:
#         try:
#             return float(num)
#         except ValueError:
#             return default
# 
# def batched(iterable, batch_size=100):
#     """
#     Sequentially yield segments of the iterable in lists of the given size.
#     """
#     batch = []
#     for idx, item in enumerate(iterable):
#         batch.append(item)
#         batch_idx = idx % batch_size
#         if batch_idx == batch_size - 1:
#             yield batch
#             batch = []
#     yield batch
# 
# def normalize_text(text):
#     """
#     Attempt to convert text to a simplified representation so it can
#     be compared ignoring diacritical marks, differences in character codes
#     for similar symbols, differences in whitespace, and similar issues.
#     If the simplified text ends up being too short to form an
#     useful representation of the text, the original text will be returned.
#     That will happen when non-latin text is used because non-latin charaters are
#     usually removed from the simplified text.
#     """
#     text = text.replace('\u2019', "'")
#     result = unicodedata.normalize('NFKD', text)\
#         .encode('ascii', 'ignore').decode()
#     result = space_punct_re.sub(' ', result).strip()
#     if len(result) < 3:
#         result = space_punct_re.sub(' ', text).strip()
#     return result


import six
import csv
import unicodecsv
import re
import sys
    import argparse
from six import BytesIO
from zipfile import ZipFile
from six.moves.urllib import request
from six.moves.urllib.error import URLError
from ..get_database_connection import get_database_connection
from ..utils import parse_number, batched, normalize_text
    ('dem', None),
    ('timezone', None),
    ('modification_date', None)
]


def read_geonames_csv():
    print("Downloading geoname data from: " + GEONAMES_ZIP_URL)
    try:
        url = request.urlopen(GEONAMES_ZIP_URL)
    except URLError:
        print("If you are operating behind a firewall, try setting the HTTP_PROXY/HTTPS_PROXY environment variables.")
        raise
    zipfile = ZipFile(BytesIO(url.read()))
    print("Download complete")
    # Loading geonames data may cause errors without setting csv.field_size_limit:
    if sys.platform == "win32":
        max_c_long_on_windows = (2**32 / 2) - 1
        csv.field_size_limit(max_c_long_on_windows)
    else:
        csv.field_size_limit(sys.maxint if six.PY2 else six.MAXSIZE)
    with zipfile.open('allCountries.txt') as f:
        reader = unicodecsv.DictReader(f,
                                       fieldnames=[
                                           k for k, v in geonames_field_mappings],
                                       encoding='utf-8',
                                       delimiter='\t',
                                       quoting=csv.QUOTE_NONE)
        for d in reader:
            d['population'] = parse_number(d['population'], 0)

# Path: epitator/get_database_connection.py
# def get_database_connection(create_database=False):
#     databse_exists = os.path.exists(ANNOTATOR_DB_PATH)
#     if databse_exists or create_database:
#         if not databse_exists:
#             print("Creating database at:", ANNOTATOR_DB_PATH)
#         connection = sqlite3.connect(ANNOTATOR_DB_PATH)
#         cur = connection.cursor()
#         cur.execute("PRAGMA foreign_keys = ON")
#         cur.execute("""
#         CREATE TABLE IF NOT EXISTS metadata (
#             property TEXT PRIMARY KEY ASC, value TEXT
#         )""")
#         if not databse_exists:
#             # Initialize the database
#             cur.execute("""
#             CREATE TABLE entities (
#                 id TEXT PRIMARY KEY, label TEXT, type TEXT, source TEXT
#             )""")
#             cur.execute("""
#             CREATE TABLE synonyms (
#                 synonym TEXT,
#                 entity_id TEXT REFERENCES entities(id),
#                 weight INTEGER
#             )""")
#             cur.execute('''
#             CREATE INDEX synonym_index ON synonyms (synonym);
#             ''')
#             cur.execute("INSERT INTO metadata VALUES ('dbversion', '0.0.1')")
#             connection.commit()
#         db_version = next(cur.execute("""
#         SELECT value AS version FROM metadata WHERE property = 'dbversion'
#         """), None)
#         if not db_version or db_version[0] != "0.0.1":
#             raise Exception("The database at " + ANNOTATOR_DB_PATH +
#                             " has a version that is not compatible by this version of EpiTator.\n"
#                             "You will need to rerun the data import scripts.")
#         return connection
#     else:
#         raise Exception("There is no EpiTator database at: " + ANNOTATOR_DB_PATH +
#                         "\nRun `python -m epitator.importers.import_all` to create a new database"
#                         "\nor set ANNOTATOR_DB_PATH to use a database at a different location.")
# 
# Path: epitator/utils.py
# def parse_number(num, default=None):
#     try:
#         return int(num)
#     except ValueError:
#         try:
#             return float(num)
#         except ValueError:
#             return default
# 
# def batched(iterable, batch_size=100):
#     """
#     Sequentially yield segments of the iterable in lists of the given size.
#     """
#     batch = []
#     for idx, item in enumerate(iterable):
#         batch.append(item)
#         batch_idx = idx % batch_size
#         if batch_idx == batch_size - 1:
#             yield batch
#             batch = []
#     yield batch
# 
# def normalize_text(text):
#     """
#     Attempt to convert text to a simplified representation so it can
#     be compared ignoring diacritical marks, differences in character codes
#     for similar symbols, differences in whitespace, and similar issues.
#     If the simplified text ends up being too short to form an
#     useful representation of the text, the original text will be returned.
#     That will happen when non-latin text is used because non-latin charaters are
#     usually removed from the simplified text.
#     """
#     text = text.replace('\u2019', "'")
#     result = unicodedata.normalize('NFKD', text)\
#         .encode('ascii', 'ignore').decode()
#     result = space_punct_re.sub(' ', result).strip()
#     if len(result) < 3:
#         result = space_punct_re.sub(' ', text).strip()
#     return result


# Path: epitator/importers/import_geonames.py
import six
import csv
import unicodecsv
import re
import sys
    import argparse
from six import BytesIO
from zipfile import ZipFile
from six.moves.urllib import request
from six.moves.urllib.error import URLError
from ..get_database_connection import get_database_connection
from ..utils import parse_number, batched, normalize_text

    ('dem', None),
    ('timezone', None),
    ('modification_date', None)
]


def read_geonames_csv():
    print("Downloading geoname data from: " + GEONAMES_ZIP_URL)
    try:
        url = request.urlopen(GEONAMES_ZIP_URL)
    except URLError:
        print("If you are operating behind a firewall, try setting the HTTP_PROXY/HTTPS_PROXY environment variables.")
        raise
    zipfile = ZipFile(BytesIO(url.read()))
    print("Download complete")
    # Loading geonames data may cause errors without setting csv.field_size_limit:
    if sys.platform == "win32":
        max_c_long_on_windows = (2**32 / 2) - 1
        csv.field_size_limit(max_c_long_on_windows)
    else:
        csv.field_size_limit(sys.maxint if six.PY2 else six.MAXSIZE)
    with zipfile.open('allCountries.txt') as f:
        reader = unicodecsv.DictReader(f,
                                       fieldnames=[
                                           k for k, v in geonames_field_mappings],
                                       encoding='utf-8',
                                       delimiter='\t',
                                       quoting=csv.QUOTE_NONE)
        for d in reader:
            d['population'] = parse_number(d['population'], 0)

            d['latitude'] = parse_number(d['latitude'], 0)