<commit_before>import ConfigParser
import logging
import os
import subprocess

import psycopg2

from .database import database as db


# Set up logging system.
logging.basicConfig()
logger = logging.getLogger(__name__)


def load_config(config_filename=None):
    """Returns a ConfigParser object.

    Configuration is loaded from these filenames, in increasing precedence:

      - ~/.spandex/user.cfg
      - SPANDEX_CFG environment variable
      - config_filename argument, if provided

    If a file cannot be opened, it will be ignored. If none of the filenames
    can be opened, the ConfigParser object will be empty.

    """
    # Build list of configuration filenames.
    config_filenames = [os.path.expanduser('~/.spandex/user.cfg')]
    config_filename_env = os.environ.get('SPANDEX_CFG')
    if config_filename_env:
        config_filenames.append(config_filename_env)
    if config_filename:
        config_filenames.append(config_filename)

    # Load configuration using ConfigParser.
    logger.debug("Loading configuration from %s" % config_filenames)
    config = ConfigParser.RawConfigParser()
    config.read(config_filenames)
    return config


def logf(level, f):
    """Log each line of a file-like object at the specified severity level."""
    for line in f:
        if line:
            logger.log(level, line.strip())


class DataLoader(object):
    """Data loader class with support for importing shapefiles.

    Some example usage:

        loader = DataLoader()

        # Load data.
        loader.load_shp('parcel/Alameda.shp', 'staging.alameda')

        # Run SQL command and commit.
        with loader.database.cursor() as cur:
            cur.execute("SELECT DISTINCT luc_desc FROM staging.alameda;")
            rows = cur.fetchall()

        # Close all connection(s).
        loader.close()

    Methods:
        close:           Close managed PostgreSQL connection(s).
        load_shp:        Load a shapefile into a PostGIS table.

    Attributes:
        database:        PostgreSQL database connection manager class.
        directory:       Path to the directory containing the shapefiles.
        srid:            Default Spatial Reference System Identifier (SRID).

    Attributes can be passed as additional constructor arguments and override
    configuration.

    Constructor arguments:
        config_filename: Path to additional configuration file.
                         If None, configuration must be provided in default
                         locations, or attributes must be passed as
                         constructor arguments in place of configuration.

    """

    def __init__(self, config_filename=None, database=None, directory=None,
                 srid=None):
        # Attempt to load configuration.
        config = load_config(config_filename)

        # Define attributes from configuration, unless overridden.
        if not database:
            database = db
        if not directory:
            directory = config.get('data', 'directory')
        if not srid:
            srid = config.get('data', 'srid')

        # Create new connection(s) using configuration, unless already
        # connected.
        try:
            database.assert_connected()
        except psycopg2.DatabaseError:
            db_config = dict(config.items('database'))
            database.connect(**db_config)

        # Assign arguments to class attributes.
        self.database = database
        if os.path.exists(directory):
            self.directory = directory
        else:
            raise IOError("Directory does not exist: %s" % directory)
        self.srid = int(srid)

    def close(self):
        return self.database.close()

    def load_shp(self, filename, table, srid=None, drop=False, append=False):
        """Load a shapefile from the directory into a PostGIS table.

        This is a Python wrapper for shp2gpsql. shp2pgsql is spawned by
        subprocess. Commands generated by shp2pgsql are executed on a<commit_msg>Add docstring to DataLoader.close<commit_after>import ConfigParser
import logging
import os
import subprocess

import psycopg2

from .database import database as db


# Set up logging system.
logging.basicConfig()
logger = logging.getLogger(__name__)


def load_config(config_filename=None):
    """Returns a ConfigParser object.

    Configuration is loaded from these filenames, in increasing precedence:

      - ~/.spandex/user.cfg
      - SPANDEX_CFG environment variable
      - config_filename argument, if provided

    If a file cannot be opened, it will be ignored. If none of the filenames
    can be opened, the ConfigParser object will be empty.

    """
    # Build list of configuration filenames.
    config_filenames = [os.path.expanduser('~/.spandex/user.cfg')]
    config_filename_env = os.environ.get('SPANDEX_CFG')
    if config_filename_env:
        config_filenames.append(config_filename_env)
    if config_filename:
        config_filenames.append(config_filename)

    # Load configuration using ConfigParser.
    logger.debug("Loading configuration from %s" % config_filenames)
    config = ConfigParser.RawConfigParser()
    config.read(config_filenames)
    return config


def logf(level, f):
    """Log each line of a file-like object at the specified severity level."""
    for line in f:
        if line:
            logger.log(level, line.strip())


class DataLoader(object):
    """Data loader class with support for importing shapefiles.

    Some example usage:

        loader = DataLoader()

        # Load data.
        loader.load_shp('parcel/Alameda.shp', 'staging.alameda')

        # Run SQL command and commit.
        with loader.database.cursor() as cur:
            cur.execute("SELECT DISTINCT luc_desc FROM staging.alameda;")
            rows = cur.fetchall()

        # Close all connection(s).
        loader.close()

    Methods:
        close:           Close managed PostgreSQL connection(s).
        load_shp:        Load a shapefile into a PostGIS table.

    Attributes:
        database:        PostgreSQL database connection manager class.
        directory:       Path to the directory containing the shapefiles.
        srid:            Default Spatial Reference System Identifier (SRID).

    Attributes can be passed as additional constructor arguments and override
    configuration.

    Constructor arguments:
        config_filename: Path to additional configuration file.
                         If None, configuration must be provided in default
                         locations, or attributes must be passed as
                         constructor arguments in place of configuration.

    """

    def __init__(self, config_filename=None, database=None, directory=None,
                 srid=None):
        # Attempt to load configuration.
        config = load_config(config_filename)

        # Define attributes from configuration, unless overridden.
        if not database:
            database = db
        if not directory:
            directory = config.get('data', 'directory')
        if not srid:
            srid = config.get('data', 'srid')

        # Create new connection(s) using configuration, unless already
        # connected.
        try:
            database.assert_connected()
        except psycopg2.DatabaseError:
            db_config = dict(config.items('database'))
            database.connect(**db_config)

        # Assign arguments to class attributes.
        self.database = database
        if os.path.exists(directory):
            self.directory = directory
        else:
            raise IOError("Directory does not exist: %s" % directory)
        self.srid = int(srid)

    def close(self):
        """Close managed PostgreSQL connection(s)."""
        return self.database.close()

    def load_shp(self, filename, table, srid=None, drop=False, append=False):
        """Load a shapefile from the directory into a PostGIS table.

        This is a Python wrapper for shp2gpsql. shp2pgsql is spawned by
        subprocess. Commands generated by shp2pgsql are executed on a
{'fraction_non_alphanumeric': 0.07407407407407407, 'fraction_numerical': 0.0012345679012345679, 'mean_word_length': 2.4590093936806148, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5239643', 'n_tokens_mistral': 2149, 'n_tokens_neox': 2046, 'n_words': 706}