<commit_before>import os
import re
import logging
import urllib2
import requests
import pandas as pd
import pydap.client
import pydap.exceptions
import xray

from bs4 import BeautifulSoup
from contextlib import closing
from datetime import datetime
from requests.exceptions import ConnectionError
from shutil import move

# Support Python 2.7 and 3.x
try:
    from io import StringIO
except ImportError:
    from cStringIO import StringIO

from exceptions import RequiredVariableNotPresent

class ArgoData(object):
    '''Collection of methods for working with Argo profiling float data.
    '''

    # Jupyter Notebook defines a root logger, use that if it exists
    if logging.getLogger().handlers:
        _notebook_handler = logging.getLogger().handlers[0]
        logger = logging.getLogger()
    else:
        logger = logging.getLogger(__name__)
        _handler = logging.StreamHandler()
        _formatter = logging.Formatter('%(levelname)s %(asctime)s %(filename)s '
                                      '%(funcName)s():%(lineno)d %(message)s')
        _handler.setFormatter(_formatter)
        logger.addHandler(_handler)

    _log_levels = (logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG)

    # Literals for groups stored in local HDF file cache
    _STATUS = 'status'
    _GLOBAL_META = 'global_meta'
    _BIO_PROFILE_INDEX = 'bio_global_index'
    _ALL_WMO_LIST = 'all_wmo_list'
    _OXY_COUNT_DF = 'oxy_count_df'
    _coordinates = {'PRES_ADJUSTED', 'LATITUDE', 'LONGITUDE', 'JULD'}

    # Names and search patterns for cache file naming/parsing
    # Make private and ignore pylint's complaints
    # No other names in this class can end in 'RE'
    _fixed_cache_base = 'biofloat_fixed_cache'
    _ageRE = 'age([0-9]+)'
    _profilesRE = 'profiles([0-9]+)'
    _pressureRE = 'pressure([0-9]+)'
    _wmoRE = 'wmo([0-9-]+)'

    _MAX_VALUE = 10000000000
    _compparms = dict(complib='zlib', complevel=9)

    # PyTables: Use non-empty minimal df to minimize HDF file size
    _blank_df = pd.DataFrame([pd.np.nan])

    def __init__(self, verbosity=0, cache_file=None, bio_list=('DOXY_ADJUSTED',),
            status_url='http://argo.jcommops.org/FTPRoot/Argo/Status/argo_all.txt',
            global_url='ftp://ftp.ifremer.fr/ifremer/argo/ar_index_global_meta.txt',
            thredds_url='http://tds0.ifremer.fr/thredds/catalog/CORIOLIS-ARGO-GDAC-OBS',
            variables=('TEMP_ADJUSTED', 'PSAL_ADJUSTED', 'DOXY_ADJUSTED')):

        '''Initialize ArgoData object.
        
        Args:
            verbosity (int): range(4), default=0
            cache_file (str): Defaults to biofloat_default_cache.hdf in users
                              home directory
            bio_list (list): List of required bio variables, e.g.:
                             ['DOXY_ADJUSTED', 'CHLA', 'BBP700', 'CDOM', 'NITRATE']
            status_url (str): Source URL for Argo status data, defaults to
                              http://argo.jcommops.org/FTPRoot/Argo/Status/argo_all.txt
            global_url (str): Source URL for DAC locations, defaults to
                              ftp://ftp.ifremer.fr/ifremer/argo/ar_index_global_meta.txt
            thredds_url (str): Base URL for THREDDS Data Server, defaults to
                               http://tds0.ifremer.fr/thredds/catalog/CORIOLIS-ARGO-GDAC-OBS
            variables (list): Variables to extract from NetCDF files and put
                              into the Pandas DataFrame

            cache_file (str):

            There are 3 kinds of cache files:

            1. The default file named biofloat_cache.hdf that is automatically
               placed in the user's home directory. It will cache whatever
               data is requested via call to get_float_dataframe().
            2. Specially named cache_files produced by the load_biofloat_cache.py
               script. These files are built with constraints and are fixed. 
               Once built they can be used in a read-only fashion to work on 
               only the data they contain. Calls to get_float_dataframe().
               will not add more data to these "fixed" cache files.
            3. Custom cache file names. These operate just like the default cache
               file, but can be named whatever the user wants. 

        '''
        self.status_url = status_url
        self.global_url = global_url
        self.thredds_url = thredds_url
        self.variables = set(variables)

        self.logger.setLevel(self._log_levels[verbosity])
        self._bio_list = bio_list

        if cache_file:
            self.cache_file_parms = self._get_cache_file_parms(cache_file)
            self.cache_file = cache_file
        else:
            # Write default cache to users home directory 
            self.cache_file = os.path.abspath(os.path.join(
                              os.path.expanduser('~'), 
                              'biofloat_default_cache.hdf'))

    def _put_df(self, df, name, metadata=None):
        '''Save Pandas DataFrame to local HDF file with optional metadata dict.
        '''
        store = pd.HDFStore(self.cache_file)
        self.logger.debug('Saving DataFrame to name "%s" in file %s',
                                              name, self.cache_file)
        if df.dropna().empty:
            store.put(name, df, format='fixed')
        else:
            ##store.append(name, df, format='table', **self._compparms)
            store.put(name, df, format='fixed')

        if metadata and store.get_storer(name):
            store.get_storer(name).attrs.metadata = metadata

        self.logger.debug('store.close()')
        store.close()

    def _get_df(self, name):
        '''Get Pandas DataFrame from local HDF file or raise KeyError.
        '''
        store = pd.HDFStore(self.cache_file)
        try:
            self.logger.debug('Getting "%s" from %s', name, self.cache_file)
            df = store[name]
            try:
                metadata = store.get_storer(name).attrs.metadata
            except AttributeError:
                metadata = None
        except (IOError, KeyError):
            raise
        finally:
            self.logger.debug('store.close()')
            store.close()<commit_msg>Update doc string for _get_df() - it returns metadata too<commit_after>import os
import re
import logging
import urllib2
import requests
import pandas as pd
import pydap.client
import pydap.exceptions
import xray

from bs4 import BeautifulSoup
from contextlib import closing
from datetime import datetime
from requests.exceptions import ConnectionError
from shutil import move

# Support Python 2.7 and 3.x
try:
    from io import StringIO
except ImportError:
    from cStringIO import StringIO

from exceptions import RequiredVariableNotPresent

class ArgoData(object):
    '''Collection of methods for working with Argo profiling float data.
    '''

    # Jupyter Notebook defines a root logger, use that if it exists
    if logging.getLogger().handlers:
        _notebook_handler = logging.getLogger().handlers[0]
        logger = logging.getLogger()
    else:
        logger = logging.getLogger(__name__)
        _handler = logging.StreamHandler()
        _formatter = logging.Formatter('%(levelname)s %(asctime)s %(filename)s '
                                      '%(funcName)s():%(lineno)d %(message)s')
        _handler.setFormatter(_formatter)
        logger.addHandler(_handler)

    _log_levels = (logging.ERROR, logging.WARN, logging.INFO, logging.DEBUG)

    # Literals for groups stored in local HDF file cache
    _STATUS = 'status'
    _GLOBAL_META = 'global_meta'
    _BIO_PROFILE_INDEX = 'bio_global_index'
    _ALL_WMO_LIST = 'all_wmo_list'
    _OXY_COUNT_DF = 'oxy_count_df'
    _coordinates = {'PRES_ADJUSTED', 'LATITUDE', 'LONGITUDE', 'JULD'}

    # Names and search patterns for cache file naming/parsing
    # Make private and ignore pylint's complaints
    # No other names in this class can end in 'RE'
    _fixed_cache_base = 'biofloat_fixed_cache'
    _ageRE = 'age([0-9]+)'
    _profilesRE = 'profiles([0-9]+)'
    _pressureRE = 'pressure([0-9]+)'
    _wmoRE = 'wmo([0-9-]+)'

    _MAX_VALUE = 10000000000
    _compparms = dict(complib='zlib', complevel=9)

    # PyTables: Use non-empty minimal df to minimize HDF file size
    _blank_df = pd.DataFrame([pd.np.nan])

    def __init__(self, verbosity=0, cache_file=None, bio_list=('DOXY_ADJUSTED',),
            status_url='http://argo.jcommops.org/FTPRoot/Argo/Status/argo_all.txt',
            global_url='ftp://ftp.ifremer.fr/ifremer/argo/ar_index_global_meta.txt',
            thredds_url='http://tds0.ifremer.fr/thredds/catalog/CORIOLIS-ARGO-GDAC-OBS',
            variables=('TEMP_ADJUSTED', 'PSAL_ADJUSTED', 'DOXY_ADJUSTED')):

        '''Initialize ArgoData object.
        
        Args:
            verbosity (int): range(4), default=0
            cache_file (str): Defaults to biofloat_default_cache.hdf in users
                              home directory
            bio_list (list): List of required bio variables, e.g.:
                             ['DOXY_ADJUSTED', 'CHLA', 'BBP700', 'CDOM', 'NITRATE']
            status_url (str): Source URL for Argo status data, defaults to
                              http://argo.jcommops.org/FTPRoot/Argo/Status/argo_all.txt
            global_url (str): Source URL for DAC locations, defaults to
                              ftp://ftp.ifremer.fr/ifremer/argo/ar_index_global_meta.txt
            thredds_url (str): Base URL for THREDDS Data Server, defaults to
                               http://tds0.ifremer.fr/thredds/catalog/CORIOLIS-ARGO-GDAC-OBS
            variables (list): Variables to extract from NetCDF files and put
                              into the Pandas DataFrame

            cache_file (str):

            There are 3 kinds of cache files:

            1. The default file named biofloat_cache.hdf that is automatically
               placed in the user's home directory. It will cache whatever
               data is requested via call to get_float_dataframe().
            2. Specially named cache_files produced by the load_biofloat_cache.py
               script. These files are built with constraints and are fixed. 
               Once built they can be used in a read-only fashion to work on 
               only the data they contain. Calls to get_float_dataframe().
               will not add more data to these "fixed" cache files.
            3. Custom cache file names. These operate just like the default cache
               file, but can be named whatever the user wants. 

        '''
        self.status_url = status_url
        self.global_url = global_url
        self.thredds_url = thredds_url
        self.variables = set(variables)

        self.logger.setLevel(self._log_levels[verbosity])
        self._bio_list = bio_list

        if cache_file:
            self.cache_file_parms = self._get_cache_file_parms(cache_file)
            self.cache_file = cache_file
        else:
            # Write default cache to users home directory 
            self.cache_file = os.path.abspath(os.path.join(
                              os.path.expanduser('~'), 
                              'biofloat_default_cache.hdf'))

    def _put_df(self, df, name, metadata=None):
        '''Save Pandas DataFrame to local HDF file with optional metadata dict.
        '''
        store = pd.HDFStore(self.cache_file)
        self.logger.debug('Saving DataFrame to name "%s" in file %s',
                                              name, self.cache_file)
        if df.dropna().empty:
            store.put(name, df, format='fixed')
        else:
            ##store.append(name, df, format='table', **self._compparms)
            store.put(name, df, format='fixed')

        if metadata and store.get_storer(name):
            store.get_storer(name).attrs.metadata = metadata

        self.logger.debug('store.close()')
        store.close()

    def _get_df(self, name):
        '''Return tuple of Pandas DataFrame and metadata dictionary.
        '''
        store = pd.HDFStore(self.cache_file)
        try:
            self.logger.debug('Getting "%s" from %s', name, self.cache_file)
            df = store[name]
            try:
                metadata = store.get_storer(name).attrs.metadata
            except AttributeError:
                metadata = None
        except (IOError, KeyError):
            raise
        finally:
            self.logger.debug('store.close()')
            store.close()
{'fraction_non_alphanumeric': 0.09139527458492976, 'fraction_numerical': 0.006066411238825032, 'mean_word_length': 2.2150372081088014, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2716277', 'n_tokens_mistral': 3718, 'n_tokens_neox': 3587, 'n_words': 985}