<commit_before>"""
    ulmo.cdec.historical.core
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module provides access to data provided by the `California Department
    of Water Resources`_ `California Data Exchange Center`_ web site.

    .. _California Department of Water Resources: http://www.water.ca.gov/
    .. _California Data Exchange Center: http://cdec.water.ca.gov


SELECTED CDEC SENSOR NUMBERS (these are not be available for all sites):

1    river stage [ft]
2    precipitation, accumulated [in]
3    SWE [in]
4    air temperature [F]
5    EC [ms/cm]
6    reservoir elevation [ft]
7    reservoir scheduled release [cfs]
8    full natural flow [cfs]
15   reservoir storage [af]
20   flow -- river discharge [cfs]
22   reservoir storage change [af]
23   reservoir outflow [cfs]
24   Evapotranspiration [in]
25   water temperature [F]
27   water turbidity [ntu]
28   chlorophyll [ug/l]
41   flow -- mean daily [cfs]
45   precipitation, incremental [in]
46   runoff volume [af]
61   water dissolved oxygen [mg/l]
62   water pH value [pH]
64   pan evaporation (incremental) [in]
65   full natural flow [af]
66   flow -- monthly volume [af]
67   accretions (estimated) [af]
71   spillway discharge [cfs]
74   lake evaporation (computed) [cfs]
76   reservoir inflow [cfs]
85   control regulating discharge [cfs]
94   top conservation storage (reservoir) [af]
100  water EC [us/cm]

CDEC DURATION CODES:

E    event
H    hourly
D    daily
M    monthly

"""

import pandas as pd
#import urllib
import requests
import re

from ulmo import util

DEFAULT_START_DATE = '01/01/1901'
DEFAULT_END_DATE   = 'Now'

def get_stations():
    """Fetches information on all CDEC sites. 

    Returns
    -------
    df : pandas DataFrame
        a pandas DataFrame (indexed on site id) with station information.
    """
        # I haven't found a better list of stations, seems pretty janky 
        # to just have them in a file, and not sure if/when it is updated.
    url = 'http://cdec.water.ca.gov/misc/all_stations.csv'
        # the csv is malformed, so some rows think there are 7 fields 
    col_names = ['id','meta_url','name','num','lat','lon','junk']
    df = pd.read_csv(url, names=col_names, header=None, quotechar="'",index_col=0)

    return df

def get_sensors(sensor_id=None):
    """
    Gets a list of sensor ids as a DataFrame indexed on sensor 
    number. Can be limited by a list of numbers.

    Usage Example:

    from ulmo import cdec
        # to get all available sensor info
    snesors = cdec.historical.get_sensors()
        # or to get just one sensor
    sensors = cdec.historical.get_sensors([1])

    Parameters
    ----------
    sites : iterable of integers or ``None``

    Returns
    -------
    df : pandas DataFrame
        a python dict with site codes mapped to site information
    """

    url = 'http://cdec.water.ca.gov/misc/senslist.html'
    df = pd.read_html(url,infer_types=False,header=0,index_col=0)[0]

    if sensor_id is None:
        return df
    else:
        return df.ix[sensor_id]

def get_station_sensors(station_ids=None, sensor_ids=None, resolutions=None):
    """
    Gets available sensors for the given stations, sensor ids and time 
    resolutions. If no station ids are provided, all available stations will 
    be used (this is not recommended, and will probably take a really long 
    time). <commit_msg>Fix indexing bug for sensor list<commit_after>"""
    ulmo.cdec.historical.core
    ~~~~~~~~~~~~~~~~~~~~~~~~~~

    This module provides access to data provided by the `California Department
    of Water Resources`_ `California Data Exchange Center`_ web site.

    .. _California Department of Water Resources: http://www.water.ca.gov/
    .. _California Data Exchange Center: http://cdec.water.ca.gov


SELECTED CDEC SENSOR NUMBERS (these are not be available for all sites):

1    river stage [ft]
2    precipitation, accumulated [in]
3    SWE [in]
4    air temperature [F]
5    EC [ms/cm]
6    reservoir elevation [ft]
7    reservoir scheduled release [cfs]
8    full natural flow [cfs]
15   reservoir storage [af]
20   flow -- river discharge [cfs]
22   reservoir storage change [af]
23   reservoir outflow [cfs]
24   Evapotranspiration [in]
25   water temperature [F]
27   water turbidity [ntu]
28   chlorophyll [ug/l]
41   flow -- mean daily [cfs]
45   precipitation, incremental [in]
46   runoff volume [af]
61   water dissolved oxygen [mg/l]
62   water pH value [pH]
64   pan evaporation (incremental) [in]
65   full natural flow [af]
66   flow -- monthly volume [af]
67   accretions (estimated) [af]
71   spillway discharge [cfs]
74   lake evaporation (computed) [cfs]
76   reservoir inflow [cfs]
85   control regulating discharge [cfs]
94   top conservation storage (reservoir) [af]
100  water EC [us/cm]

CDEC DURATION CODES:

E    event
H    hourly
D    daily
M    monthly

"""

import pandas as pd
#import urllib
import requests
import re

from ulmo import util

DEFAULT_START_DATE = '01/01/1901'
DEFAULT_END_DATE   = 'Now'

def get_stations():
    """Fetches information on all CDEC sites. 

    Returns
    -------
    df : pandas DataFrame
        a pandas DataFrame (indexed on site id) with station information.
    """
        # I haven't found a better list of stations, seems pretty janky 
        # to just have them in a file, and not sure if/when it is updated.
    url = 'http://cdec.water.ca.gov/misc/all_stations.csv'
        # the csv is malformed, so some rows think there are 7 fields 
    col_names = ['id','meta_url','name','num','lat','lon','junk']
    df = pd.read_csv(url, names=col_names, header=None, quotechar="'",index_col=0)

    return df

def get_sensors(sensor_id=None):
    """
    Gets a list of sensor ids as a DataFrame indexed on sensor 
    number. Can be limited by a list of numbers.

    Usage Example:

    from ulmo import cdec
        # to get all available sensor info
    snesors = cdec.historical.get_sensors()
        # or to get just one sensor
    sensors = cdec.historical.get_sensors([1])

    Parameters
    ----------
    sites : iterable of integers or ``None``

    Returns
    -------
    df : pandas DataFrame
        a python dict with site codes mapped to site information
    """

    url = 'http://cdec.water.ca.gov/misc/senslist.html'
    df = pd.read_html(url,infer_types=False,header=0)[0]
    df.set_index('Sensor No')

    if sensor_id is None:
        return df
    else:
        return df.ix[sensor_id]

def get_station_sensors(station_ids=None, sensor_ids=None, resolutions=None):
    """
    Gets available sensors for the given stations, sensor ids and time 
    resolutions. If no station ids are provided, all available stations will 
    be used (this is not recommended, and will probably take a really long 
    time). 
{'fraction_non_alphanumeric': 0.09898215075969907, 'fraction_numerical': 0.02020947042336628, 'mean_word_length': 2.957968476357268, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '492472', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2082, 'n_words': 801}