<commit_before># -*- coding: utf-8 -*-
"""
Created on Sun Jan  3 00:30:36 2016

@author: p
"""
import urllib2
import xmltodict
import pandas as pd
from datetime import datetime 
from httplib import BadStatusLine
import matplotlib.pyplot as plt
import numpy as np
import avgMeths
import requests
import json

class usgs:
    @staticmethod 
    def getelev(x):
        '''
        Uses USGS elevation service to retrieve elevation
        
        Input
        -----
        x, y = longitude and latitude of point where elevation is desired        
    
        Output
        ------
        ned float elevation of location in meters
        '''
        elev = "http://ned.usgs.gov/epqs/pqs.php?x="+str(x[0])+"&y="+str(x[1])+"&units=Meters&output=xml"
        try:        
            response = urllib2.urlopen(elev)
            html = response.read()
            d = xmltodict.parse(html)
            g = float(d['USGS_Elevation_Point_Query_Service']['Elevation_Query']['Elevation'])
        except(BadStatusLine):
            try:        
                response = urllib2.urlopen(elev)
                html = response.read()
                d = xmltodict.parse(html)
                g = float(d['USGS_Elevation_Point_Query_Service']['Elevation_Query']['Elevation'])
            except(BadStatusLine):
                print "could not fetch %s" % html
                g = 0                
                pass                 
        return g
    
    @staticmethod     
    def USGSID(x):
        def dms(dec):
            DD = str(int(abs(dec)))
            MM = str(int((abs(dec) - int(DD))*60)).zfill(2)
            SS = str(int(round((((abs(dec) - int(DD))*60) - int(MM))*60, 0))).zfill(2)
            return DD+MM+SS
        return dms(x[1])+dms(x[0])+'01'
    
    def get_nwis(self, val_list, selectType ='dv_site', start_date = '1800-01-01', end_date = ''):
        """
        Request stream gauge data from the USGS NWIS.
        Args:
            val_list (str or int or list of either):
                can be a list or 8-digit hucs or stations or single 
                a valid site is 01585200; a valid huc is 16010204
            selectType (str):
                options: 'dv_site','dv_huc','gw_site',or 'gw_huc'
            start_date (str):
               should take on the form yyyy-mm-dd; default is '1800-01-01'
            end_date (str):
                should take on the form yyyy-mm-dd; default is today
        Returns:
            a station and a data Pandas dataframe.
        Raises:
            ConnectionError  due to connection problems like refused connection
            or DNS Error.

        Example::
            >>> import wellapplication as wa
            >>> USGS = wa.usgs()
            >>> site, data = USGS.get_nwis('01585200', 'dv_site', '2012-06-01', '2012-07-01')

        The specification for this service is located here:
        http://waterservices.usgs.gov/rest/IV-Service.html
        
        This function was adapted from: https://github.com/mroberge/hydrofunctions
        """
        val_list = self.parsesitelist(val_list)

        if end_date == '':
            dy = datetime.today()
            end_date = str(dy.year)+'-'+str(dy.month)+'-'+str(dy.day)

        header = {'Accept-encoding': 'gzip'}

        
        valdict = {
            'dv_site':{'format': 'json', 'sites': val_list, 'parameterCd': '00060',
                      'startDT': start_date, 'endDT': end_date},
            'dv_huc':{'format': 'json', 'huc': val_list, 'parameterCd': '00060',
                      'startDT': start_date, 'endDT': end_date},
            'gw_site':{'format': 'json', 'sites': val_list, 'siteType':'GW','siteStatus':'all',
                      'startDT': start_date, 'endDT': end_date},
            'gw_huc':{'format': 'json', 'huc': val_list, 'siteType':'GW','siteStatus':'all',
                      'startDT': start_date, 'endDT': end_date}
            }


        url = 'http://waterservices.usgs.gov/nwis/'
            
        if selectType == 'dv_site' or selectType == 'dv_huc':
            service = 'dv'
        elif selectType == 'gw_site' or selectType == 'gw_huc':
            service = 'gwlevels'
        url = url + service + '/?'
        response_ob = requests.get(url, params=valdict[selectType], headers=header)

        nwis_dict = response_ob.json()

        dt = nwis_dict['value']['timeSeries']

        station_id,lat,lon,srs,station_type,station_nm =  [],[],[],[],[],[]
        f = {}
        for i in range(len(dt)):

            station_id.append(dt[i]['sourceInfo']['siteCode'][0]['value'])
            lat.append(dt[i]['sourceInfo']['geoLocation'][u'geogLocation']['latitude'])
            lon.append(dt[i]['sourceInfo']['geoLocation'][u'geogLocation']['longitude'])
            srs.append(dt[i]['sourceInfo']['geoLocation'][u'geogLocation']['srs'])
            station_type.append(dt[i]['sourceInfo']['siteProperty'][0]['value'])
            station_nm .append(dt[i]['sourceInfo'][ u'siteName'])

            df = pd.DataFrame(dt[i]['values'][0]['value'], columns=['dateTime', 'value'])
            df.index = pd.to_datetime(df.pop('dateTime'))
            df.value = df.value.astype(float)
            df.index.name = 'datetime'
            df.replace(to_replace='-999999', value=np.nan)
            f[dt[i]['sourceInfo']['siteCode'][0]['value']] = df
        stat_dict = {'station_id':station_id,'lat':lat,'lon':lon,'srs':srs,'station_nm':station_nm,'stat_type':station_type}
        stations = pd.DataFrame(stat_dict)
        if len(dt) > 1:
            data = pd.concat(f)
        else:
            data = f[dt[0]['sourceInfo']['siteCode'][0]['value']]
            data['station_id'] = dt[0]['sourceInfo']['siteCode'][0]['value']
        return stations,data
    
    
    def getInfo(self, html):
        '''
        Input
        -----
        html = location of data to be queried <http://waterservices.usgs.gov>         
        
        Output
        ------
        df = Pandas Dataframe containing data downloaded from USGS
        '''
        
        try:
            linefile = urllib2.urlopen(html).readlines()
            numlist=[]
            num=0
            for line in linefile:<commit_msg>Update usgsGis.py<commit_after># -*- coding: utf-8 -*-
"""
Created on Sun Jan  3 00:30:36 2016

@author: p
"""
import urllib2
import xmltodict
import pandas as pd
from datetime import datetime 
from httplib import BadStatusLine
import matplotlib.pyplot as plt
import numpy as np
import avgMeths
import requests
import json

class usgs:
    @staticmethod 
    def getelev(x):
        '''
        Uses USGS elevation service to retrieve elevation
        
        Input
        -----
        x, y = longitude and latitude of point where elevation is desired        
    
        Output
        ------
        ned float elevation of location in meters
        '''
        elev = "http://ned.usgs.gov/epqs/pqs.php?x="+str(x[0])+"&y="+str(x[1])+"&units=Meters&output=xml"
        try:        
            response = urllib2.urlopen(elev)
            html = response.read()
            d = xmltodict.parse(html)
            g = float(d['USGS_Elevation_Point_Query_Service']['Elevation_Query']['Elevation'])
        except(BadStatusLine):
            try:        
                response = urllib2.urlopen(elev)
                html = response.read()
                d = xmltodict.parse(html)
                g = float(d['USGS_Elevation_Point_Query_Service']['Elevation_Query']['Elevation'])
            except(BadStatusLine):
                print "could not fetch %s" % html
                g = 0                
                pass                 
        return g
    
    @staticmethod     
    def USGSID(x):
        def dms(dec):
            DD = str(int(abs(dec)))
            MM = str(int((abs(dec) - int(DD))*60)).zfill(2)
            SS = str(int(round((((abs(dec) - int(DD))*60) - int(MM))*60, 0))).zfill(2)
            return DD+MM+SS
        return dms(x[1])+dms(x[0])+'01'
    
    def get_nwis(self, val_list, selectType ='dv_site', start_date = '1800-01-01', end_date = ''):
        """
        Request stream gauge data from the USGS NWIS.
        Args:
            val_list (str or int or list of either):
                can be a list or 8-digit hucs or stations or single 
                a valid site is 01585200; a valid huc is 16010204
            selectType (str):
                options: 'dv_site','dv_huc','gw_site',or 'gw_huc'
            start_date (str):
               should take on the form yyyy-mm-dd; default is '1800-01-01'
            end_date (str):
                should take on the form yyyy-mm-dd; default is today
        Returns:
            a station and a data Pandas dataframe.
        Raises:
            ConnectionError  due to connection problems like refused connection
            or DNS Error.

        Example::
            >>> import wellapplication as wa
            >>> USGS = wa.usgs()
            >>> site, data = USGS.get_nwis('01585200', 'dv_site', '2012-06-01', '2012-07-01')

        The specification for this service is located here:
        http://waterservices.usgs.gov/rest/IV-Service.html
        
        This function was adapted from: https://github.com/mroberge/hydrofunctions
        """
        val_list = self.parsesitelist(val_list)

        if end_date == '':
            dy = datetime.today()
            end_date = str(dy.year)+'-'+str(dy.month)+'-'+str(dy.day)

        header = {'Accept-encoding': 'gzip'}

        
        valdict = {
            'dv_site':{'format': 'json', 'sites': val_list, 'parameterCd': '00060',
                      'startDT': start_date, 'endDT': end_date},
            'dv_huc':{'format': 'json', 'huc': val_list, 'parameterCd': '00060',
                      'startDT': start_date, 'endDT': end_date},
            'gw_site':{'format': 'json', 'sites': val_list, 'siteType':'GW','siteStatus':'all',
                      'startDT': start_date, 'endDT': end_date},
            'gw_huc':{'format': 'json', 'huc': val_list, 'siteType':'GW','siteStatus':'all',
                      'startDT': start_date, 'endDT': end_date}
            }


        url = 'http://waterservices.usgs.gov/nwis/'
            
        if selectType == 'dv_site' or selectType == 'dv_huc':
            service = 'dv'
        elif selectType == 'gw_site' or selectType == 'gw_huc':
            service = 'gwlevels'
        url = url + service + '/?'
        response_ob = requests.get(url, params=valdict[selectType], headers=header)

        nwis_dict = response_ob.json()

        dt = nwis_dict['value']['timeSeries']

        station_id,lat,lon,srs,station_type,station_nm =  [],[],[],[],[],[]
        f = {}
        for i in range(len(dt)):

            station_id.append(dt[i]['sourceInfo']['siteCode'][0]['value'])
            lat.append(dt[i]['sourceInfo']['geoLocation'][u'geogLocation']['latitude'])
            lon.append(dt[i]['sourceInfo']['geoLocation'][u'geogLocation']['longitude'])
            srs.append(dt[i]['sourceInfo']['geoLocation'][u'geogLocation']['srs'])
            station_type.append(dt[i]['sourceInfo']['siteProperty'][0]['value'])
            station_nm .append(dt[i]['sourceInfo'][ u'siteName'])

            df = pd.DataFrame(dt[i]['values'][0]['value'], columns=['dateTime', 'value'])
            df.index = pd.to_datetime(df.pop('dateTime'))
            df.value = df.value.astype(float)
            df.index.name = 'datetime'
            df.replace(to_replace='-999999', value=np.nan)
            f[dt[i]['sourceInfo']['siteCode'][0]['value']] = df
        stat_dict = {'station_id':station_id,'lat':lat,'lon':lon,'srs':srs,'station_nm':station_nm,'stat_type':station_type}
        stations = pd.DataFrame(stat_dict)
        if len(dt) > 1:
            data = pd.concat(f)
            data.index.set_names('station_id',level=0,inplace=True)
        else:
            data = f[dt[0]['sourceInfo']['siteCode'][0]['value']]
            data['station_id'] = dt[0]['sourceInfo']['siteCode'][0]['value']
        return stations,data
    
    
    def getInfo(self, html):
        '''
        Input
        -----
        html = location of data to be queried <http://waterservices.usgs.gov>         
        
        Output
        ------
        df = Pandas Dataframe containing data downloaded from USGS
        '''
        
        try:
            linefile = urllib2.urlopen(html).readlines()
            numlist=[]
            num=0
            for line in linefile:
{'fraction_non_alphanumeric': 0.1477181745396317, 'fraction_numerical': 0.018494795836669336, 'mean_word_length': 1.9940076701821667, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '487942', 'n_tokens_mistral': 4032, 'n_tokens_neox': 3799, 'n_words': 801}