<commit_before>#!/usr/bin/env python

import os
import io
import sys
import string
import pymongo
import json
from bson import json_util
from bson import objectid
import re
import urllib2
from time import time
from time import sleep

#setup the connection to the gauges database
conn = pymongo.Connection(os.environ['OPENSHIFT_MONGODB_DB_URL'])
db = conn.gauges
    
    
# USGS requires a major filter. I'm using huc (hydrological area)
# Range from 01 to 21

#This does not seem to complete reliably

for i in range(1,3):

    if i < 10:
        hucstring = "0" + str(i)
    else:
        hucstring = str(i)
        
    


    
    requesturl = "http://waterservices.usgs.gov/nwis/iv/?format=json,1.1&huc="+ hucstring + "&parameterCd=00060,00065&siteType=ST"

    print "Loading HUC ",hucstring

    try:
        #code
        req = urllib2.Request(requesturl)
        opener = urllib2.build_opener()
        f = opener.open(req)
    except:
        continue
    
    entry = json.loads(f.read())
    
    print "Loaded HUC ",hucstring


    count = int (len(entry['value']['timeSeries']) - 1)

    while count >= 0:
#We construct an array of the relevant values associated with a guage number
#Note that gage height and discharge are in separate entries
#Right here we're just filling out the "permanent" values<commit_msg>Trying just NY<commit_after>#!/usr/bin/env python

import os
import io
import sys
import string
import pymongo
import json
from bson import json_util
from bson import objectid
import re
import urllib2
from time import time
from time import sleep

#setup the connection to the gauges database
conn = pymongo.Connection(os.environ['OPENSHIFT_MONGODB_DB_URL'])
db = conn.gauges
    
    
# USGS requires a major filter. I'm using huc (hydrological area)
# Range from 01 to 21

#This does not seem to complete reliably

for i in range(1,2):

    if i < 10:
        hucstring = "0" + str(i)
    else:
        hucstring = str(i)
        
    


    
#    requesturl = "http://waterservices.usgs.gov/nwis/iv/?format=json,1.1&huc="+ hucstring + "&parameterCd=00060,00065&siteType=ST"

    requestURL = "http://waterservices.usgs.gov/nwis/iv/?format=json,1.1&stateCd=ny&parameterCd=00060,00065&siteType=ST"

    print "Loading request"

    try:
        #code
        req = urllib2.Request(requesturl)
        opener = urllib2.build_opener()
        f = opener.open(req)
    except:
        continue
    
    entry = json.loads(f.read())
    
    print "Loaded request "


    count = int (len(entry['value']['timeSeries']) - 1)

    while count >= 0:
#We construct an array of the relevant values associated with a guage number
#Note that gage height and discharge are in separate entries
#Right here we're just filling out the "permanent" values
{'fraction_non_alphanumeric': 0.09439421338155515, 'fraction_numerical': 0.02314647377938517, 'mean_word_length': 3.1160714285714284, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3698099', 'n_tokens_mistral': 967, 'n_tokens_neox': 895, 'n_words': 287}