<commit_before>#!/bin/env python
# -*- coding: utf-8 -*-

import urllib2, lxml.html, os
from datetime import datetime
from utils import timeadjust
import time
import sys


def getAUDCNY():
    API = "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s=AUDCNY=x"
    try:
        response = urllib2.urlopen(API)
        content = response.read()
        content = content.split(',')
        print content
        name = content[0].split('=')[0].strip('"')
        report_t = "%s %s" % (content[2].strip('"'), content[3].replace('\r\n','').strip('"'))
        dt = datetime.strptime(report_t, "%m/%d/%Y %I:%M%p")
        t = time.mktime(timeadjust(dt).timetuple())
        value = content[1]
                
    except:
        print "Error:getAUDCNY()"
        return None
    print "name: %s, time:%s, value:%s" % (name, t, value)
    return (name, t, value)


def petrolCapture(imgstorepath):
    BASEURL = 'http://www.accc.gov.au'
    IMGSTORE = imgstorepath
    
    #fetch the source data of petrol price from accc australia
    accchttp = urllib2.urlopen('http://www.accc.gov.au/consumers/petrol-diesel-and-lpg/recent-city-petrol-prices')
    data = accchttp.read()
    accchttp.close()
    doc = lxml.html.document_fromstring(data)
    
    
    #get the data of cheapest in last week
    t1 = doc.xpath('/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/table/tbody/tr/td/text()')
    cheapest = [item for item in t1 if item != '\n']  
    db.savePetrolCheapest(cheapest)
    
    #get the charts of major cities
    xpaths = {'sydney':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[10]/img/@src',
              'melbourne': '/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[13]/img/@src',
              'brisbane':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[16]/img/@src',
              'adelaide':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[19]/img/@src',
              'perth':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[22]/img/@src'
    }
    
    for city in xpaths:
        img = doc.xpath(xpaths[city])
        img1url = "%s%s" % ( BASEURL, img[0])<commit_msg>unicode processing<commit_after>#!/bin/env python
# -*- coding: utf-8 -*-

import urllib2, lxml.html, os
from datetime import datetime
from utils import timeadjust
import time
import sys


def getAUDCNY():
    API = "http://download.finance.yahoo.com/d/quotes.csv?e=.csv&f=sl1d1t1&s=AUDCNY=x"
    try:
        response = urllib2.urlopen(API)
        content = response.read()
        content = content.split(',')
        print content
        name = content[0].split('=')[0].strip('"')
        report_t = "%s %s" % (content[2].strip('"'), content[3].replace('\r\n','').strip('"'))
        dt = datetime.strptime(report_t, "%m/%d/%Y %I:%M%p")
        t = time.mktime(timeadjust(dt).timetuple())
        value = content[1]
                
    except:
        print "Error:getAUDCNY()"
        return None
    print "name: %s, time:%s, value:%s" % (name, t, value)
    return (name, t, value)


def petrolCapture(imgstorepath):
    BASEURL = 'http://www.accc.gov.au'
    IMGSTORE = imgstorepath
    
    #fetch the source data of petrol price from accc australia
    accchttp = urllib2.urlopen('http://www.accc.gov.au/consumers/petrol-diesel-and-lpg/recent-city-petrol-prices')
    data = accchttp.read()
    accchttp.close()
    doc = lxml.html.document_fromstring(data)
    
    
    #get the data of cheapest in last week
    t1 = doc.xpath('/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/table/tbody/tr/td/text()')
    cheapest = [item for item in t1 if item.strip('\n\t') != '']  
    cheapest[0] = cheapest[0].encode('utf-8').replace('\xc2\xa0', ' ') # remove special char
    db.savePetrolCheapest(cheapest)
    
    #get the charts of major cities
    xpaths = {'sydney':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[10]/img/@src',
              'melbourne': '/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[13]/img/@src',
              'brisbane':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[16]/img/@src',
              'adelaide':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[19]/img/@src',
              'perth':'/html/body/div/div/div[3]/div/div[2]/div/div/div/div/div/div/div/div/div/div/p[22]/img/@src'
    }
    
    for city in xpaths:
        img = doc.xpath(xpaths[city])
        img1url = "%s%s" % ( BASEURL, img[0])
{'fraction_non_alphanumeric': 0.1946222791293214, 'fraction_numerical': 0.017285531370038413, 'mean_word_length': 2.66171875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '25367', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1811, 'n_words': 266}