<commit_before>import sys
import logging
import datetime
import sys
import os
import pymongo
import csv
import re
import copy
import datetime
import time
import pytz
import shutil
import urllib
import subprocess
from pymongo import ASCENDING, DESCENDING
from zipline.daemon import Daemon
import zipline.util as qutil
import zipline.db as db
import host_settings

class FinancialDataLoader():
    """
    Load trade and quote data from tickdata extracts into the db.
    Dates and times in the extracts must be in GMT.
    
    All data extract files are expected to be in $HOME/fdl/. The expected directory layout is::
        /benchmark.csv      -- this will be created from yahoo data each time load_bench_marks is run
        /interest_rates.csv --
    """
    BATCH_SIZE = 100

    def __init__(self):
        self.conn, self.db = db.DbConnection.get()
        self.data_file_path = os.environ['HOME'] + "/fdl/" 
        subprocess.call("mkdir {data_dir}".format(data_dir=self.data_file_path), shell=True)
        self.last_bm_close = None
        
    def load_bench_marks(self):
        """Fetches the S&P end of day pricing history from yahoo, loads it to db.bench_marks"""
        start = time.time()
        start_date = datetime.datetime(year=1950, month=1, day=3)
        end_date = datetime.datetime.utcnow()
        file_path = self.data_file_path + "benchmark.csv"
        fp = open(file_path + ".tmp", "wb")
        
        #create benchmark files
        #^GSPC 19500103
        query = {}
        query['s'] = "^GSPC" #the s&p 500
        query['d'] = end_date.month - 1 # end_date month, zero indexed
        query['e'] = end_date.day # end_date day str(int(todate[6:8])) #day
        query['f'] = end_date.year #end_date year str(int(todate[0:4]))
        query['g'] = "d" #daily frequency
        query['a'] = start_date.month - 1 #start_date month, zero indexed
        query['b'] = start_date.day #start_date day 
        query['c'] = start_date.year #start_date year
        
        #print query
        params = urllib.urlencode(query)
        params += "&ignore=.csv"

        url = "http://ichart.yahoo.com/table.csv?%s" % params
        qutil.LOGGER.info("fetching {url}".format(url=url))
        f = urllib.urlopen(url)
        fp.write(f.read())
        fp.close()
        qutil.LOGGER.info("fetched {url} Reversing.".format(url=url))
        
        tmp_file = file_path + ".tmp"
        reversed_tmp_file = file_path + ".rev"
        
        rcode = subprocess.call("tac {oldfile} > {newfile}".format(oldfile=tmp_file, newfile=reversed_tmp_file), shell=True)
        #on mac, there is no tac command, so use tail -r (which isn't available on debian)
        if rcode != 0:
            rcode = subprocess.call("tail -r {oldfile} > {newfile}".format(oldfile=tmp_file, newfile=reversed_tmp_file), shell=True)
        
        #tail -1 benchmark.csv.rev > benchmark.csv
        subprocess.call("echo \"date,open,high,low,close,volume,adj_close\" > {result}".format(newfile=reversed_tmp_file, result=self.data_file_path), shell=True)
        #sed '$d' < ~/fdl/benchmark.csv.rev >> ~/fdl/benchmark.csv   
        subprocess.call("sed '$d' < {newfile} >> {result}".format(newfile=reversed_tmp_file, result=self.data_file_path), shell=True)
        #clean up working files
        subprocess.call("rm {tmp} {reversed}".format(tmp=tmp_file, reversed=reversed_tmp_file), shell=True)
        
        #load the records into mongodb
        self.db.bench_marks.drop()
        qutil.LOGGER.info("processing benchmark info")
        self.parse_file(self.db.bench_marks, 
                        self.bench_mark_cb, 
                        file_path, 
                        ['date','open','high','low','close','volume','adj_close'], 
                        None,
                        0)
        qutil.LOGGER.info("benchmark info complete")
        total = time.time() - start
        qutil.LOGGER.info("%d seconds to load benchmark history" % total)
        
    def load_treasuries(self):
        """fetches data from the treasury.gov yield curve website, and populates the treasury_curves table.
        
        to explore data available from the treasury:
        http://www.treasury.gov/resource-center/data-chart-center/interest-rates/Pages/TextView.aspx?data=yield
        
        to fetch xml of all daily yield curves:
        http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData
        """
        
        from xml.dom.minidom import parse
        self.db.treasury_curves.drop()<commit_msg>fixed bug with benchmark dataloader - file path names were wrong.<commit_after>import sys
import logging
import datetime
import sys
import os
import pymongo
import csv
import re
import copy
import datetime
import time
import pytz
import shutil
import urllib
import subprocess
from pymongo import ASCENDING, DESCENDING
from zipline.daemon import Daemon
import zipline.util as qutil
import zipline.db as db
import host_settings

class FinancialDataLoader():
    """
    Load trade and quote data from tickdata extracts into the db.
    Dates and times in the extracts must be in GMT.
    
    All data extract files are expected to be in $HOME/fdl/. The expected directory layout is::
        /benchmark.csv      -- this will be created from yahoo data each time load_bench_marks is run
        /interest_rates.csv --
    """
    BATCH_SIZE = 100

    def __init__(self):
        self.conn, self.db = db.DbConnection.get()
        self.data_file_path = os.environ['HOME'] + "/fdl/" 
        subprocess.call("mkdir {data_dir}".format(data_dir=self.data_file_path), shell=True)
        self.last_bm_close = None
        
    def load_bench_marks(self):
        """Fetches the S&P end of day pricing history from yahoo, loads it to db.bench_marks"""
        start = time.time()
        start_date = datetime.datetime(year=1950, month=1, day=3)
        end_date = datetime.datetime.utcnow()
        file_path = os.path.join(self.data_file_path, "benchmark.csv")
        fp = open(file_path + ".tmp", "wb")
        
        #create benchmark files
        #^GSPC 19500103
        query = {}
        query['s'] = "^GSPC" #the s&p 500
        query['d'] = end_date.month - 1 # end_date month, zero indexed
        query['e'] = end_date.day # end_date day str(int(todate[6:8])) #day
        query['f'] = end_date.year #end_date year str(int(todate[0:4]))
        query['g'] = "d" #daily frequency
        query['a'] = start_date.month - 1 #start_date month, zero indexed
        query['b'] = start_date.day #start_date day 
        query['c'] = start_date.year #start_date year
        
        #print query
        params = urllib.urlencode(query)
        params += "&ignore=.csv"

        url = "http://ichart.yahoo.com/table.csv?%s" % params
        qutil.LOGGER.info("fetching {url}".format(url=url))
        f = urllib.urlopen(url)
        fp.write(f.read())
        fp.close()
        qutil.LOGGER.info("fetched {url} Reversing.".format(url=url))
        
        tmp_file = file_path + ".tmp"
        reversed_tmp_file = file_path + ".rev"
        
        rcode = subprocess.call("tac {oldfile} > {newfile}".format(oldfile=tmp_file, newfile=reversed_tmp_file), shell=True)
        #on mac, there is no tac command, so use tail -r (which isn't available on debian)
        if rcode != 0:
            rcode = subprocess.call("tail -r {oldfile} > {newfile}".format(oldfile=tmp_file, newfile=reversed_tmp_file), shell=True)
        
        #tail -1 benchmark.csv.rev > benchmark.csv
        subprocess.call("echo \"date,open,high,low,close,volume,adj_close\" > {result}".format(newfile=reversed_tmp_file, result=file_path), shell=True)
        #sed '$d' < ~/fdl/benchmark.csv.rev >> ~/fdl/benchmark.csv   
        subprocess.call("sed '$d' < {newfile} >> {result}".format(newfile=reversed_tmp_file, result=file_path), shell=True)
        #clean up working files
        subprocess.call("rm {file}".format(file=tmp_file), shell=True)
        subprocess.call("rm {file}".format(file=reversed_tmp_file), shell=True)
        
        #load the records into mongodb
        self.db.bench_marks.drop()
        qutil.LOGGER.info("processing benchmark info")
        self.parse_file(self.db.bench_marks, 
                        self.bench_mark_cb, 
                        file_path, 
                        ['date','open','high','low','close','volume','adj_close'], 
                        None,
                        0)
        qutil.LOGGER.info("benchmark info complete")
        total = time.time() - start
        qutil.LOGGER.info("%d seconds to load benchmark history" % total)
        
    def load_treasuries(self):
        """fetches data from the treasury.gov yield curve website, and populates the treasury_curves table.
        
        to explore data available from the treasury:
        http://www.treasury.gov/resource-center/data-chart-center/interest-rates/Pages/TextView.aspx?data=yield
        
        to fetch xml of all daily yield curves:
        http://data.treasury.gov/feed.svc/DailyTreasuryYieldCurveRateData
        """
        
        from xml.dom.minidom import parse
        self.db.treasury_curves.drop()
{'fraction_non_alphanumeric': 0.11608894702419882, 'fraction_numerical': 0.00632221495530848, 'mean_word_length': 2.538372541457771, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3272880', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2829, 'n_words': 697}