# Path: utility/datafilepath.py
# class DataFilePath:
#     def __init__(self):
#     def getOrderDir_Train(self):  
#     def getOrderDir_Test1(self):  
#     def getTest1Dir(self):
#     def getTest2Dir(self):
#     def getTrainDir(self):
#     def getGapCsv_Train(self):
#     def getGapCsv_Test1(self):
#     def getTestset1Readme(self):
#     def getTestset2Readme(self):
#     def getGapFilename(self):
#     def getGapPredictionFileName(self):
#     def getPrevGapFileName(self):
#     def get_dir_name(self, data_dir):
# 
# Path: utility/dumpload.py
# class DumpLoad:
#     def __init__(self, pickle_filepath):
#         self.pickle_filepath = pickle_filepath
#         dir_path = os.path.dirname(pickle_filepath)
#         if not os.path.exists(dir_path):
#             os.makedirs(dir_path)
#         
#         return
#     def load(self):
#         with open(self.pickle_filepath, 'rb') as handle:
#             dataset = pickle.load(handle)
#         return dataset
#     def isExisiting(self):
#         return os.path.exists(self.pickle_filepath) 
#     def run(self):
#         self.dump((np.arange(10), np.arange(20)))
#         print(self.load())
#         return
#     
#     def dump(self, dataset):
#         with open(self.pickle_filepath, 'wb') as f:
#             pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
#         return


import pandas as pd
import os.path
import numpy as np
from  districtid import singletonDistricId
from timeslot import singletonTimeslot
from os import walk
from utility.datafilepath import g_singletonDataFilePath
from time import time
from utility.dumpload import DumpLoad
from seaborn.utils import iqr
        for (dirpath, _, filenames) in walk(rootpath):
            #remove all files with file name starting with .
            filenames = [dirpath+x for x in filenames if x[0] != '.']
            f.extend(filenames)
            break
        return f
    def saveOrderCsv(self, filename):
        print("loading file {}".format(filename))
        df = pd.read_csv(filename, delimiter='\t', header=None, names =['order_id','driver_id','passenger_id', 'start_district_hash','dest_district_hash','Price','Time'])
        df['start_district_id'] = df['start_district_hash'].map(singletonDistricId.convertToId)
        df['time_slotid'] = df['Time'].map(singletonTimeslot.convertToSlot)
#         df['dest_district_id'] = df['dest_district_hash'].map(singletonDistricId.convertToId)
        df.to_csv(os.path.dirname(filename) + '/temp/'+ os.path.basename(filename) + '.csv')
        print df.describe()
        return df, filename
    def saveGapCSV(self, df, filename):
        items = []
        grouped  = df.groupby(['start_district_id','time_slotid'])
        for name, group in grouped:
            timeslot = singletonTimeslot.convertToStr(name[1])
            missedReqs = group['driver_id'].isnull().sum()
            allReqs = group.shape[0]
            items.append(list(name) + [timeslot]+[missedReqs] + [allReqs])
        resDf = pd.DataFrame(items, columns=['start_district_id','time_slotid','time_slot','gap', 'all_requests'])
        resDf.to_csv(os.path.dirname(filename) + '/temp/'+ os.path.basename(filename) + '_gap.csv')
        print resDf.describe()
        return

    def load_gapdf(self, data_dir):
        filename = "../data_raw/" + data_dir.split('/')[-2]+ "_gap.csv"

# Path: utility/datafilepath.py
# class DataFilePath:
#     def __init__(self):
#     def getOrderDir_Train(self):  
#     def getOrderDir_Test1(self):  
#     def getTest1Dir(self):
#     def getTest2Dir(self):
#     def getTrainDir(self):
#     def getGapCsv_Train(self):
#     def getGapCsv_Test1(self):
#     def getTestset1Readme(self):
#     def getTestset2Readme(self):
#     def getGapFilename(self):
#     def getGapPredictionFileName(self):
#     def getPrevGapFileName(self):
#     def get_dir_name(self, data_dir):
# 
# Path: utility/dumpload.py
# class DumpLoad:
#     def __init__(self, pickle_filepath):
#         self.pickle_filepath = pickle_filepath
#         dir_path = os.path.dirname(pickle_filepath)
#         if not os.path.exists(dir_path):
#             os.makedirs(dir_path)
#         
#         return
#     def load(self):
#         with open(self.pickle_filepath, 'rb') as handle:
#             dataset = pickle.load(handle)
#         return dataset
#     def isExisiting(self):
#         return os.path.exists(self.pickle_filepath) 
#     def run(self):
#         self.dump((np.arange(10), np.arange(20)))
#         print(self.load())
#         return
#     
#     def dump(self, dataset):
#         with open(self.pickle_filepath, 'wb') as f:
#             pickle.dump(dataset, f, pickle.HIGHEST_PROTOCOL)
#         return


# Path: exploredata/order.py
import pandas as pd
import os.path
import numpy as np
from  districtid import singletonDistricId
from timeslot import singletonTimeslot
from os import walk
from utility.datafilepath import g_singletonDataFilePath
from time import time
from utility.dumpload import DumpLoad
from seaborn.utils import iqr

        for (dirpath, _, filenames) in walk(rootpath):
            #remove all files with file name starting with .
            filenames = [dirpath+x for x in filenames if x[0] != '.']
            f.extend(filenames)
            break
        return f
    def saveOrderCsv(self, filename):
        print("loading file {}".format(filename))
        df = pd.read_csv(filename, delimiter='\t', header=None, names =['order_id','driver_id','passenger_id', 'start_district_hash','dest_district_hash','Price','Time'])
        df['start_district_id'] = df['start_district_hash'].map(singletonDistricId.convertToId)
        df['time_slotid'] = df['Time'].map(singletonTimeslot.convertToSlot)
#         df['dest_district_id'] = df['dest_district_hash'].map(singletonDistricId.convertToId)
        df.to_csv(os.path.dirname(filename) + '/temp/'+ os.path.basename(filename) + '.csv')
        print df.describe()
        return df, filename
    def saveGapCSV(self, df, filename):
        items = []
        grouped  = df.groupby(['start_district_id','time_slotid'])
        for name, group in grouped:
            timeslot = singletonTimeslot.convertToStr(name[1])
            missedReqs = group['driver_id'].isnull().sum()
            allReqs = group.shape[0]
            items.append(list(name) + [timeslot]+[missedReqs] + [allReqs])
        resDf = pd.DataFrame(items, columns=['start_district_id','time_slotid','time_slot','gap', 'all_requests'])
        resDf.to_csv(os.path.dirname(filename) + '/temp/'+ os.path.basename(filename) + '_gap.csv')
        print resDf.describe()
        return

    def load_gapdf(self, data_dir):
        filename = "../data_raw/" + data_dir.split('/')[-2]+ "_gap.csv"

        df = pd.read_csv(filename, index_col= 0)