# Path: Data/msd.py
# class MillionSongDataset:
#     def __init__(self, features, lyrics, lyric_mappings, tracks):
#         """
# 
#         :param features:
#         :param lyrics:
#         :param lyric_mappings:
#         :param tracks:
#         """
#         # Initialize lyric and feature databases.
#         self._fdb = MillionSongFeatureDatabase(features)
#         self._ldb = MillionSongLyricDatabase(lyrics, lyric_mappings)
# 
#         # Load track.
#         self.tracks = []
#         for line in [line.rstrip() for line in open(tracks)]:
#             self.tracks.append(line)
# 
#         self.train = []
#         self.validate = []
#         self.test = []
# 
#     def close(self):
#         self._fdb.close()
#         self._ldb.close()
#         
# 
#     def generate_track_list(self):
#         """
#         Generates a list of songs with both full features and lyrics and
#         saves to a target file.
#         :return: None
#         """
#         with_lyrics = set(self._ldb.get_songs_with_lyrics())
#         with_features = set(self._fdb.get_songs_with_all_features())
#         with_both = with_lyrics.intersection(with_features)
# 
#         with open('tracks.txt', 'wb') as f:
#             for t in with_both:
#                 f.write('%s\n' % t)
# 
#     def load_track_list(self, f):
#         """
# 
#         :param f:
#         :return: None
#         """
#         for line in [line.rstrip for line in open(f)]:
#             self.tracks += [line]
# 
#     def get_features(self, track_ids):
#         """
# 
#         :param track_ids:
#         :return:
#         """
#         return self._fdb.get_features_by_msd_list(track_ids)
# 
#     '''
#     def get_bow(self, track_ids):
#         count = 0
#         for t in track_ids:
#             count += 1
#             print count
#             yield self._ldb.get_bow_by_msd(t)
#     '''
# 
#     def get_bow(self, track_ids):
#         """
# 
#         :param track_ids:
#         :return:
#         """
#         return self._ldb.get_bow_by_msd_batch(track_ids)
# 
#     def generate_split(self, train, validate, test, total=147000):
#         """
#         Generates a train/validate/test split of the data.
#         :param train: [0,1] portion used for training
#         :param validate: [0,1] portion used for validate
#         :param test: [0,1] portion used for test
#         :param total: total number of examples used from all 3.
#         :return: None
#         """
#         assert(train + test + validate == 1.0)
# 
#         total = min(total, len(self.tracks))
#         end_train = int(total*train)
#         end_validate = int(total*validate) + end_train
#         self.train = self.tracks[:end_train]
#         self.validate = self.tracks[end_train:end_validate]
#         self.test = self.tracks[end_validate:total]
# 
# Path: utils/data_utils/labels.py
# class Labels(object):
#     """
#     Given a list of features, assigns each an ID starting at zero.
#     """
# 
#     def __init__(self, labels):
#         """
# 
#         :param labels: Collection of labels to create IDs for.
#         """
#         self.count = 0
#         self.ids = {}
#         for l in labels:
#             self.ids.update({ l : self.count })
#             self.count += 1
# 
#     def get(self, label):
#         """
# 
#         :return: The ID associated with a label, or None if there is not one.
#         """
#         try:
#             return self.ids[label]
#         except:
#             return None
# 
# Path: utils/training_utils/params.py
# TRAIN_FRACTION = 0.45
# 
# TEST_FRACTION = 0.45
# 
# VALIDATE_FRACTION = 0.10
# 
# TOTAL_NUM_EXAMPLES = 60000
# 
# Path: utils/training_utils/params.py
# EXPT_DIRECTORY_PATH = "./Experiments"  # Path of the Experiments directory
# 
# Path: utils/network_utils/params.py
# class LossTypes(Enum):
#     mse = "mse"
#     cross_entropy = "cross-entropy"


from Data.msd import MillionSongDataset
from params import FEATURES, LYRICS, LYRIC_MAPPINGS, TRACKS
from utils.data_utils.labels import Labels
from utils.training_utils.params import TRAIN_FRACTION, TEST_FRACTION, VALIDATE_FRACTION, TOTAL_NUM_EXAMPLES
from utils.training_utils.params import EXPT_DIRECTORY_PATH
from utils.network_utils.params import LossTypes
import numpy as np
import os
import sys


def fetch_data(tasks):
    """
    Fetches the dataset from the Database and then divides it into training / testing / validation data

    :param task_ids: Dictionary of task identifiers-loss type pairs indexed by task-id.
    :return: split data, and labels
    """
    # Load the data set in memory.
    db = MillionSongDataset(FEATURES, LYRICS, LYRIC_MAPPINGS, TRACKS)
    # Generate train/validation/test splits.

# Path: Data/msd.py
# class MillionSongDataset:
#     def __init__(self, features, lyrics, lyric_mappings, tracks):
#         """
# 
#         :param features:
#         :param lyrics:
#         :param lyric_mappings:
#         :param tracks:
#         """
#         # Initialize lyric and feature databases.
#         self._fdb = MillionSongFeatureDatabase(features)
#         self._ldb = MillionSongLyricDatabase(lyrics, lyric_mappings)
# 
#         # Load track.
#         self.tracks = []
#         for line in [line.rstrip() for line in open(tracks)]:
#             self.tracks.append(line)
# 
#         self.train = []
#         self.validate = []
#         self.test = []
# 
#     def close(self):
#         self._fdb.close()
#         self._ldb.close()
#         
# 
#     def generate_track_list(self):
#         """
#         Generates a list of songs with both full features and lyrics and
#         saves to a target file.
#         :return: None
#         """
#         with_lyrics = set(self._ldb.get_songs_with_lyrics())
#         with_features = set(self._fdb.get_songs_with_all_features())
#         with_both = with_lyrics.intersection(with_features)
# 
#         with open('tracks.txt', 'wb') as f:
#             for t in with_both:
#                 f.write('%s\n' % t)
# 
#     def load_track_list(self, f):
#         """
# 
#         :param f:
#         :return: None
#         """
#         for line in [line.rstrip for line in open(f)]:
#             self.tracks += [line]
# 
#     def get_features(self, track_ids):
#         """
# 
#         :param track_ids:
#         :return:
#         """
#         return self._fdb.get_features_by_msd_list(track_ids)
# 
#     '''
#     def get_bow(self, track_ids):
#         count = 0
#         for t in track_ids:
#             count += 1
#             print count
#             yield self._ldb.get_bow_by_msd(t)
#     '''
# 
#     def get_bow(self, track_ids):
#         """
# 
#         :param track_ids:
#         :return:
#         """
#         return self._ldb.get_bow_by_msd_batch(track_ids)
# 
#     def generate_split(self, train, validate, test, total=147000):
#         """
#         Generates a train/validate/test split of the data.
#         :param train: [0,1] portion used for training
#         :param validate: [0,1] portion used for validate
#         :param test: [0,1] portion used for test
#         :param total: total number of examples used from all 3.
#         :return: None
#         """
#         assert(train + test + validate == 1.0)
# 
#         total = min(total, len(self.tracks))
#         end_train = int(total*train)
#         end_validate = int(total*validate) + end_train
#         self.train = self.tracks[:end_train]
#         self.validate = self.tracks[end_train:end_validate]
#         self.test = self.tracks[end_validate:total]
# 
# Path: utils/data_utils/labels.py
# class Labels(object):
#     """
#     Given a list of features, assigns each an ID starting at zero.
#     """
# 
#     def __init__(self, labels):
#         """
# 
#         :param labels: Collection of labels to create IDs for.
#         """
#         self.count = 0
#         self.ids = {}
#         for l in labels:
#             self.ids.update({ l : self.count })
#             self.count += 1
# 
#     def get(self, label):
#         """
# 
#         :return: The ID associated with a label, or None if there is not one.
#         """
#         try:
#             return self.ids[label]
#         except:
#             return None
# 
# Path: utils/training_utils/params.py
# TRAIN_FRACTION = 0.45
# 
# TEST_FRACTION = 0.45
# 
# VALIDATE_FRACTION = 0.10
# 
# TOTAL_NUM_EXAMPLES = 60000
# 
# Path: utils/training_utils/params.py
# EXPT_DIRECTORY_PATH = "./Experiments"  # Path of the Experiments directory
# 
# Path: utils/network_utils/params.py
# class LossTypes(Enum):
#     mse = "mse"
#     cross_entropy = "cross-entropy"


# Path: utils/data_utils/data_handler.py
from Data.msd import MillionSongDataset
from params import FEATURES, LYRICS, LYRIC_MAPPINGS, TRACKS
from utils.data_utils.labels import Labels
from utils.training_utils.params import TRAIN_FRACTION, TEST_FRACTION, VALIDATE_FRACTION, TOTAL_NUM_EXAMPLES
from utils.training_utils.params import EXPT_DIRECTORY_PATH
from utils.network_utils.params import LossTypes
import numpy as np
import os
import sys



def fetch_data(tasks):
    """
    Fetches the dataset from the Database and then divides it into training / testing / validation data

    :param task_ids: Dictionary of task identifiers-loss type pairs indexed by task-id.
    :return: split data, and labels
    """
    # Load the data set in memory.
    db = MillionSongDataset(FEATURES, LYRICS, LYRIC_MAPPINGS, TRACKS)
    # Generate train/validation/test splits.

    db.generate_split(TRAIN_FRACTION, VALIDATE_FRACTION, TEST_FRACTION, TOTAL_NUM_EXAMPLES)