# Path: betago/dataloader/index_processor.py
# class KGSIndex(object):
# 
#     def __init__(self,
#                  kgs_url='http://u-go.net/gamerecords/',
#                  index_page='kgs_index.html',
#                  data_directory='data'):
#         '''
#         Create an index of zip files containing SGF data of actual Go Games on KGS.
# 
#         Parameters:
#         -----------
#         kgs_url: URL with links to zip files of games
#         index_page: Name of local html file of kgs_url
#         data_directory: name of directory relative to current path to store SGF data
#         '''
#         self.kgs_url = kgs_url
#         self.index_page = index_page
#         self.data_directory = data_directory
#         self.file_info = []
#         self.urls = []
#         self.load_index()  # Load index on creation
# 
#     def download_files(self):
#         '''
#         Download zip files by distributing work on all available CPUs
#         '''
#         if not os.path.isdir(self.data_directory):
#             os.makedirs(self.data_directory)
# 
#         urls_to_download = []
#         for file_info in self.file_info:
#             url = file_info['url']
#             file_name = file_info['filename']
#             if not os.path.isfile(self.data_directory + '/' + file_name):
#                 urls_to_download.append((url, self.data_directory + '/' + file_name))
#         cores = multiprocessing.cpu_count()
#         pool = multiprocessing.Pool(processes=cores)
#         try:
#             it = pool.imap(worker, urls_to_download)
#             for i in it:
#                 pass
#             pool.close()
#             pool.join()
#         except KeyboardInterrupt:
#             print(">>> Caught KeyboardInterrupt, terminating workers")
#             pool.terminate()
#             pool.join()
#             sys.exit(-1)
# 
#     def create_index_page(self):
#         '''
#         If there is no local html containing links to files, create one.
#         '''
#         if os.path.isfile(self.index_page):
#             print('>>> Reading cached index page')
#             index_file = open(self.index_page, 'r')
#             index_contents = index_file.read()
#             index_file.close()
#         else:
#             print('>>> Downloading index page')
#             fp = urlopen(self.kgs_url)
#             data = six.text_type(fp.read())
#             fp.close()
#             index_contents = data
#             index_file = open(self.index_page, 'w')
#             index_file.write(index_contents)
#             index_file.close()
#         return index_contents
# 
#     def load_index(self):
#         '''
#         Create the actual index representation from the previously downloaded or cached html.
#         '''
#         index_contents = self.create_index_page()
#         split_page = [item for item in index_contents.split('<a href="') if item.startswith("https://")]
#         for item in split_page:
#             download_url = item.split('">Download')[0]
#             if download_url.endswith('.tar.gz'):
#                 self.urls.append(download_url)
#         for url in self.urls:
#             filename = os.path.basename(url)
#             split_file_name = filename.split('-')
#             num_games = int(split_file_name[len(split_file_name) - 2])
#             print(filename + ' ' + str(num_games))
#             self.file_info.append({'url': url, 'filename': filename, 'num_games': num_games})


import os
import random
from .index_processor import KGSIndex
from six.moves import range
    def draw_samples(self, num_sample_games):
        '''
        Draw num_sample_games many training games from index.
        '''
        available_games = []
        index = KGSIndex(data_directory=self.data_dir)

        for fileinfo in index.file_info:
            filename = fileinfo['filename']
            year = int(filename.split('-')[1].split('_')[0])
            if year > self.cap_year:
                continue
            num_games = fileinfo['num_games']
            for i in range(num_games):
                available_games.append((filename, i))
        print('>>> Total number of games used: ' + str(len(available_games)))

        sample_set = set()
        while len(sample_set) < num_sample_games:
            sample = random.choice(available_games)
            if sample not in sample_set:
                sample_set.add(sample)
        print('Drawn ' + str(num_sample_games) + ' samples:')
        return list(sample_set)

    def draw_training_games(self):
        '''
        Get list of all non-test games, that are no later than dec 2014
        Ignore games after cap_year to keep training data stable
        '''

# Path: betago/dataloader/index_processor.py
# class KGSIndex(object):
# 
#     def __init__(self,
#                  kgs_url='http://u-go.net/gamerecords/',
#                  index_page='kgs_index.html',
#                  data_directory='data'):
#         '''
#         Create an index of zip files containing SGF data of actual Go Games on KGS.
# 
#         Parameters:
#         -----------
#         kgs_url: URL with links to zip files of games
#         index_page: Name of local html file of kgs_url
#         data_directory: name of directory relative to current path to store SGF data
#         '''
#         self.kgs_url = kgs_url
#         self.index_page = index_page
#         self.data_directory = data_directory
#         self.file_info = []
#         self.urls = []
#         self.load_index()  # Load index on creation
# 
#     def download_files(self):
#         '''
#         Download zip files by distributing work on all available CPUs
#         '''
#         if not os.path.isdir(self.data_directory):
#             os.makedirs(self.data_directory)
# 
#         urls_to_download = []
#         for file_info in self.file_info:
#             url = file_info['url']
#             file_name = file_info['filename']
#             if not os.path.isfile(self.data_directory + '/' + file_name):
#                 urls_to_download.append((url, self.data_directory + '/' + file_name))
#         cores = multiprocessing.cpu_count()
#         pool = multiprocessing.Pool(processes=cores)
#         try:
#             it = pool.imap(worker, urls_to_download)
#             for i in it:
#                 pass
#             pool.close()
#             pool.join()
#         except KeyboardInterrupt:
#             print(">>> Caught KeyboardInterrupt, terminating workers")
#             pool.terminate()
#             pool.join()
#             sys.exit(-1)
# 
#     def create_index_page(self):
#         '''
#         If there is no local html containing links to files, create one.
#         '''
#         if os.path.isfile(self.index_page):
#             print('>>> Reading cached index page')
#             index_file = open(self.index_page, 'r')
#             index_contents = index_file.read()
#             index_file.close()
#         else:
#             print('>>> Downloading index page')
#             fp = urlopen(self.kgs_url)
#             data = six.text_type(fp.read())
#             fp.close()
#             index_contents = data
#             index_file = open(self.index_page, 'w')
#             index_file.write(index_contents)
#             index_file.close()
#         return index_contents
# 
#     def load_index(self):
#         '''
#         Create the actual index representation from the previously downloaded or cached html.
#         '''
#         index_contents = self.create_index_page()
#         split_page = [item for item in index_contents.split('<a href="') if item.startswith("https://")]
#         for item in split_page:
#             download_url = item.split('">Download')[0]
#             if download_url.endswith('.tar.gz'):
#                 self.urls.append(download_url)
#         for url in self.urls:
#             filename = os.path.basename(url)
#             split_file_name = filename.split('-')
#             num_games = int(split_file_name[len(split_file_name) - 2])
#             print(filename + ' ' + str(num_games))
#             self.file_info.append({'url': url, 'filename': filename, 'num_games': num_games})


# Path: betago/dataloader/sampling.py
import os
import random
from .index_processor import KGSIndex
from six.moves import range

    def draw_samples(self, num_sample_games):
        '''
        Draw num_sample_games many training games from index.
        '''
        available_games = []
        index = KGSIndex(data_directory=self.data_dir)

        for fileinfo in index.file_info:
            filename = fileinfo['filename']
            year = int(filename.split('-')[1].split('_')[0])
            if year > self.cap_year:
                continue
            num_games = fileinfo['num_games']
            for i in range(num_games):
                available_games.append((filename, i))
        print('>>> Total number of games used: ' + str(len(available_games)))

        sample_set = set()
        while len(sample_set) < num_sample_games:
            sample = random.choice(available_games)
            if sample not in sample_set:
                sample_set.add(sample)
        print('Drawn ' + str(num_sample_games) + ' samples:')
        return list(sample_set)

    def draw_training_games(self):
        '''
        Get list of all non-test games, that are no later than dec 2014
        Ignore games after cap_year to keep training data stable
        '''

        index = KGSIndex(data_directory=self.data_dir)