<commit_before># -*- coding: utf-8 -*-

import os, re
import data_entity

class DataPool():
    
    def __init__(self, section_set=[], data_path="./penn-wsj-deps/"):
        """
        Initialize the Data set
        
        :param section_set: the sections to be used. 
        It can be specifiled by a list of either integer or tuple.
        for tuple (a,b), the section would be from a to b including a and b
        :type section_set: list(int/tuple)
        
        :param data_path: the relative or absolute path to the 'penn-wsj-deps' folder
        (include "penn-wsj-deps")
        :type data_path: str
        
        """  
        self.data_path = data_path
        self.set_section_list(section_set)
        
        self.reset_whole()
        self.load()

    def reset_whole(self):
        """
        Reset the index variables and the data list
        data list is the list of data in the feature form
        """
        self.reset()
        self.data_list = []

    def reset(self):
        """
        Reset only the index variables
        """
        self.current_index = -1

    def has_next_data(self):
        i = self.current_index + 1
        if i >= 0 and i < len(self.data_list):
            return True
        else:
            return False
        
    def get_next_data(self):
        """
            make sure to use function has_next_data function before calling this function
        """
        self.current_index += 1
        return self.data_list[self.current_index]
        
    def load(self):
        """
        Load the trainning data
        """
        print "Loading data..."
        for section in self.section_list:
            for file_name in os.listdir(self.data_path + "%02d" % section):
                file_path = self.data_path + "%02d/" % section + file_name
                self.data_list = self.data_list + self.get_data_list(file_path)

    def get_data_list(self, file_path):
        """
        Form the DependencyTree list from the specified file
        
        :param file_path: the path to the data file
        :type: str
        
        :return: a list of DependencyTree in the specified file
        :rtype: list(DependencyTree)
        """
        f = open(file_path)
        data_list = []
        word_list = []
        pos_list = []
        edge_set = {}
        current_index = 0

        for line in f:
            line = line[:-1]
            if line != '':
                current_index = current_index + 1
                entry = line.split()
                if len(entry) != 4:
                    print "invalid data!!"
                else:
                    word_list.append(entry[0])
                    pos_list.append(entry[1])
                    edge_set[(int(entry[2]), current_index)] = entry[3]
            else:
                if word_list != []:
                    d_entity = data_entity.DataEntity(word_list,pos_list,edge_set)
                    data_list.append(d_entity)
                    #print d_tree.get_word_list()
                word_list = []
                pos_list = []
                edge_set = {}
                current_index = 0
        return data_list
    
    def set_section_list(self, section_set):
        """
        Set the section list from section set
        the section set is a list contains:
            tuples --   representing the range of the section,
                        i.e. (1,3) means range 1,2,3
            int    --   single section number
        
        :param section_set: the sections to be used
        :type section_set: list(tuple/int) 
        """
        self.section_list = [section for section in section_set if isinstance(section, int)]
        
        section_sets = \
            map(lambda section_tuple: range(section_tuple[0], section_tuple[1]+1),
                [st for st in section_set if isinstance(st, tuple)])
        for s_set in section_sets:<commit_msg>Classes Reorganized<commit_after># -*- coding: utf-8 -*-

import os, re
from sentence import *

class DataPool():
    
    def __init__(self, section_set=[], data_path="./penn-wsj-deps/"):
        """
        Initialize the Data set
        
        :param section_set: the sections to be used. 
        It can be specifiled by a list of either integer or tuple.
        for tuple (a,b), the section would be from a to b including a and b
        :type section_set: list(int/tuple)
        
        :param data_path: the relative or absolute path to the 'penn-wsj-deps' folder
        (include "penn-wsj-deps")
        :type data_path: str
        
        """  
        self.data_path = data_path
        self.set_section_list(section_set)
        
        self.reset_whole()
        self.load()

    def reset_whole(self):
        """
        Reset the index variables and the data list
        data list is the list of data in the feature form
        """
        self.reset()
        self.data_list = []

    def reset(self):
        """
        Reset only the index variables
        """
        self.current_index = -1

    def has_next_data(self):
        i = self.current_index + 1
        if i >= 0 and i < len(self.data_list):
            return True
        else:
            return False
        
    def get_next_data(self):
        """
            make sure to use function has_next_data function before calling this function
        """
        self.current_index += 1
        return self.data_list[self.current_index]
        
    def load(self):
        """
        Load the trainning data
        """
        print "Loading data..."
        for section in self.section_list:
            for file_name in os.listdir(self.data_path + "%02d" % section):
                file_path = self.data_path + "%02d/" % section + file_name
                self.data_list = self.data_list + self.get_data_list(file_path)

    def get_data_list(self, file_path):
        """
        Form the DependencyTree list from the specified file
        
        :param file_path: the path to the data file
        :type: str
        
        :return: a list of DependencyTree in the specified file
        :rtype: list(DependencyTree)
        """
        f = open(file_path)
        data_list = []
        word_list = []
        pos_list = []
        edge_set = {}
        current_index = 0

        for line in f:
            line = line[:-1]
            if line != '':
                current_index = current_index + 1
                entity = line.split()
                if len(entity) != 4:
                    print "invalid data!!"
                else:
                    word_list.append(entity[0])
                    pos_list.append(entity[1])
                    edge_set[(int(entity[2]), current_index)] = entity[3]
            else:
                if word_list != []:
                    sent = Sentence(word_list,pos_list,edge_set)
                    data_list.append(sent)
                    #print d_tree.get_word_list()
                word_list = []
                pos_list = []
                edge_set = {}
                current_index = 0
        return data_list
    
    def set_section_list(self, section_set):
        """
        Set the section list from section set
        the section set is a list contains:
            tuples --   representing the range of the section,
                        i.e. (1,3) means range 1,2,3
            int    --   single section number
        
        :param section_set: the sections to be used
        :type section_set: list(tuple/int) 
        """
        self.section_list = [section for section in section_set if isinstance(section, int)]
        
        section_sets = \
            map(lambda section_tuple: range(section_tuple[0], section_tuple[1]+1),
                [st for st in section_set if isinstance(st, tuple)])
        for s_set in section_sets:
{'fraction_non_alphanumeric': 0.08825025432349949, 'fraction_numerical': 0.006612410986775178, 'mean_word_length': 1.5428386679599095, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1267524', 'n_tokens_mistral': 2233, 'n_tokens_neox': 2152, 'n_words': 635}