<commit_before># -*- coding: utf-8 -*-
import feature_set, data_set, feature_vector, dependency_tree
import eisner

class WeightLearner():
    """
    Learns the weight of the features using maximum perceptron algorithm
    """
    
    def __init__(self):
        """
        Initialize the WeightLearner
        """
        self.MAX_ITERATE = 20
        self.fset = feature_set.FeatureSet(
                    dependency_tree.DependencyTree(),
                    'weight.db')
        return
    
    def learn_weight_sections(self, section_set=None, data_path=None):
        """
        Given the path and specified sections, 
        for each dep_tree in the source
        learn the weight for each feature in that feature set
        
        :param section_set: see section_set in DataSet
        :type section_set: list(int/tuple)
        
        :param data_path: see data_path in DataSet
        :type data_path: str
        """
        dataset = data_set.DataSet(section_set, data_path)
        while dataset.has_next_data():
            self.update_weight(dataset.get_next_data())  
        return
            
    def learn_weight_sentence(self, file_name, index=0, data_path=None):
        """
        Given the location for one sentence in the source
        update the weight of the features in that sentence
        
        :param file_name: name of the file that contains the sentence
        :type file_name: str
        
        :param index: location of the sentence in that file, 
                      default 0 (1st sentence)
        :type index: int
        
        :param data_path: path to the data source
        :type data_path: int
        """
        dataset = data_set.DataSet(section_set, data_path)
        self.update_weight(dataset.get_data(file_name, index))
        return
        
    def update_weight(self,dep_tree):
        """
        Given one dependency tree, the function update the weight 
        for each feature in this sentence
        
        :param dep_tree: a class contains the information about the
        word_list, pos_list, edge and the information of edge type
        :type dep_tree: DependencyTree
        
        :return: updated feature set
        :rtype: FeatureSet
        """
        self.fset.switch_tree(dep_tree)
        word_list = dep_tree.get_word_list()
        gold_edge_set = \
            set([(head_index,dep_index) for head_index,dep_index,_ in dep_tree.get_edge_list()])
        print "gold set:", gold_edge_set
        
        for i in range(self.MAX_ITERATE):
            _, current_edge_set = \
               eisner.EisnerParser(word_list).parse(self.fset.get_edge_score)
            # guarantee to converge ???????
            # will the eisner calculate the different trees?
            if current_edge_set == gold_edge_set:
                break
            
            # calculate the global score
            # assume the length of each local vector in the same sentanse is the same
            # the truth_global_vector will change because of the change in weights
            current_global_vector = self.get_global_vector(current_edge_set)
            print current_edge_set
            gold_global_vector = self.get_global_vector(gold_edge_set)
            gold_global_vector.eliminate(current_global_vector)
	      # print gold_global_vector.feature_dict
            self.fset.update_weight_vector(gold_global_vector)
        self.fset.dump()
        return 

    def get_global_vector(self, edge_set):
        """
        Calculate the global vector with the current weight, the order of the feature
        score is the same order as the feature set

        :param edge_set: the set of edges represented as tuples
        :type: list(tuple(integer, integer))
        
        :return: The global vector of the sentence with the current weight
        :rtype: list
        """
        global_vector = feature_vector.FeatureVector()
        for head_index,dep_index in edge_set:
            local_vector = self.fset.get_local_vector(head_index,dep_index)
            global_vector.aggregate(local_vector)
        return global_vector
        
        
    def get_feature_set(self):
        """
        :return: the updated feature set
        :rtype: FeatureSet
        """
        return self.fset<commit_msg>update for dump and change dep_tree<commit_after># -*- coding: utf-8 -*-
import feature_set, data_set, feature_vector, dependency_tree
import eisner

class WeightLearner():
    """
    Learns the weight of the features using maximum perceptron algorithm
    """
    
    def __init__(self):
        """
        Initialize the WeightLearner
        """
        self.MAX_ITERATE = 20
        self.fset = feature_set.FeatureSet(
                    dependency_tree.DependencyTree(),
                    'weight.db')
        return
    
    def learn_weight_sections(self, section_set=None, data_path=None):
        """
        Given the path and specified sections, 
        for each dep_tree in the source
        learn the weight for each feature in that feature set
        
        :param section_set: see section_set in DataSet
        :type section_set: list(int/tuple)
        
        :param data_path: see data_path in DataSet
        :type data_path: str
        """
        dataset = data_set.DataSet(section_set, data_path)
        while dataset.has_next_data():
            self.update_weight(dataset.get_next_data())  
        self.fset.dump()
        self.fset.close()
        return
            
    def learn_weight_sentence(self,file_name,index=0,data_path=None):
        """
        Given the location for one sentence in the source
        update the weight of the features in that sentence
        
        :param file_name: name of the file that contains the sentence
        :type file_name: str
        
        :param index: location of the sentence in that file, 
                      default 0 (1st sentence)
        :type index: int
        
        :param data_path: path to the data source
        :type data_path: int
        """
        dataset = data_set.DataSet(data_path=data_path)
        self.update_weight(dataset.get_data(file_name, index))
        self.fset.dump()
        self.fset.close()
        return
        
    def update_weight(self,dep_tree):
        """
        Given one dependency tree, the function update the weight 
        for each feature in this sentence
        
        :param dep_tree: a class contains the information about the
        word_list, pos_list, edge and the information of edge type
        :type dep_tree: DependencyTree
        
        :return: updated feature set
        :rtype: FeatureSet
        """
        self.fset.switch_tree(dep_tree)
        word_list = dep_tree.get_word_list()
        gold_edge_set = \
            set([(head_index,dep_index) for head_index,dep_index,_ in dep_tree.get_edge_list()])
        print "gold set:", gold_edge_set
        
        for i in range(self.MAX_ITERATE):
            _, current_edge_set = \
               eisner.EisnerParser(word_list).parse(self.fset.get_edge_score)
            # guarantee to converge ???????
            # will the eisner calculate the different trees?
            if current_edge_set == gold_edge_set:
                break
            
            # calculate the global score
            # assume the length of each local vector in the same sentanse is the same
            # the truth_global_vector will change because of the change in weights
            current_global_vector = self.get_global_vector(current_edge_set)
            print current_edge_set
            gold_global_vector = self.get_global_vector(gold_edge_set)
            gold_global_vector.eliminate(current_global_vector)
	      # print gold_global_vector.feature_dict
            self.fset.update_weight_vector(gold_global_vector)
        return 

    def get_global_vector(self, edge_set):
        """
        Calculate the global vector with the current weight, the order of the feature
        score is the same order as the feature set

        :param edge_set: the set of edges represented as tuples
        :type: list(tuple(integer, integer))
        
        :return: The global vector of the sentence with the current weight
        :rtype: list
        """
        global_vector = feature_vector.FeatureVector()
        for head_index,dep_index in edge_set:
            local_vector = self.fset.get_local_vector(head_index,dep_index)
            global_vector.aggregate(local_vector)
        return global_vector
        
        
    def get_feature_set(self):
        """
        :return: the updated feature set
        :rtype: FeatureSet
        """
        return self.fset
{'fraction_non_alphanumeric': 0.06385353873632507, 'fraction_numerical': 0.0013395847287340924, 'mean_word_length': 1.9068786502271253, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3467368', 'n_tokens_mistral': 2527, 'n_tokens_neox': 2330, 'n_words': 687}