<commit_before>from sklearn.neighbors import NearestNeighbors, LSHForest
import numpy as np
from sklearn.base import BaseEstimator
import subprocess
from scipy.sparse.csr import csr_matrix
from nltk.translate import IBMModel1
from nltk.translate import AlignedSent


class KNeighborsListNetClassifier(BaseEstimator):
    """
    This classifier first examines the labels that appear in the n-nearest documents and then generates
    features for each of the labels. Given these features, the ranking algorithm 'ListNET' then ranks the labels.
    Finally, the k highest ranked labels are assigned as output.
    
    The features that are generated are the following:
        - the sum of the cosine similarities to all the documents in the n-neighborhood that the label is actually assigned to
        - the total count of appearances of the label in the document
        - the probability that the label translates to the document's title according to the IBM1-translation-model.
        - the count of how often the label directly appears in the title
        
    This algorithm uses the ListNET implementation from [RankLib] in the background. Therefore, it assumes a Java-
    installation to be available and the RankLib-2.5.jar file to be in the same folder as this file.
        
    The whole procedure's main idea was taken from [MLA]. However, as some of their techniques cannot be applied
    in the same way to our application, some of the implementation details differ.
    
    References
    ----------
    [MLA]   Huang, Minlie, <NAME>, and <NAME>. 
            "Recommending MeSH terms for annotating biomedical articles." 
            Journal of the American Medical Informatics Association 18.5 (2011): 660-667.
    [RankLib] https://sourceforge.net/p/lemur/wiki/RankLib/
    
    Parameters
    ----------
    n_neighbors: int, default=20
        The size of the neighborhood, from which labels are considered as input to the ranking algorithm
    topk: int, default=5
        The number of labels taken from the top of the list after running the ranking algorithm
    max_iterations: int, default=300
        Given as input to the RankLib library. Determines the number of iterations to train the ranking classifier with.
    count_concepts: boolean, default=False
        Must indicate whether the feature vector of a document contains concepts or not.
    number_of_concepts: boolean, default=0
        Must indicate the number of concepts contained in the document's feature vector.
    count_terms: terms, default=True
        Must indicate whether the feature vector of a document contains terms or not.
    """
    def __init__(self, use_lsh_forest=False, n_neighbors=20, topk=5, max_iterations = 300, count_concepts = False, number_of_concepts = 0, count_terms = False, **kwargs):
        
        self.n_neighbors = n_neighbors
        self.knn = LSHForest(n_neighbors=n_neighbors, **kwargs) if use_lsh_forest else NearestNeighbors(
            n_neighbors=n_neighbors, **kwargs)
        self.y = None
        self.topk = topk
        self.max_iterations = max_iterations
        self.count_concepts = count_concepts
        self.count_terms = count_terms
        self.number_of_concepts = number_of_concepts

    def fit(self, X, y):
        self.y = y
        self.knn.fit(X)
        
        distances_to_neighbors,neighbor_id_lists = self.knn.kneighbors()
        
        self._train_translation_model(X,y)
        
        # create features for each label in a documents neighborhood and write them in a file
        self._extract_and_write(X, neighbor_id_lists, distances_to_neighbors, y = y, sum = True)
        # use the RankLib libraries ListNet algorithm to create a ranking model
        subprocess.call(["java", "-jar", "RankLib-2.5.jar", "-train", "listnet_train", "-save", "listnet_model", "-ranker" ,"7", "-epoch", str(self.max_iterations)])


    def _train_translation_model(self, X, y):
        translations = []
        print("Creating the translations.")
        
        for row in range(0, X.shape[0] - 1):
            
            title = self._recompose_title(X, row)
            for label in self._get_labels_of_row(row, y):
                translations.append(AlignedSent(title, [label]))
        
    
        #for i,title_string in enumerate(titles):
        #   for label in labels[i]:
        # asent = AlignedSent(title_string.split(),[label])
        # translations.append(asent)
        
        print("Done creating the translations.")
        ibm1 = IBMModel1(translations, 5)
        self.ibm1 = ibm1
    
    def _recompose_title(self, X, row):
        title = []
        for word in X[row].nonzero():
            title.append(str(word))
        
        return title
    
    def _extract_and_write(self, X, neighbor_id_lists, distances_to_neighbors, fileName = "listnet_train", y = None, sum = False):
        listnet_training_samples = open(fileName, "w", encoding='utf-8')
        qid = 1
        
        sum_intersection_percent = 0
        sum_union_percent = 0
        doc_to_neighborhood_dict = {}
        for curr_doc, neighbor_list in enumerate(neighbor_id_lists):
            
            labels_in_neighborhood = self.y[neighbor_list]
            
            labels_per_neighbor = []
            for i in range(0,labels_in_neighborhood.shape[0] - 1):
                cols = self._get_labels_of_row(i,labels_in_neighborhood)
                labels_per_neighbor.append(cols)
                
            all_labels_in_neighborhood = [label for labels in labels_per_neighbor for label in labels]
            all_labels_in_neighborhood = list(set(all_labels_in_neighborhood))
            
            sum_intersection_percent += self._intersection_percent(set(all_labels_in_neighborhood), set(self._get_labels_of_row(0,self.y[curr_doc])))
            sum_union_percent += len(set(all_labels_in_neighborhood)) / len(set(self._get_labels_of_row(0,self.y[curr_doc])))
            
            for label in all_labels_in_neighborhood:
                listnet_training_samples.write(self._generate_line_for_label(label, qid, X, y, curr_doc, labels_per_neighbor, neighbor_list, distances_to_neighbors[curr_doc]))
            
            doc_to_neighborhood_dict[str(qid)] = all_labels_in_neighborhood
            
            qid += 1
        
        if sum:
            self.sum_intersection_percent = 100 * sum_intersection_percent / X.shape[0]
            self.sum_union_percent = 100 * sum_union_percent / X.shape[0]
        return doc_to_neighborhood_dict
                         
    def _intersection_percent(self, all_labels, curr_labels):
        return len(all_labels.intersection(curr_labels)) / len(curr_labels)
    
    def predict(self, X):
        distances, neighbor_id_lists = self.knn.kneighbors(X, n_neighbors=self.n_neighbors)
        doc_to_neighborhood_dict = self._extract_and_write(X, neighbor_id_lists, distances, fileName="listnet_test", y = None)
        
        # use the created ListNET model to get a ranking java -jar RankLib-2.5.jar -rank listnet_test -load listnet_model -ranker 7 -score test_scores
        subprocess.call(["java", "-jar", "RankLib-2.5.jar", "-rank", "listnet_test", "-score", "test_scores", "-load", "listnet_model", "-ranker" ,"7"])
        
        scoresFile = open("test_scores", "r")
        
        results = {}
        for line in scoresFile:
            qid, score = self._extract_score(line)
            if str(qid) in results:
                labels = results[str(qid)]
                labels.append(score)
                results[str(qid)] = labels
            else :
                results[str(qid)] = [score]
            
        predictions = csr_matrix((X.shape[0], self.y.shape[1]))
        
        doc_to_neighborhood_dict = self._extract_topk_score(doc_to_neighborhood_dict, results)
        
        for i in range(0,X.shape[0] - 1):
            for label in doc_to_neighborhood_dict[str(i + 1)]:
                predictions[i, label] = 1
        
        print("In the " + str(self.n_neighbors) + " nearest neighbors there are " + str(self.sum_intersection_percent) + " %% of the labels.")
        print("The number of labels in the " + str(self.n_neighbors) + " nearest neighbors is " + str(self.sum_union_percent))<commit_msg>Fixed minor bug in listnet classifier.<commit_after>from sklearn.neighbors import NearestNeighbors, LSHForest
import numpy as np
from sklearn.base import BaseEstimator
import subprocess
from scipy.sparse.csr import csr_matrix
from nltk.translate import IBMModel1
from nltk.translate import AlignedSent


class KNeighborsListNetClassifier(BaseEstimator):
    """
    This classifier first examines the labels that appear in the n-nearest documents and then generates
    features for each of the labels. Given these features, the ranking algorithm 'ListNET' then ranks the labels.
    Finally, the k highest ranked labels are assigned as output.
    
    The features that are generated are the following:
        - the sum of the cosine similarities to all the documents in the n-neighborhood that the label is actually assigned to
        - the total count of appearances of the label in the document
        - the probability that the label translates to the document's title according to the IBM1-translation-model.
        - the count of how often the label directly appears in the title
        
    This algorithm uses the ListNET implementation from [RankLib] in the background. Therefore, it assumes a Java-
    installation to be available and the RankLib-2.5.jar file to be in the same folder as this file.
        
    The whole procedure's main idea was taken from [MLA]. However, as some of their techniques cannot be applied
    in the same way to our application, some of the implementation details differ.
    
    References
    ----------
    [MLA]   Huang, Minlie, <NAME>, and <NAME>. 
            "Recommending MeSH terms for annotating biomedical articles." 
            Journal of the American Medical Informatics Association 18.5 (2011): 660-667.
    [RankLib] https://sourceforge.net/p/lemur/wiki/RankLib/
    
    Parameters
    ----------
    n_neighbors: int, default=20
        The size of the neighborhood, from which labels are considered as input to the ranking algorithm
    topk: int, default=5
        The number of labels taken from the top of the list after running the ranking algorithm
    max_iterations: int, default=300
        Given as input to the RankLib library. Determines the number of iterations to train the ranking classifier with.
    count_concepts: boolean, default=False
        Must indicate whether the feature vector of a document contains concepts or not.
    number_of_concepts: boolean, default=0
        Must indicate the number of concepts contained in the document's feature vector.
    count_terms: terms, default=True
        Must indicate whether the feature vector of a document contains terms or not.
    """
    def __init__(self, use_lsh_forest=False, n_neighbors=20, topk=5, max_iterations = 300, count_concepts = False, number_of_concepts = 0, count_terms = False, **kwargs):
        
        self.n_neighbors = n_neighbors
        self.knn = LSHForest(n_neighbors=n_neighbors, **kwargs) if use_lsh_forest else NearestNeighbors(
            n_neighbors=n_neighbors, **kwargs)
        self.y = None
        self.topk = topk
        self.max_iterations = max_iterations
        self.count_concepts = count_concepts
        self.count_terms = count_terms
        self.number_of_concepts = number_of_concepts

    def fit(self, X, y):
        self.y = y
        self.knn.fit(X)
        
        distances_to_neighbors,neighbor_id_lists = self.knn.kneighbors()
        
        self._train_translation_model(X,y)
        
        # create features for each label in a documents neighborhood and write them in a file
        self._extract_and_write(X, neighbor_id_lists, distances_to_neighbors, y = y, sum = True)
        # use the RankLib libraries ListNet algorithm to create a ranking model
        subprocess.call(["java", "-jar", "RankLib-2.5.jar", "-train", "listnet_train", "-save", "listnet_model", "-ranker" ,"7", "-epoch", str(self.max_iterations)])


    def _train_translation_model(self, X, y):
        translations = []
        print("Creating the translations.")
        
        for row in range(0, X.shape[0]):
            
            title = self._recompose_title(X, row)
            for label in self._get_labels_of_row(row, y):
                translations.append(AlignedSent(title, [label]))
        
    
        #for i,title_string in enumerate(titles):
        #   for label in labels[i]:
        # asent = AlignedSent(title_string.split(),[label])
        # translations.append(asent)
        
        print("Done creating the translations.")
        ibm1 = IBMModel1(translations, 5)
        self.ibm1 = ibm1
    
    def _recompose_title(self, X, row):
        title = []
        for word in X[row].nonzero():
            title.append(str(word))
        
        return title
    
    def _extract_and_write(self, X, neighbor_id_lists, distances_to_neighbors, fileName = "listnet_train", y = None, sum = False):
        listnet_training_samples = open(fileName, "w", encoding='utf-8')
        qid = 1
        
        sum_intersection_percent = 0
        sum_union_percent = 0
        doc_to_neighborhood_dict = {}
        for curr_doc, neighbor_list in enumerate(neighbor_id_lists):
            
            labels_in_neighborhood = self.y[neighbor_list]
            
            labels_per_neighbor = []
            for i in range(0,labels_in_neighborhood.shape[0] - 1):
                cols = self._get_labels_of_row(i,labels_in_neighborhood)
                labels_per_neighbor.append(cols)
                
            all_labels_in_neighborhood = [label for labels in labels_per_neighbor for label in labels]
            all_labels_in_neighborhood = list(set(all_labels_in_neighborhood))
            
            sum_intersection_percent += self._intersection_percent(set(all_labels_in_neighborhood), set(self._get_labels_of_row(0,self.y[curr_doc])))
            sum_union_percent += len(set(all_labels_in_neighborhood)) / len(set(self._get_labels_of_row(0,self.y[curr_doc])))
            
            for label in all_labels_in_neighborhood:
                listnet_training_samples.write(self._generate_line_for_label(label, qid, X, y, curr_doc, labels_per_neighbor, neighbor_list, distances_to_neighbors[curr_doc]))
            
            doc_to_neighborhood_dict[str(qid)] = all_labels_in_neighborhood
            
            qid += 1
        
        if sum:
            self.sum_intersection_percent = 100 * sum_intersection_percent / X.shape[0]
            self.sum_union_percent = 100 * sum_union_percent / X.shape[0]
        return doc_to_neighborhood_dict
                         
    def _intersection_percent(self, all_labels, curr_labels):
        return len(all_labels.intersection(curr_labels)) / len(curr_labels)
    
    def predict(self, X):
        distances, neighbor_id_lists = self.knn.kneighbors(X, n_neighbors=self.n_neighbors)
        doc_to_neighborhood_dict = self._extract_and_write(X, neighbor_id_lists, distances, fileName="listnet_test", y = None)
        
        # use the created ListNET model to get a ranking java -jar RankLib-2.5.jar -rank listnet_test -load listnet_model -ranker 7 -score test_scores
        subprocess.call(["java", "-jar", "RankLib-2.5.jar", "-rank", "listnet_test", "-score", "test_scores", "-load", "listnet_model", "-ranker" ,"7"])
        
        scoresFile = open("test_scores", "r")
        
        results = {}
        for line in scoresFile:
            qid, score = self._extract_score(line)
            if str(qid) in results:
                labels = results[str(qid)]
                labels.append(score)
                results[str(qid)] = labels
            else :
                results[str(qid)] = [score]
            
        predictions = csr_matrix((X.shape[0], self.y.shape[1]))
        
        doc_to_neighborhood_dict = self._extract_topk_score(doc_to_neighborhood_dict, results)
        
        for i in range(0,X.shape[0]):
            for label in doc_to_neighborhood_dict[str(i + 1)]:
                predictions[i, label] = 1
        
        print("In the " + str(self.n_neighbors) + " nearest neighbors there are " + str(self.sum_intersection_percent) + " %% of the labels.")
        print("The number of labels in the " + str(self.n_neighbors) + " nearest neighbors is " + str(self.sum_union_percent))
{'fraction_non_alphanumeric': 0.08318754901369367, 'fraction_numerical': 0.008686734632321893, 'mean_word_length': 2.614126880313931, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3106022', 'n_tokens_mistral': 5014, 'n_tokens_neox': 4764, 'n_words': 1364}