<commit_before>
from pygraph.classes.digraph import digraph as pydigraph
from pagerank_weighted import pagerank_weighted as pagerank
from math import log10

SUMMARY_LENGTH = .2
TEST_FILE = "test_data/textrank_example.txt"


def textrank(filename=None, text=None):
    # Creates the graph and calculates the simmilarity coefficient
    # for every pair of nodes.
    if text != None:
        text = text.split('\n')
    
    if filename != None:
        text = get_text_from_file(filename)
    
    graph = get_graph(text)
    set_graph_edge_weights(graph)

    # Ranks the sentences using the PageRank algorithm.
    scores = pagerank(graph)

    # Extracts the most important sentences.
    extracted_sentences = extract_sentences(graph.nodes(), scores)

    # Sorts the extracted sentences by apparition order in the
    # original text.
    summary = sort_by_apparition(extracted_sentences, text)
    
    return "\n".join(summary)


def sort_by_apparition(extracted_sentences, text):
    summary = []

    for sentence in text:
        if sentence in extracted_sentences:
            summary.append(sentence)

    return summary


def extract_sentences(sentences, scores):
    sentences.sort(key=lambda s: scores[s], reverse=True)
    length = len(sentences) * SUMMARY_LENGTH
    return sentences[:int(length)]


def get_text_from_file(filename):
    with open(filename) as fp:
        return fp.readlines()


def get_graph(text):
    graph = pydigraph()

    # Creates the graph.
    for line in text:
        graph.add_node(line)

    return graph


def set_graph_edge_weights(graph):
    for sentence_1 in graph.nodes():
        for sentence_2 in graph.nodes():
            if sentence_1 == sentence_2:
                continue

            edge_1 = (sentence_1, sentence_2)
            edge_2 = (sentence_2, sentence_1)

            if graph.has_edge(edge_1) or graph.has_edge(edge_2):
                continue

            similarity = get_similarity(sentence_1, sentence_2)

            graph.add_edge(edge_1, similarity)
            graph.add_edge(edge_2, similarity)


def get_similarity(s1, s2):
    s1_list = s1.split()
    s2_list = s2.split()

    common_word_count = get_common_word_count(s1_list, s2_list)

    log_s1 = log10(len(s1_list))
    log_s2 = log10(len(s2_list))

    return common_word_count / (log_s1 + log_s2)


def get_common_word_count(s1_list, s2_list):
    return sum(1 for w in set(s1_list) if w in set(s2_list))


if __name__ == "__main__":
    print textrank(filename=TEST_FILE)
<commit_msg>Subo modificaciones, no anda porq las listas no son hasheables, hay que hacer que clean_text devuelva cadenas y no listas<commit_after>
from pygraph.classes.digraph import digraph as pydigraph
from pagerank_weighted import pagerank_weighted as pagerank
from textcleaner import clean_text
from math import log10

SUMMARY_LENGTH = .2
TEST_FILE = "samples/textrank_example.txt"


def textrank(text):
    # Gets a dict of  original_sentence -> processed_sentence
    sentences = clean_text(text)

    # Creates the graph and calculates the simmilarity coefficient for every pair of nodes.
    graph = get_graph(sentences.values())
    set_graph_edge_weights(graph)

    # Ranks the sentences using the PageRank algorithm.
    scores = pagerank(graph)

    # Extracts the most important sentences.
    extracted_sentences = extract_sentences(graph.nodes(), scores)

    # Sorts the extracted sentences by apparition order in the
    # original text.
    summary = sort_by_apparition(extracted_sentences, sentences, text)
    
    return "\n".join(summary)


def sort_by_apparition(extracted_sentences, sentences, text):
    summary = []

    for key in sentences:
        if sentences[key] in extracted_sentences:
            summary.append(key)

    return summary


def extract_sentences(sentences, scores):
    sentences.sort(key=lambda s: scores[s], reverse=True)
    length = len(sentences) * SUMMARY_LENGTH
    return sentences[:int(length)]


def get_graph(text):
    graph = pydigraph()

    # Creates the graph.
    for line in text:
        graph.add_node(line)

    return graph


def set_graph_edge_weights(graph):
    for sentence_1 in graph.nodes():
        for sentence_2 in graph.nodes():
            if sentence_1 == sentence_2:
                continue

            edge_1 = (sentence_1, sentence_2)
            edge_2 = (sentence_2, sentence_1)

            if graph.has_edge(edge_1) or graph.has_edge(edge_2):
                continue

            similarity = get_similarity(sentence_1, sentence_2)

            graph.add_edge(edge_1, similarity)
            graph.add_edge(edge_2, similarity)


def get_similarity(s1_list, s2_list):
    common_word_count = get_common_word_count(s1_list, s2_list)

    log_s1 = log10(len(s1_list))
    log_s2 = log10(len(s2_list))

    return common_word_count / (log_s1 + log_s2)


def get_common_word_count(s1_list, s2_list):
    return sum(1 for w in set(s1_list) if w in set(s2_list))


if __name__ == "__main__":
    main()

def main():
    with open(TEST_FILE) as file:
        text = file.read()

    print textrank(text)

{'fraction_non_alphanumeric': 0.0812451057165231, 'fraction_numerical': 0.015661707126076743, 'mean_word_length': 3.16721044045677, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3769291', 'n_tokens_mistral': 1705, 'n_tokens_neox': 1611, 'n_words': 419}