<commit_before>
import sys
import os
from itertools import combinations
from rouge_calculator import MODEL_DIRECTORY
from rouge_calculator import MODEL_FILENAME
from rouge_calculator import TEXT_FILENAME_FORMAT
from rouge_calculator import GOLD_REFERENCES_DIR_FORMAT

# Imports files from a parent directory.
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'textrank'))
from textcleaner import get_sentences

SUMMARY_LENGHT = 0.2
DATASET = 'summa'


def get_sentences_from_text(text_number):
    text_filename = TEXT_FILENAME_FORMAT.format(dataset=DATASET, text_number=text_number)
    with open(text_filename) as fp:
        text = fp.read()
    return list(get_sentences(text))


def get_optimum_summary(text_number):
    """ Creates the best possible summary of a set length trying
    all posible combinations.
    """

    # Temporary stores the best results so far.
    best_sentences = None
    best_scores = None

    sentences = get_sentences_from_text(text_number)
    summary_lenght = int(len(sentences) * SUMMARY_LENGHT)
    # Creates a summary for each combination of sentences.
    for combination in combinations(sentences, summary_lenght):
        #print combination
        pass

    # gold_references_dir = GOLD_REFERENCES_DIR_FORMAT.format(dataset=DATASET, text_number=text_number)


get_optimum_summary(1)<commit_msg>Finished with the best summary logic.<commit_after>
import sys
import os
from itertools import combinations
from rouge_calculator import evaluate_summary
from rouge_calculator import MODEL_DIRECTORY
from rouge_calculator import MODEL_FILENAME
from rouge_calculator import TEXT_FILENAME_FORMAT
from rouge_calculator import GOLD_REFERENCES_DIR_FORMAT

# Imports files from a parent directory.
sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, 'textrank'))
from textcleaner import get_sentences

SUMMARY_LENGHT = 0.2
DATASET = 'summa'
OPTIMUM_FILENAME_FORMAT = 'results/opt_{text_number:02d}.txt'


def get_sentences_from_text(text_number):
    text_filename = TEXT_FILENAME_FORMAT.format(dataset=DATASET, text_number=text_number)
    with open(text_filename) as fp:
        text = fp.read()
    return list(get_sentences(text))


def get_score_for_summay(text_number, summary):
    with open(os.path.join(MODEL_DIRECTORY, MODEL_FILENAME), 'w') as fp:
        fp.write(summary)

    gold_references_dir = GOLD_REFERENCES_DIR_FORMAT.format(dataset=DATASET, text_number=text_number)
    return evaluate_summary(gold_references_dir)


def get_optimum_summary(text_number):
    """ Creates the best possible summary of a set length trying
    all posible combinations.
    """

    # Temporary stores the best results so far.
    best_summary = None
    best_score = 0

    sentences = get_sentences_from_text(text_number)
    summary_lenght = int(len(sentences) * SUMMARY_LENGHT)

    # Creates a summary for each combination of sentences.
    for combination in combinations(sentences, summary_lenght):
        summary = "\n".join(combination)
        result = get_score_for_summay(text_number, summary)

        # We consider the average of several scores.
        score = (result['rouge_1_f_score'] + result['rouge_2_f_score'] + result['rouge_su*_f_score']) / 3

        if score > best_score:
            best_score = score
            best_summary = summary

        print "summary with score", score, ". best score so far:", best_score
        print ""
        print ""
        print ""

    # The optimum summary is written to hard disk.
    output_filename = OPTIMUM_FILENAME_FORMAT.format(text_number=text_number)
    with open(output_filename, 'w') as fp:
        fp.write(best_summary)


get_optimum_summary(1)
{'fraction_non_alphanumeric': 0.07465091299677766, 'fraction_numerical': 0.00322234156820623, 'mean_word_length': 4.437956204379562, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2440002', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1152, 'n_words': 284}