<commit_before>#! /usr/bin/env python
#-*- coding: utf-8 -*-

from datetime import timedelta

import itertools
import utils


def compare_documents(feed):
    """
    Compare a list of documents by pair.
    """
    duplicates = []
    for pair in itertools.combinations(feed.articles, 2):
        date1 = pair[0].date
        date2 = pair[1].date
        if pair[0].content != "" and \
            utils.clear_string(pair[0].title) == utils.clear_string(pair[1].title) and \
            (date1 - date2) < timedelta(days = 1):
            duplicates.append(pair)
    return duplicates<commit_msg>Do not check at all the content.<commit_after>#! /usr/bin/env python
#-*- coding: utf-8 -*-

import itertools
from datetime import timedelta

import utils

def compare_documents(feed):
    """
    Compare a list of documents by pair.
    """
    duplicates = []
    for pair in itertools.combinations(feed.articles, 2):
        date1 = pair[0].date
        date2 = pair[1].date
        if utils.clear_string(pair[0].title) == utils.clear_string(pair[1].title) and \
                                        (date1 - date2) < timedelta(days = 1):
            duplicates.append(pair)
    return duplicates
{'fraction_non_alphanumeric': 0.12437395659432388, 'fraction_numerical': 0.019198664440734557, 'mean_word_length': 2.3679775280898876, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6686733', 'n_tokens_mistral': 382, 'n_tokens_neox': 369, 'n_words': 93}