<commit_before>from datetime import timedelta
from django.utils import timezone

import difflib

from newswall.models import Story

VERIFY_CROSSPOST_DAYS = 2

class ProviderBase(object):
    def __init__(self, source, config):
        self.source = source
        self.config = config

    def update(self):
        raise NotImplementedError

    def create_story(self, object_url, **kwargs):
        force_update = kwargs.get('force_update', False)
        try:
            story = Story.objects.get(object_url=object_url)
        except Story.DoesNotExist:
            pass
        else:
            if not force_update:
                return story
            else:
                story.delete()

        defaults = {'source': self.source }
        defaults.update(kwargs)

        if defaults.get('title'):
            recent_stories = Story.objects.filter(is_active=True,
                              timestamp__gte=timezone.now()
                            - timedelta(days=VERIFY_CROSSPOST_DAYS))
            # check if the titles are similar
            for story in recent_stories:
                match = difflib.SequenceMatcher(None,
                                                defaults['title'], story.title)
                if match.ratio() > 0.6:
                    # the two stories are similar:
                    if self.source.priority <= story.source.priority:
                        defaults['is_active'] = False
                    else:
                        # deactivate the other story
                        story.deactivate()

        return Story.objects.get_or_create(object_url=object_url,
                                           defaults=defaults)
<commit_msg>remove tracking information on long URLs<commit_after>from datetime import timedelta
from django.utils import timezone

import difflib

from newswall.models import Story

VERIFY_CROSSPOST_DAYS = 2

class ProviderBase(object):
    def __init__(self, source, config):
        self.source = source
        self.config = config

    def update(self):
        raise NotImplementedError

    def create_story(self, object_url, **kwargs):
        force_update = kwargs.get('force_update', False)
        # The URL can only be 255 characters long to be indexed
        if len(object_url) > 255:
            object_url = object_url.split('?')[0]
        try:
            story = Story.objects.get(object_url=object_url)
        except Story.DoesNotExist:
            pass
        else:
            if not force_update:
                return story
            else:
                story.delete()

        defaults = {'source': self.source }
        defaults.update(kwargs)

        if defaults.get('title'):
            recent_stories = Story.objects.filter(is_active=True,
                              timestamp__gte=timezone.now()
                            - timedelta(days=VERIFY_CROSSPOST_DAYS))
            # check if the titles are similar
            for story in recent_stories:
                match = difflib.SequenceMatcher(None,
                                                defaults['title'], story.title)
                if match.ratio() > 0.6:
                    # the two stories are similar:
                    if self.source.priority <= story.source.priority:
                        defaults['is_active'] = False
                    else:
                        # deactivate the other story
                        story.deactivate()
        return Story.objects.get_or_create(object_url=object_url,
                                           defaults=defaults)

{'fraction_non_alphanumeric': 0.07413600891861762, 'fraction_numerical': 0.0036231884057971015, 'mean_word_length': 1.5471965933286018, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7027904', 'n_tokens_mistral': 906, 'n_tokens_neox': 860, 'n_words': 215}