<commit_before>from django.conf import settings
from robots.models import Url
from cms.sitemaps import CMSSitemap
from robots.settings import ADMIN
from django.db.models import Q
from itertools import chain
from itertools import ifilterfalse, imap, izip

ID_PREFIX = 'disallowed'


def get_site_id(data, instance, sites_field):
    if instance and instance.id:
        id = instance.sites.all()[0].id
    else:
        id = sites_field.choices.queryset[0].id
    return data.get('sites', id)


def get_url(pattern):
    try:
        return Url.objects.get_or_create(pattern=pattern)[0]
    except Url.MultipleObjectsReturned:
        return Url.objects.filter(pattern=pattern)[0]


def get_choices(site, protocol):
    """
    Returns a list with the urls patterns for the site parameter
    The list will be in this format required by the disallowed field widget:
    [['1', '/pattern1/'], ['2', '/pattern2/'], ['disallowed_3', '/pattern4/'], ...]

    The patterns are taken from the sitemap for the site param.
    Some of the ids are real db ids, and others (like disallowed_3) are fake ones
    (generated here).
    """
    # Make sure that the '/admin/' pattern is allways present
    #  in the choice list
    get_url(ADMIN)

    #generate patterns from the sitemap
    saved_site = settings.__class__.SITE_ID.value
    settings.__class__.SITE_ID.value = site.id
    urls = CMSSitemap().get_urls(site=site, protocol=protocol)
    all_sitemap_patterns = map(lambda item: item['location'].replace("%s://%s" % (protocol, site.domain), ''), urls)
    settings.__class__.SITE_ID.value = saved_site

    #Some patterns are already present in the db and I need their real ids
    #This processing step could have been avoided, but I need the sitemap
    #    patterns to be displayed first in the left side box.
    f = Q(pattern__in=all_sitemap_patterns)<commit_msg>LUN-819 -- some refactoring<commit_after>from django.conf import settings
from robots.models import Url
from cms.sitemaps import CMSSitemap
from robots.settings import ADMIN
from django.db.models import Q
from itertools import chain
from itertools import ifilterfalse, imap, izip

ID_PREFIX = 'disallowed'


def get_site_id(data, instance, sites_field):
    if instance and instance.id:
        id = instance.sites.all()[0].id
    else:
        id = sites_field.choices.queryset[0].id
    return data.get('sites', id)


def get_url(pattern):
    try:
        return Url.objects.get_or_create(pattern=pattern)[0]
    except Url.MultipleObjectsReturned:
        return Url.objects.filter(pattern=pattern)[0]


def get_choices(site, protocol):
    """
    Returns a list with the urls patterns for the site parameter
    The list will be in this format required by the disallowed field widget:
    [['1', '/pattern1/'], ['2', '/pattern2/'], ['disallowed_3', '/pattern4/'], ...]

    The patterns are taken from the sitemap for the site param.
    Some of the ids are real db ids, and others (like disallowed_3) are fake ones
    (generated here).
    """
    def get_slug(url):
        return url['location'].replace("%s://%s" % (protocol, site.domain), '')

    # Make sure that the '/admin/' pattern is allways present
    #  in the choice list
    get_url(ADMIN)

    #generate patterns from the sitemap
    saved_site = settings.__class__.SITE_ID.value
    settings.__class__.SITE_ID.value = site.id
    urls = CMSSitemap().get_urls(site=site, protocol=protocol)
    all_sitemap_patterns = map(lambda item: get_slug(item), urls)
    settings.__class__.SITE_ID.value = saved_site

    #Some patterns are already present in the db and I need their real ids
    #This processing step could have been avoided, but I need the sitemap
    #    patterns to be displayed first in the left side box.
    f = Q(pattern__in=all_sitemap_patterns)
{'fraction_non_alphanumeric': 0.09781751249013937, 'fraction_numerical': 0.006573757559821194, 'mean_word_length': 3.5611510791366907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1308670', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1157, 'n_words': 387}