<commit_before>import re 
from taggit.models import Tag 
from taggit.contrib.suggest.models import TagKeyword, TagRegExp
from django.conf import settings 


HAS_NLTK = True 
try: 
    from nltk.stemmer.porter import PorterStemmer
except ImportError: 
    HAS_NLTK = False 

def _suggest_keywords(content=None):
    """ Suggest by keywords """ 
    suggested_keywords = set()
    keywords = TagKeyword.objects.values_list('keyword', 'tag')

    for k in keywords: 
        if k[0] in content: 
            suggested_keywords.add(k[1])

    return suggested_keywords 

def _suggest_regexps(content=None): 
    """ Suggest by regular expressions """ 
    # Grab all regular expressions and compile them 
    suggested_regexps = set()
    regexps = set()
    regexp_keywords = TagRegExp.objects.values_list(
                            'regexp', 
                            'tag', 
                            )

    for r in regexp_keywords: 
        regexps.add((re.compile(r[0]), r[1]))

    # Look for our regular expressions in the content 
    for r in regexps:
        if r[0].search(content):
            suggested_regexps.add(r[1])

    return suggested_regexps 

def suggest_tags(content=None):
    """ Suggest tags based on text content """ 
    suggested_keywords = _suggest_keywords(content)
    suggested_regexps  = _suggest_regexps(content) 
    suggested_tag_ids  = suggested_keywords | suggested_regexps

    # Turn the found IDs into tags 
    where_string = 'id IN (%s)' % ','.join([str(x) for x in suggested_tag_ids])
    tags = Tag.objects.extra(where=[where_string])

    return tags<commit_msg>Fixed suggest_tags() to properly use the ORM<commit_after>import re 
from taggit.models import Tag 
from taggit.contrib.suggest.models import TagKeyword, TagRegExp
from django.conf import settings 


HAS_NLTK = True 
try: 
    from nltk.stemmer.porter import PorterStemmer
except ImportError: 
    HAS_NLTK = False 

def _suggest_keywords(content=None):
    """ Suggest by keywords """ 
    suggested_keywords = set()
    keywords = TagKeyword.objects.values_list('keyword', 'tag')

    for k in keywords: 
        if k[0] in content: 
            suggested_keywords.add(k[1])

    return suggested_keywords 

def _suggest_regexps(content=None): 
    """ Suggest by regular expressions """ 
    # Grab all regular expressions and compile them 
    suggested_regexps = set()
    regexps = set()
    regexp_keywords = TagRegExp.objects.values_list(
                            'regexp', 
                            'tag', 
                            )

    for r in regexp_keywords: 
        regexps.add((re.compile(r[0]), r[1]))

    # Look for our regular expressions in the content 
    for r in regexps:
        if r[0].search(content):
            suggested_regexps.add(r[1])

    return suggested_regexps 

def suggest_tags(content=None):
    """ Suggest tags based on text content """ 
    suggested_keywords = _suggest_keywords(content)
    suggested_regexps  = _suggest_regexps(content) 
    suggested_tag_ids  = suggested_keywords | suggested_regexps

    return Tag.objects.filter(id__in=suggested_tag_ids)
{'fraction_non_alphanumeric': 0.08719259022676461, 'fraction_numerical': 0.0038326413286489938, 'mean_word_length': 2.7599039615846337, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3199577', 'n_tokens_mistral': 973, 'n_tokens_neox': 898, 'n_words': 238}