<commit_before># TODO - cleanup regex to do proper string subs

import logging
import os
import re
import sys
from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ObjectDoesNotExist

from distutils.version import LooseVersion as versioner

from core.models import BaseModel
from package.fields import CreationDateTimeField, ModificationDateTimeField
from package.repos import github
from package.pypi import fetch_releases
from package.repos import get_repo_for_repo_url
from package.signals import signal_fetch_latest_metadata

repo_url_help_text = settings.PACKAGINATOR_HELP_TEXT['REPO_URL']
pypi_url_help_text = settings.PACKAGINATOR_HELP_TEXT['PYPI_URL']

class NoPyPiVersionFound(Exception):
    pass
    

class Category(BaseModel):
    
    title = models.CharField(_("Title"), max_length="50")
    slug  = models.SlugField(_("slug"))
    description = models.TextField(_("description"), blank=True)
    title_plural = models.CharField(_("Title Plural"), max_length="50", blank=True) 
    show_pypi = models.BooleanField(_("Show pypi stats & version"), default=True)
    
    class Meta:
        ordering = ['title']
        verbose_name_plural = 'Categories'<commit_msg>Removing unnecessary import<commit_after># TODO - cleanup regex to do proper string subs

import logging
import os
import re
import sys
from datetime import datetime, timedelta

from django.conf import settings
from django.contrib.auth.models import User
from django.core.urlresolvers import reverse
from django.db import models
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ObjectDoesNotExist

from distutils.version import LooseVersion as versioner

from core.models import BaseModel
from package.repos import github
from package.pypi import fetch_releases
from package.repos import get_repo_for_repo_url
from package.signals import signal_fetch_latest_metadata

repo_url_help_text = settings.PACKAGINATOR_HELP_TEXT['REPO_URL']
pypi_url_help_text = settings.PACKAGINATOR_HELP_TEXT['PYPI_URL']

class NoPyPiVersionFound(Exception):
    pass
    

class Category(BaseModel):
    
    title = models.CharField(_("Title"), max_length="50")
    slug  = models.SlugField(_("slug"))
    description = models.TextField(_("description"), blank=True)
    title_plural = models.CharField(_("Title Plural"), max_length="50", blank=True) 
    show_pypi = models.BooleanField(_("Show pypi stats & version"), default=True)
    
    class Meta:
        ordering = ['title']
        verbose_name_plural = 'Categories'
{'fraction_non_alphanumeric': 0.0766501064584812, 'fraction_numerical': 0.0028388928317956, 'mean_word_length': 5.363431151241535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2604703', 'n_tokens_mistral': 829, 'n_tokens_neox': 799, 'n_words': 228}