<commit_before>import sys

from django.core.exceptions import FieldError, ValidationError
from django.db import models
from django.utils.encoding import smart_text
from collections import deque

PY2 = sys.version_info[0] == 2
if not PY2:
    string_types = (str,)
else:
    string_types = basestring
<commit_msg>Fix django < 1.5 smart_text import<commit_after>import sys

from django.core.exceptions import FieldError, ValidationError
from django.db import models
try:
    from django.utils.encoding import smart_text
except ImportError:
    from django.utils.encoding import smart_unicode as smart_text
from collections import deque

PY2 = sys.version_info[0] == 2
if not PY2:
    string_types = (str,)
else:
    string_types = basestring

{'fraction_non_alphanumeric': 0.06756756756756757, 'fraction_numerical': 0.013513513513513514, 'mean_word_length': 4.880952380952381, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3345826', 'n_tokens_mistral': 229, 'n_tokens_neox': 220, 'n_words': 73}