<commit_before>#!/usr/bin/env python

from __future__ import print_function
_IS_PYTHON_3 = sys.version_info.major == 3

import codecs
import nltk
from nltk.corpus import stopwords
import re
import string
import sys

stop_words = stopwords.words('english')

# The low end of shared words to consider
LOWER_BOUND = .20

# The high end, since anything above this is probably SEO garbage or a
# duplicate sentence
UPPER_BOUND = .90


def u(s):
    """Ensure our string is unicode independent of Python version, since Python 3 versions < 3.3 do not support the u"..." prefix"""
    if _IS_PYTHON_3:
        return s
    else:
        # not well documented but seems to work
        return codecs.unicode_escape_decode(s)[0]


def is_unimportant(word):
    """Decides if a word is ok to toss out for the sentence comparisons"""
    return word in ['.', '!', ',', ] or '\'' in word or word in stop_words
<commit_msg>Move variable so code actually runs<commit_after>#!/usr/bin/env python

from __future__ import print_function

import codecs
import nltk
from nltk.corpus import stopwords
import re
import string
import sys

_IS_PYTHON_3 = sys.version_info.major == 3

stop_words = stopwords.words('english')

# The low end of shared words to consider
LOWER_BOUND = .20

# The high end, since anything above this is probably SEO garbage or a
# duplicate sentence
UPPER_BOUND = .90


def u(s):
    """Ensure our string is unicode independent of Python version, since Python 3 versions < 3.3 do not support the u"..." prefix"""
    if _IS_PYTHON_3:
        return s
    else:
        # not well documented but seems to work
        return codecs.unicode_escape_decode(s)[0]


def is_unimportant(word):
    """Decides if a word is ok to toss out for the sentence comparisons"""
    return word in ['.', '!', ',', ] or '\'' in word or word in stop_words

{'fraction_non_alphanumeric': 0.08690928843020097, 'fraction_numerical': 0.011950027159152634, 'mean_word_length': 3.5935162094763093, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5249998', 'n_tokens_mistral': 583, 'n_tokens_neox': 566, 'n_words': 231}