<commit_before>from django.conf import settings
import re
import os
import telemeta
import mimetypes

PAGES_ROOT = os.path.join(os.path.dirname(telemeta.__file__), 'pages')

class PageTextContent(object):
    def __init__(self, filename, path):
        self.filename = filename
        self.path = path

    def __iter__(self):
        file = open(self.filename, 'r')
        for line in file:
            yield line.rstrip('\r\n')
        file.close()

    def __unicode__(self):
        file = open(self.filename, 'r')
        data = file.read()
        file.close()
        return data

class PageAttachment(object):
    def __init__(self, filename, path):
        self.filename = filename
        self.path     = path

    def mimetype(self):
        type, encoding = mimetypes.guess_type(self.filename)
        return type

    def __iter__(self):
        file = open(self.filename, 'rb')
        buffer_size = 0x10000
        while True:
            chunk = file.read(buffer_size)
            yield chunk
            if len(chunk) < buffer_size:
                break

        file.close()

def language_code(request=None):
    code = (request and getattr(request, 'LANGUAGE_CODE', None)) or settings.LANGUAGE_CODE
    cut = re.split('[_-]', code)
    code = cut[0]
    return code.lower()

def project_dir():
    import settings as settings_mod
    if '__init__.py' in settings_mod.__file__:
        p = os.path.dirname(settings_mod.__file__)
    else:
        p = settings_mod.__file__
    project_directory, settings_filename = os.path.split(p)
    if project_directory == os.curdir or not project_directory:
        project_directory = os.getcwd()<commit_msg>fix import settings (might fix #16)<commit_after>from django.conf import settings
import re
import os
import telemeta
import mimetypes

PAGES_ROOT = os.path.join(os.path.dirname(telemeta.__file__), 'pages')

class PageTextContent(object):
    def __init__(self, filename, path):
        self.filename = filename
        self.path = path

    def __iter__(self):
        file = open(self.filename, 'r')
        for line in file:
            yield line.rstrip('\r\n')
        file.close()

    def __unicode__(self):
        file = open(self.filename, 'r')
        data = file.read()
        file.close()
        return data

class PageAttachment(object):
    def __init__(self, filename, path):
        self.filename = filename
        self.path     = path

    def mimetype(self):
        type, encoding = mimetypes.guess_type(self.filename)
        return type

    def __iter__(self):
        file = open(self.filename, 'rb')
        buffer_size = 0x10000
        while True:
            chunk = file.read(buffer_size)
            yield chunk
            if len(chunk) < buffer_size:
                break

        file.close()

def language_code(request=None):
    code = (request and getattr(request, 'LANGUAGE_CODE', None)) or settings.LANGUAGE_CODE
    cut = re.split('[_-]', code)
    code = cut[0]
    return code.lower()

def project_dir():
    from django.conf import settings as settings_mod
    if '__init__.py' in settings_mod.__file__:
        p = os.path.dirname(settings_mod.__file__)
    else:
        p = settings_mod.__file__
    project_directory, settings_filename = os.path.split(p)
    if project_directory == os.curdir or not project_directory:
        project_directory = os.getcwd()
{'fraction_non_alphanumeric': 0.1030500444181226, 'fraction_numerical': 0.004737933076695291, 'mean_word_length': 2.486068111455108, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1744898', 'n_tokens_mistral': 1033, 'n_tokens_neox': 1003, 'n_words': 256}