<commit_before># -*- coding: utf-8 -*-
import os

from logya.compat import file_open as open
from logya.compat import is3
from logya.encoder import JSONEncoder
from logya.globals import allowed_exts


class FileWriter(object):
    """Class for writing site files."""

    def get_canonical_filename(self, name):
        """Get file name from given path or file.

        If name is not recognized as a file name a /index.html is added.
        To be recognized as a file name it must end in one of self.allowed_exts.
        Leading slashes are stripped off.
        """

        # TODO explain this
        if not name.startswith('/'):
            name = '/%s' % name

        # only allowed extension will be written to a file, otherwise a
        # directory with the name is created and content written to index.html
        fext = os.path.splitext(name)[1]
        if not fext or fext.lstrip('.') not in allowed_exts:
            name = os.path.join(name, 'index.html')

        return name.lstrip('/')

    def getfile(self, dir_dst, path):
        """Determine file to create and return an open file handle for writing.

        Paths pointing to a file name will be created as they are. When a path
        points to a directory a file named index.html will be created in that
        directory.
        """

        filename = self.get_canonical_filename(path)
        # create target directory if it doesn't exist
        dir_target = os.path.join(dir_dst, os.path.dirname(filename))
        if not os.path.exists(dir_target):
            os.makedirs(dir_target)
        return open(os.path.join(dir_dst, filename), 'w', encoding='utf-8')

    def write(self, file, content):
        """Write content to file and close it."""

        if not is3:
            content = content.encode('utf-8')
        file.write(content)
        file.close()


class DocWriter(FileWriter):
    """Class for writing site documents."""

    def __init__(self, dir_dst, template):
        """Set required properties."""

        self.dir_dst = dir_dst
        self.template = template
        self.encoder = JSONEncoder()

    def set_template_vars(self, doc):
        """Set template variables."""

        # empty doc vars dictionary to not retain previous doc values
        self.template.empty_doc_vars()
        for field, val in list(doc.items()):
            if isinstance(val, str) and not is3:
                val = val.decode('utf-8')
            self.template.add_doc_var(field, val)

    def write(self, doc, template):
        """Render and write document to created file.

        Returns False if template is False.
        """

        if not template:
            print(('Warning: doc %s has no template set and won\'t be created.'
                % doc['url']))<commit_msg>removed JSONencoder from writer<commit_after># -*- coding: utf-8 -*-
import os

from logya.compat import file_open as open
from logya.compat import is3
from logya.globals import allowed_exts


class FileWriter(object):
    """Class for writing site files."""

    def get_canonical_filename(self, name):
        """Get file name from given path or file.

        If name is not recognized as a file name a /index.html is added.
        To be recognized as a file name it must end in one of self.allowed_exts.
        Leading slashes are stripped off.
        """

        # TODO explain this
        if not name.startswith('/'):
            name = '/%s' % name

        # only allowed extension will be written to a file, otherwise a
        # directory with the name is created and content written to index.html
        fext = os.path.splitext(name)[1]
        if not fext or fext.lstrip('.') not in allowed_exts:
            name = os.path.join(name, 'index.html')

        return name.lstrip('/')

    def getfile(self, dir_dst, path):
        """Determine file to create and return an open file handle for writing.

        Paths pointing to a file name will be created as they are. When a path
        points to a directory a file named index.html will be created in that
        directory.
        """

        filename = self.get_canonical_filename(path)
        # create target directory if it doesn't exist
        dir_target = os.path.join(dir_dst, os.path.dirname(filename))
        if not os.path.exists(dir_target):
            os.makedirs(dir_target)
        return open(os.path.join(dir_dst, filename), 'w', encoding='utf-8')

    def write(self, file, content):
        """Write content to file and close it."""

        if not is3:
            content = content.encode('utf-8')
        file.write(content)
        file.close()


class DocWriter(FileWriter):
    """Class for writing site documents."""

    def __init__(self, dir_dst, template):
        """Set required properties."""

        self.dir_dst = dir_dst
        self.template = template

    def set_template_vars(self, doc):
        """Set template variables."""

        # empty doc vars dictionary to not retain previous doc values
        self.template.empty_doc_vars()
        for field, val in list(doc.items()):
            if isinstance(val, str) and not is3:
                val = val.decode('utf-8')
            self.template.add_doc_var(field, val)

    def write(self, doc, template):
        """Render and write document to created file.

        Returns False if template is False.
        """

        if not template:
            print(('Warning: doc %s has no template set and won\'t be created.'
                % doc['url']))
{'fraction_non_alphanumeric': 0.09682107175295186, 'fraction_numerical': 0.0029064486830154405, 'mean_word_length': 2.2676557863501485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1016684', 'n_tokens_mistral': 1525, 'n_tokens_neox': 1475, 'n_words': 570}