<commit_before># -*- coding: utf-8 -*-

# Copyright (C) 2014 <NAME> <iv at altlinux dot org>
#
# Author: <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import errno
import os

import six

from docutils import frontend
from docutils import parsers as docutils_parser
from docutils import utils


class ParsedFile(object):
    def __init__(self, filename, encoding='utf8'):
        self._filename = filename
        self._content = None
        self._raw_content = None
        self._encoding = encoding
        self._doc = None

    @property
    def document(self):
        if self._doc is None:
            # Use the rst parsers document output to do as much of the
            # validation as we can without resorting to custom logic (this
            # parser is what sphinx and others use anyway so it's hopefully
            # mature).
            parser_cls = docutils_parser.get_parser_class("rst")
            parser = parser_cls()
            defaults = {
                'input_encoding': self.encoding,
                'halt_level': 5,
                'report_level': 5,
                'quiet': True,
                'file_insertion_enabled': False,
                'traceback': True,
            }
            opt = frontend.OptionParser(components=[parser], defaults=defaults)
            self._doc = utils.new_document(source_path=self.filename,
                                           settings=opt.get_default_values())
            parser.parse(self.raw_contents, self._doc)
        return self._doc

    def lines_iter(self, remove_trailing_newline=True):
        with open(self.filename, 'rb') as fh:
            for line in fh:
                line = six.text_type(line, encoding=self.encoding)
                if remove_trailing_newline and line.endswith("\n"):
                    line = line[0:-1]
                yield line

    @property
    def filename(self):
        return self._filename

    @property
    def encoding(self):
        return self._encoding

    @property
    def raw_contents(self):
        if self._raw_content is None:
            with open(self.filename, 'rb') as fh:
                self._raw_content = fh.read()
        return self._raw_content

    @property
    def contents(self):
        if self._content is None:
            self._content = six.text_type(self.raw_contents,
                                          encoding=self.encoding)
        return self._content<commit_msg>Use the non-raw contents to ensure 3.3+ compat.<commit_after># -*- coding: utf-8 -*-

# Copyright (C) 2014 <NAME> <iv at altlinux dot org>
#
# Author: <NAME> <<EMAIL>>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import errno
import os

import six

from docutils import frontend
from docutils import parsers as docutils_parser
from docutils import utils


class ParsedFile(object):
    def __init__(self, filename, encoding='utf8'):
        self._filename = filename
        self._content = None
        self._raw_content = None
        self._encoding = encoding
        self._doc = None

    @property
    def document(self):
        if self._doc is None:
            # Use the rst parsers document output to do as much of the
            # validation as we can without resorting to custom logic (this
            # parser is what sphinx and others use anyway so it's hopefully
            # mature).
            parser_cls = docutils_parser.get_parser_class("rst")
            parser = parser_cls()
            defaults = {
                'input_encoding': self.encoding,
                'halt_level': 5,
                'report_level': 5,
                'quiet': True,
                'file_insertion_enabled': False,
                'traceback': True,
            }
            opt = frontend.OptionParser(components=[parser], defaults=defaults)
            self._doc = utils.new_document(source_path=self.filename,
                                           settings=opt.get_default_values())
            parser.parse(self.contents, self._doc)
        return self._doc

    def lines_iter(self, remove_trailing_newline=True):
        with open(self.filename, 'rb') as fh:
            for line in fh:
                line = six.text_type(line, encoding=self.encoding)
                if remove_trailing_newline and line.endswith("\n"):
                    line = line[0:-1]
                yield line

    @property
    def filename(self):
        return self._filename

    @property
    def encoding(self):
        return self._encoding

    @property
    def raw_contents(self):
        if self._raw_content is None:
            with open(self.filename, 'rb') as fh:
                self._raw_content = fh.read()
        return self._raw_content

    @property
    def contents(self):
        if self._content is None:
            self._content = six.text_type(self.raw_contents,
                                          encoding=self.encoding)
        return self._content
{'fraction_non_alphanumeric': 0.08179064288118479, 'fraction_numerical': 0.005048805116122518, 'mean_word_length': 2.0840685002594705, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4067090', 'n_tokens_mistral': 1580, 'n_tokens_neox': 1518, 'n_words': 522}