<commit_before>from __future__ import unicode_literals
from __future__ import print_function

import weakref
from random import choice
import logging
import re

from fs.errors import FSError
from fs.opener import open_fs
from fs import walk
from fs import wrap
from collections import defaultdict

from . import errors
from .parser import Parser
from . import xmlreport
from .settings import SettingContainer, SettingsContainer, SettingsSectionContainer
from .importer import fs_import
from .elements import ElementBase
from . import pyversion
from .tools import textual_list, split_commas
from . import translations
from .versioning import Version, VersionSpec
from .compat import iteritems, implements_to_string, string_types, text_type, xrange, PY2
from .context import Context


log = logging.getLogger("moya")
startup_log = logging.getLogger("moya.startup")
tests_log = logging.getLogger("moya.tests")


def _make_id():
    """A random token to make default libnames unique"""
    _ID_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
    return ''.join(choice(_ID_CHARS) for _ in xrange(6))


class FailedDocument(object):
    def __init__(self, path, code, line, col, msg, diagnosis=None):
        self.path = path
        self.code = code
        self.line = line
        self.col = col
        self.msg = msg
        self.diagnosis = diagnosis
        super(FailedDocument, self).__init__()

    def __repr__(self):
        return "<faileddocument '{}', line {} ({})>".format(self.path, self.line, self.msg)

    def render(self):
        error = xmlreport.render_error(self.code,
                                       self.line,
                                       col=None,
                                       col_text="here")
        return "[Error parsing %s]\n%s:\n\n%s\n" % (self.path, self.msg, error)


@implements_to_string
class Library(object):
    """A collection of documents"""

    def __init__(self, archive, fs=None, settings_path=None, long_name=None, namespace=None, rebuild=False):
        self._archive = weakref.ref(archive)
        self.documents = []
        self.elements_by_name = {}
        self.elements_by_type = defaultdict(list)
        self.long_name = long_name
        self.no_py = rebuild
        self.namespace = namespace
        self.settings = SettingsSectionContainer()
        self.system_settings = {}
        self.templates_info = {}
        self.data_info = {}
        self.data_fs = None
        self.media = {}
        self.author = {}
        self.libinfo = {}
        self.version = None
        self.load_fs = None
        self.loaded_ini = None

        # A list of the documents that failed to import
        self.failed_documents = []
        self._libnames = defaultdict(int)
        self.replace_nodes = defaultdict(list)

        self.py = {}
        self.filters = {}
        self.loaded = False
        self.imported_documents = False
        self.documentation_location = None
        self.translations_location = None
        self.default_language = "en"
        self.languages = []
        self.translations = translations.Translations()

        self.priority = 0
        self.template_priority = 0
        self.data_priority = None
        self.built = False
        self.finalized = False
        self.imported_tests = False

        self._cfg = None

        if fs:
            self.load(fs, settings_path=settings_path)

    @property
    def archive(self):
        return self._archive()

    @property
    def enum(self):
        return self.archive.get_lib_enums(self.long_name)

    @property
    def install_location(self):
        try:
            return self.load_fs.getsyspath('/')
        except FSError:
            return None

    @property
    def cfg(self):
        return self._cfg

    @property
    def version_spec(self):
        """Get a version spec for precisely this version"""
        return VersionSpec("{}=={}".format(self.long_name, self.version))

    @property
    def version_name(self):
        return "{} {}".format(self.long_name, self.version)

    def __str__(self):
        return '<library {}>'.format(self.long_name)

    def __repr__(self):
        return '<library {}>'.format(self.long_name)

    def add_replacement_node(self, element):
        self.replace_nodes[element.libname].append(element)

    def get_element(self, element_ref, app=None):
        return self.archive.get_element(element_ref, app=app, lib=self.lib)

    def allocate_libname(self, docid):
        """Allocates a name, unique to the library"""
        new_libname = docid
        while new_libname in self.elements_by_name:
            new_libname = "{}.{}".format(docid, _make_id())
        return new_libname

        if docid not in self.elements_by_name:
            return docid

    def qualify_libname(self, libname):
        """Qualifies the libname to identify the container library (if it isn't already qualified)"""
        if '#' not in libname:
            return "%s#%s" % (self.long_name, libname)
        else:
            if libname.startswith('#'):
                return self.long_name + libname
            return libname

    def import_document(self, fs, path):
        """Import a single document"""
        return self.import_documents(fs, files=(path,))

    def import_documents(self, fs, wildcard="*.xml", recurse=False, files=None):
        """Imports a number of documents in to the library"""

        if isinstance(fs, string_types):
            fs = open_fs(fs)
        #fs = wrap.DirCache(fs)
        if files is None:
            if recurse:
                files = sorted(fs.walk.files(filter=[wildcard]))
            else:
                files = sorted(
                    name
                    for name, _is_dir in fs.filterdir(
                        files=[wildcard],
                        exclude_dirs=True
                    )
                )
        else:
            files = sorted(files)

        documents = {}
        import_count = 0
        for filepath in files:
            parser = Parser(self.archive, fs, filepath, library=self)
            try:
                document = parser.parse()
                if document is not None:
                    self.documents.append(document)
                    import_count += 1
                    documents[filepath] = document.structure

            except errors.ParseError as parse_error:
                line, col = parse_error.position

                if fs.hassyspath(filepath):
                    path = fs.getsyspath(filepath)
                else:
                    path = fs.desc(filepath)
                failed_doc = FailedDocument(path=path,
                                            code=parser.xml,<commit_msg>import files fix<commit_after>from __future__ import unicode_literals
from __future__ import print_function

import weakref
from random import choice
import logging
import re

from fs.errors import FSError
from fs.opener import open_fs
from fs import walk
from fs import wrap
from collections import defaultdict

from . import errors
from .parser import Parser
from . import xmlreport
from .settings import SettingContainer, SettingsContainer, SettingsSectionContainer
from .importer import fs_import
from .elements import ElementBase
from . import pyversion
from .tools import textual_list, split_commas
from . import translations
from .versioning import Version, VersionSpec
from .compat import iteritems, implements_to_string, string_types, text_type, xrange, PY2
from .context import Context


log = logging.getLogger("moya")
startup_log = logging.getLogger("moya.startup")
tests_log = logging.getLogger("moya.tests")


def _make_id():
    """A random token to make default libnames unique"""
    _ID_CHARS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
    return ''.join(choice(_ID_CHARS) for _ in xrange(6))


class FailedDocument(object):
    def __init__(self, path, code, line, col, msg, diagnosis=None):
        self.path = path
        self.code = code
        self.line = line
        self.col = col
        self.msg = msg
        self.diagnosis = diagnosis
        super(FailedDocument, self).__init__()

    def __repr__(self):
        return "<faileddocument '{}', line {} ({})>".format(self.path, self.line, self.msg)

    def render(self):
        error = xmlreport.render_error(self.code,
                                       self.line,
                                       col=None,
                                       col_text="here")
        return "[Error parsing %s]\n%s:\n\n%s\n" % (self.path, self.msg, error)


@implements_to_string
class Library(object):
    """A collection of documents"""

    def __init__(self, archive, fs=None, settings_path=None, long_name=None, namespace=None, rebuild=False):
        self._archive = weakref.ref(archive)
        self.documents = []
        self.elements_by_name = {}
        self.elements_by_type = defaultdict(list)
        self.long_name = long_name
        self.no_py = rebuild
        self.namespace = namespace
        self.settings = SettingsSectionContainer()
        self.system_settings = {}
        self.templates_info = {}
        self.data_info = {}
        self.data_fs = None
        self.media = {}
        self.author = {}
        self.libinfo = {}
        self.version = None
        self.load_fs = None
        self.loaded_ini = None

        # A list of the documents that failed to import
        self.failed_documents = []
        self._libnames = defaultdict(int)
        self.replace_nodes = defaultdict(list)

        self.py = {}
        self.filters = {}
        self.loaded = False
        self.imported_documents = False
        self.documentation_location = None
        self.translations_location = None
        self.default_language = "en"
        self.languages = []
        self.translations = translations.Translations()

        self.priority = 0
        self.template_priority = 0
        self.data_priority = None
        self.built = False
        self.finalized = False
        self.imported_tests = False

        self._cfg = None

        if fs:
            self.load(fs, settings_path=settings_path)

    @property
    def archive(self):
        return self._archive()

    @property
    def enum(self):
        return self.archive.get_lib_enums(self.long_name)

    @property
    def install_location(self):
        try:
            return self.load_fs.getsyspath('/')
        except FSError:
            return None

    @property
    def cfg(self):
        return self._cfg

    @property
    def version_spec(self):
        """Get a version spec for precisely this version"""
        return VersionSpec("{}=={}".format(self.long_name, self.version))

    @property
    def version_name(self):
        return "{} {}".format(self.long_name, self.version)

    def __str__(self):
        return '<library {}>'.format(self.long_name)

    def __repr__(self):
        return '<library {}>'.format(self.long_name)

    def add_replacement_node(self, element):
        self.replace_nodes[element.libname].append(element)

    def get_element(self, element_ref, app=None):
        return self.archive.get_element(element_ref, app=app, lib=self.lib)

    def allocate_libname(self, docid):
        """Allocates a name, unique to the library"""
        new_libname = docid
        while new_libname in self.elements_by_name:
            new_libname = "{}.{}".format(docid, _make_id())
        return new_libname

        if docid not in self.elements_by_name:
            return docid

    def qualify_libname(self, libname):
        """Qualifies the libname to identify the container library (if it isn't already qualified)"""
        if '#' not in libname:
            return "%s#%s" % (self.long_name, libname)
        else:
            if libname.startswith('#'):
                return self.long_name + libname
            return libname

    def import_document(self, fs, path):
        """Import a single document"""
        return self.import_documents(fs, files=(path,))

    def import_documents(self, fs, wildcard="*.xml", recurse=False, files=None):
        """Imports a number of documents in to the library"""

        if isinstance(fs, string_types):
            fs = open_fs(fs)
        #fs = wrap.DirCache(fs)
        if files is None:
            if recurse:
                files = sorted(fs.walk.files(filter=[wildcard]))
            else:
                files = sorted(
                    info.name
                    for info in fs.filterdir(
                        files=[wildcard],
                        exclude_dirs=['*']
                    )
                )
        else:
            files = sorted(files)

        documents = {}
        import_count = 0
        for filepath in files:
            parser = Parser(self.archive, fs, filepath, library=self)
            try:
                document = parser.parse()
                if document is not None:
                    self.documents.append(document)
                    import_count += 1
                    documents[filepath] = document.structure

            except errors.ParseError as parse_error:
                line, col = parse_error.position

                if fs.hassyspath(filepath):
                    path = fs.getsyspath(filepath)
                else:
                    path = fs.desc(filepath)
                failed_doc = FailedDocument(path=path,
                                            code=parser.xml,
{'fraction_non_alphanumeric': 0.09442793462109955, 'fraction_numerical': 0.00237741456166419, 'mean_word_length': 2.0782071804253373, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6109621', 'n_tokens_mistral': 3810, 'n_tokens_neox': 3635, 'n_words': 948}