<commit_before># -*- coding: utf-8 -*-

"""
Metadata Parser
~~~~~~~~~~~~~~~
This module supports the relation parser by handling statements.
"""

import logging
import re

from pyparsing import Suppress, And, Word, Optional, MatchFirst
from pyparsing import pyparsing_common as ppc

from pybel.parser.utils import word, quote, delimitedSet
from .baseparser import BaseParser
from .parse_exceptions import *
from ..constants import *

__all__ = ['MetadataParser']

log = logging.getLogger(__name__)

as_tag = Suppress(BEL_KEYWORD_AS)
url_tag = Suppress(BEL_KEYWORD_URL)
list_tag = Suppress(BEL_KEYWORD_LIST)
owl_tag = Suppress(BEL_KEYWORD_OWL)
set_tag = Suppress(BEL_KEYWORD_SET)
define_tag = Suppress(BEL_KEYWORD_DEFINE)

function_tags = Word(''.join(belns_encodings))

value = quote | ppc.identifier

SEMANTIC_VERSION_STRING_RE = re.compile(
    '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<release>[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+(?P<build>[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?')


class MetadataParser(BaseParser):
    """A parser for the document and definitions section of a BEL document.

    .. seealso::

        BEL 1.0 Specification for the `DEFINE <http://openbel.org/language/web/version_1.0/bel_specification_version_1.0.html#_define>`_ keyword
    """

    def __init__(self, cache_manager, namespace_dict=None, annotation_dict=None, namespace_regex=None,
                 annotations_regex=None, default_namespace=None):
        """
        :param cache_manager: A cache manager
        :type cache_manager: pybel.manager.cache.CacheManager
        :param namespace_dict: A dictionary of pre-loaded, enumerated namespaces from 
                                {namespace keyword: set of valid values}
        :type namespace_dict: dict
        :param annotation_dict: A dictionary of pre-loaded, enumerated annotations from
                                {annotation keyword: set of valid values}
        :type annotation_dict: dict
        :param namespace_regex: A dictionary of pre-loaded, regular expression namespaces from
                                {namespace keyword: regex string}
        :type namespace_regex: dict
        :param annotations_regex: A dictionary of pre-loaded, regular expression annotations from 
                                {annotation keyword: regex string}
        :type annotations_regex: dict
        :param default_namespace: A set of strings that can be used without a namespace
        :type default_namespace: set of str
        """
        #: This metadata parser's internal definition cache manager
        self.cache_manager = cache_manager
        #: A dictionary of cached {namespace keyword: set of values}
        self.namespace_dict = {} if namespace_dict is None else namespace_dict
        #: A dictionary of cached {annotation keyword: set of values}
        self.annotations_dict = {} if annotation_dict is None else annotation_dict
        #: A dictionary of {namespace keyword: regular expression string}
        self.namespace_regex = {} if namespace_regex is None else namespace_regex
        #: A dictionary of {namespace keyword: compiled regular expression}
        self.namespace_regex_compiled = {ns: re.compile(pat) for ns, pat in self.namespace_regex.items()}
        #: A set of names that can be used without a namespace
        self.default_namespace = set(default_namespace) if default_namespace is not None else None
        #: A dictionary of {annotation keyword: regular expression string}
        self.annotations_regex = {} if annotations_regex is None else annotations_regex
        #: A dictionary of {annotation keyword: compiled regular expression}
        self.annotations_regex_compiled = {ns: re.compile(pat) for ns, pat in self.annotations_regex.items()}

        #: A dictionary containing the document metadata
        self.document_metadata = {}

        #: A dictionary from {namespace keyword: BEL namespace URL}
        self.namespace_url_dict = {}
        #: A dictionary from {namespace keyword: OWL namespace URL}
        self.namespace_owl_dict = {}
        #: A dictionary from {annotation keyword: BEL annotation URL}
        self.annotation_url_dict = {}
        #: A dictionary from {annotation keyword: OWL annotation URL}
        self.annotations_owl_dict = {}
        #: A list of annotations that are defined ad-hoc in the BEL script
        self.annotation_list_list = []  # TODO switch to set

        self.document = And([set_tag, Suppress(BEL_KEYWORD_DOCUMENT), word('key'), Suppress('='), value('value')])

        namespace_tag = And([define_tag, Suppress(BEL_KEYWORD_NAMESPACE), ppc.identifier('name'), as_tag])
        self.namespace_url = And([namespace_tag, url_tag, quote('url')])
        self.namespace_owl = And([namespace_tag, owl_tag, Optional(function_tags('functions')), quote('url')])
        self.namespace_pattern = And([namespace_tag, Suppress(BEL_KEYWORD_PATTERN), quote('value')])

        annotation_tag = And([define_tag, Suppress(BEL_KEYWORD_ANNOTATION), ppc.identifier('name'), as_tag])
        self.annotation_url = And([annotation_tag, url_tag, quote('url')])
        self.annotation_owl = And([annotation_tag, owl_tag, quote('url')])
        self.annotation_list = And([annotation_tag, list_tag, delimitedSet('values')])
        self.annotation_pattern = And([annotation_tag, Suppress(BEL_KEYWORD_PATTERN), quote('value')])

        self.document.setParseAction(self.handle_document)
        self.namespace_url.setParseAction(self.handle_namespace_url)
        self.namespace_owl.setParseAction(self.handle_namespace_owl)
        self.namespace_pattern.setParseAction(self.handle_namespace_pattern)
        self.annotation_url.setParseAction(self.handle_annotations_url)
        self.annotation_owl.setParseAction(self.handle_annotation_owl)
        self.annotation_list.setParseAction(self.handle_annotation_list)
        self.annotation_pattern.setParseAction(self.handle_annotation_pattern)

        self.language = MatchFirst([
            self.document,
            self.namespace_url,
            self.namespace_owl,
            self.annotation_url,
            self.annotation_list,
            self.annotation_owl,
            self.annotation_pattern,
            self.namespace_pattern
        ])

        BaseParser.__init__(self, self.language)

        log.info('Metadata parser connected to %s', self.cache_manager.connection)

    def handle_document(self, s, l, tokens):
        key = tokens['key']
        value = tokens['value']

        if key not in DOCUMENT_KEYS:
            raise InvalidMetadataException(key, value)

        norm_key = DOCUMENT_KEYS[key]

        if norm_key in self.document_metadata:
            log.warning('Tried to overwrite metadata: %s', key)
            return tokens

        self.document_metadata[norm_key] = value

        if norm_key == METADATA_VERSION and not SEMANTIC_VERSION_STRING_RE.match(value):
            raise NotSemanticVersionException(value)

        return tokens

    def handle_namespace_url(self, s, l, tokens):
        namespace = tokens['name']

        if self.has_namespace(namespace):
            log.warning('Tried to overwrite namespace: %s', namespace)
            return tokens

        url = tokens['url']
        terms = self.cache_manager.get_namespace(url)

        self.namespace_dict[namespace] = terms
        self.namespace_url_dict[namespace] = url

        return tokens

    def handle_namespace_owl(self, s, l, tokens):
        namespace = tokens['name']

        if self.has_namespace(namespace):
            log.warning('Tried to overwrite owl namespace: %s', namespace)
            return tokens

        functions = set(tokens['functions'] if 'functions' in tokens else belns_encodings)

        url = tokens['url']

        terms = self.cache_manager.get_namespace_owl_terms(url)

        self.namespace_dict[namespace] = {term: functions for term in terms}
        self.namespace_owl_dict[namespace] = url

        return tokens

    def handle_namespace_pattern(self, s, l, tokens):
        namespace = tokens['name']

        if self.has_namespace(namespace):
            log.warning('Tried to overwrite namespace: {}'.format(namespace))
            return tokens

        value = tokens['value']

        self.namespace_regex[namespace] = value
        self.namespace_regex_compiled[namespace] = re.compile(value)

        return tokens

    def handle_annotation_owl(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: {}'.format(annotation))
            return tokens

        url = tokens['url']

        terms = self.cache_manager.get_annotation_owl_terms(url)

        self.annotations_dict[annotation] = set(terms)
        self.annotations_owl_dict[annotation] = url

        return tokens

    def handle_annotations_url(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: %s', annotation)
            return tokens

        url = tokens['url']

        self.annotations_dict[annotation] = self.cache_manager.get_annotation(url)
        self.annotation_url_dict[annotation] = url

        return tokens

    def handle_annotation_list(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: {}'.format(annotation))
            return tokens

        values = set(tokens['values'])

        self.annotations_dict[annotation] = values
        self.annotation_list_list.append(annotation)

        return tokens

    def handle_annotation_pattern(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: {}'.format(annotation))
            return tokens

        value = tokens['value']

        self.annotations_regex[annotation] = value
        self.annotations_regex_compiled[annotation] = re.compile(value)

        return tokens

    def has_enumerated_annotation(self, annotation):
        return annotation in self.annotations_dict

    def has_regex_annotation(self, annotation):
        return annotation in self.annotations_regex

    def has_annotation(self, annotation):
        return self.has_enumerated_annotation(annotation) or self.has_regex_annotation(annotation)

    def raise_for_missing_annotation(self, annotation):
        if not self.has_annotation(annotation):
            raise UndefinedAnnotationWarning(annotation)

    def has_enumerated_annotation_value(self, annotation, value):
        return annotation in self.annotations_dict and value in self.annotations_dict[annotation]

    def has_regex_annotation_value(self, annotation, value):
        return annotation in self.annotations_regex and self.annotations_regex_compiled[annotation].match(value)

    def has_annotation_value(self, annotation, value):
        self.raise_for_missing_annotation(annotation)

        return self.has_enumerated_annotation_value(annotation, value) or self.has_regex_annotation_value(annotation,
                                                                                                          value)

    def raise_for_missing_annotation_value(self, annotation, value):
        self.raise_for_missing_annotation(annotation)

        if self.has_enumerated_annotation(annotation) and not self.has_enumerated_annotation_value(annotation, value):
            raise IllegalAnnotationValueWarning(value, annotation)

        if self.has_regex_annotation(annotation) and not self.has_regex_annotation_value(annotation, value):
            raise MissingAnnotationRegexWarning(value, annotation)

    def has_enumerated_namespace(self, namespace):
        return namespace in self.namespace_dict

    def has_regex_namespace(self, namespace):
        return namespace in self.namespace_regex

    def has_namespace(self, namespace):
        return self.has_enumerated_namespace(namespace) or self.has_regex_namespace(namespace)

    def raise_for_missing_namespace(self, namespace):
        if not self.has_namespace(namespace):
            raise UndefinedNamespaceWarning(namespace)

    def has_enumerated_namespace_name(self, namespace, name):
        return self.has_enumerated_namespace(namespace) and name in self.namespace_dict[namespace]

    def has_regex_namespace_name(self, namespace, name):
        return namespace in self.namespace_regex_compiled and self.namespace_regex_compiled[namespace].match(name)

    def has_namespace_name(self, namespace, name):
        self.raise_for_missing_namespace(namespace)

        return self.has_enumerated_namespace_name(namespace, name) or self.has_regex_namespace_name(namespace, name)

    def raise_for_missing_name(self, namespace, name):
        self.raise_for_missing_namespace(namespace)

        if self.has_enumerated_namespace(namespace) and not self.has_enumerated_namespace_name(namespace, name):
            raise MissingNamespaceNameWarning(name, namespace)

        if self.has_regex_namespace(namespace) and not self.has_regex_namespace_name(namespace, name):
            raise MissingNamespaceRegexWarning(name, namespace)
<commit_msg>Removed extra logging from metadata parser<commit_after># -*- coding: utf-8 -*-

"""
Metadata Parser
~~~~~~~~~~~~~~~
This module supports the relation parser by handling statements.
"""

import logging
import re

from pyparsing import Suppress, And, Word, Optional, MatchFirst
from pyparsing import pyparsing_common as ppc

from pybel.parser.utils import word, quote, delimitedSet
from .baseparser import BaseParser
from .parse_exceptions import *
from ..constants import *

__all__ = ['MetadataParser']

log = logging.getLogger(__name__)

as_tag = Suppress(BEL_KEYWORD_AS)
url_tag = Suppress(BEL_KEYWORD_URL)
list_tag = Suppress(BEL_KEYWORD_LIST)
owl_tag = Suppress(BEL_KEYWORD_OWL)
set_tag = Suppress(BEL_KEYWORD_SET)
define_tag = Suppress(BEL_KEYWORD_DEFINE)

function_tags = Word(''.join(belns_encodings))

value = quote | ppc.identifier

SEMANTIC_VERSION_STRING_RE = re.compile(
    '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<release>[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?(?:\+(?P<build>[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*))?')


class MetadataParser(BaseParser):
    """A parser for the document and definitions section of a BEL document.

    .. seealso::

        BEL 1.0 Specification for the `DEFINE <http://openbel.org/language/web/version_1.0/bel_specification_version_1.0.html#_define>`_ keyword
    """

    def __init__(self, cache_manager, namespace_dict=None, annotation_dict=None, namespace_regex=None,
                 annotations_regex=None, default_namespace=None):
        """
        :param cache_manager: A cache manager
        :type cache_manager: pybel.manager.cache.CacheManager
        :param namespace_dict: A dictionary of pre-loaded, enumerated namespaces from 
                                {namespace keyword: set of valid values}
        :type namespace_dict: dict
        :param annotation_dict: A dictionary of pre-loaded, enumerated annotations from
                                {annotation keyword: set of valid values}
        :type annotation_dict: dict
        :param namespace_regex: A dictionary of pre-loaded, regular expression namespaces from
                                {namespace keyword: regex string}
        :type namespace_regex: dict
        :param annotations_regex: A dictionary of pre-loaded, regular expression annotations from 
                                {annotation keyword: regex string}
        :type annotations_regex: dict
        :param default_namespace: A set of strings that can be used without a namespace
        :type default_namespace: set of str
        """
        #: This metadata parser's internal definition cache manager
        self.cache_manager = cache_manager
        #: A dictionary of cached {namespace keyword: set of values}
        self.namespace_dict = {} if namespace_dict is None else namespace_dict
        #: A dictionary of cached {annotation keyword: set of values}
        self.annotations_dict = {} if annotation_dict is None else annotation_dict
        #: A dictionary of {namespace keyword: regular expression string}
        self.namespace_regex = {} if namespace_regex is None else namespace_regex
        #: A dictionary of {namespace keyword: compiled regular expression}
        self.namespace_regex_compiled = {ns: re.compile(pat) for ns, pat in self.namespace_regex.items()}
        #: A set of names that can be used without a namespace
        self.default_namespace = set(default_namespace) if default_namespace is not None else None
        #: A dictionary of {annotation keyword: regular expression string}
        self.annotations_regex = {} if annotations_regex is None else annotations_regex
        #: A dictionary of {annotation keyword: compiled regular expression}
        self.annotations_regex_compiled = {ns: re.compile(pat) for ns, pat in self.annotations_regex.items()}

        #: A dictionary containing the document metadata
        self.document_metadata = {}

        #: A dictionary from {namespace keyword: BEL namespace URL}
        self.namespace_url_dict = {}
        #: A dictionary from {namespace keyword: OWL namespace URL}
        self.namespace_owl_dict = {}
        #: A dictionary from {annotation keyword: BEL annotation URL}
        self.annotation_url_dict = {}
        #: A dictionary from {annotation keyword: OWL annotation URL}
        self.annotations_owl_dict = {}
        #: A list of annotations that are defined ad-hoc in the BEL script
        self.annotation_list_list = []  # TODO switch to set

        self.document = And([set_tag, Suppress(BEL_KEYWORD_DOCUMENT), word('key'), Suppress('='), value('value')])

        namespace_tag = And([define_tag, Suppress(BEL_KEYWORD_NAMESPACE), ppc.identifier('name'), as_tag])
        self.namespace_url = And([namespace_tag, url_tag, quote('url')])
        self.namespace_owl = And([namespace_tag, owl_tag, Optional(function_tags('functions')), quote('url')])
        self.namespace_pattern = And([namespace_tag, Suppress(BEL_KEYWORD_PATTERN), quote('value')])

        annotation_tag = And([define_tag, Suppress(BEL_KEYWORD_ANNOTATION), ppc.identifier('name'), as_tag])
        self.annotation_url = And([annotation_tag, url_tag, quote('url')])
        self.annotation_owl = And([annotation_tag, owl_tag, quote('url')])
        self.annotation_list = And([annotation_tag, list_tag, delimitedSet('values')])
        self.annotation_pattern = And([annotation_tag, Suppress(BEL_KEYWORD_PATTERN), quote('value')])

        self.document.setParseAction(self.handle_document)
        self.namespace_url.setParseAction(self.handle_namespace_url)
        self.namespace_owl.setParseAction(self.handle_namespace_owl)
        self.namespace_pattern.setParseAction(self.handle_namespace_pattern)
        self.annotation_url.setParseAction(self.handle_annotations_url)
        self.annotation_owl.setParseAction(self.handle_annotation_owl)
        self.annotation_list.setParseAction(self.handle_annotation_list)
        self.annotation_pattern.setParseAction(self.handle_annotation_pattern)

        self.language = MatchFirst([
            self.document,
            self.namespace_url,
            self.namespace_owl,
            self.annotation_url,
            self.annotation_list,
            self.annotation_owl,
            self.annotation_pattern,
            self.namespace_pattern
        ])

        BaseParser.__init__(self, self.language)

    def handle_document(self, s, l, tokens):
        key = tokens['key']
        value = tokens['value']

        if key not in DOCUMENT_KEYS:
            raise InvalidMetadataException(key, value)

        norm_key = DOCUMENT_KEYS[key]

        if norm_key in self.document_metadata:
            log.warning('Tried to overwrite metadata: %s', key)
            return tokens

        self.document_metadata[norm_key] = value

        if norm_key == METADATA_VERSION and not SEMANTIC_VERSION_STRING_RE.match(value):
            raise NotSemanticVersionException(value)

        return tokens

    def handle_namespace_url(self, s, l, tokens):
        namespace = tokens['name']

        if self.has_namespace(namespace):
            log.warning('Tried to overwrite namespace: %s', namespace)
            return tokens

        url = tokens['url']
        terms = self.cache_manager.get_namespace(url)

        self.namespace_dict[namespace] = terms
        self.namespace_url_dict[namespace] = url

        return tokens

    def handle_namespace_owl(self, s, l, tokens):
        namespace = tokens['name']

        if self.has_namespace(namespace):
            log.warning('Tried to overwrite owl namespace: %s', namespace)
            return tokens

        functions = set(tokens['functions'] if 'functions' in tokens else belns_encodings)

        url = tokens['url']

        terms = self.cache_manager.get_namespace_owl_terms(url)

        self.namespace_dict[namespace] = {term: functions for term in terms}
        self.namespace_owl_dict[namespace] = url

        return tokens

    def handle_namespace_pattern(self, s, l, tokens):
        namespace = tokens['name']

        if self.has_namespace(namespace):
            log.warning('Tried to overwrite namespace: {}'.format(namespace))
            return tokens

        value = tokens['value']

        self.namespace_regex[namespace] = value
        self.namespace_regex_compiled[namespace] = re.compile(value)

        return tokens

    def handle_annotation_owl(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: {}'.format(annotation))
            return tokens

        url = tokens['url']

        terms = self.cache_manager.get_annotation_owl_terms(url)

        self.annotations_dict[annotation] = set(terms)
        self.annotations_owl_dict[annotation] = url

        return tokens

    def handle_annotations_url(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: %s', annotation)
            return tokens

        url = tokens['url']

        self.annotations_dict[annotation] = self.cache_manager.get_annotation(url)
        self.annotation_url_dict[annotation] = url

        return tokens

    def handle_annotation_list(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: {}'.format(annotation))
            return tokens

        values = set(tokens['values'])

        self.annotations_dict[annotation] = values
        self.annotation_list_list.append(annotation)

        return tokens

    def handle_annotation_pattern(self, s, l, tokens):
        annotation = tokens['name']

        if self.has_annotation(annotation):
            log.warning('Tried to overwrite annotation: {}'.format(annotation))
            return tokens

        value = tokens['value']

        self.annotations_regex[annotation] = value
        self.annotations_regex_compiled[annotation] = re.compile(value)

        return tokens

    def has_enumerated_annotation(self, annotation):
        return annotation in self.annotations_dict

    def has_regex_annotation(self, annotation):
        return annotation in self.annotations_regex

    def has_annotation(self, annotation):
        return self.has_enumerated_annotation(annotation) or self.has_regex_annotation(annotation)

    def raise_for_missing_annotation(self, annotation):
        if not self.has_annotation(annotation):
            raise UndefinedAnnotationWarning(annotation)

    def has_enumerated_annotation_value(self, annotation, value):
        return annotation in self.annotations_dict and value in self.annotations_dict[annotation]

    def has_regex_annotation_value(self, annotation, value):
        return annotation in self.annotations_regex and self.annotations_regex_compiled[annotation].match(value)

    def has_annotation_value(self, annotation, value):
        self.raise_for_missing_annotation(annotation)

        return self.has_enumerated_annotation_value(annotation, value) or self.has_regex_annotation_value(annotation,
                                                                                                          value)

    def raise_for_missing_annotation_value(self, annotation, value):
        self.raise_for_missing_annotation(annotation)

        if self.has_enumerated_annotation(annotation) and not self.has_enumerated_annotation_value(annotation, value):
            raise IllegalAnnotationValueWarning(value, annotation)

        if self.has_regex_annotation(annotation) and not self.has_regex_annotation_value(annotation, value):
            raise MissingAnnotationRegexWarning(value, annotation)

    def has_enumerated_namespace(self, namespace):
        return namespace in self.namespace_dict

    def has_regex_namespace(self, namespace):
        return namespace in self.namespace_regex

    def has_namespace(self, namespace):
        return self.has_enumerated_namespace(namespace) or self.has_regex_namespace(namespace)

    def raise_for_missing_namespace(self, namespace):
        if not self.has_namespace(namespace):
            raise UndefinedNamespaceWarning(namespace)

    def has_enumerated_namespace_name(self, namespace, name):
        return self.has_enumerated_namespace(namespace) and name in self.namespace_dict[namespace]

    def has_regex_namespace_name(self, namespace, name):
        return namespace in self.namespace_regex_compiled and self.namespace_regex_compiled[namespace].match(name)

    def has_namespace_name(self, namespace, name):
        self.raise_for_missing_namespace(namespace)

        return self.has_enumerated_namespace_name(namespace, name) or self.has_regex_namespace_name(namespace, name)

    def raise_for_missing_name(self, namespace, name):
        self.raise_for_missing_namespace(namespace)

        if self.has_enumerated_namespace(namespace) and not self.has_enumerated_namespace_name(namespace, name):
            raise MissingNamespaceNameWarning(name, namespace)

        if self.has_regex_namespace(namespace) and not self.has_regex_namespace_name(namespace, name):
            raise MissingNamespaceRegexWarning(name, namespace)

{'fraction_non_alphanumeric': 0.08879436895396545, 'fraction_numerical': 0.0011292204614747618, 'mean_word_length': 3.218482057796126, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1668039', 'n_tokens_mistral': 7309, 'n_tokens_neox': 6901, 'n_words': 1707}