<commit_before>from __future__ import absolute_import

import copy
import os
import logging
from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
                    Optional, Set, Tuple, Union)

from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from rdflib import Graph, URIRef  # pylint: disable=unused-import
from rdflib.namespace import OWL, RDFS
from ruamel.yaml.comments import CommentedMap
from schema_salad import validate
from schema_salad.schema import Names, convert_to_dict
from schema_salad.avro.schema import make_avsc_object, Schema
from schema_salad.sourceline import SourceLine
from schema_salad.ref_resolver import uri_file_path
from six import iteritems, string_types
from typing import IO
from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
                               Text, Type)
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from . import expression
from .errors import WorkflowException
from .loghandler import _logger
from .mutation import MutationManager  # pylint: disable=unused-import
from .pathmapper import PathMapper  # pylint: disable=unused-import
from .pathmapper import CONTENT_LIMIT, get_listing, normalizeFilesDirs, visit_class
from .stdfsaccess import StdFsAccess  # pylint: disable=unused-import
from .utils import aslist, docker_windows_path_adjust, json_dumps, onWindows



if TYPE_CHECKING:
    from .provenance import ProvenanceProfile  # pylint: disable=unused-import


def content_limit_respected_read_bytes(f):  # type: (IO) -> bytes
    contents = f.read(CONTENT_LIMIT + 1)
    if len(contents) > CONTENT_LIMIT:
        raise WorkflowException("loadContents handling encountered buffer that is exceeds maximum lenght of %d bytes" % CONTENT_LIMIT)
    return contents


def content_limit_respected_read(f):  # type: (IO) -> Text
    return content_limit_respected_read_bytes(f).decode("utf-8")


def substitute(value, replace):  # type: (Text, Text) -> Text
    if replace[0] == "^":
        return substitute(value[0:value.rindex('.')], replace[1:])
    return value + replace

def formatSubclassOf(fmt, cls, ontology, visited):
    # type: (Text, Text, Optional[Graph], Set[Text]) -> bool
    """Determine if `fmt` is a subclass of `cls`."""

    if URIRef(fmt) == URIRef(cls):
        return True

    if ontology is None:
        return False

    if fmt in visited:
        return False

    visited.add(fmt)

    uriRefFmt = URIRef(fmt)

    for s, p, o in ontology.triples((uriRefFmt, RDFS.subClassOf, None)):
        # Find parent classes of `fmt` and search upward
        if formatSubclassOf(o, cls, ontology, visited):
            return True

    for s, p, o in ontology.triples((uriRefFmt, OWL.equivalentClass, None)):
        # Find equivalent classes of `fmt` and search horizontally
        if formatSubclassOf(o, cls, ontology, visited):
            return True

    for s, p, o in ontology.triples((None, OWL.equivalentClass, uriRefFmt)):<commit_msg>Fix crash with carets in secondaryFiles (#826)<commit_after>from __future__ import absolute_import

import copy
import os
import logging
from typing import (Any, Callable, Dict, List, MutableMapping, MutableSequence,
                    Optional, Set, Tuple, Union)

from typing_extensions import Text, Type, TYPE_CHECKING  # pylint: disable=unused-import
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from rdflib import Graph, URIRef  # pylint: disable=unused-import
from rdflib.namespace import OWL, RDFS
from ruamel.yaml.comments import CommentedMap
from schema_salad import validate
from schema_salad.schema import Names, convert_to_dict
from schema_salad.avro.schema import make_avsc_object, Schema
from schema_salad.sourceline import SourceLine
from schema_salad.ref_resolver import uri_file_path
from six import iteritems, string_types
from typing import IO
from typing_extensions import (TYPE_CHECKING,  # pylint: disable=unused-import
                               Text, Type)
# move to a regular typing import when Python 3.3-3.6 is no longer supported

from . import expression
from .errors import WorkflowException
from .loghandler import _logger
from .mutation import MutationManager  # pylint: disable=unused-import
from .pathmapper import PathMapper  # pylint: disable=unused-import
from .pathmapper import CONTENT_LIMIT, get_listing, normalizeFilesDirs, visit_class
from .stdfsaccess import StdFsAccess  # pylint: disable=unused-import
from .utils import aslist, docker_windows_path_adjust, json_dumps, onWindows



if TYPE_CHECKING:
    from .provenance import ProvenanceProfile  # pylint: disable=unused-import


def content_limit_respected_read_bytes(f):  # type: (IO) -> bytes
    contents = f.read(CONTENT_LIMIT + 1)
    if len(contents) > CONTENT_LIMIT:
        raise WorkflowException("loadContents handling encountered buffer that is exceeds maximum lenght of %d bytes" % CONTENT_LIMIT)
    return contents


def content_limit_respected_read(f):  # type: (IO) -> Text
    return content_limit_respected_read_bytes(f).decode("utf-8")


def substitute(value, replace):  # type: (Text, Text) -> Text
    if replace.startswith("^"):
        try:
            return substitute(value[0:value.rindex('.')], replace[1:])
        except ValueError:
            # No extension to remove
            return value + replace.lstrip("^")
    return value + replace

def formatSubclassOf(fmt, cls, ontology, visited):
    # type: (Text, Text, Optional[Graph], Set[Text]) -> bool
    """Determine if `fmt` is a subclass of `cls`."""

    if URIRef(fmt) == URIRef(cls):
        return True

    if ontology is None:
        return False

    if fmt in visited:
        return False

    visited.add(fmt)

    uriRefFmt = URIRef(fmt)

    for s, p, o in ontology.triples((uriRefFmt, RDFS.subClassOf, None)):
        # Find parent classes of `fmt` and search upward
        if formatSubclassOf(o, cls, ontology, visited):
            return True

    for s, p, o in ontology.triples((uriRefFmt, OWL.equivalentClass, None)):
        # Find equivalent classes of `fmt` and search horizontally
        if formatSubclassOf(o, cls, ontology, visited):
            return True

    for s, p, o in ontology.triples((None, OWL.equivalentClass, uriRefFmt)):
{'fraction_non_alphanumeric': 0.08212334113973459, 'fraction_numerical': 0.004371584699453552, 'mean_word_length': 3.8640850417615793, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1258438', 'n_tokens_mistral': 1942, 'n_tokens_neox': 1875, 'n_words': 625}