<commit_before># Copyright (c) 2016 Civic Knowledge. This file is licensed under the terms of the
# Revised BSD License, included in this distribution as LICENSE

"""
Special term subclasses
"""


from os.path import split, basename
from metatab.util import slugify

from metatab.exc import GenerateError
from metatab.parser import ROOT_TERM, ELIDED_TERM


EMPTY_SOURCE_HEADER = '_NONE_'  # Marker for a column that is in the destination table but not in the source


class Term(object):
    """Term object represent a row in a Metatab file, and handle interpeting the
        row into the parts of a term

        Public attributes. These are set externally to the constructor.

        file_name Filename or URL of faile that contains term
        row: Row number of term
        col Column number of term
        term_parent Term was generated from arguments of parent
        child_property_type What datatype to use in dict conversion
        valid Did term pass validation tests? Usually based on DeclaredTerm values.

    """

    _common_properties = 'url name description schema'.split()


    def __init__(self, term, value, term_args=False,
                 row=None, col=None, file_name=None, file_type=None,
                 parent=None, doc=None, section=None):
        """

        :param term: Simple or compoint term name
        :param value: Term value, from second column of spreadsheet
        :param term_args: Colums 2+ from term row, which become properties
        :param row: Row number in the file where the term appears. Starts at 0
        :param col: Column number, for arg children ( properties )
        :param file_name: The name or url of the file where the term appears.
        :param file_type: Usually None, but may be 'declare' for terms that appear in declare docs.
        :param parent: If set, the term is an arg child, and the term_parent is the parent term.
        :param doc: a MetatabDoc object, for the document the term is being parsed into.


        """

        def strip_if_str(v):
            try:
                return v.strip()
            except AttributeError:
                return v

        self.parent = parent  # If set, term was generated from term args

        self._term = None
        self.term = term  # A lot going on in this setter!

        self._orig_term = term

        self.value = strip_if_str(value) if value else None
        self.args = [strip_if_str(x) for x in (term_args or [])]

        self._section = None
        self.section = section
        self.doc = doc

        try:
            self.file_name = slugify(basename(file_name))
        except TypeError:
            self.file_name = file_name

        self.file_type = file_type
        self.row = row
        self.col = col

        if self.parent:
            assert self.parent.record_term_lc == self.parent_term_lc, (
            term, self.parent.record_term_lc, self.parent_term_lc)

        # When converting to a dict, what dict to to use for the self.value value
        self.term_value_name = '@value'  # May be change in term parsing

        # When converting to a dict, what datatype should be used for this term.
        # Can be forced to list, scalar, dict or other types.
        self.child_property_type = 'any'
        self.valid = None
<commit_msg>3.5 -> 3.6 version problem<commit_after># Copyright (c) 2016 Civic Knowledge. This file is licensed under the terms of the
# Revised BSD License, included in this distribution as LICENSE

"""
Special term subclasses
"""


from os.path import split, basename
from metatab.util import slugify

from metatab.exc import GenerateError
from metatab.parser import ROOT_TERM, ELIDED_TERM


EMPTY_SOURCE_HEADER = '_NONE_'  # Marker for a column that is in the destination table but not in the source


class Term(object):
    """Term object represent a row in a Metatab file, and handle interpeting the
        row into the parts of a term

        Public attributes. These are set externally to the constructor.

        file_name Filename or URL of faile that contains term
        row: Row number of term
        col Column number of term
        term_parent Term was generated from arguments of parent
        child_property_type What datatype to use in dict conversion
        valid Did term pass validation tests? Usually based on DeclaredTerm values.

    """

    _common_properties = 'url name description schema'.split()


    def __init__(self, term, value, term_args=False,
                 row=None, col=None, file_name=None, file_type=None,
                 parent=None, doc=None, section=None):
        """

        :param term: Simple or compoint term name
        :param value: Term value, from second column of spreadsheet
        :param term_args: Colums 2+ from term row, which become properties
        :param row: Row number in the file where the term appears. Starts at 0
        :param col: Column number, for arg children ( properties )
        :param file_name: The name or url of the file where the term appears.
        :param file_type: Usually None, but may be 'declare' for terms that appear in declare docs.
        :param parent: If set, the term is an arg child, and the term_parent is the parent term.
        :param doc: a MetatabDoc object, for the document the term is being parsed into.


        """

        def strip_if_str(v):
            try:
                return v.strip()
            except AttributeError:
                return v

        self.parent = parent  # If set, term was generated from term args

        self._term = None
        self.term = term  # A lot going on in this setter!

        self._orig_term = term

        self.value = strip_if_str(value) if value else None
        self.args = [strip_if_str(x) for x in (term_args or [])]

        self._section = None
        self.section = section
        self.doc = doc

        # Tried this the "pythonic" way, catching exceptions gnerated when file_name is None,
        # but python3.5 raises AttributeError, and python3.6 raises TypeError, and catching two different exceptions
        # based on the version of python is *really* not pythonic
        if file_name:
            self.file_name = slugify(basename(file_name))
        else:
            self.file_name = '<none>'

        self.file_type = file_type
        self.row = row
        self.col = col

        if self.parent:
            assert self.parent.record_term_lc == self.parent_term_lc, (
            term, self.parent.record_term_lc, self.parent_term_lc)

        # When converting to a dict, what dict to to use for the self.value value
        self.term_value_name = '@value'  # May be change in term parsing

        # When converting to a dict, what datatype should be used for this term.
        # Can be forced to list, scalar, dict or other types.
        self.child_property_type = 'any'
        self.valid = None

{'fraction_non_alphanumeric': 0.06123335263462652, 'fraction_numerical': 0.0028951939779965257, 'mean_word_length': 2.410167818361303, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6491200', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1816, 'n_words': 806}