<commit_before>#!/usr/bin/python
#
# Copyright (c) 2008 rPath, Inc.
#

import re

import descriptor_errors as errors

class _DescriptorDataField(object):
    __slots__ = [ '_node', '_nodeDescriptor' ]
    def __init__(self, node, nodeDescriptor, checkConstraints = True):
        self._node = node
        self._nodeDescriptor = nodeDescriptor
        if checkConstraints:
            self.checkConstraints()

    def checkConstraints(self):
        errorList = []
        descriptions = self._nodeDescriptor.get_descriptions()
        defaultLangDesc = descriptions.asDict()[None]
        if self._nodeDescriptor.multiple:
            # Get the node's children as values
            values = [ x.text for x in self._node
                if hasattr(x, 'tag') and x.tag == 'item' ]
            if self._nodeDescriptor.required and not values:
                errorList.append("Missing field: '%s'" %
                    self._nodeDescriptor.name)
            elif isinstance(self._nodeDescriptor.type, list):
                errorList.extend(_validateEnumeratedValue(values,
                                 self._nodeDescriptor.type,
                                 defaultLangDesc))
            else:
                # It is conceivable that one has a multi-valued field with a
                # simple type
                errorList.extend(_validateMultiValue(values,
                                 self._nodeDescriptor.type,
                                 defaultLangDesc,
                                 self._nodeDescriptor.constraints))
        else:
            value = self._node.text<commit_msg>Fixed default lang description retrieval to not fail if descriptions are not available<commit_after>#!/usr/bin/python
#
# Copyright (c) 2008 rPath, Inc.
#

import re

import descriptor_errors as errors

class _DescriptorDataField(object):
    __slots__ = [ '_node', '_nodeDescriptor' ]
    def __init__(self, node, nodeDescriptor, checkConstraints = True):
        self._node = node
        self._nodeDescriptor = nodeDescriptor
        if checkConstraints:
            self.checkConstraints()

    def checkConstraints(self):
        errorList = []
        descriptions = self._nodeDescriptor.get_descriptions()
        defaultLangDesc = descriptions.asDict().get(None)
        if self._nodeDescriptor.multiple:
            # Get the node's children as values
            values = [ x.text for x in self._node
                if hasattr(x, 'tag') and x.tag == 'item' ]
            if self._nodeDescriptor.required and not values:
                errorList.append("Missing field: '%s'" %
                    self._nodeDescriptor.name)
            elif isinstance(self._nodeDescriptor.type, list):
                errorList.extend(_validateEnumeratedValue(values,
                                 self._nodeDescriptor.type,
                                 defaultLangDesc))
            else:
                # It is conceivable that one has a multi-valued field with a
                # simple type
                errorList.extend(_validateMultiValue(values,
                                 self._nodeDescriptor.type,
                                 defaultLangDesc,
                                 self._nodeDescriptor.constraints))
        else:
            value = self._node.text
{'fraction_non_alphanumeric': 0.07794965119805884, 'fraction_numerical': 0.002426448286320898, 'mean_word_length': 1.7529215358931554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2247894', 'n_tokens_mistral': 792, 'n_tokens_neox': 760, 'n_words': 197}