<commit_before>##===--------------------------------------------------*- coding: utf-8 -*-===##
##
## This source file is part of the Swift.org open source project
##
## Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
## Licensed under Apache License v2.0 with Runtime Library Exception
##
## See http://swift.org/LICENSE.txt for license information
## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
##
##===----------------------------------------------------------------------===##

import re

class UnicodeProperty(object):
    """Abstract base class for Unicode properties."""

    def __init__(self):
        raise NotImplemented

    def get_default_value(self):
        raise NotImplemented

    def get_value(self, cp):
        raise NotImplemented

    def to_numeric_value(self, value):
        raise NotImplemented

    def get_numeric_value(self, cp):
        raise NotImplemented

class GraphemeClusterBreakPropertyTable(UnicodeProperty):
    """Grapheme_Cluster_Break property."""

    # An array of tuples (start_code_point, end_code_point, value).
    property_value_ranges = []

    property_values = [ None for i in range(0, 0x110000) ]

    # Note: Numeric values should be consistent with
    # '_GraphemeClusterBreakPropertyValue' enum on the Swift side, and with
    # 'GraphemeClusterBreakProperty' in the compiler C++ code.  If there is a
    # reason for eithor of those to differ, then this mapping can be overriden
    # after an instance of this class is created.
    numeric_value_table = {
      'Other': 0,
      'CR': 1,
      'LF': 2,
      'Control': 3,
      'Extend': 4,
      'Regional_Indicator': 5,
      'Prepend': 6,
      'SpacingMark': 7,
      'L': 8,
      'V': 9,
      'T': 10,
      'LV': 11,
      'LVT': 12,
    }

    symbolic_values = [
      'Other', 'CR', 'LF', 'Control', 'Extend', 'Regional_Indicator',
      'Prepend', 'SpacingMark', 'L', 'V', 'T', 'LV', 'LVT',
    ]

    def __init__(self, grapheme_break_property_file_name):
        # Load the data file.
        with open(grapheme_break_property_file_name, 'rb') as f:
            for line in f:
                # Strip comments.
                line = re.sub('#.*', '', line)

                # Single code point?
                m = re.match('([0-9A-F]+) +; +([a-zA-Z]+) ', line)
                if m:
                    code_point = int(m.group(1), 16)
                    value = m.group(2)
                    self.property_value_ranges += \
                        [(code_point, code_point, value)]
                    continue

                # Range of code points?
                m = re.match('([0-9A-F]+)..([0-9A-F]+) +; +([a-zA-Z_]+) ', line)
                if m:
                    start_code_point = int(m.group(1), 16)
                    end_code_point = int(m.group(2), 16)
                    value = m.group(3)
                    self.property_value_ranges += \
                        [(start_code_point, end_code_point, value)]

        # Prepare a flat lookup table for fast access.
        for cp in range(0, 0x110000):
            self.property_values[cp] = self.get_default_value()

        for start_code_point,end_code_point,value in self.property_value_ranges:
            for cp in range(start_code_point, end_code_point + 1):
                self.property_values[cp] = value<commit_msg>Unicode tables: DRY in GraphemeClusterBreakPropertyTable.symbolic_values<commit_after>##===--------------------------------------------------*- coding: utf-8 -*-===##
##
## This source file is part of the Swift.org open source project
##
## Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors
## Licensed under Apache License v2.0 with Runtime Library Exception
##
## See http://swift.org/LICENSE.txt for license information
## See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
##
##===----------------------------------------------------------------------===##

import re

class UnicodeProperty(object):
    """Abstract base class for Unicode properties."""

    def __init__(self):
        raise NotImplemented

    def get_default_value(self):
        raise NotImplemented

    def get_value(self, cp):
        raise NotImplemented

    def to_numeric_value(self, value):
        raise NotImplemented

    def get_numeric_value(self, cp):
        raise NotImplemented

class GraphemeClusterBreakPropertyTable(UnicodeProperty):
    """Grapheme_Cluster_Break property."""

    # An array of tuples (start_code_point, end_code_point, value).
    property_value_ranges = []

    property_values = [ None for i in range(0, 0x110000) ]

    # Note: Numeric values should be consistent with
    # '_GraphemeClusterBreakPropertyValue' enum on the Swift side, and with
    # 'GraphemeClusterBreakProperty' in the compiler C++ code.  If there is a
    # reason for eithor of those to differ, then this mapping can be overriden
    # after an instance of this class is created.
    numeric_value_table = {
      'Other': 0,
      'CR': 1,
      'LF': 2,
      'Control': 3,
      'Extend': 4,
      'Regional_Indicator': 5,
      'Prepend': 6,
      'SpacingMark': 7,
      'L': 8,
      'V': 9,
      'T': 10,
      'LV': 11,
      'LVT': 12,
    }

    def __init__(self, grapheme_break_property_file_name):
        # Build 'self.symbolic_values' -- an array that maps numeric property
        # values to symbolic values.
        self.symbolic_values = \
            [ None ] * (max(self.numeric_value_table.values()) + 1)
        for k,v in self.numeric_value_table.iteritems():
            self.symbolic_values[v] = k

        # Load the data file.
        with open(grapheme_break_property_file_name, 'rb') as f:
            for line in f:
                # Strip comments.
                line = re.sub('#.*', '', line)

                # Single code point?
                m = re.match('([0-9A-F]+) +; +([a-zA-Z]+) ', line)
                if m:
                    code_point = int(m.group(1), 16)
                    value = m.group(2)
                    self.property_value_ranges += \
                        [(code_point, code_point, value)]
                    continue

                # Range of code points?
                m = re.match('([0-9A-F]+)..([0-9A-F]+) +; +([a-zA-Z_]+) ', line)
                if m:
                    start_code_point = int(m.group(1), 16)
                    end_code_point = int(m.group(2), 16)
                    value = m.group(3)
                    self.property_value_ranges += \
                        [(start_code_point, end_code_point, value)]

        # Prepare a flat lookup table for fast access.
        for cp in range(0, 0x110000):
            self.property_values[cp] = self.get_default_value()

        for start_code_point,end_code_point,value in self.property_value_ranges:
            for cp in range(start_code_point, end_code_point + 1):
                self.property_values[cp] = value
{'fraction_non_alphanumeric': 0.14759036144578314, 'fraction_numerical': 0.017641996557659207, 'mean_word_length': 1.9200167504187604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6585968', 'n_tokens_mistral': 2192, 'n_tokens_neox': 2015, 'n_words': 563}