<commit_before>/* ------------------------------------------------------------------------

   unicodedata -- Provides access to the Unicode 5.2 data base.

   Data was extracted from the Unicode 5.2 UnicodeData.txt file.

   Written by <NAME> (<EMAIL>).
   Modified for Python 2.0 by <NAME> (<EMAIL>)
   Modified by <NAME> (<EMAIL>)

   Copyright (c) Corporation for National Research Initiatives.

   ------------------------------------------------------------------------ */

#include "Python.h"
#include "ucnhash.h"
#include "structmember.h"

/* character properties */

typedef struct {
    const unsigned char category;       /* index into
                                           _PyUnicode_CategoryNames */
    const unsigned char combining;      /* combining class value 0 - 255 */
    const unsigned char bidirectional;  /* index into
                                           _PyUnicode_BidirectionalNames */
    const unsigned char mirrored;       /* true if mirrored in bidir mode */
    const unsigned char east_asian_width;       /* index into
                                                   _PyUnicode_EastAsianWidth */
    const unsigned char normalization_quick_check; /* see is_normalized() */
} _PyUnicode_DatabaseRecord;

typedef struct change_record {
    /* sequence of fields should be the same as in merge_old_version */
    const unsigned char bidir_changed;
    const unsigned char category_changed;
    const unsigned char decimal_changed;
    const unsigned char mirrored_changed;
    const double numeric_changed;
} change_record;

/* data file generated by Tools/unicode/makeunicodedata.py */
#include "unicodedata_db.h"

static const _PyUnicode_DatabaseRecord*<commit_msg>Py_TOUPPER missing on Python 2.6 and Win32 pre 2.7.3<commit_after>/* ------------------------------------------------------------------------

   unicodedata -- Provides access to the Unicode 5.2 data base.

   Data was extracted from the Unicode 5.2 UnicodeData.txt file.

   Written by <NAME> (<EMAIL>).
   Modified for Python 2.0 by <NAME> (<EMAIL>)
   Modified by <NAME> (<EMAIL>.de)

   Copyright (c) Corporation for National Research Initiatives.

   ------------------------------------------------------------------------ */

#include "Python.h"
#include "ucnhash.h"
#include "structmember.h"

#if PY_MAJOR_VERSION == 2 && (PY_MINOR_VERSION < 7 || PY_MICRO_VERSION < 3)
#define Py_TOUPPER(c) toupper(c)
#endif

/* character properties */

typedef struct {
    const unsigned char category;       /* index into
                                           _PyUnicode_CategoryNames */
    const unsigned char combining;      /* combining class value 0 - 255 */
    const unsigned char bidirectional;  /* index into
                                           _PyUnicode_BidirectionalNames */
    const unsigned char mirrored;       /* true if mirrored in bidir mode */
    const unsigned char east_asian_width;       /* index into
                                                   _PyUnicode_EastAsianWidth */
    const unsigned char normalization_quick_check; /* see is_normalized() */
} _PyUnicode_DatabaseRecord;

typedef struct change_record {
    /* sequence of fields should be the same as in merge_old_version */
    const unsigned char bidir_changed;
    const unsigned char category_changed;
    const unsigned char decimal_changed;
    const unsigned char mirrored_changed;
    const double numeric_changed;
} change_record;

/* data file generated by Tools/unicode/makeunicodedata.py */
#include "unicodedata_db.h"

static const _PyUnicode_DatabaseRecord*
{'fraction_non_alphanumeric': 0.149399609047752, 'fraction_numerical': 0.008377548170901982, 'mean_word_length': 1.9603305785123968, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4905092', 'n_tokens_mistral': 936, 'n_tokens_neox': 867, 'n_words': 300}