<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* $Id$ */

package org.apache.fop.complexscripts.util;

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.fop.util.CharUtilities;

/**
 * <p>Script related utilities.</p>
 *
 * <p>This work was originally authored by <NAME> (<EMAIL>).</p>
 */
public final class CharScript {

    // CSOFF: LineLength

    //
    // The following script codes are based on ISO 15924. Codes less than 1000 are
    // official assignments from 15924; those equal to or greater than 1000 are FOP
    // implementation specific.
    //
    /** hebrew script constant */
    public static final int SCRIPT_HEBREW                               = 125;  // 'hebr'
    /** mongolian script constant */
    public static final int SCRIPT_MONGOLIAN                            = 145;  // 'mong'
    /** arabic script constant */
    public static final int SCRIPT_ARABIC                               = 160;  // 'arab'
    /** greek script constant */
    public static final int SCRIPT_GREEK                                = 200;  // 'grek'
    /** latin script constant */
    public static final int SCRIPT_LATIN                                = 215;  // 'latn'
    /** cyrillic script constant */
    public static final int SCRIPT_CYRILLIC                             = 220;  // 'cyrl'
    /** georgian script constant */
    public static final int SCRIPT_GEORGIAN                             = 240;  // 'geor'
    /** bopomofo script constant */
    public static final int SCRIPT_BOPOMOFO                             = 285;  // 'bopo'
    /** hangul script constant */
    public static final int SCRIPT_HANGUL                               = 286;  // 'hang'
    /** gurmukhi script constant */
    public static final int SCRIPT_GURMUKHI                             = 310;  // 'guru'
    /** gurmukhi 2 script constant */
    public static final int SCRIPT_GURMUKHI_2                           = 1310; // 'gur2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** devanagari script constant */
    public static final int SCRIPT_DEVANAGARI                           = 315;  // 'deva'
    /** devanagari 2 script constant */
    public static final int SCRIPT_DEVANAGARI_2                         = 1315; // 'dev2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** gujarati script constant */
    public static final int SCRIPT_GUJARATI                             = 320;  // 'gujr'
    /** gujarati 2 script constant */
    public static final int SCRIPT_GUJARATI_2                           = 1320; // 'gjr2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** bengali script constant */
    public static final int SCRIPT_BENGALI                              = 326;  // 'beng'
    /** bengali 2 script constant */
    public static final int SCRIPT_BENGALI_2                            = 1326; // 'bng2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** oriya script constant */
    public static final int SCRIPT_ORIYA                                = 327;  // 'orya'
    /** oriya 2 script constant */
    public static final int SCRIPT_ORIYA_2                              = 1327; // 'ory2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** tibetan script constant */
    public static final int SCRIPT_TIBETAN                              = 330;  // 'tibt'
    /** telugu script constant */
    public static final int SCRIPT_TELUGU                               = 340;  // 'telu'
    /** telugu 2 script constant */
    public static final int SCRIPT_TELUGU_2                             = 1340; // 'tel2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** kannada script constant */
    public static final int SCRIPT_KANNADA                              = 345;  // 'knda'
    /** kannada 2 script constant */
    public static final int SCRIPT_KANNADA_2                            = 1345; // 'knd2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** tamil script constant */
    public static final int SCRIPT_TAMIL                                = 346;  // 'taml'
    /** tamil 2 script constant */
    public static final int SCRIPT_TAMIL_2                              = 1346; // 'tml2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** malayalam script constant */
    public static final int SCRIPT_MALAYALAM                            = 347;  // 'mlym'
    /** malayalam 2 script constant */
    public static final int SCRIPT_MALAYALAM_2                          = 1347; // 'mlm2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** sinhalese script constant */
    public static final int SCRIPT_SINHALESE                            = 348;  // 'sinh'
    /** burmese script constant */
    public static final int SCRIPT_BURMESE                              = 350;  // 'mymr'
    /** thai script constant */
    public static final int SCRIPT_THAI                                 = 352;  // 'thai'
    /** khmer script constant */
    public static final int SCRIPT_KHMER                                = 355;  // 'khmr'
    /** lao script constant */
    public static final int SCRIPT_LAO                                  = 356;  // 'laoo'
    /** hiragana script constant */
    public static final int SCRIPT_HIRAGANA                             = 410;  // 'hira'
    /** ethiopic script constant */
    public static final int SCRIPT_ETHIOPIC                             = 430;  // 'ethi'
    /** han script constant */
    public static final int SCRIPT_HAN                                  = 500;  // 'hani'
    /** katakana script constant */
    public static final int SCRIPT_KATAKANA                             = 410;  // 'kana'
    /** math script constant */
    public static final int SCRIPT_MATH                                 = 995;  // 'zmth'
    /** symbol script constant */
    public static final int SCRIPT_SYMBOL                               = 996;  // 'zsym'
    /** undetermined script constant */
    public static final int SCRIPT_UNDETERMINED                         = 998;  // 'zyyy'
    /** uncoded script constant */
    public static final int SCRIPT_UNCODED                              = 999;  // 'zzzz'

    /**
      * A static (class) parameter indicating whether V2 indic shaping
      * rules apply or not, with default being <code>true</code>.
      */
    private static final boolean USE_V2_INDIC = true;

    private CharScript() {
    }

    /**
     * Determine if character c is punctuation.
     * @param c a character represented as a unicode scalar value
     * @return true if character is punctuation
     */
    public static boolean isPunctuation(int c) {
        if ((c >= 0x0021) && (c <= 0x002F)) {             // basic latin punctuation
            return true;
        } else if ((c >= 0x003A) && (c <= 0x0040)) {      // basic latin punctuation
            return true;
        } else if ((c >= 0x005F) && (c <= 0x0060)) {      // basic latin punctuation
            return true;
        } else if ((c >= 0x007E) && (c <= 0x007E)) {      // basic latin punctuation
            return true;
        } else if ((c >= 0x007E) && (c <= 0x007E)) {      // basic latin punctuation
            return true;<commit_msg>Minor: removed duplicate test<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/* $Id$ */

package org.apache.fop.complexscripts.util;

import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;

import org.apache.fop.util.CharUtilities;

/**
 * <p>Script related utilities.</p>
 *
 * <p>This work was originally authored by <NAME> (<EMAIL>).</p>
 */
public final class CharScript {

    // CSOFF: LineLength

    //
    // The following script codes are based on ISO 15924. Codes less than 1000 are
    // official assignments from 15924; those equal to or greater than 1000 are FOP
    // implementation specific.
    //
    /** hebrew script constant */
    public static final int SCRIPT_HEBREW                               = 125;  // 'hebr'
    /** mongolian script constant */
    public static final int SCRIPT_MONGOLIAN                            = 145;  // 'mong'
    /** arabic script constant */
    public static final int SCRIPT_ARABIC                               = 160;  // 'arab'
    /** greek script constant */
    public static final int SCRIPT_GREEK                                = 200;  // 'grek'
    /** latin script constant */
    public static final int SCRIPT_LATIN                                = 215;  // 'latn'
    /** cyrillic script constant */
    public static final int SCRIPT_CYRILLIC                             = 220;  // 'cyrl'
    /** georgian script constant */
    public static final int SCRIPT_GEORGIAN                             = 240;  // 'geor'
    /** bopomofo script constant */
    public static final int SCRIPT_BOPOMOFO                             = 285;  // 'bopo'
    /** hangul script constant */
    public static final int SCRIPT_HANGUL                               = 286;  // 'hang'
    /** gurmukhi script constant */
    public static final int SCRIPT_GURMUKHI                             = 310;  // 'guru'
    /** gurmukhi 2 script constant */
    public static final int SCRIPT_GURMUKHI_2                           = 1310; // 'gur2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** devanagari script constant */
    public static final int SCRIPT_DEVANAGARI                           = 315;  // 'deva'
    /** devanagari 2 script constant */
    public static final int SCRIPT_DEVANAGARI_2                         = 1315; // 'dev2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** gujarati script constant */
    public static final int SCRIPT_GUJARATI                             = 320;  // 'gujr'
    /** gujarati 2 script constant */
    public static final int SCRIPT_GUJARATI_2                           = 1320; // 'gjr2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** bengali script constant */
    public static final int SCRIPT_BENGALI                              = 326;  // 'beng'
    /** bengali 2 script constant */
    public static final int SCRIPT_BENGALI_2                            = 1326; // 'bng2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** oriya script constant */
    public static final int SCRIPT_ORIYA                                = 327;  // 'orya'
    /** oriya 2 script constant */
    public static final int SCRIPT_ORIYA_2                              = 1327; // 'ory2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** tibetan script constant */
    public static final int SCRIPT_TIBETAN                              = 330;  // 'tibt'
    /** telugu script constant */
    public static final int SCRIPT_TELUGU                               = 340;  // 'telu'
    /** telugu 2 script constant */
    public static final int SCRIPT_TELUGU_2                             = 1340; // 'tel2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** kannada script constant */
    public static final int SCRIPT_KANNADA                              = 345;  // 'knda'
    /** kannada 2 script constant */
    public static final int SCRIPT_KANNADA_2                            = 1345; // 'knd2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** tamil script constant */
    public static final int SCRIPT_TAMIL                                = 346;  // 'taml'
    /** tamil 2 script constant */
    public static final int SCRIPT_TAMIL_2                              = 1346; // 'tml2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** malayalam script constant */
    public static final int SCRIPT_MALAYALAM                            = 347;  // 'mlym'
    /** malayalam 2 script constant */
    public static final int SCRIPT_MALAYALAM_2                          = 1347; // 'mlm2'       -- MSFT (pseudo) script tag for variant shaping semantics
    /** sinhalese script constant */
    public static final int SCRIPT_SINHALESE                            = 348;  // 'sinh'
    /** burmese script constant */
    public static final int SCRIPT_BURMESE                              = 350;  // 'mymr'
    /** thai script constant */
    public static final int SCRIPT_THAI                                 = 352;  // 'thai'
    /** khmer script constant */
    public static final int SCRIPT_KHMER                                = 355;  // 'khmr'
    /** lao script constant */
    public static final int SCRIPT_LAO                                  = 356;  // 'laoo'
    /** hiragana script constant */
    public static final int SCRIPT_HIRAGANA                             = 410;  // 'hira'
    /** ethiopic script constant */
    public static final int SCRIPT_ETHIOPIC                             = 430;  // 'ethi'
    /** han script constant */
    public static final int SCRIPT_HAN                                  = 500;  // 'hani'
    /** katakana script constant */
    public static final int SCRIPT_KATAKANA                             = 410;  // 'kana'
    /** math script constant */
    public static final int SCRIPT_MATH                                 = 995;  // 'zmth'
    /** symbol script constant */
    public static final int SCRIPT_SYMBOL                               = 996;  // 'zsym'
    /** undetermined script constant */
    public static final int SCRIPT_UNDETERMINED                         = 998;  // 'zyyy'
    /** uncoded script constant */
    public static final int SCRIPT_UNCODED                              = 999;  // 'zzzz'

    /**
      * A static (class) parameter indicating whether V2 indic shaping
      * rules apply or not, with default being <code>true</code>.
      */
    private static final boolean USE_V2_INDIC = true;

    private CharScript() {
    }

    /**
     * Determine if character c is punctuation.
     * @param c a character represented as a unicode scalar value
     * @return true if character is punctuation
     */
    public static boolean isPunctuation(int c) {
        if ((c >= 0x0021) && (c <= 0x002F)) {             // basic latin punctuation
            return true;
        } else if ((c >= 0x003A) && (c <= 0x0040)) {      // basic latin punctuation
            return true;
        } else if ((c >= 0x005F) && (c <= 0x0060)) {      // basic latin punctuation
            return true;
        } else if ((c >= 0x007E) && (c <= 0x007E)) {      // basic latin punctuation
            return true;
{'fraction_non_alphanumeric': 0.08894614157772053, 'fraction_numerical': 0.027235921972764077, 'mean_word_length': 1.6320632870519858, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6718355', 'n_tokens_mistral': 4953, 'n_tokens_neox': 4289, 'n_words': 1537}