<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one   *
 * or more contributor license agreements.  See the NOTICE file *
 * distributed with this work for additional information        *
 * regarding copyright ownership.  The ASF licenses this file   *
 * to you under the Apache License, Version 2.0 (the            *
 * "License"); you may not use this file except in compliance   *
 * with the License.  You may obtain a copy of the License at   *
 *                                                              *
 *   http://www.apache.org/licenses/LICENSE-2.0                 *
 *                                                              *
 * Unless required by applicable law or agreed to in writing,   *
 * software distributed under the License is distributed on an  *
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
 * KIND, either express or implied.  See the License for the    *
 * specific language governing permissions and limitations      *
 * under the License.                                           *
 */ 
package org.apache.rat.annotation;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Writer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

/**
 * Add a licence header to a document. This appender does not check for the
 * existence of an existing licence header, it is assumed that either a second
 * licence header is intentional or that there is no licence header present
 * already.
 * 
 */
public abstract class AbstractLicenceAppender {
    private static final int TYPE_UNKNOWN = 0;
    private static final int TYPE_JAVA = 1;
    private static final int TYPE_XML = 2;
    private static final int TYPE_HTML = 3;
    private static final int TYPE_CSS = 4;
    private static final int TYPE_JAVASCRIPT = 5;
    private static final int TYPE_APT = 6;
    private static final int TYPE_PROPERTIES = 7;
    private static final int TYPE_PYTHON = 8;
    private static final int TYPE_C = 9;
    private static final int TYPE_H = 10;
    private static final int TYPE_SH = 11;
    private static final int TYPE_BAT = 12;
    private static final int TYPE_VM = 13;
    private static final int TYPE_SCALA = 14;
    private static final int TYPE_RUBY = 15;
    private static final int TYPE_PERL = 16;
    private static final int TYPE_TCL = 17;
    private static final int TYPE_CPP = 18;
    private static final int TYPE_CSHARP = 19;
    private static final int TYPE_PHP = 20;
    private static final int TYPE_GROOVY = 21;
    private static final int TYPE_VISUAL_STUDIO_SOLUTION = 22;

    /** the line separator for this OS */
    private static final String LINE_SEP = System.getProperty("line.separator");

    private static final int[] FAMILY_C = new int[] {
        TYPE_JAVA, TYPE_JAVASCRIPT, TYPE_C, TYPE_H, TYPE_SCALA,
        TYPE_CSS, TYPE_CPP, TYPE_CSHARP, TYPE_PHP, TYPE_GROOVY,
    };
    private static final int[] FAMILY_SGML = new int[] {
        TYPE_XML, TYPE_HTML,
    };
    private static final int[] FAMILY_SH = new int[] {
        TYPE_PROPERTIES, TYPE_PYTHON, TYPE_SH, TYPE_RUBY, TYPE_PERL,
        TYPE_TCL, TYPE_VISUAL_STUDIO_SOLUTION,
    };
    private static final int[] FAMILY_BAT = new int[] {
        TYPE_BAT,
    };
    private static final int[] FAMILY_APT = new int[] {
        TYPE_APT,
    };
    private static final int[] FAMILY_VELOCITY = new int[] {
        TYPE_VM,
    };
    private static final int[] EXPECTS_HASH_PLING = new int[] {
        TYPE_PYTHON, TYPE_SH, TYPE_RUBY, TYPE_PERL, TYPE_TCL
    };
    private static final int[] EXPECTS_AT_ECHO = new int[] {
        TYPE_BAT,
    };
    private static final int[] EXPECTS_PACKAGE = new int[] {
        TYPE_JAVA,
    };
    private static final int[] EXPECTS_XML_DECL = new int[] {
        TYPE_XML,
    };
    private static final int[] EXPECTS_PHP_PI = new int[] {
        TYPE_PHP,
    };
    private static final int[] EXPECTS_MSVSSF_HEADER = new int[] {
        TYPE_VISUAL_STUDIO_SOLUTION,
    };

    private static final Map<String, Integer> EXT2TYPE = new HashMap();

    static {
        // these arrays are used in Arrays.binarySearch so they must
        // be sorted
        Arrays.sort(FAMILY_C);
        Arrays.sort(FAMILY_SGML);
        Arrays.sort(FAMILY_SH);
        Arrays.sort(FAMILY_BAT);
        Arrays.sort(FAMILY_APT);
        Arrays.sort(FAMILY_VELOCITY);

        Arrays.sort(EXPECTS_HASH_PLING);
        Arrays.sort(EXPECTS_AT_ECHO);
        Arrays.sort(EXPECTS_PACKAGE);
        Arrays.sort(EXPECTS_XML_DECL);
        Arrays.sort(EXPECTS_MSVSSF_HEADER);

        EXT2TYPE.put("apt", Integer.valueOf(TYPE_APT));
        EXT2TYPE.put("asax", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("ascx", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("aspx", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("bat", Integer.valueOf(TYPE_BAT));
        EXT2TYPE.put("c", Integer.valueOf(TYPE_C));
        EXT2TYPE.put("cc", Integer.valueOf(TYPE_CPP));
        EXT2TYPE.put("cmd", Integer.valueOf(TYPE_BAT));
        EXT2TYPE.put("config", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("cpp", Integer.valueOf(TYPE_CPP));
        EXT2TYPE.put("cs", Integer.valueOf(TYPE_CSHARP));
        EXT2TYPE.put("csdproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("csproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("css", Integer.valueOf(TYPE_CSS));
        EXT2TYPE.put("groovy", Integer.valueOf(TYPE_GROOVY));
        EXT2TYPE.put("h", Integer.valueOf(TYPE_H));
        EXT2TYPE.put("hh", Integer.valueOf(TYPE_H));
        EXT2TYPE.put("hpp", Integer.valueOf(TYPE_H));
        EXT2TYPE.put("htm", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("html", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("java", Integer.valueOf(TYPE_JAVA));
        EXT2TYPE.put("js", Integer.valueOf(TYPE_JAVASCRIPT));
        EXT2TYPE.put("ndoc", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("php", Integer.valueOf(TYPE_PHP));
        EXT2TYPE.put("pl", Integer.valueOf(TYPE_PERL));
        EXT2TYPE.put("properties", Integer.valueOf(TYPE_PROPERTIES));
        EXT2TYPE.put("py", Integer.valueOf(TYPE_PYTHON));
        EXT2TYPE.put("rb", Integer.valueOf(TYPE_RUBY));
        EXT2TYPE.put("rdf", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("resx", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("scala", Integer.valueOf(TYPE_SCALA));
        EXT2TYPE.put("sh", Integer.valueOf(TYPE_SH));
        EXT2TYPE.put("sln", Integer.valueOf(TYPE_VISUAL_STUDIO_SOLUTION));
        EXT2TYPE.put("tcl", Integer.valueOf(TYPE_TCL));
        EXT2TYPE.put("vbdproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("vbproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("vcproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("vm", Integer.valueOf(TYPE_VM));
        EXT2TYPE.put("vsdisco", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("webinfo", Integer.valueOf(TYPE_XML));<commit_msg>Additional .NET stuff used by Lucene.NET<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one   *
 * or more contributor license agreements.  See the NOTICE file *
 * distributed with this work for additional information        *
 * regarding copyright ownership.  The ASF licenses this file   *
 * to you under the Apache License, Version 2.0 (the            *
 * "License"); you may not use this file except in compliance   *
 * with the License.  You may obtain a copy of the License at   *
 *                                                              *
 *   http://www.apache.org/licenses/LICENSE-2.0                 *
 *                                                              *
 * Unless required by applicable law or agreed to in writing,   *
 * software distributed under the License is distributed on an  *
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY       *
 * KIND, either express or implied.  See the License for the    *
 * specific language governing permissions and limitations      *
 * under the License.                                           *
 */ 
package org.apache.rat.annotation;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.FilterInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Writer;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;

/**
 * Add a licence header to a document. This appender does not check for the
 * existence of an existing licence header, it is assumed that either a second
 * licence header is intentional or that there is no licence header present
 * already.
 * 
 */
public abstract class AbstractLicenceAppender {
    private static final int TYPE_UNKNOWN = 0;
    private static final int TYPE_JAVA = 1;
    private static final int TYPE_XML = 2;
    private static final int TYPE_HTML = 3;
    private static final int TYPE_CSS = 4;
    private static final int TYPE_JAVASCRIPT = 5;
    private static final int TYPE_APT = 6;
    private static final int TYPE_PROPERTIES = 7;
    private static final int TYPE_PYTHON = 8;
    private static final int TYPE_C = 9;
    private static final int TYPE_H = 10;
    private static final int TYPE_SH = 11;
    private static final int TYPE_BAT = 12;
    private static final int TYPE_VM = 13;
    private static final int TYPE_SCALA = 14;
    private static final int TYPE_RUBY = 15;
    private static final int TYPE_PERL = 16;
    private static final int TYPE_TCL = 17;
    private static final int TYPE_CPP = 18;
    private static final int TYPE_CSHARP = 19;
    private static final int TYPE_PHP = 20;
    private static final int TYPE_GROOVY = 21;
    private static final int TYPE_VISUAL_STUDIO_SOLUTION = 22;

    /** the line separator for this OS */
    private static final String LINE_SEP = System.getProperty("line.separator");

    private static final int[] FAMILY_C = new int[] {
        TYPE_JAVA, TYPE_JAVASCRIPT, TYPE_C, TYPE_H, TYPE_SCALA,
        TYPE_CSS, TYPE_CPP, TYPE_CSHARP, TYPE_PHP, TYPE_GROOVY,
    };
    private static final int[] FAMILY_SGML = new int[] {
        TYPE_XML, TYPE_HTML,
    };
    private static final int[] FAMILY_SH = new int[] {
        TYPE_PROPERTIES, TYPE_PYTHON, TYPE_SH, TYPE_RUBY, TYPE_PERL,
        TYPE_TCL, TYPE_VISUAL_STUDIO_SOLUTION,
    };
    private static final int[] FAMILY_BAT = new int[] {
        TYPE_BAT,
    };
    private static final int[] FAMILY_APT = new int[] {
        TYPE_APT,
    };
    private static final int[] FAMILY_VELOCITY = new int[] {
        TYPE_VM,
    };
    private static final int[] EXPECTS_HASH_PLING = new int[] {
        TYPE_PYTHON, TYPE_SH, TYPE_RUBY, TYPE_PERL, TYPE_TCL
    };
    private static final int[] EXPECTS_AT_ECHO = new int[] {
        TYPE_BAT,
    };
    private static final int[] EXPECTS_PACKAGE = new int[] {
        TYPE_JAVA,
    };
    private static final int[] EXPECTS_XML_DECL = new int[] {
        TYPE_XML,
    };
    private static final int[] EXPECTS_PHP_PI = new int[] {
        TYPE_PHP,
    };
    private static final int[] EXPECTS_MSVSSF_HEADER = new int[] {
        TYPE_VISUAL_STUDIO_SOLUTION,
    };

    private static final Map<String, Integer> EXT2TYPE = new HashMap();

    static {
        // these arrays are used in Arrays.binarySearch so they must
        // be sorted
        Arrays.sort(FAMILY_C);
        Arrays.sort(FAMILY_SGML);
        Arrays.sort(FAMILY_SH);
        Arrays.sort(FAMILY_BAT);
        Arrays.sort(FAMILY_APT);
        Arrays.sort(FAMILY_VELOCITY);

        Arrays.sort(EXPECTS_HASH_PLING);
        Arrays.sort(EXPECTS_AT_ECHO);
        Arrays.sort(EXPECTS_PACKAGE);
        Arrays.sort(EXPECTS_XML_DECL);
        Arrays.sort(EXPECTS_MSVSSF_HEADER);

        EXT2TYPE.put("apt", Integer.valueOf(TYPE_APT));
        EXT2TYPE.put("asax", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("ascx", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("aspx", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("bat", Integer.valueOf(TYPE_BAT));
        EXT2TYPE.put("c", Integer.valueOf(TYPE_C));
        EXT2TYPE.put("cc", Integer.valueOf(TYPE_CPP));
        EXT2TYPE.put("cmd", Integer.valueOf(TYPE_BAT));
        EXT2TYPE.put("config", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("cpp", Integer.valueOf(TYPE_CPP));
        EXT2TYPE.put("cs", Integer.valueOf(TYPE_CSHARP));
        EXT2TYPE.put("csdproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("csproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("css", Integer.valueOf(TYPE_CSS));
        EXT2TYPE.put("fxcop", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("groovy", Integer.valueOf(TYPE_GROOVY));
        EXT2TYPE.put("h", Integer.valueOf(TYPE_H));
        EXT2TYPE.put("hh", Integer.valueOf(TYPE_H));
        EXT2TYPE.put("hpp", Integer.valueOf(TYPE_H));
        EXT2TYPE.put("htm", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("html", Integer.valueOf(TYPE_HTML));
        EXT2TYPE.put("java", Integer.valueOf(TYPE_JAVA));
        EXT2TYPE.put("js", Integer.valueOf(TYPE_JAVASCRIPT));
        EXT2TYPE.put("ndoc", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("nunit", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("php", Integer.valueOf(TYPE_PHP));
        EXT2TYPE.put("pl", Integer.valueOf(TYPE_PERL));
        EXT2TYPE.put("properties", Integer.valueOf(TYPE_PROPERTIES));
        EXT2TYPE.put("py", Integer.valueOf(TYPE_PYTHON));
        EXT2TYPE.put("rb", Integer.valueOf(TYPE_RUBY));
        EXT2TYPE.put("rdf", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("resx", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("scala", Integer.valueOf(TYPE_SCALA));
        EXT2TYPE.put("sh", Integer.valueOf(TYPE_SH));
        EXT2TYPE.put("shfbproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("sln", Integer.valueOf(TYPE_VISUAL_STUDIO_SOLUTION));
        EXT2TYPE.put("stylecop", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("tcl", Integer.valueOf(TYPE_TCL));
        EXT2TYPE.put("vbdproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("vbproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("vcproj", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("vm", Integer.valueOf(TYPE_VM));
        EXT2TYPE.put("vsdisco", Integer.valueOf(TYPE_XML));
        EXT2TYPE.put("webinfo", Integer.valueOf(TYPE_XML));
{'fraction_non_alphanumeric': 0.10816396830864623, 'fraction_numerical': 0.011436445056837753, 'mean_word_length': 2.9381443298969074, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '143677', 'n_tokens_mistral': 4698, 'n_tokens_neox': 4419, 'n_words': 1132}