<commit_before>/*

   Derby - Class org.apache.derbyTesting.functionTests.harness.Sed

   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

 */

package org.apache.derbyTesting.functionTests.harness;

/***
 * Sed.java
 *
 * This is a version of "sed" in Java for the Derby Function Tests,
 * written using the java.util.regex regular expression classes.
 * The substitutions/deletions are based on the original kornshell tests.
 *
 ***/

import java.io.*;
import java.util.Vector;
import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.derbyTesting.functionTests.util.TestUtil;

public class Sed
{
	private	static	final	String	SQL_EXCEPTION_FILTERED_SUBSTITUTION = 
        "java.sql.SQLException:";

    public Sed()
    {
    }

    public static void main(String[] args) throws Exception {
        if (args == null || args.length != 2) {
            System.err.println("Usage: Sed sourcefile targetfile");
            System.exit(1);
        }
        File src = new File(args[0]);
        File tgt = new File(args[1]);
        new Sed().exec(src,tgt,null, false, false,false);
    }

    // The arguments should be the names of the input and output files
    public void exec
		(File srcFile, File dstFile, InputStream isSed, boolean isJCC, boolean isI18N, boolean isJDBC4)
        throws IOException
    {
    	String hostName = TestUtil.getHostName();
    	
        // Vector for storing lines to be deleted
        Vector<String> deleteLines = new Vector<String>();
        deleteLines.addElement("^ij version.*$");
        deleteLines.addElement("^\\*\\*\\*\\* Test Run Started .* \\*\\*\\*\\*$");
        deleteLines.addElement("^\\*\\*\\*\\* Test Run Completed .* \\*\\*\\*\\*$");
        deleteLines.addElement("^ELAPSED TIME = [0-9]* milliseconds$");
        deleteLines.addElement("^\\^\\?$");
        //deleteLines.addElement("^\\.$"); // originally to remove lines with a dot
        deleteLines.addElement("^S.*ij> $");
        deleteLines.addElement("^ *$");
        deleteLines.addElement("^Server StackTrace:$");
        deleteLines.addElement("^\\[ *$");
        deleteLines.addElement("^\\] *$");
        deleteLines.addElement("^\\[$");
        deleteLines.addElement("^\\]$");
        deleteLines.addElement("^<not available>\\]$");
        deleteLines.addElement("^(.*at .*)\\(.*:[0-9].*\\)$");
        deleteLines.addElement("^(.*at .*)\\(*.java\\)$");
        deleteLines.addElement("^(.*at .*)\\(Compiled Code\\)$");
        deleteLines.addElement("^.*at java.*\\<init\\>\\(.*\\(Compiled Code\\)\\)$");
        deleteLines.addElement("^(.*at .*)\\(Interpreted Code\\)$");
        deleteLines.addElement("^(.*at .*)\\(Unknown Source\\)$");
        deleteLines.addElement("^(.*at .*)\\(Native Method\\)$");
        deleteLines.addElement("^\\tat $"); // rare case of incomplete stack trace line
        deleteLines.addElement("optimizer estimated cost");
        deleteLines.addElement("optimizer estimated row count");
        deleteLines.addElement("Using executables built for native_threads");
        deleteLines.addElement("Estimate of memory used");
        deleteLines.addElement("Size of merge runs");
        deleteLines.addElement("Number of merge runs");
        deleteLines.addElement("Sort type");
        deleteLines.addElement("Optimization started at .*$");
        deleteLines.addElement("WARNING 02000: No row was found for FETCH, UPDATE or DELETE");
        // deleteLines for stack traces from j9 jvm to match those above for other jvms
        deleteLines.addElement("Stack trace:");	
        deleteLines.addElement("^.*java/.*\\<init\\>\\(.*\\)V");
        deleteLines.addElement("^.*org/apache/derby/.*\\(.*\\).*$");	
        // next for j9 stack trace with jarfiles test run.
        deleteLines.addElement("^.*java/.*\\(.*\\).*$");
        deleteLines.addElement("^\\[.*db2jcc.jar\\] [0-9].[1-9] - .*$");	
        deleteLines.addElement("^\\[.*db2jcc_license_c.jar\\] [1-9].[0-9] - .*$");	
        deleteLines.addElement("^XSDB.*$");
<commit_msg>DERBY-6881: Test failures with JDK 9-ea b111<commit_after>/*

   Derby - Class org.apache.derbyTesting.functionTests.harness.Sed

   Licensed to the Apache Software Foundation (ASF) under one or more
   contributor license agreements.  See the NOTICE file distributed with
   this work for additional information regarding copyright ownership.
   The ASF licenses this file to You under the Apache License, Version 2.0
   (the "License"); you may not use this file except in compliance with
   the License.  You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

 */

package org.apache.derbyTesting.functionTests.harness;

/***
 * Sed.java
 *
 * This is a version of "sed" in Java for the Derby Function Tests,
 * written using the java.util.regex regular expression classes.
 * The substitutions/deletions are based on the original kornshell tests.
 *
 ***/

import java.io.*;
import java.util.Vector;
import java.util.List;
import java.util.Properties;
import java.util.StringTokenizer;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.derbyTesting.functionTests.util.TestUtil;

public class Sed
{
	private	static	final	String	SQL_EXCEPTION_FILTERED_SUBSTITUTION = 
        "java.sql.SQLException:";

    public Sed()
    {
    }

    public static void main(String[] args) throws Exception {
        if (args == null || args.length != 2) {
            System.err.println("Usage: Sed sourcefile targetfile");
            System.exit(1);
        }
        File src = new File(args[0]);
        File tgt = new File(args[1]);
        new Sed().exec(src,tgt,null, false, false,false);
    }

    // The arguments should be the names of the input and output files
    public void exec
		(File srcFile, File dstFile, InputStream isSed, boolean isJCC, boolean isI18N, boolean isJDBC4)
        throws IOException
    {
    	String hostName = TestUtil.getHostName();
    	
        // Vector for storing lines to be deleted
        Vector<String> deleteLines = new Vector<String>();
        deleteLines.addElement("^ij version.*$");
        deleteLines.addElement("^\\*\\*\\*\\* Test Run Started .* \\*\\*\\*\\*$");
        deleteLines.addElement("^\\*\\*\\*\\* Test Run Completed .* \\*\\*\\*\\*$");
        deleteLines.addElement("^ELAPSED TIME = [0-9]* milliseconds$");
        deleteLines.addElement("^\\^\\?$");
        //deleteLines.addElement("^\\.$"); // originally to remove lines with a dot
        deleteLines.addElement("^S.*ij> $");
        deleteLines.addElement("^ *$");
        deleteLines.addElement("^Server StackTrace:$");
        deleteLines.addElement("^\\[ *$");
        deleteLines.addElement("^\\] *$");
        deleteLines.addElement("^\\[$");
        deleteLines.addElement("^\\]$");
        deleteLines.addElement("^<not available>\\]$");
        deleteLines.addElement("^(.*at .*)\\(.*:[0-9].*\\)$");
        deleteLines.addElement("^(.*at .*)\\(*.java\\)$");
        deleteLines.addElement("^(.*at .*)\\(Compiled Code\\)$");
        deleteLines.addElement("^.*at java.*\\<init\\>\\(.*\\(Compiled Code\\)\\)$");
        deleteLines.addElement("^(.*at .*)\\(Interpreted Code\\)$");
        deleteLines.addElement("^(.*at .*)\\(Unknown Source\\)$");
        deleteLines.addElement("^(.*at .*)\\(.*Native Method\\)$");
        deleteLines.addElement("^\\tat $"); // rare case of incomplete stack trace line
        deleteLines.addElement("optimizer estimated cost");
        deleteLines.addElement("optimizer estimated row count");
        deleteLines.addElement("Using executables built for native_threads");
        deleteLines.addElement("Estimate of memory used");
        deleteLines.addElement("Size of merge runs");
        deleteLines.addElement("Number of merge runs");
        deleteLines.addElement("Sort type");
        deleteLines.addElement("Optimization started at .*$");
        deleteLines.addElement("WARNING 02000: No row was found for FETCH, UPDATE or DELETE");
        // deleteLines for stack traces from j9 jvm to match those above for other jvms
        deleteLines.addElement("Stack trace:");	
        deleteLines.addElement("^.*java/.*\\<init\\>\\(.*\\)V");
        deleteLines.addElement("^.*org/apache/derby/.*\\(.*\\).*$");	
        // next for j9 stack trace with jarfiles test run.
        deleteLines.addElement("^.*java/.*\\(.*\\).*$");
        deleteLines.addElement("^\\[.*db2jcc.jar\\] [0-9].[1-9] - .*$");	
        deleteLines.addElement("^\\[.*db2jcc_license_c.jar\\] [1-9].[0-9] - .*$");	
        deleteLines.addElement("^XSDB.*$");

{'fraction_non_alphanumeric': 0.16052172126938483, 'fraction_numerical': 0.00739447468419431, 'mean_word_length': 2.907704654895666, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6440411', 'n_tokens_mistral': 2905, 'n_tokens_neox': 2814, 'n_words': 766}