<commit_before>package org.ferris.riviera.console.research;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.Scanner;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import org.ferris.riviera.console.script.ScriptBuilder;
import org.junit.Before;
import org.junit.Test;

/**
 *
 * @author <NAME> <EMAIL> @mjremijan
 */
public class ImportSql {

    ScriptBuilder builder;

    @Before
    public void before() {
        builder = new ScriptBuilder();
    }

    @Test
    public void readFromJar() throws Exception
    {
        File f = new File("src/test/misc/ddl-1.0-SNAPSHOT.jar");
        JarFile jarFile = new JarFile(f);

        Enumeration<JarEntry> jarEntries
            = jarFile.entries();

        jarFile.stream().forEach(v -> {
                try {
                    InputStream is = jarFile.getInputStream(v);
                    String result = new BufferedReader(new InputStreamReader(is)).lines()
                            .parallel().collect(Collectors.joining("\n"));

                    //System.out.printf("---------------------%n%s---------------------%n",result);

                    List<String> l
                            = split(result);

                    l.forEach(x -> System.out.printf(">>> %s%n",x));

                } catch (IOException ex) {
                   throw new RuntimeException(ex);
                }
            }
        );

    }


    /**
     * http://stackoverflow.com/questions/1497569/how-to-execute-sql-script-file-using-jdbc
     *
     * @param conn
     * @param in
     * @throws SQLException
     */
    public List<String> split(String sql)
    {
        Scanner s = new Scanner(sql);
        //s.useDelimiter("(;(\r)?\n)|(--\n)");
        s.useDelimiter("(;(\r)?\n)|((\r)?\n)?(--)?.*(--(\r)?\n)");

        List<String> l = new LinkedList<>();
        try {
            while (s.hasNext()) {
                String line = s.next();
                if (line.startsWith("/*!") && line.endsWith("*/")) {
                    int i = line.indexOf(' ');
                    line = line.substring(i + 1, line.length() - " */".length());
                }

                if (line.trim().length() > 0) {
                    l.add(line);
                }
            }
        } finally {
        }
        return l;
    }

    /**
     * http://stackoverflow.com/questions/1497569/how-to-execute-sql-script-file-using-jdbc
     *
     * @param conn
     * @param in
     * @throws SQLException
     */
    public static void importSQL(Connection conn, InputStream in) throws SQLException {
        Scanner s = new Scanner(in);
        //s.useDelimiter("(;(\r)?\n)|(--\n)");
        s.useDelimiter("(;(\r)?\n)|((\r)?\n)?(--)?.*(--(\r)?\n)");
        Statement st = null;
        try {
            st = conn.createStatement();
            while (s.hasNext()) {
                String line = s.next();
                if (line.startsWith("/*!") && line.endsWith("*/")) {
                    int i = line.indexOf(' ');
                    line = line.substring(i + 1, line.length() - " */".length());
                }

                if (line.trim().length() > 0) {
                    st.execute(line);
                }
            }
        } finally {
            if (st != null) {
                st.close();
            }
        }
    }
}
<commit_msg>RIVIERA-13: refactoring the old 'scripts' package unit tests<commit_after>package org.ferris.riviera.console.research;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Enumeration;
import java.util.LinkedList;
import java.util.List;
import java.util.Scanner;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;
import java.util.stream.Collectors;
import org.junit.Test;

/**
 *
 * @author <NAME> <EMAIL> @mjremijan
 */
public class ImportSql {


    @Test
    public void readFromJar() throws Exception
    {
        File f = new File("src/test/misc/ddl-1.0-SNAPSHOT.jar");
        JarFile jarFile = new JarFile(f);

        Enumeration<JarEntry> jarEntries
            = jarFile.entries();

        jarFile.stream().forEach(v -> {
                try {
                    InputStream is = jarFile.getInputStream(v);
                    String result = new BufferedReader(new InputStreamReader(is)).lines()
                            .parallel().collect(Collectors.joining("\n"));

                    //System.out.printf("---------------------%n%s---------------------%n",result);

                    List<String> l
                            = split(result);

                    l.forEach(x -> System.out.printf(">>> %s%n",x));

                } catch (IOException ex) {
                   throw new RuntimeException(ex);
                }
            }
        );

    }


    /**
     * http://stackoverflow.com/questions/1497569/how-to-execute-sql-script-file-using-jdbc
     *
     * @param conn
     * @param in
     * @throws SQLException
     */
    public List<String> split(String sql)
    {
        Scanner s = new Scanner(sql);
        //s.useDelimiter("(;(\r)?\n)|(--\n)");
        s.useDelimiter("(;(\r)?\n)|((\r)?\n)?(--)?.*(--(\r)?\n)");

        List<String> l = new LinkedList<>();
        try {
            while (s.hasNext()) {
                String line = s.next();
                if (line.startsWith("/*!") && line.endsWith("*/")) {
                    int i = line.indexOf(' ');
                    line = line.substring(i + 1, line.length() - " */".length());
                }

                if (line.trim().length() > 0) {
                    l.add(line);
                }
            }
        } finally {
        }
        return l;
    }

    /**
     * http://stackoverflow.com/questions/1497569/how-to-execute-sql-script-file-using-jdbc
     *
     * @param conn
     * @param in
     * @throws SQLException
     */
    public static void importSQL(Connection conn, InputStream in) throws SQLException {
        Scanner s = new Scanner(in);
        //s.useDelimiter("(;(\r)?\n)|(--\n)");
        s.useDelimiter("(;(\r)?\n)|((\r)?\n)?(--)?.*(--(\r)?\n)");
        Statement st = null;
        try {
            st = conn.createStatement();
            while (s.hasNext()) {
                String line = s.next();
                if (line.startsWith("/*!") && line.endsWith("*/")) {
                    int i = line.indexOf(' ');
                    line = line.substring(i + 1, line.length() - " */".length());
                }

                if (line.trim().length() > 0) {
                    st.execute(line);
                }
            }
        } finally {
            if (st != null) {
                st.close();
            }
        }
    }
}

{'fraction_non_alphanumeric': 0.1716960044705225, 'fraction_numerical': 0.00586756077116513, 'mean_word_length': 1.5270031768443346, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6892588', 'n_tokens_mistral': 2206, 'n_tokens_neox': 2106, 'n_words': 380}