<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.client.solrj.io.sql;

import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.Properties;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.ZkStateReader;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;

/**
 * All base tests will be done with CloudSolrStream. Under the covers CloudSolrStream uses SolrStream so
 * SolrStream will get fully exercised through these tests.
 **/

@Slow
@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45"})
public class JdbcTest extends SolrCloudTestCase {

  private static final String COLLECTIONORALIAS = "collection1";

  private static final String id = "id";

  private static String zkHost;

  @BeforeClass
  public static void setupCluster() throws Exception {
    configureCluster(2)
        .addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
        .configure();

    String collection;
    boolean useAlias = random().nextBoolean();
    if (useAlias) {
      collection = COLLECTIONORALIAS + "_collection";
    } else {
      collection = COLLECTIONORALIAS;
    }
    CollectionAdminRequest.createCollection(collection, "conf", 2, 1).process(cluster.getSolrClient());
    AbstractDistribZkTestBase.waitForRecoveriesToFinish(collection, cluster.getSolrClient().getZkStateReader(),
        false, true, DEFAULT_TIMEOUT);
    if (useAlias) {
      CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection).process(cluster.getSolrClient());
    }

    new UpdateRequest()
        .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "testnull_i", null)
        .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "testnull_i", "2")
        .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "testnull_i", null)
        .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "testnull_i", "4")
        .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "testnull_i", null)
        .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6", "testnull_i", "6")
        .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7", "testnull_i", null)
        .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8", "testnull_i", "8")
        .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9", "testnull_i", null)
        .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10", "testnull_i", "10")
        .commit(cluster.getSolrClient(), collection);

    zkHost = cluster.getZkServer().getZkAddress();
  }

  @Test
  public void doTest() throws Exception {

    Properties props = new Properties();

    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i desc limit 2")) {
          assertTrue(rs.next());

          assertEquals(14, rs.getLong("a_i"));
          assertEquals(14, rs.getLong(2));
          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(3));
          assertEquals(10, rs.getDouble("a_f"), 0);
          assertEquals(10, rs.getDouble(4), 0);

          assertTrue(rs.next());

          assertEquals(13, rs.getLong("a_i"));
          assertEquals(13, rs.getLong(2));
          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(3));
          assertEquals(9, rs.getDouble("a_f"), 0);
          assertEquals(9, rs.getDouble(4), 0);

          assertFalse(rs.next());
        }

        //Test statement reuse
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i asc limit 2")) {
          assertTrue(rs.next());

          assertEquals(0, rs.getLong("a_i"));
          assertEquals(0, rs.getLong(2));
          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(3));
          assertEquals(1, rs.getDouble("a_f"), 0);
          assertEquals(1, rs.getDouble(4), 0);

          assertTrue(rs.next());

          assertEquals(1, rs.getLong("a_i"));
          assertEquals(1, rs.getLong(2));
          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(3));
          assertEquals(5, rs.getDouble("a_f"), 0);
          assertEquals(5, rs.getDouble(4), 0);

          assertFalse(rs.next());
        }
      }

      //Test connection reuse
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i desc limit 2")) {
          assertTrue(rs.next());

          assertEquals(14, rs.getLong("a_i"));
          assertEquals(14, rs.getLong(2));

          assertTrue(rs.next());

          assertEquals(13, rs.getLong("a_i"));
          assertEquals(13, rs.getLong(2));

          assertFalse(rs.next());
        }

        //Test statement reuse
        stmt.setMaxRows(2);
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i asc")) {
          assertTrue(rs.next());

          assertEquals(0, rs.getLong("a_i"));
          assertEquals(0, rs.getLong(2));

          assertTrue(rs.next());

          assertEquals(1, rs.getLong("a_i"));
          assertEquals(1, rs.getLong(2));

          assertFalse(rs.next());
        }

        //Test simple loop. Since limit is set it will override the statement maxRows.
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i asc    LIMIT   100")) {
          int count = 0;
          while (rs.next()) {
            ++count;
          }
          assertEquals(10, count);
        }
      }
    }

  }

  @Test
  public void testFacetAggregation() throws Exception {

    //Test facet aggregation
    Properties props = new Properties();
    props.put("aggregationMode", "facet");
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testMapReduceAggregation() throws Exception {

    //Test map / reduce aggregation
    Properties props = new Properties();
    props.put("aggregationMode", "map_reduce");
    props.put("numWorkers", "2");
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testConnectionParams() throws Exception {

    //Test params on the url
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost +
        "?collection=" + COLLECTIONORALIAS + "&aggregationMode=map_reduce&numWorkers=2")) {

      Properties p = ((ConnectionImpl) con).getProperties();

      assert (p.getProperty("aggregationMode").equals("map_reduce"));
      assert (p.getProperty("numWorkers").equals("2"));

      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testJDBCUrlParameters() throws Exception {

    // Test JDBC paramters in URL
    try (Connection con = DriverManager.getConnection(
        "jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS + "&username=&password=&testKey1=testValue&testKey2")) {

      Properties p = ((ConnectionImpl) con).getProperties();
      assertEquals("", p.getProperty("username"));
      assertEquals("", p.getProperty("password"));
      assertEquals("testValue", p.getProperty("testKey1"));
      assertEquals("", p.getProperty("testKey2"));

      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testJDBCPropertiesParameters() throws Exception {

    // Test JDBC paramters in properties
    Properties providedProperties = new Properties();
    providedProperties.put("collection", COLLECTIONORALIAS);
    providedProperties.put("username", "");
    providedProperties.put("password", "");
    providedProperties.put("testKey1", "testValue");
    providedProperties.put("testKey2", "");

    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost, providedProperties)) {
      Properties p = ((ConnectionImpl) con).getProperties();
      assert (p.getProperty("username").equals(""));
      assert (p.getProperty("password").equals(""));
      assert (p.getProperty("testKey1").equals("testValue"));
      assert (p.getProperty("testKey2").equals(""));

      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }
  }

  @Ignore("Fix error checking")
  @Test
  public void testErrorPropagation() throws Exception {
    //Test error propagation
    Properties props = new Properties();
    props.put("aggregationMode", "facet");
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select crap from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {
        } catch (Exception e) {
          String errorMessage = e.getMessage();
          assertTrue(errorMessage.contains("Group by queries must include at least one aggregate function"));
        }
      }
    }
  }

  @Test
  public void testSQLExceptionThrownWhenQueryAndConnUseDiffCollections() throws Exception  {
    String badCollection = COLLECTIONORALIAS + "bad";
    String connectionString = "jdbc:solr://" + zkHost + "?collection=" + badCollection;
    String sql = "select id, a_i, a_s, a_f from " + badCollection + " order by a_i desc limit 2";

    //Bad connection string: wrong collection name
    try(Connection connection = DriverManager.getConnection(connectionString)) {
      try (Statement statement = connection.createStatement()) {
        try (ResultSet ignored = statement.executeQuery(sql)) {
          fail("Expected query against wrong collection to throw a SQLException.");
        }
      }
    } catch (SQLException ignore) {
      // Expected exception due to miss matched collection
    }
  }

  @Test
  public void testDriverMetadata() throws Exception {
    String collection = COLLECTIONORALIAS;

    String connectionString1 = "jdbc:solr://" + zkHost + "?collection=" + collection +
        "&username=&password=&testKey1=testValue&testKey2";
    Properties properties1 = new Properties();

    String sql = "select id, a_i, a_s, a_f as my_float_col, testnull_i from " + collection +
        " order by a_i desc";

    String connectionString2 = "jdbc:solr://" + zkHost + "?collection=" + collection +
        "&aggregationMode=map_reduce&numWorkers=2&username=&password=&testKey1=testValue&testKey2";
    Properties properties2 = new Properties();

    String sql2 = sql + " limit 2";

    //testJDBCMethods(collection, connectionString1, properties1, sql);
    //testJDBCMethods(collection, connectionString2, properties2, sql);
    testJDBCMethods(collection, connectionString1, properties1, sql2);
    testJDBCMethods(collection, connectionString2, properties2, sql2);
  }

  private void testJDBCMethods(String collection, String connectionString, Properties properties, String sql) throws Exception {
    try (Connection con = DriverManager.getConnection(connectionString, properties)) {
      assertTrue(con.isValid(DEFAULT_CONNECTION_TIMEOUT));

      assertEquals(zkHost, con.getCatalog());
      con.setCatalog(zkHost);
      assertEquals(zkHost, con.getCatalog());

      assertEquals(null, con.getSchema());
      con.setSchema("myschema");
      assertEquals(null, con.getSchema());

      DatabaseMetaData databaseMetaData = con.getMetaData();
      assertNotNull(databaseMetaData);

      assertEquals(con, databaseMetaData.getConnection());
      assertEquals(connectionString, databaseMetaData.getURL());

      assertEquals(4, databaseMetaData.getJDBCMajorVersion());
      assertEquals(0, databaseMetaData.getJDBCMinorVersion());

      assertEquals("Apache Solr", databaseMetaData.getDatabaseProductName());

      // The following tests require package information that is not available when running via Maven
//      assertEquals(this.getClass().getPackage().getSpecificationVersion(), databaseMetaData.getDatabaseProductVersion());
//      assertEquals(0, databaseMetaData.getDatabaseMajorVersion());
//      assertEquals(0, databaseMetaData.getDatabaseMinorVersion());

//      assertEquals(this.getClass().getPackage().getSpecificationTitle(), databaseMetaData.getDriverName());
//      assertEquals(this.getClass().getPackage().getSpecificationVersion(), databaseMetaData.getDriverVersion());
//      assertEquals(0, databaseMetaData.getDriverMajorVersion());
//      assertEquals(0, databaseMetaData.getDriverMinorVersion());

      try(ResultSet rs = databaseMetaData.getSchemas()) {
        assertTrue(rs.next());
        assertEquals(zkHost, rs.getString("tableSchem"));
        assertNull(rs.getString("tableCat"));
        assertTrue(rs.next());
        assertEquals("metadata", rs.getString("tableSchem"));
        assertNull(rs.getString("tableCat"));
        assertFalse(rs.next());
      }

      try(ResultSet rs = databaseMetaData.getCatalogs()) {
        assertTrue(rs.next());
        assertNull(rs.getString("tableCat"));
        assertFalse(rs.next());
      }

      ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();

      SortedSet<String> tables = new TreeSet<>();
      Set<String> collections = zkStateReader.getClusterState().getCollectionsMap().keySet();
      Set<String> aliases = zkStateReader.getAliases().getCollectionAliasMap().keySet();
      tables.addAll(collections);
      tables.addAll(aliases);

      try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) {
        for(String table : tables) {
          assertTrue(rs.next());
          assertNull(rs.getString("tableCat"));
          assertEquals(zkHost, rs.getString("tableSchem"));
          assertEquals(table, rs.getString("tableName"));
          assertEquals("TABLE", rs.getString("tableType"));
          assertNull(rs.getString("remarks"));
        }
        assertFalse(rs.next());<commit_msg>Fix two test issues an NPE and ordering issue<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.client.solrj.io.sql;

import java.sql.Connection;
import java.sql.DatabaseMetaData;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;

import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.cloud.AbstractDistribZkTestBase;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.Aliases;
import org.apache.solr.common.cloud.ZkStateReader;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;

/**
 * All base tests will be done with CloudSolrStream. Under the covers CloudSolrStream uses SolrStream so
 * SolrStream will get fully exercised through these tests.
 **/

@Slow
@LuceneTestCase.SuppressCodecs({"Lucene3x", "Lucene40", "Lucene41", "Lucene42", "Lucene45"})
public class JdbcTest extends SolrCloudTestCase {

  private static final String COLLECTIONORALIAS = "collection1";

  private static final String id = "id";

  private static String zkHost;

  @BeforeClass
  public static void setupCluster() throws Exception {
    configureCluster(2)
        .addConfig("conf", getFile("solrj").toPath().resolve("solr").resolve("configsets").resolve("streaming").resolve("conf"))
        .configure();

    String collection;
    boolean useAlias = random().nextBoolean();
    if (useAlias) {
      collection = COLLECTIONORALIAS + "_collection";
    } else {
      collection = COLLECTIONORALIAS;
    }
    CollectionAdminRequest.createCollection(collection, "conf", 2, 1).process(cluster.getSolrClient());
    AbstractDistribZkTestBase.waitForRecoveriesToFinish(collection, cluster.getSolrClient().getZkStateReader(),
        false, true, DEFAULT_TIMEOUT);
    if (useAlias) {
      CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection).process(cluster.getSolrClient());
    }

    new UpdateRequest()
        .add(id, "0", "a_s", "hello0", "a_i", "0", "a_f", "1", "testnull_i", null)
        .add(id, "2", "a_s", "hello0", "a_i", "2", "a_f", "2", "testnull_i", "2")
        .add(id, "3", "a_s", "hello3", "a_i", "3", "a_f", "3", "testnull_i", null)
        .add(id, "4", "a_s", "hello4", "a_i", "4", "a_f", "4", "testnull_i", "4")
        .add(id, "1", "a_s", "hello0", "a_i", "1", "a_f", "5", "testnull_i", null)
        .add(id, "5", "a_s", "hello3", "a_i", "10", "a_f", "6", "testnull_i", "6")
        .add(id, "6", "a_s", "hello4", "a_i", "11", "a_f", "7", "testnull_i", null)
        .add(id, "7", "a_s", "hello3", "a_i", "12", "a_f", "8", "testnull_i", "8")
        .add(id, "8", "a_s", "hello3", "a_i", "13", "a_f", "9", "testnull_i", null)
        .add(id, "9", "a_s", "hello0", "a_i", "14", "a_f", "10", "testnull_i", "10")
        .commit(cluster.getSolrClient(), collection);

    zkHost = cluster.getZkServer().getZkAddress();
  }

  @Test
  public void doTest() throws Exception {

    Properties props = new Properties();

    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i desc limit 2")) {
          assertTrue(rs.next());

          assertEquals(14, rs.getLong("a_i"));
          assertEquals(14, rs.getLong(2));
          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(3));
          assertEquals(10, rs.getDouble("a_f"), 0);
          assertEquals(10, rs.getDouble(4), 0);

          assertTrue(rs.next());

          assertEquals(13, rs.getLong("a_i"));
          assertEquals(13, rs.getLong(2));
          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(3));
          assertEquals(9, rs.getDouble("a_f"), 0);
          assertEquals(9, rs.getDouble(4), 0);

          assertFalse(rs.next());
        }

        //Test statement reuse
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i asc limit 2")) {
          assertTrue(rs.next());

          assertEquals(0, rs.getLong("a_i"));
          assertEquals(0, rs.getLong(2));
          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(3));
          assertEquals(1, rs.getDouble("a_f"), 0);
          assertEquals(1, rs.getDouble(4), 0);

          assertTrue(rs.next());

          assertEquals(1, rs.getLong("a_i"));
          assertEquals(1, rs.getLong(2));
          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(3));
          assertEquals(5, rs.getDouble("a_f"), 0);
          assertEquals(5, rs.getDouble(4), 0);

          assertFalse(rs.next());
        }
      }

      //Test connection reuse
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i desc limit 2")) {
          assertTrue(rs.next());

          assertEquals(14, rs.getLong("a_i"));
          assertEquals(14, rs.getLong(2));

          assertTrue(rs.next());

          assertEquals(13, rs.getLong("a_i"));
          assertEquals(13, rs.getLong(2));

          assertFalse(rs.next());
        }

        //Test statement reuse
        stmt.setMaxRows(2);
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i asc")) {
          assertTrue(rs.next());

          assertEquals(0, rs.getLong("a_i"));
          assertEquals(0, rs.getLong(2));

          assertTrue(rs.next());

          assertEquals(1, rs.getLong("a_i"));
          assertEquals(1, rs.getLong(2));

          assertFalse(rs.next());
        }

        //Test simple loop. Since limit is set it will override the statement maxRows.
        try (ResultSet rs = stmt.executeQuery("select id, a_i, a_s, a_f from " + COLLECTIONORALIAS + " order by a_i asc    LIMIT   100")) {
          int count = 0;
          while (rs.next()) {
            ++count;
          }
          assertEquals(10, count);
        }
      }
    }

  }

  @Test
  public void testFacetAggregation() throws Exception {

    //Test facet aggregation
    Properties props = new Properties();
    props.put("aggregationMode", "facet");
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testMapReduceAggregation() throws Exception {

    //Test map / reduce aggregation
    Properties props = new Properties();
    props.put("aggregationMode", "map_reduce");
    props.put("numWorkers", "2");
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testConnectionParams() throws Exception {

    //Test params on the url
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost +
        "?collection=" + COLLECTIONORALIAS + "&aggregationMode=map_reduce&numWorkers=2")) {

      Properties p = ((ConnectionImpl) con).getProperties();

      assert (p.getProperty("aggregationMode").equals("map_reduce"));
      assert (p.getProperty("numWorkers").equals("2"));

      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testJDBCUrlParameters() throws Exception {

    // Test JDBC paramters in URL
    try (Connection con = DriverManager.getConnection(
        "jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS + "&username=&password=&testKey1=testValue&testKey2")) {

      Properties p = ((ConnectionImpl) con).getProperties();
      assertEquals("", p.getProperty("username"));
      assertEquals("", p.getProperty("password"));
      assertEquals("testValue", p.getProperty("testKey1"));
      assertEquals("", p.getProperty("testKey2"));

      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }

  }

  @Test
  public void testJDBCPropertiesParameters() throws Exception {

    // Test JDBC paramters in properties
    Properties providedProperties = new Properties();
    providedProperties.put("collection", COLLECTIONORALIAS);
    providedProperties.put("username", "");
    providedProperties.put("password", "");
    providedProperties.put("testKey1", "testValue");
    providedProperties.put("testKey2", "");

    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost, providedProperties)) {
      Properties p = ((ConnectionImpl) con).getProperties();
      assert (p.getProperty("username").equals(""));
      assert (p.getProperty("password").equals(""));
      assert (p.getProperty("testKey1").equals("testValue"));
      assert (p.getProperty("testKey2").equals(""));

      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select a_s, sum(a_f) from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {

          assertTrue(rs.next());

          assertEquals("hello3", rs.getString("a_s"));
          assertEquals("hello3", rs.getString(1));
          assertEquals(26, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(26, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello0", rs.getString("a_s"));
          assertEquals("hello0", rs.getString(1));
          assertEquals(18, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(18, rs.getDouble(2), 0);

          assertTrue(rs.next());

          assertEquals("hello4", rs.getString("a_s"));
          assertEquals("hello4", rs.getString(1));
          assertEquals(11, rs.getDouble("EXPR$1"), 0); //sum(a_f)
          assertEquals(11, rs.getDouble(2), 0);

          assertFalse(rs.next());
        }
      }
    }
  }

  @Ignore("Fix error checking")
  @Test
  public void testErrorPropagation() throws Exception {
    //Test error propagation
    Properties props = new Properties();
    props.put("aggregationMode", "facet");
    try (Connection con = DriverManager.getConnection("jdbc:solr://" + zkHost + "?collection=" + COLLECTIONORALIAS, props)) {
      try (Statement stmt = con.createStatement()) {
        try (ResultSet rs = stmt.executeQuery("select crap from " + COLLECTIONORALIAS + " group by a_s " +
            "order by sum(a_f) desc")) {
        } catch (Exception e) {
          String errorMessage = e.getMessage();
          assertTrue(errorMessage.contains("Group by queries must include at least one aggregate function"));
        }
      }
    }
  }

  @Test
  public void testSQLExceptionThrownWhenQueryAndConnUseDiffCollections() throws Exception  {
    String badCollection = COLLECTIONORALIAS + "bad";
    String connectionString = "jdbc:solr://" + zkHost + "?collection=" + badCollection;
    String sql = "select id, a_i, a_s, a_f from " + badCollection + " order by a_i desc limit 2";

    //Bad connection string: wrong collection name
    try(Connection connection = DriverManager.getConnection(connectionString)) {
      try (Statement statement = connection.createStatement()) {
        try (ResultSet ignored = statement.executeQuery(sql)) {
          fail("Expected query against wrong collection to throw a SQLException.");
        }
      }
    } catch (SQLException ignore) {
      // Expected exception due to miss matched collection
    }
  }

  @Test
  public void testDriverMetadata() throws Exception {
    String collection = COLLECTIONORALIAS;

    String connectionString1 = "jdbc:solr://" + zkHost + "?collection=" + collection +
        "&username=&password=&testKey1=testValue&testKey2";
    Properties properties1 = new Properties();

    String sql = "select id, a_i, a_s, a_f as my_float_col, testnull_i from " + collection +
        " order by a_i desc";

    String connectionString2 = "jdbc:solr://" + zkHost + "?collection=" + collection +
        "&aggregationMode=map_reduce&numWorkers=2&username=&password=&testKey1=testValue&testKey2";
    Properties properties2 = new Properties();

    String sql2 = sql + " limit 2";

    //testJDBCMethods(collection, connectionString1, properties1, sql);
    //testJDBCMethods(collection, connectionString2, properties2, sql);
    testJDBCMethods(collection, connectionString1, properties1, sql2);
    testJDBCMethods(collection, connectionString2, properties2, sql2);
  }

  private void testJDBCMethods(String collection, String connectionString, Properties properties, String sql) throws Exception {
    try (Connection con = DriverManager.getConnection(connectionString, properties)) {
      assertTrue(con.isValid(DEFAULT_CONNECTION_TIMEOUT));

      assertEquals(zkHost, con.getCatalog());
      con.setCatalog(zkHost);
      assertEquals(zkHost, con.getCatalog());

      assertEquals(null, con.getSchema());
      con.setSchema("myschema");
      assertEquals(null, con.getSchema());

      DatabaseMetaData databaseMetaData = con.getMetaData();
      assertNotNull(databaseMetaData);

      assertEquals(con, databaseMetaData.getConnection());
      assertEquals(connectionString, databaseMetaData.getURL());

      assertEquals(4, databaseMetaData.getJDBCMajorVersion());
      assertEquals(0, databaseMetaData.getJDBCMinorVersion());

      assertEquals("Apache Solr", databaseMetaData.getDatabaseProductName());

      // The following tests require package information that is not available when running via Maven
//      assertEquals(this.getClass().getPackage().getSpecificationVersion(), databaseMetaData.getDatabaseProductVersion());
//      assertEquals(0, databaseMetaData.getDatabaseMajorVersion());
//      assertEquals(0, databaseMetaData.getDatabaseMinorVersion());

//      assertEquals(this.getClass().getPackage().getSpecificationTitle(), databaseMetaData.getDriverName());
//      assertEquals(this.getClass().getPackage().getSpecificationVersion(), databaseMetaData.getDriverVersion());
//      assertEquals(0, databaseMetaData.getDriverMajorVersion());
//      assertEquals(0, databaseMetaData.getDriverMinorVersion());


      List<String> tableSchemas = new ArrayList<>(Arrays.asList(zkHost, "metadata"));
      try(ResultSet rs = databaseMetaData.getSchemas()) {
        assertTrue(rs.next());
        assertTrue(tableSchemas.contains(rs.getString("tableSchem")));
        tableSchemas.remove(rs.getString("tableSchem"));
        assertNull(rs.getString("tableCat"));
        assertTrue(rs.next());
        assertTrue(tableSchemas.contains(rs.getString("tableSchem")));
        tableSchemas.remove(rs.getString("tableSchem"));
        assertNull(rs.getString("tableCat"));
        assertFalse(rs.next());
        assertTrue(tableSchemas.isEmpty());
      }

      try(ResultSet rs = databaseMetaData.getCatalogs()) {
        assertTrue(rs.next());
        assertNull(rs.getString("tableCat"));
        assertFalse(rs.next());
      }

      CloudSolrClient solrClient = cluster.getSolrClient();
      solrClient.connect();
      ZkStateReader zkStateReader = solrClient.getZkStateReader();

      SortedSet<String> tables = new TreeSet<>();

      Set<String> collectionsSet = zkStateReader.getClusterState().getCollectionsMap().keySet();
      tables.addAll(collectionsSet);

      Aliases aliases = zkStateReader.getAliases();
      if(aliases != null) {
        Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap();
        if(collectionAliasMap != null) {
          Set<String> aliasesSet = collectionAliasMap.keySet();
          tables.addAll(aliasesSet);
        }
      }

      try(ResultSet rs = databaseMetaData.getTables(null, zkHost, "%", null)) {
        for(String table : tables) {
          assertTrue(rs.next());
          assertNull(rs.getString("tableCat"));
          assertEquals(zkHost, rs.getString("tableSchem"));
          assertEquals(table, rs.getString("tableName"));
          assertEquals("TABLE", rs.getString("tableType"));
          assertNull(rs.getString("remarks"));
        }
        assertFalse(rs.next());
{'fraction_non_alphanumeric': 0.15005711022272986, 'fraction_numerical': 0.016609556443936798, 'mean_word_length': 3.207127840624687, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4824789', 'n_tokens_mistral': 12847, 'n_tokens_neox': 12477, 'n_words': 2676}