<commit_before>package com.quasardb.spark

import java.nio.ByteBuffer

import org.apache.spark.sql.{SQLContext, Row, SaveMode}
import org.apache.spark.{SparkContext, SparkException}

import org.scalatest.{BeforeAndAfterAll, FunSuite}

import net.quasardb.qdb.QdbCluster;

import com.quasardb.spark._
import com.quasardb.spark.rdd._

import scala.language.implicitConversions

class QdbTagsSuite extends FunSuite with BeforeAndAfterAll {

  private var qdbUri: String = "qdb://127.0.0.1:2836"
  private var sqlContext: SQLContext = _
  private val key1: String = java.util.UUID.randomUUID.toString
  private val key2: String = java.util.UUID.randomUUID.toString
  private val key3: String = java.util.UUID.randomUUID.toString

  private val tag1: String = java.util.UUID.randomUUID.toString
  private val tag2: String = java.util.UUID.randomUUID.toString

  override protected def beforeAll(): Unit = {
    super.beforeAll()
    sqlContext = new SQLContext(new SparkContext("local[2]", "QdbTagsSuite"))

    // Store a few default entries
    val entry1 = new QdbCluster(qdbUri).integer(key1)
    val entry2 = new QdbCluster(qdbUri).integer(key2)
    val entry3 = new QdbCluster(qdbUri).blob(key3)

    entry1.put(123)
    entry2.put(124)
    entry3.put(ByteBuffer.allocateDirect(3).put(new String("125").getBytes()))

    entry1.attachTag(tag1)
    entry2.attachTag(tag1)
    entry3.attachTag(tag2)
  }

  override protected def afterAll(): Unit = {
    try {
      sqlContext
        .sparkContext
        .stop()

    } finally {
      super.afterAll()
    }
  }

  test("searching for tags") {
    val results1 = sqlContext
      .tagAsDataFrame(qdbUri, tag1)
      .collect().sorted

    val results2 = sqlContext
      .tagAsDataFrame(qdbUri, tag2)
      .collect().sorted

    assert(results1.size == 2)

    assert(results1.head == key1)
    assert(results1.last == key2)

    assert(results2.size == 1)
    assert(results2.last == key3)
  }

  test("searching for integers by tag") {
    val results = sqlContext
      .tagAsDataFrame(qdbUri, tag1)
      .getInteger()
      .collect().sorted

    assert(results.size == 2)
    assert(results.head._2 == 123)
    assert(results.last._2 == 124)
  }

  test("searching for string by tag") {
    val results = sqlContext
      .tagAsDataFrame(qdbUri, tag2)
      .getString()
      .collect().sorted

    assert(results.size == 1)
    assert(results.last._2 == "125")
  }

}<commit_msg>Don't rely on ordering of results<commit_after>package com.quasardb.spark

import java.nio.ByteBuffer

import org.apache.spark.sql.{SQLContext, Row, SaveMode}
import org.apache.spark.{SparkContext, SparkException}

import org.scalatest.{BeforeAndAfterAll, FunSuite}

import net.quasardb.qdb.QdbCluster;

import com.quasardb.spark._
import com.quasardb.spark.rdd._

import scala.language.implicitConversions

class QdbTagsSuite extends FunSuite with BeforeAndAfterAll {

  private var qdbUri: String = "qdb://127.0.0.1:2836"
  private var sqlContext: SQLContext = _
  private val key1: String = java.util.UUID.randomUUID.toString
  private val key2: String = java.util.UUID.randomUUID.toString
  private val key3: String = java.util.UUID.randomUUID.toString

  private val tag1: String = java.util.UUID.randomUUID.toString
  private val tag2: String = java.util.UUID.randomUUID.toString

  override protected def beforeAll(): Unit = {
    super.beforeAll()
    sqlContext = new SQLContext(new SparkContext("local[2]", "QdbTagsSuite"))

    // Store a few default entries
    val entry1 = new QdbCluster(qdbUri).integer(key1)
    val entry2 = new QdbCluster(qdbUri).integer(key2)
    val entry3 = new QdbCluster(qdbUri).blob(key3)

    entry1.put(123)
    entry2.put(124)
    entry3.put(ByteBuffer.allocateDirect(3).put(new String("125").getBytes()))

    entry1.attachTag(tag1)
    entry2.attachTag(tag1)
    entry3.attachTag(tag2)
  }

  override protected def afterAll(): Unit = {
    try {
      sqlContext
        .sparkContext
        .stop()

    } finally {
      super.afterAll()
    }
  }

  test("searching for tags") {
    val results1 = sqlContext
      .tagAsDataFrame(qdbUri, tag1)
      .collect().sorted

    val results2 = sqlContext
      .tagAsDataFrame(qdbUri, tag2)
      .collect().sorted

    assert(results1.size == 2)

    assert(results1.head == key1 || results1.head == key2)
    assert(results1.last == key1 || results1.last == key2)

    assert(results2.size == 1)
    assert(results2.last == key3)
  }

  test("searching for integers by tag") {
    val results = sqlContext
      .tagAsDataFrame(qdbUri, tag1)
      .getInteger()
      .collect().sorted

    assert(results.size == 2)
    assert(results.head._2 == 123 || results.head._2 == 124)
    assert(results.last._2 == 123 || results.last._2 == 124)
  }

  test("searching for string by tag") {
    val results = sqlContext
      .tagAsDataFrame(qdbUri, tag2)
      .getString()
      .collect().sorted

    assert(results.size == 1)
    assert(results.last._2 == "125")
  }

}
{'fraction_non_alphanumeric': 0.11970423661071143, 'fraction_numerical': 0.030775379696243006, 'mean_word_length': 3.292452830188679, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3012633', 'n_tokens_mistral': 1758, 'n_tokens_neox': 1671, 'n_words': 347}