<commit_before>/*
 *  ____    ____    _____    ____    ___     ____ 
 * |  _ \  |  _ \  | ____|  / ___|  / _/    / ___|        Precog (R)
 * | |_) | | |_) | |  _|   | |     | |  /| | |  _         Advanced Analytics Engine for NoSQL Data
 * |  __/  |  _ <  | |___  | |___  |/ _| | | |_| |        Copyright (C) 2010 - 2013 SlamData, Inc.
 * |_|     |_| \_\ |_____|  \____|   /__/   \____|        All Rights Reserved.
 *
 * This program is free software: you can redistribute it and/or modify it under the terms of the 
 * GNU Affero General Public License as published by the Free Software Foundation, either version 
 * 3 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 
 * the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with this 
 * program. If not, see <http://www.gnu.org/licenses/>.
 *
 */
package com.precog
package muspelheim

import yggdrasil._
import com.precog.common._
import org.specs2.mutable._

trait EvalStackSpecs extends Specification {
  def eval(str: String, debug: Boolean = false): Set[SValue]
  def evalE(str: String, debug: Boolean = false): Set[SEvent]

  "the full stack" should {
    "count a filtered clicks dataset" in {
      val input = """
        | clicks := //clicks
        | count(clicks where clicks.time > 0)""".stripMargin
        
      eval(input) mustEqual Set(SDecimal(100))
    }

    "count the campaigns dataset" >> {
      "<root>" >> {
        eval("count(//campaigns)") mustEqual Set(SDecimal(100))
      }
      
      "gender" >> {
        eval("count(//campaigns.gender)") mustEqual Set(SDecimal(100))
      }
      
      "platform" >> {
        eval("count(//campaigns.platform)") mustEqual Set(SDecimal(100))
      }
      
      "campaign" >> {
        eval("count(//campaigns.campaign)") mustEqual Set(SDecimal(100))
      }
      
      "cpm" >> {
        eval("count(//campaigns.cpm)") mustEqual Set(SDecimal(100))
      }

      "ageRange" >> {
        eval("count(//campaigns.ageRange)") mustEqual Set(SDecimal(100))
      }
    }

    "reduce the obnoxiously large dataset" >> {
      "<root>" >> {
        eval("mean(//obnoxious.v)", true) mustEqual Set(SDecimal(50000.5))
      }
    }

    "accept !true and !false" >> {
      "!true" >> {
        eval("!true") mustEqual Set(SBoolean(false))
      }

      "!false" >> {
        eval("!false") mustEqual Set(SBoolean(true))
      }
    }

    "accept a dereferenced array" >> {
      "non-empty array" >> {
        eval("[1,2,3].foo") mustEqual Set()
      }

      "empty array" >> {
        eval("[].foo") mustEqual Set()
      }
    }

    "accept a dereferenced object" >> {
      "non-empty object" >> {
        eval("{a: 42}[1]") mustEqual Set()
      }

      "empty object" >> {
        eval("{}[0]") mustEqual Set()
      }
    }    
    
    "accept a where'd empty array and empty object" >> {
      "empty object (left)" >> {
        eval("{} where true") mustEqual Set(SObject(Map()))
      }

      "empty object (right)" >> {
        eval("true where {}") mustEqual Set()
      }
      
      "empty array (left)" >> {
        eval("[] where true") mustEqual Set(SArray(Vector()))
      }

      "empty array (right)" >> {
        eval("true where []") mustEqual Set()
      }
    }    
    
    "accept a with'd empty array and empty object" >> {
      "empty object (left)" >> {
        eval("{} with true") mustEqual Set()
      }

      "empty object (right)" >> {
        eval("true with {}") mustEqual Set()
      }
      
      "empty array (left)" >> {
        eval("[] with true") mustEqual Set()
      }

      "empty array (right)" >> {
        eval("true with []") mustEqual Set()
      }
    }

    "perform a simple join by value sorting" in {
      val input = """
        | clicks := //clicks
        | views := //views
        |
        | clicks ~ views
        |   std::string::concat(clicks.pageId, views.pageId) where clicks.userId = views.userId
        """.stripMargin

      val resultsE = evalE(input)

      resultsE must haveSize(473)

      val results = resultsE collect {
        case (ids, str) if ids.length == 2 => str
      }

      results must contain(SString("page-2page-2"))
      results must contain(SString("page-2page-1"))
      results must contain(SString("page-4page-3"))
      results must contain(SString("page-4page-4"))
      results must contain(SString("page-3page-4"))
      results must contain(SString("page-3page-0"))
      results must contain(SString("page-0page-2"))
      results must contain(SString("page-0page-4"))
      results must contain(SString("page-0page-0"))
      results must contain(SString("page-0page-1"))
      results must contain(SString("page-4page-2"))
      results must contain(SString("page-0page-3"))
      results must contain(SString("page-1page-1"))
      results must contain(SString("page-1page-4"))
      results must contain(SString("page-1page-0"))
      results must contain(SString("page-1page-2"))
      results must contain(SString("page-1page-3"))
      results must contain(SString("page-3page-3"))
      results must contain(SString("page-3page-1"))
      results must contain(SString("page-4page-0"))
      results must contain(SString("page-4page-1"))
      results must contain(SString("page-3page-2"))
      results must contain(SString("page-2page-3"))
      results must contain(SString("page-2page-4"))
      results must contain(SString("page-2page-0")) 
    }

    "union sets coming out of a solve" >> {
      val input = """
        clicks := //clicks
        foobar := solve 'a {userId: 'a, size: count(clicks where clicks.userId = 'a)}
        foobaz := solve 'b {pageId: 'b, size: count(clicks where clicks.pageId = 'b)}
        foobar union foobaz
      """.stripMargin

      val results = evalE(input)

      results must haveSize(26)

      forall(results) {
        case (ids, SObject(obj)) => {
          ids must haveSize(1)
          obj must haveSize(2)
          obj must haveKey("userId") or haveKey("pageId")
          obj must haveKey("size")
        }
      }
    }
    
    "have the correct number of identities and values in a relate" >> {
      "with the sum plus the LHS" >> {
        val input = """
          | //clicks ~ //campaigns
          | sum := //clicks.time + //campaigns.cpm
          | sum + //clicks.time""".stripMargin

        val results = evalE(input)

        results must haveSize(10000)

        forall(results) {
          case (ids, _) => ids must haveSize(2)
        }
      }
      
      "with the sum plus the RHS" >> {
        val input = """
          | //clicks ~ //campaigns
          | sum := //clicks.time + //campaigns.cpm
          | sum + //campaigns.cpm""".stripMargin
<commit_msg>Added test to EvalStackSpecs.<commit_after>/*
 *  ____    ____    _____    ____    ___     ____ 
 * |  _ \  |  _ \  | ____|  / ___|  / _/    / ___|        Precog (R)
 * | |_) | | |_) | |  _|   | |     | |  /| | |  _         Advanced Analytics Engine for NoSQL Data
 * |  __/  |  _ <  | |___  | |___  |/ _| | | |_| |        Copyright (C) 2010 - 2013 SlamData, Inc.
 * |_|     |_| \_\ |_____|  \____|   /__/   \____|        All Rights Reserved.
 *
 * This program is free software: you can redistribute it and/or modify it under the terms of the 
 * GNU Affero General Public License as published by the Free Software Foundation, either version 
 * 3 of the License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; 
 * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See 
 * the GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with this 
 * program. If not, see <http://www.gnu.org/licenses/>.
 *
 */
package com.precog
package muspelheim

import yggdrasil._
import com.precog.common._
import org.specs2.mutable._

trait EvalStackSpecs extends Specification {
  def eval(str: String, debug: Boolean = false): Set[SValue]
  def evalE(str: String, debug: Boolean = false): Set[SEvent]

  "the full stack" should {
    "count a filtered clicks dataset" in {
      val input = """
        | clicks := //clicks
        | count(clicks where clicks.time > 0)""".stripMargin
        
      eval(input) mustEqual Set(SDecimal(100))
    }

    "count the campaigns dataset" >> {
      "<root>" >> {
        eval("count(//campaigns)") mustEqual Set(SDecimal(100))
      }
      
      "gender" >> {
        eval("count(//campaigns.gender)") mustEqual Set(SDecimal(100))
      }
      
      "platform" >> {
        eval("count(//campaigns.platform)") mustEqual Set(SDecimal(100))
      }
      
      "campaign" >> {
        eval("count(//campaigns.campaign)") mustEqual Set(SDecimal(100))
      }
      
      "cpm" >> {
        eval("count(//campaigns.cpm)") mustEqual Set(SDecimal(100))
      }

      "ageRange" >> {
        eval("count(//campaigns.ageRange)") mustEqual Set(SDecimal(100))
      }
    }

    "reduce the obnoxiously large dataset" >> {
      "<root>" >> {
        eval("mean(//obnoxious.v)", true) mustEqual Set(SDecimal(50000.5))
      }
    }

    "accept !true and !false" >> {
      "!true" >> {
        eval("!true") mustEqual Set(SBoolean(false))
      }

      "!false" >> {
        eval("!false") mustEqual Set(SBoolean(true))
      }
    }

    "accept a dereferenced array" >> {
      "non-empty array" >> {
        eval("[1,2,3].foo") mustEqual Set()
      }

      "empty array" >> {
        eval("[].foo") mustEqual Set()
      }
    }

    "accept a dereferenced object" >> {
      "non-empty object" >> {
        eval("{a: 42}[1]") mustEqual Set()
      }

      "empty object" >> {
        eval("{}[0]") mustEqual Set()
      }
    }    
    
    "accept a where'd empty array and empty object" >> {
      "empty object (left)" >> {
        eval("{} where true") mustEqual Set(SObject(Map()))
      }

      "empty object (right)" >> {
        eval("true where {}") mustEqual Set()
      }
      
      "empty array (left)" >> {
        eval("[] where true") mustEqual Set(SArray(Vector()))
      }

      "empty array (right)" >> {
        eval("true where []") mustEqual Set()
      }
    }    
    
    "accept a with'd empty array and empty object" >> {
      "empty object (left)" >> {
        eval("{} with true") mustEqual Set()
      }

      "empty object (right)" >> {
        eval("true with {}") mustEqual Set()
      }
      
      "empty array (left)" >> {
        eval("[] with true") mustEqual Set()
      }

      "empty array (right)" >> {
        eval("true with []") mustEqual Set()
      }
    }

    "perform a simple join by value sorting" in {
      val input = """
        | clicks := //clicks
        | views := //views
        |
        | clicks ~ views
        |   std::string::concat(clicks.pageId, views.pageId) where clicks.userId = views.userId
        """.stripMargin

      val resultsE = evalE(input)

      resultsE must haveSize(473)

      val results = resultsE collect {
        case (ids, str) if ids.length == 2 => str
      }

      results must contain(SString("page-2page-2"))
      results must contain(SString("page-2page-1"))
      results must contain(SString("page-4page-3"))
      results must contain(SString("page-4page-4"))
      results must contain(SString("page-3page-4"))
      results must contain(SString("page-3page-0"))
      results must contain(SString("page-0page-2"))
      results must contain(SString("page-0page-4"))
      results must contain(SString("page-0page-0"))
      results must contain(SString("page-0page-1"))
      results must contain(SString("page-4page-2"))
      results must contain(SString("page-0page-3"))
      results must contain(SString("page-1page-1"))
      results must contain(SString("page-1page-4"))
      results must contain(SString("page-1page-0"))
      results must contain(SString("page-1page-2"))
      results must contain(SString("page-1page-3"))
      results must contain(SString("page-3page-3"))
      results must contain(SString("page-3page-1"))
      results must contain(SString("page-4page-0"))
      results must contain(SString("page-4page-1"))
      results must contain(SString("page-3page-2"))
      results must contain(SString("page-2page-3"))
      results must contain(SString("page-2page-4"))
      results must contain(SString("page-2page-0")) 
    }

    "union sets coming out of a solve" >> {
      val input = """
        clicks := //clicks
        foobar := solve 'a {userId: 'a, size: count(clicks where clicks.userId = 'a)}
        foobaz := solve 'b {pageId: 'b, size: count(clicks where clicks.pageId = 'b)}
        foobar union foobaz
      """.stripMargin

      val results = evalE(input)

      results must haveSize(26)

      forall(results) {
        case (ids, SObject(obj)) => {
          ids must haveSize(1)
          obj must haveSize(2)
          obj must haveKey("userId") or haveKey("pageId")
          obj must haveKey("size")
        }
      }
    }

    "accept covariance inside an object with'd with another object" >> {
      val input = """
        clicks := //clicks
        counts := solve 'time
          {count: count(clicks where clicks.time = 'time) }

        cov := std::stats::cov(counts.count, counts.count)
        counts with {covariance: cov}
      """.stripMargin

      val results = evalE(input)

      results must haveSize(81)  

      forall(results) {
        case (ids, SObject(obj)) =>
          ids must haveSize(1)
          obj must haveKey("covariance")
          obj must haveKey("count")
      }
    }
    
    "have the correct number of identities and values in a relate" >> {
      "with the sum plus the LHS" >> {
        val input = """
          | //clicks ~ //campaigns
          | sum := //clicks.time + //campaigns.cpm
          | sum + //clicks.time""".stripMargin

        val results = evalE(input)

        results must haveSize(10000)

        forall(results) {
          case (ids, _) => ids must haveSize(2)
        }
      }
      
      "with the sum plus the RHS" >> {
        val input = """
          | //clicks ~ //campaigns
          | sum := //clicks.time + //campaigns.cpm
          | sum + //campaigns.cpm""".stripMargin

{'fraction_non_alphanumeric': 0.14456588987420085, 'fraction_numerical': 0.01519213583556747, 'mean_word_length': 2.1557483731019524, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 148, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '139462', 'n_tokens_mistral': 4665, 'n_tokens_neox': 4466, 'n_words': 1151}