<commit_before>/*
 * Copyright 2015 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.netflix.atlas.core.model

import com.netflix.atlas.core.model.DataExpr.AggregateFunction
import com.netflix.atlas.core.stacklang.SimpleWord
import com.netflix.atlas.core.stacklang.StandardVocabulary.Macro
import com.netflix.atlas.core.stacklang.Vocabulary
import com.netflix.atlas.core.stacklang.Word

object MathVocabulary extends Vocabulary {

  import com.netflix.atlas.core.model.ModelExtractors._

  val name: String = "math"

  val dependsOn: List[Vocabulary] = List(DataVocabulary)

  val words: List[Word] = List(
    GroupBy,
    Const,
    Random,
    Time,
    CommonQuery,
    Abs, Negate, Sqrt, PerStep,

    Add, Subtract, Multiply, Divide,
    GreaterThan,
    GreaterThanEqual,
    LessThan,
    LessThanEqual,
    FAdd, FSubtract, FMultiply, FDivide,

    And, Or,

    Sum, Count, Min, Max,

    Macro("avg", List(":dup", ":sum", ":swap", ":count", ":div"), List("name,sps,:eq,(,nf.cluster,),:by")),

    Macro("pct", List(":dup", ":sum", ":div", "100", ":mul"), List("name,sps,:eq,(,nf.cluster,),:by")),

    Macro("dist-avg", List(
        "statistic", "(", "totalTime", "totalAmount", ")", ":in", ":sum",
        "statistic", "count", ":eq", ":sum",
        ":div"
      ),
      List(":dist-avg,name,sps,:eq,:cq,(,name,),:by")),

    Macro("dist-stddev", List(
        // N
        "statistic", "count", ":eq", ":sum",

        // sum(x^2)
        "statistic", "totalOfSquares", ":eq", ":sum",

        // N * sum(x^2)
        ":mul",

        // sum(x)
        "statistic", "(", "totalAmount", "totalTime", ")", ":in", ":sum",

        // sum(x)^2
        ":dup", ":mul",

        // N * sum(x^2) - sum(x)^2
        ":sub",

        // N^2
        "statistic", "count", ":eq", ":sum", ":dup", ":mul",

        // v = (N * sum(x^2) - sum(x)^2) / N^2
        ":div",

        // stddev = sqrt(v)
        ":sqrt"
      ),
      List(":dist-stddev,name,sps,:eq,:cq"))
  )<commit_msg>change dist-avg and dist-stddev to<commit_after>/*
 * Copyright 2015 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.netflix.atlas.core.model

import com.netflix.atlas.core.model.DataExpr.AggregateFunction
import com.netflix.atlas.core.stacklang.SimpleWord
import com.netflix.atlas.core.stacklang.StandardVocabulary.Macro
import com.netflix.atlas.core.stacklang.Vocabulary
import com.netflix.atlas.core.stacklang.Word

object MathVocabulary extends Vocabulary {

  import com.netflix.atlas.core.model.ModelExtractors._

  val name: String = "math"

  val dependsOn: List[Vocabulary] = List(DataVocabulary)

  val words: List[Word] = List(
    GroupBy,
    Const,
    Random,
    Time,
    CommonQuery,
    Abs, Negate, Sqrt, PerStep,

    Add, Subtract, Multiply, Divide,
    GreaterThan,
    GreaterThanEqual,
    LessThan,
    LessThanEqual,
    FAdd, FSubtract, FMultiply, FDivide,

    And, Or,

    Sum, Count, Min, Max,

    Macro("avg", List(":dup", ":sum", ":swap", ":count", ":div"), List("name,sps,:eq,(,nf.cluster,),:by")),

    Macro("pct", List(":dup", ":sum", ":div", "100", ":mul"), List("name,sps,:eq,(,nf.cluster,),:by")),

    Macro("dist-avg", List(
        "statistic", "(", "totalTime", "totalAmount", ")", ":in", ":sum",
        "statistic", "count", ":eq", ":sum",
        ":div",
        ":swap", ":cq"
      ),
      List("name,sps,:eq")),

    Macro("dist-stddev", List(
        // N
        "statistic", "count", ":eq", ":sum",

        // sum(x^2)
        "statistic", "totalOfSquares", ":eq", ":sum",

        // N * sum(x^2)
        ":mul",

        // sum(x)
        "statistic", "(", "totalAmount", "totalTime", ")", ":in", ":sum",

        // sum(x)^2
        ":dup", ":mul",

        // N * sum(x^2) - sum(x)^2
        ":sub",

        // N^2
        "statistic", "count", ":eq", ":sum", ":dup", ":mul",

        // v = (N * sum(x^2) - sum(x)^2) / N^2
        ":div",

        // stddev = sqrt(v)
        ":sqrt",

        // Swap and use :cq to apply a common query
        ":swap", ":cq"
      ),
      List("name,sps,:eq"))
  )
{'fraction_non_alphanumeric': 0.17894736842105263, 'fraction_numerical': 0.007797270955165692, 'mean_word_length': 2.840568862275449, 'pattern_counts': {'":': 66, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1375583', 'n_tokens_mistral': 1783, 'n_tokens_neox': 1723, 'n_words': 441}