<commit_before>(:
  Copyright 2012-2018 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace trace = "http://marklogic.com/data-hub/trace";

import module namespace config = "http://marklogic.com/data-hub/config"
  at "/com.marklogic.hub/config.xqy";

import module namespace consts = "http://marklogic.com/data-hub/consts"
  at "/MarkLogic/data-hub-framework/impl/consts.xqy";

import module namespace err = "http://marklogic.com/data-hub/err"
  at "/MarkLogic/data-hub-framework/impl/error-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/MarkLogic/data-hub-framework/impl/hub-utils-lib.xqy";

import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";

import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
  at "/MarkLogic/data-hub-framework/impl/run-flow-context.xqy";

import module namespace search = "http://marklogic.com/appservices/search"
  at "/MarkLogic/appservices/search/search.xqy";

declare option xdmp:mapping "false";

(: new trace-settings are initialized for each transaction :)
declare variable $current-trace-settings := map:map();

declare function trace:new-trace() as map:map
{
  map:new((
    map:entry("traceId", xdmp:random()),
    map:entry("created", fn:current-dateTime())
  ))
};

declare function trace:enable-tracing($enabled as xs:boolean)
{
  xdmp:eval('
    declare namespace trace = "http://marklogic.com/data-hub/trace";
    declare variable $enabled external;
    xdmp:document-insert(
      "/com.marklogic.hub/settings/__tracing_enabled__.xml",
      element trace:is-tracing-enabled { if ($enabled) then 1 else 0 },
      xdmp:default-permissions(),
      "hub-core-module")
    ',
    map:new((map:entry("enabled", $enabled))),
    map:new((map:entry("database", xdmp:modules-database()), map:entry("ignoreAmps", fn:true())))
  ),
  hul:invalidate-field-cache("tracing-enabled")
};

declare function trace:enabled() as xs:boolean
{
  let $key := "tracing-enabled"
  let $flag :=  hul:from-field-cache-or-empty($key, ())
  return
    if (exists($flag)) then
      $flag
    else
      hul:set-field-cache(
        $key,
        xdmp:eval('
          declare namespace trace = "http://marklogic.com/data-hub/trace";
          fn:exists(
            cts:search(
              fn:doc("/com.marklogic.hub/settings/__tracing_enabled__.xml"),
              cts:element-value-query(xs:QName("trace:is-tracing-enabled"), "1", ("exact")),
              ("unfiltered", "score-zero", "unchecked", "unfaceted")
            )
          )
        ',(), map:new(map:entry("database", xdmp:modules-database()))),
        xs:dayTimeDuration("PT1M")
      )
};

declare function trace:has-errors() as xs:boolean
{
  (map:get($current-trace-settings, "_has_errors"), fn:false())[1] eq fn:true()
};

declare %private function trace:increment-error-count()
{
  map:put($current-trace-settings, "error-count", trace:get-error-count() + 1)
};

declare function trace:get-error-count()
{
  (map:get($current-trace-settings, "error-count"), 0)[1]
};

declare %private function trace:add-failed-item($item as xs:string)
{
  json:array-push(trace:get-failed-items(), $item)
};

declare %private function trace:add-completed-item($item as xs:string)
{
  json:array-push(trace:get-completed-items(), $item)
};

declare function trace:set-plugin-label(
  $label as xs:string)
{
  trace:set-plugin-label(
    rfc:get-trace($rfc:item-context),
    $label
  )
};

(:
 : Sets the label of the currently running plugin
 :
 : @param $label - the label of the running plugin
 :)
declare function trace:set-plugin-label(
  $current-trace as map:map,
  $label as xs:string)
{
  map:put($current-trace, "plugin-label", $label)
};

declare function trace:get-plugin-label(
  $current-trace as map:map)
as xs:string
{
  map:get($current-trace, "plugin-label")
};

declare function trace:reset-plugin-input()
{
  trace:reset-plugin-input(rfc:get-trace($rfc:item-context))
};

declare function trace:reset-plugin-input(
  $current-trace as map:map)
{
  map:put($current-trace, "plugin-input", json:object())
};

declare %private function trace:get-plugin-input(
  $current-trace as map:map)
{
  let $o := map:get($current-trace, "plugin-input")
  where fn:exists($o)
  return
    if (rfc:is-json()) then
      let $oo := json:object()
      let $_ :=
        for $key in map:keys($o)
        let $value := map:get($o, $key)
        let $value := trace:sanitize-data($value)
        return
         map:put($oo, $key, $value)
      return $oo
    else
      for $key in map:keys($o)
      return
        element { $key } {
          let $value := map:get($o, $key)
          return
            trace:sanitize-data($value)
        }
};

(:
 : Registers an input with the trace library
 :
 : @param $label - the label of the input being registered
 : @param $input - the value of the input being registered
 :)
declare function trace:set-plugin-input(
  $label as xs:string,
  $input)
{
  trace:set-plugin-input(rfc:get-trace($rfc:item-context), $label, $input)
};

declare function trace:set-plugin-input(
  $current-trace as map:map,
  $label as xs:string,
  $input)
{
  let $existing := (map:get($current-trace, "plugin-input"), json:object())[1]
  let $_ := map:put($existing, $label, $input)
  return
    map:put($current-trace, "plugin-input", $existing)
};

declare function trace:get-completed-items()
{
  if (map:contains($current-trace-settings, "completed-items")) then
    map:get($current-trace-settings, "completed-items")
  else
    let $value := json:array()
    let $_ := map:put($current-trace-settings, "completed-items", $value)
    return
      $value
};

declare function trace:get-failed-items()
{
  if (map:contains($current-trace-settings, "failed-items")) then
    map:get($current-trace-settings, "failed-items")
  else
    let $value := json:array()
    let $_ := map:put($current-trace-settings, "failed-items", $value)
    return
      $value
};

declare function trace:write-trace(
  $item-context as map:map)
{
  let $identifier := rfc:get-id($item-context)
  where $identifier instance of xs:string
  return
    trace:add-completed-item($identifier),
  trace:write-error-trace($item-context)
};

declare %private function trace:write-error-trace(
  $item-context as map:map)
{
  let $current-trace := rfc:get-trace($item-context)
  return
    if (trace:enabled() or trace:has-errors()) then (
      let $trace :=
        if (rfc:is-json()) then
          xdmp:to-json((
            map:entry("trace",
              map:new((
                map:entry("jobId", rfc:get-job-id()),
                map:entry("format", rfc:get-data-format()),
                map:entry("traceId", map:get($current-trace, "traceId")),
                map:entry("created", map:get($current-trace, "created")),
                map:entry("identifier", rfc:get-id($item-context)),
                map:entry("flowType", rfc:get-flow-type()),
                map:entry("hasError", trace:has-errors()),
                let $steps := json:array()
                let $_ :=
                  for $step in map:get($current-trace, "traceSteps")
                  return
                    json:array-push($steps, $step)
                return
                  map:entry("steps", $steps)
              ))
            )
          ))
        else
          document {
            element trace {
              element jobId { rfc:get-job-id() },
              element format { rfc:get-data-format() },
              element traceId { map:get($current-trace, "traceId") },
              element created { map:get($current-trace, "created") },
              element identifier { rfc:get-id($item-context) },
              element flowType { rfc:get-flow-type() },
              element hasError { trace:has-errors() },
              element steps {
                for $step in map:get($current-trace, "traceSteps")
                return
                  element step {
                    element label { map:get($step, "label") },
                    element input { map:get($step, "input") },
                    element output { map:get($step, "output") },
                    element error { map:get($step, "error") },
                    element duration { map:get($step, "duration") },
                    element options { map:get($step, "options") }
                  }
              }
            }
          }
      return
        xdmp:eval('
          xquery version "1.0-ml";

          declare option xdmp:mapping "false";

          declare variable $trace external;
          declare variable $extension external;

          xdmp:document-insert(
            "/" || $trace/*:trace/*:traceId || $extension,
            $trace,
            xdmp:default-permissions(),
            ("trace", $trace/*:trace/*:flow-type)
          )
        ',
        map:new((
          map:entry("trace", $trace),
          map:entry("extension", if (rfc:is-json()) then ".json" else ".xml")
        )),<commit_msg>Fixed missing flow type collection from xqy json output traces<commit_after>(:
  Copyright 2012-2018 MarkLogic Corporation

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License.
:)
xquery version "1.0-ml";

module namespace trace = "http://marklogic.com/data-hub/trace";

import module namespace config = "http://marklogic.com/data-hub/config"
  at "/com.marklogic.hub/config.xqy";

import module namespace consts = "http://marklogic.com/data-hub/consts"
  at "/MarkLogic/data-hub-framework/impl/consts.xqy";

import module namespace err = "http://marklogic.com/data-hub/err"
  at "/MarkLogic/data-hub-framework/impl/error-lib.xqy";

import module namespace hul = "http://marklogic.com/data-hub/hub-utils-lib"
  at "/MarkLogic/data-hub-framework/impl/hub-utils-lib.xqy";

import module namespace json="http://marklogic.com/xdmp/json"
  at "/MarkLogic/json/json.xqy";

import module namespace rfc = "http://marklogic.com/data-hub/run-flow-context"
  at "/MarkLogic/data-hub-framework/impl/run-flow-context.xqy";

import module namespace search = "http://marklogic.com/appservices/search"
  at "/MarkLogic/appservices/search/search.xqy";

declare option xdmp:mapping "false";

(: new trace-settings are initialized for each transaction :)
declare variable $current-trace-settings := map:map();

declare function trace:new-trace() as map:map
{
  map:new((
    map:entry("traceId", xdmp:random()),
    map:entry("created", fn:current-dateTime())
  ))
};

declare function trace:enable-tracing($enabled as xs:boolean)
{
  xdmp:eval('
    declare namespace trace = "http://marklogic.com/data-hub/trace";
    declare variable $enabled external;
    xdmp:document-insert(
      "/com.marklogic.hub/settings/__tracing_enabled__.xml",
      element trace:is-tracing-enabled { if ($enabled) then 1 else 0 },
      xdmp:default-permissions(),
      "hub-core-module")
    ',
    map:new((map:entry("enabled", $enabled))),
    map:new((map:entry("database", xdmp:modules-database()), map:entry("ignoreAmps", fn:true())))
  ),
  hul:invalidate-field-cache("tracing-enabled")
};

declare function trace:enabled() as xs:boolean
{
  let $key := "tracing-enabled"
  let $flag :=  hul:from-field-cache-or-empty($key, ())
  return
    if (exists($flag)) then
      $flag
    else
      hul:set-field-cache(
        $key,
        xdmp:eval('
          declare namespace trace = "http://marklogic.com/data-hub/trace";
          fn:exists(
            cts:search(
              fn:doc("/com.marklogic.hub/settings/__tracing_enabled__.xml"),
              cts:element-value-query(xs:QName("trace:is-tracing-enabled"), "1", ("exact")),
              ("unfiltered", "score-zero", "unchecked", "unfaceted")
            )
          )
        ',(), map:new(map:entry("database", xdmp:modules-database()))),
        xs:dayTimeDuration("PT1M")
      )
};

declare function trace:has-errors() as xs:boolean
{
  (map:get($current-trace-settings, "_has_errors"), fn:false())[1] eq fn:true()
};

declare %private function trace:increment-error-count()
{
  map:put($current-trace-settings, "error-count", trace:get-error-count() + 1)
};

declare function trace:get-error-count()
{
  (map:get($current-trace-settings, "error-count"), 0)[1]
};

declare %private function trace:add-failed-item($item as xs:string)
{
  json:array-push(trace:get-failed-items(), $item)
};

declare %private function trace:add-completed-item($item as xs:string)
{
  json:array-push(trace:get-completed-items(), $item)
};

declare function trace:set-plugin-label(
  $label as xs:string)
{
  trace:set-plugin-label(
    rfc:get-trace($rfc:item-context),
    $label
  )
};

(:
 : Sets the label of the currently running plugin
 :
 : @param $label - the label of the running plugin
 :)
declare function trace:set-plugin-label(
  $current-trace as map:map,
  $label as xs:string)
{
  map:put($current-trace, "plugin-label", $label)
};

declare function trace:get-plugin-label(
  $current-trace as map:map)
as xs:string
{
  map:get($current-trace, "plugin-label")
};

declare function trace:reset-plugin-input()
{
  trace:reset-plugin-input(rfc:get-trace($rfc:item-context))
};

declare function trace:reset-plugin-input(
  $current-trace as map:map)
{
  map:put($current-trace, "plugin-input", json:object())
};

declare %private function trace:get-plugin-input(
  $current-trace as map:map)
{
  let $o := map:get($current-trace, "plugin-input")
  where fn:exists($o)
  return
    if (rfc:is-json()) then
      let $oo := json:object()
      let $_ :=
        for $key in map:keys($o)
        let $value := map:get($o, $key)
        let $value := trace:sanitize-data($value)
        return
         map:put($oo, $key, $value)
      return $oo
    else
      for $key in map:keys($o)
      return
        element { $key } {
          let $value := map:get($o, $key)
          return
            trace:sanitize-data($value)
        }
};

(:
 : Registers an input with the trace library
 :
 : @param $label - the label of the input being registered
 : @param $input - the value of the input being registered
 :)
declare function trace:set-plugin-input(
  $label as xs:string,
  $input)
{
  trace:set-plugin-input(rfc:get-trace($rfc:item-context), $label, $input)
};

declare function trace:set-plugin-input(
  $current-trace as map:map,
  $label as xs:string,
  $input)
{
  let $existing := (map:get($current-trace, "plugin-input"), json:object())[1]
  let $_ := map:put($existing, $label, $input)
  return
    map:put($current-trace, "plugin-input", $existing)
};

declare function trace:get-completed-items()
{
  if (map:contains($current-trace-settings, "completed-items")) then
    map:get($current-trace-settings, "completed-items")
  else
    let $value := json:array()
    let $_ := map:put($current-trace-settings, "completed-items", $value)
    return
      $value
};

declare function trace:get-failed-items()
{
  if (map:contains($current-trace-settings, "failed-items")) then
    map:get($current-trace-settings, "failed-items")
  else
    let $value := json:array()
    let $_ := map:put($current-trace-settings, "failed-items", $value)
    return
      $value
};

declare function trace:write-trace(
  $item-context as map:map)
{
  let $identifier := rfc:get-id($item-context)
  where $identifier instance of xs:string
  return
    trace:add-completed-item($identifier),
  trace:write-error-trace($item-context)
};

declare %private function trace:write-error-trace(
  $item-context as map:map)
{
  let $current-trace := rfc:get-trace($item-context)
  return
    if (trace:enabled() or trace:has-errors()) then (
      let $trace :=
        if (rfc:is-json()) then
          xdmp:to-json((
            map:entry("trace",
              map:new((
                map:entry("jobId", rfc:get-job-id()),
                map:entry("format", rfc:get-data-format()),
                map:entry("traceId", map:get($current-trace, "traceId")),
                map:entry("created", map:get($current-trace, "created")),
                map:entry("identifier", rfc:get-id($item-context)),
                map:entry("flowType", rfc:get-flow-type()),
                map:entry("hasError", trace:has-errors()),
                let $steps := json:array()
                let $_ :=
                  for $step in map:get($current-trace, "traceSteps")
                  return
                    json:array-push($steps, $step)
                return
                  map:entry("steps", $steps)
              ))
            )
          ))
        else
          document {
            element trace {
              element jobId { rfc:get-job-id() },
              element format { rfc:get-data-format() },
              element traceId { map:get($current-trace, "traceId") },
              element created { map:get($current-trace, "created") },
              element identifier { rfc:get-id($item-context) },
              element flowType { rfc:get-flow-type() },
              element hasError { trace:has-errors() },
              element steps {
                for $step in map:get($current-trace, "traceSteps")
                return
                  element step {
                    element label { map:get($step, "label") },
                    element input { map:get($step, "input") },
                    element output { map:get($step, "output") },
                    element error { map:get($step, "error") },
                    element duration { map:get($step, "duration") },
                    element options { map:get($step, "options") }
                  }
              }
            }
          }
      return
        xdmp:eval('
          xquery version "1.0-ml";

          declare option xdmp:mapping "false";

          declare variable $trace external;
          declare variable $extension external;

          xdmp:document-insert(
            "/" || $trace/*:trace/*:traceId || $extension,
            $trace,
            xdmp:default-permissions(),
            ("trace", $trace/*:trace/*:flow-type, $trace/*:trace/*:flowType)
          )
        ',
        map:new((
          map:entry("trace", $trace),
          map:entry("extension", if (rfc:is-json()) then ".json" else ".xml")
        )),
{'fraction_non_alphanumeric': 0.1529815797689666, 'fraction_numerical': 0.0026017275470912684, 'mean_word_length': 2.7588499902210053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5446617', 'n_tokens_mistral': 6106, 'n_tokens_neox': 5901, 'n_words': 1306}