<commit_before>(:
 Copyright 2002-2016 MarkLogic Corporation.  All Rights Reserved. 

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
:)
xquery version "1.0-ml";


(: This module supports code generation for entity services.
 : This module has no public API.  See entity-services.xqy for
 : the public interface.
 :)
module namespace es-codegen = "http://marklogic.com/entity-services-codegen";

import module namespace esi = "http://marklogic.com/entity-services-impl"
    at "entity-services-impl.xqy";

import module namespace functx   = "http://www.functx.com" at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

declare namespace es = "http://marklogic.com/entity-services";
declare namespace tde = "http://marklogic.com/xdmp/tde";


declare function es-codegen:conversion-module-generate(
    $entity-type as map:map
) as document-node()
{
    let $default-base-uri := "http://example.org/"
    let $info := map:get($entity-type, "info")
    let $title := map:get($info, "title")
    let $prefix := lower-case(substring($title,1,1)) || substring($title,2)
    let $version:= map:get($info, "version")
    let $definitions := map:get($entity-type, "definitions")
    let $base-uri := fn:head((map:get($info, "baseUri"), $default-base-uri))
    let $base-uri := 
            if (fn:matches($base-uri, "[#/]$")) 
            then $base-uri 
            else concat($base-uri, "#")
    return
document {
<conversion-module>xquery version "1.0-ml";

(: 
 : This module was generated by MarkLogic Entity Services. 
 : The source entity type document was {$title}-{$version}
 :
 : To use this module, examine how you wish to extract data from sources,
 : and modify the various extract-instance-{{X}} functions to 
 : create the instances you wish.
 :
 : You may wish to/need to alter
 : 1.  values.  For example, creating duration values from decimal months.
 : 2.  references.  This conversion module assumes you want to denormalize 
 :     instances when storing them in documents.  You may choose to remove 
 :     code that denormalizes, and just include reference values in your instances 
 :     instead.
 : 3.  Source XPath expressions.  The data coming into the extract-instance-{{X}} 
 :     functions will probably not be exactly what this module predicts.
 :
 : After modifying this file, put it in your project for deployment to the modules 
 : database of your application, and check it into your source control system.
 :
 : Modification History:
 :   Generated at timestamp: {fn:current-dateTime()}
 :   Persisted by AUTHOR
 :   Date: DATE
 :)
module namespace {$prefix} = "{$base-uri}{$title}-{$version}";

import module namespace es = "http://marklogic.com/entity-services" 
    at "/MarkLogic/entity-services/entity-services.xqy";


(:
 :  extract-instance-{{entity-type}} functions 
 :
 :  These functions take together take a source document and create a nested
 :  map structure from it.
 :  The resulting map is used by instance-to-canonical-xml to create documents
 :  in the database.
 :  
 :  It is expected that an implementer will edit at least XPath expressions in
 :  the extraction functions.  It is less likely that you will want to edit
 :  the instance-to-canonical-xml or envelope functions.
 :)
{ 
    for $entity-type-key in map:keys(map:get($entity-type, "definitions"))
    return 
    <extract-instance>
(:~
 : Creates a map:map representation of an entity instance from some source
 : document.
 : @param $source-node  A document or node that contains data for populating a {$entity-type-key}
 : @return A map:map instance that holds the data for this entity type.
 :)
declare function {$prefix}:extract-instance-{$entity-type-key}(
    $source-node as node()
) as map:map
{{
    json:object()
        (: This line identifies the type of this instance.  Do not change it. :)
        =>map:with('$type', '{ $entity-type-key }')
        (: This line adds the original source document as an attachment.
         : If this entity type is not the root of a document, you should remove this.
         : If the source document is JSON, use 
         : =>map:with('$attachments', xdmp:quote($source-node))
         : because you cannot preserve JSON nodes with the XML envelope verbatim.
         :)
        =>map:with('$attachments', $source-node)
        (: The following lines are generated from the '{ $entity-type-key }' entity type 
         : You need to ensure that all of the property paths are correct for your source
         : data to populate instances.  The general pattern is
         : =>map:with('keyName', data($source-node/path/to/data/in/the/source))
         : but you may also wish to convert values
         : =>map:with('dateKeyName', xdmp:parse-dateTime("[Y0001]-[M01]-[D01]T[h01]:[m01]:[s01].[f1][Z]", $source-node/path/to/data/in/the/source))
         : You can also implement lookup functions, 
         : =>map:with('lookupKey', cts:search( collection('customers'), string($source-node/path/to/lookup/key))/id
         : or populate the instance with constants.
         : =>map:with('constantValue', 10)
         : Once you've customized this function, write a test with expected inputs, and a test instance document
         : created with es:entity-type-get-test-instances($entity-type)
         :)
    {
    (: Begin code generation block :)
    let $this-type := map:get($definitions, $entity-type-key)
    let $properties-map := map:get($this-type, "properties")
    let $properties-keys := map:keys($properties-map)
    for $property-key in map:keys($properties-map)
    let $is-array := 
            map:get(map:get($properties-map, $property-key), "datatype") 
            eq "array"
    let $wrap-if-array := function($str) {
            if ($is-array)
            then concat("json:to-array(", $str, " ! data(.))")
            else concat("data(", $str, ")")
        }
    let $ref :=
        if ($is-array)
        then map:get(map:get(map:get($properties-map, $property-key), "items"), "$ref")
        else map:get(map:get($properties-map, $property-key), "$ref")
    let $path-to-property := concat("$source-node/", $entity-type-key, "/", $property-key)
    let $value := 
        if (empty($ref))
        then 
            $wrap-if-array($path-to-property)
        else 
            if(contains($ref, "#/definitions"))
            then
            concat("&#10;            if (", $path-to-property, "/element())",
                   "&#10;            then json:to-array(", 
                    $path-to-property, " ! ", $prefix, ":extract-instance-", replace($ref, "#/definitions/", ""), "(.))",
                   "&#10;            else data(",
                   $path-to-property, 
                   ")")
            else
               concat(
                '(: The following assignment comes from an external reference.  Its value probably requires developer attention :)',
                $path-to-property, "/node()")
                
    (: if a property is required, use map:with to force inclusion :)
    let $function-call-string :=
        if ($property-key = ( map:get($this-type, "primaryKey"), json:array-values( map:get($this-type, "required")) ))
        then "       =>map:with("
        else "    =>es:optional("
    return
    concat($function-call-string, 
            functx:pad-string-to-length("'" || $property-key || "',", " ", max((  (string-length($property-key)+4), 25) )+1 ),
            $value, 
            ")&#10;   "
          )
    (: end code generation block :)
    }
}};
    </extract-instance>/text()
}

(:~
 : Turns an entity instance into an XML structure.
 : This out-of-the box implementation traverses a map structure
 : and turns it deterministically into an XML tree.
 : Using this function as-is should be sufficient for most use<commit_msg>bug:40485 move comment to more sensible place<commit_after>(:
 Copyright 2002-2016 MarkLogic Corporation.  All Rights Reserved. 

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
:)
xquery version "1.0-ml";


(: This module supports code generation for entity services.
 : This module has no public API.  See entity-services.xqy for
 : the public interface.
 :)
module namespace es-codegen = "http://marklogic.com/entity-services-codegen";

import module namespace esi = "http://marklogic.com/entity-services-impl"
    at "entity-services-impl.xqy";

import module namespace functx   = "http://www.functx.com" at "/MarkLogic/functx/functx-1.0-nodoc-2007-01.xqy";

declare namespace es = "http://marklogic.com/entity-services";
declare namespace tde = "http://marklogic.com/xdmp/tde";


declare function es-codegen:conversion-module-generate(
    $entity-type as map:map
) as document-node()
{
    let $default-base-uri := "http://example.org/"
    let $info := map:get($entity-type, "info")
    let $title := map:get($info, "title")
    let $prefix := lower-case(substring($title,1,1)) || substring($title,2)
    let $version:= map:get($info, "version")
    let $definitions := map:get($entity-type, "definitions")
    let $base-uri := fn:head((map:get($info, "baseUri"), $default-base-uri))
    let $base-uri := 
            if (fn:matches($base-uri, "[#/]$")) 
            then $base-uri 
            else concat($base-uri, "#")
    return
document {
<conversion-module>xquery version "1.0-ml";

(: 
 : This module was generated by MarkLogic Entity Services. 
 : The source entity type document was {$title}-{$version}
 :
 : To use this module, examine how you wish to extract data from sources,
 : and modify the various extract-instance-{{X}} functions to 
 : create the instances you wish.
 :
 : You may wish to/need to alter
 : 1.  values.  For example, creating duration values from decimal months.
 : 2.  references.  This conversion module assumes you want to denormalize 
 :     instances when storing them in documents.  You may choose to remove 
 :     code that denormalizes, and just include reference values in your instances 
 :     instead.
 : 3.  Source XPath expressions.  The data coming into the extract-instance-{{X}} 
 :     functions will probably not be exactly what this module predicts.
 :
 : After modifying this file, put it in your project for deployment to the modules 
 : database of your application, and check it into your source control system.
 :
 : Modification History:
 :   Generated at timestamp: {fn:current-dateTime()}
 :   Persisted by AUTHOR
 :   Date: DATE
 :)
module namespace {$prefix} = "{$base-uri}{$title}-{$version}";

import module namespace es = "http://marklogic.com/entity-services" 
    at "/MarkLogic/entity-services/entity-services.xqy";


(:
 :  extract-instance-{{entity-type}} functions 
 :
 :  These functions take together take a source document and create a nested
 :  map structure from it.
 :  The resulting map is used by instance-to-canonical-xml to create documents
 :  in the database.
 :  
 :  It is expected that an implementer will edit at least XPath expressions in
 :  the extraction functions.  It is less likely that you will want to edit
 :  the instance-to-canonical-xml or envelope functions.
 :)
{ 
    for $entity-type-key in map:keys(map:get($entity-type, "definitions"))
    return 
    <extract-instance>
(:~
 : Creates a map:map representation of an entity instance from some source
 : document.
 : @param $source-node  A document or node that contains data for populating a {$entity-type-key}
 : @return A map:map instance that holds the data for this entity type.
 :)
declare function {$prefix}:extract-instance-{$entity-type-key}(
    $source-node as node()
) as map:map
{{
    json:object()
        (: This line identifies the type of this instance.  Do not change it. :)
        =>map:with('$type', '{ $entity-type-key }')
        (: This line adds the original source document as an attachment.
         : If this entity type is not the root of a document, you should remove this.
         : If the source document is JSON, use 
         : =>map:with('$attachments', xdmp:quote($source-node))
         : because you cannot preserve JSON nodes with the XML envelope verbatim.
         :)
        =>map:with('$attachments', $source-node)
        (: The following lines are generated from the '{ $entity-type-key }' entity type 
         : You need to ensure that all of the property paths are correct for your source
         : data to populate instances.  The general pattern is
         : =>map:with('keyName', data($source-node/path/to/data/in/the/source))
         : but you may also wish to convert values
         : =>map:with('dateKeyName', xdmp:parse-dateTime("[Y0001]-[M01]-[D01]T[h01]:[m01]:[s01].[f1][Z]", $source-node/path/to/data/in/the/source))
         : You can also implement lookup functions, 
         : =>map:with('lookupKey', cts:search( collection('customers'), string($source-node/path/to/lookup/key))/id
         : or populate the instance with constants.
         : =>map:with('constantValue', 10)
         : Once you've customized this function, write a test with expected inputs, and a test instance document
         : created with es:entity-type-get-test-instances($entity-type)
         :)
    {
    (: Begin code generation block :)
    let $this-type := map:get($definitions, $entity-type-key)
    let $properties-map := map:get($this-type, "properties")
    let $properties-keys := map:keys($properties-map)
    for $property-key in map:keys($properties-map)
    let $is-array := 
            map:get(map:get($properties-map, $property-key), "datatype") 
            eq "array"
    let $wrap-if-array := function($str) {
            if ($is-array)
            then concat("json:to-array(", $str, " ! data(.))")
            else concat("data(", $str, ")")
        }
    let $ref :=
        if ($is-array)
        then map:get(map:get(map:get($properties-map, $property-key), "items"), "$ref")
        else map:get(map:get($properties-map, $property-key), "$ref")
    let $path-to-property := concat("$source-node/", $entity-type-key, "/", $property-key)
    let $property-comment :=
        if (empty($ref))
        then ""
        else if (contains($ref, "#/definitions"))
        then ""
        else concat(
      '    (: The following property assigment comes from an external reference.',
'&#10;         : Its generated value probably requires developer attention.',
'&#10;         :)',
'&#10;    ')
    let $value :=
        if (empty($ref))
        then 
            $wrap-if-array($path-to-property)
        else 
            if(contains($ref, "#/definitions"))
            then
            concat("&#10;            if (", $path-to-property, "/element())",
                   "&#10;            then json:to-array(",
                    $path-to-property, " ! ", $prefix, ":extract-instance-", replace($ref, "#/definitions/", ""), "(.))",
                   "&#10;            else data(",
                   $path-to-property,
                   ")")
            else
               concat(
                $path-to-property, "/node()")
                
    (: if a property is required, use map:with to force inclusion :)
    let $function-call-string :=
        if ($property-key = ( map:get($this-type, "primaryKey"), json:array-values( map:get($this-type, "required")) ))
        then "       =>map:with("
        else "    =>es:optional("
    return
    concat($property-comment,
            $function-call-string, 
            functx:pad-string-to-length("'" || $property-key || "',", " ", max((  (string-length($property-key)+4), 25) )+1 ),
            $value, 
            ")&#10;   "
          )
    (: end code generation block :)
    }
}};
    </extract-instance>/text()
}

(:~
 : Turns an entity instance into an XML structure.
 : This out-of-the box implementation traverses a map structure
 : and turns it deterministically into an XML tree.
 : Using this function as-is should be sufficient for most use
{'fraction_non_alphanumeric': 0.13203050524308865, 'fraction_numerical': 0.007685891325071497, 'mean_word_length': 2.81044267877412, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '377460', 'n_tokens_mistral': 4909, 'n_tokens_neox': 4699, 'n_words': 1569}