<commit_before><?xml version="1.0" encoding="UTF-8"?>
<p:declare-step xmlns:p="http://www.w3.org/ns/xproc" 
  xmlns:c="http://www.w3.org/ns/xproc-step"
  xmlns:cx="http://xmlcalabash.com/ns/extensions"
  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
  xmlns:xs="http://www.w3.org/2001/XMLSchema"
  xmlns:docx2hub="http://transpect.io/docx2hub"
  xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"
  xmlns:o="urn:schemas-microsoft-com:office:office"
  xmlns:rel="http://schemas.openxmlformats.org/package/2006/relationships"
  xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"
  xmlns:tr="http://transpect.io"
  version="1.0" 
  name="single-tree"
  type="docx2hub:single-tree">

  <p:input port="source" primary="true">
    <p:documentation>This is to prevent any other default readable port to be connected with the xslt port.</p:documentation>
    <p:empty/>
  </p:input>
  <p:input port="schematron" primary="false">
    <p:document href="../sch/single-tree.sch.xml"/>
  </p:input>
  <p:input port="xslt" primary="false">
    <p:document href="../xsl/main.xsl"/>
  </p:input>
  
  <p:output port="result" primary="true">
    <p:pipe port="result" step="add-xml-base-attr"/>
  </p:output>
  <p:output port="params">
    <p:pipe port="result" step="params"/>
  </p:output>
  <p:output port="zip-manifest">
    <p:pipe port="result" step="zip-manifest"/>
  </p:output>
  <p:output port="report" sequence="true">
    <p:pipe port="result" step="check"/>
  </p:output>

  <p:serialization port="result" omit-xml-declaration="false"/>
  
  <p:option name="docx" required="true">
    <p:documentation>OS name (preferably with full path, may not resolve if only a relative path is given), file:, http:, or
      https: URL. The file will be fetched first if it is an HTTP URL.</p:documentation>
  </p:option>
  <p:option name="debug" required="false" select="'no'"/>
  <p:option name="debug-dir-uri" required="false" select="'file:/tmp/debug'"/>
  <p:option name="srcpaths" required="false" select="'no'"/>
  <p:option name="unwrap-tooltip-links" required="false" select="'no'"/>
  <p:option name="mml-space-handling" select="'mspace'">
    <p:documentation>see corresponding documentation for docx2hub</p:documentation>
  </p:option>
  <p:option name="hub-version" required="false" select="'1.2'"/>
  <p:option name="fail-on-error" select="'no'"/>
  <p:option name="field-vars" required="false" select="'no'"/>
  <p:option name="extract-dir" required="false" select="''">
    <p:documentation>Directory (OS path, not file: URL) to which the file will be unzipped. If option is empty string, will be
      '.tmp' appended to OS file path.</p:documentation>
  </p:option>
  <p:option name="no-srcpaths-for-text-runs-threshold" select="'40000'">
    <p:documentation>In order to speed up conversion for long documents, if more w:r elements are found, they won’t receive 
      a srcpath of their own. In principle, srcpath generation may be sped up by computing them more efficiently,
      building on a tunnelled parameter that contains the parent element’s already-computed srcpath.</p:documentation>
  </p:option>
  <p:option name="apply-changemarkup" required="false" select="'yes'">
    <p:documentation xmlns="http://www.w3.org/1999/xhtml">
      <p>Apply all change markup on the compound word document.</p>
    </p:documentation>
  </p:option>  

  <p:import href="http://xmlcalabash.com/extension/steps/library-1.0.xpl"/>
  
  <p:import href="http://transpect.io/calabash-extensions/unzip-extension/unzip-declaration.xpl"/>
  <p:import href="http://transpect.io/xproc-util/store-debug/xpl/store-debug.xpl"/>
  <p:import href="http://transpect.io/xproc-util/file-uri/xpl/file-uri.xpl"/>
  <p:import href="http://transpect.io/xproc-util/xslt-mode/xpl/xslt-mode.xpl"/>

  <p:variable name="basename" select="replace($docx, '^(.+?)([^/\\]+)\.do[ct][mx]$', '$2')"><p:empty/></p:variable>

  <tr:file-uri name="locate-docx">
    <p:with-option name="filename" select="$docx"/>
  </tr:file-uri>
  
  <tr:store-debug>
    <p:with-option name="pipeline-step" select="concat('docx2hub/', $basename, '/00-file-uri')"/>
    <p:with-option name="active" select="$debug"/>
    <p:with-option name="base-uri" select="$debug-dir-uri"/>
  </tr:store-debug><commit_msg>apply-changemarkup is only an option of single-tree-enhanced<commit_after><?xml version="1.0" encoding="UTF-8"?>
<p:declare-step xmlns:p="http://www.w3.org/ns/xproc" 
  xmlns:c="http://www.w3.org/ns/xproc-step"
  xmlns:cx="http://xmlcalabash.com/ns/extensions"
  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
  xmlns:xs="http://www.w3.org/2001/XMLSchema"
  xmlns:docx2hub="http://transpect.io/docx2hub"
  xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"
  xmlns:o="urn:schemas-microsoft-com:office:office"
  xmlns:rel="http://schemas.openxmlformats.org/package/2006/relationships"
  xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"
  xmlns:tr="http://transpect.io"
  version="1.0" 
  name="single-tree"
  type="docx2hub:single-tree">

  <p:input port="source" primary="true">
    <p:documentation>This is to prevent any other default readable port to be connected with the xslt port.</p:documentation>
    <p:empty/>
  </p:input>
  <p:input port="schematron" primary="false">
    <p:document href="../sch/single-tree.sch.xml"/>
  </p:input>
  <p:input port="xslt" primary="false">
    <p:document href="../xsl/main.xsl"/>
  </p:input>
  
  <p:output port="result" primary="true">
    <p:pipe port="result" step="add-xml-base-attr"/>
  </p:output>
  <p:output port="params">
    <p:pipe port="result" step="params"/>
  </p:output>
  <p:output port="zip-manifest">
    <p:pipe port="result" step="zip-manifest"/>
  </p:output>
  <p:output port="report" sequence="true">
    <p:pipe port="result" step="check"/>
  </p:output>

  <p:serialization port="result" omit-xml-declaration="false"/>
  
  <p:option name="docx" required="true">
    <p:documentation>OS name (preferably with full path, may not resolve if only a relative path is given), file:, http:, or
      https: URL. The file will be fetched first if it is an HTTP URL.</p:documentation>
  </p:option>
  <p:option name="debug" required="false" select="'no'"/>
  <p:option name="debug-dir-uri" required="false" select="'file:/tmp/debug'"/>
  <p:option name="srcpaths" required="false" select="'no'"/>
  <p:option name="unwrap-tooltip-links" required="false" select="'no'"/>
  <p:option name="mml-space-handling" select="'mspace'">
    <p:documentation>see corresponding documentation for docx2hub</p:documentation>
  </p:option>
  <p:option name="hub-version" required="false" select="'1.2'"/>
  <p:option name="fail-on-error" select="'no'"/>
  <p:option name="field-vars" required="false" select="'no'"/>
  <p:option name="extract-dir" required="false" select="''">
    <p:documentation>Directory (OS path, not file: URL) to which the file will be unzipped. If option is empty string, will be
      '.tmp' appended to OS file path.</p:documentation>
  </p:option>
  <p:option name="no-srcpaths-for-text-runs-threshold" select="'40000'">
    <p:documentation>In order to speed up conversion for long documents, if more w:r elements are found, they won’t receive 
      a srcpath of their own. In principle, srcpath generation may be sped up by computing them more efficiently,
      building on a tunnelled parameter that contains the parent element’s already-computed srcpath.</p:documentation>
  </p:option>

  <p:import href="http://xmlcalabash.com/extension/steps/library-1.0.xpl"/>
  
  <p:import href="http://transpect.io/calabash-extensions/unzip-extension/unzip-declaration.xpl"/>
  <p:import href="http://transpect.io/xproc-util/store-debug/xpl/store-debug.xpl"/>
  <p:import href="http://transpect.io/xproc-util/file-uri/xpl/file-uri.xpl"/>
  <p:import href="http://transpect.io/xproc-util/xslt-mode/xpl/xslt-mode.xpl"/>

  <p:variable name="basename" select="replace($docx, '^(.+?)([^/\\]+)\.do[ct][mx]$', '$2')"><p:empty/></p:variable>

  <tr:file-uri name="locate-docx">
    <p:with-option name="filename" select="$docx"/>
  </tr:file-uri>
  
  <tr:store-debug>
    <p:with-option name="pipeline-step" select="concat('docx2hub/', $basename, '/00-file-uri')"/>
    <p:with-option name="active" select="$debug"/>
    <p:with-option name="base-uri" select="$debug-dir-uri"/>
  </tr:store-debug>
{'fraction_non_alphanumeric': 0.19513064133016628, 'fraction_numerical': 0.01152019002375297, 'mean_word_length': 4.6516778523489934, 'pattern_counts': {'":': 0, '<': 139, '<?xml version=': 2, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 9, 'xml': 41}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1959207', 'n_tokens_mistral': 3106, 'n_tokens_neox': 2944, 'n_words': 562}