<commit_before><!--
  Copyright (C) 2011 Orbeon, Inc.

  This program is free software; you can redistribute it and/or modify it under the terms of the
  GNU Lesser General Public License as published by the Free Software Foundation; either version
  2.1 of the License, or (at your option) any later version.

  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  See the GNU Lesser General Public License for more details.

  The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
  -->
<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline"
          xmlns:oxf="http://www.orbeon.com/oxf/processors"
          xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
          xmlns:xs="http://www.w3.org/2001/XMLSchema"
          xmlns:xh="http://www.w3.org/1999/xhtml"
          xmlns:xf="http://www.w3.org/2002/xforms"
          xmlns:ev="http://www.w3.org/2001/xml-events"
          xmlns:xxf="http://orbeon.org/oxf/xml/xforms"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xmlns:saxon="http://saxon.sf.net/"
          xmlns:fr="http://orbeon.org/oxf/xml/form-runner"
          xmlns:utils="java:org.orbeon.oxf.xml.SaxonUtils">

    <p:param type="input" name="instance"/>
    <p:param type="output" name="data"/>

    <!-- Apply unzip/XForms model/zip/result -->
    <p:processor name="oxf:pipeline">
        <p:input name="instance" href="#instance"/>
        <p:input name="config" href="apply.xpl"/>
        <p:input name="xforms-model">
            <!-- XForms model that gets inserted into Form Runner to perform import upon initialization -->
            <xf:model
                id="fr-batch-validation-model"
                xxf:xpath-analysis="true"
                xxf:function-library="org.orbeon.oxf.fr.library.FormRunnerFunctionLibrary">

                <xf:instance src="input:data" id="fr-batch-data" xxf:readonly="true"/>
                <xf:instance id="fr-empty-data"><dummy/></xf:instance>
                <xf:instance id="fr-import-stats" xxf:exclude-result-prefixes="#all">
                    <stats>
                        <total>0</total>
                        <processed>0</processed>
                        <succeeded>0</succeeded>
                    </stats>
                </xf:instance>

                <xf:submission
                    id="fr-send-stats"
                    ref="instance('fr-import-stats')"
                    method="post"
                    action="echo:"
                    replace="all"/>

                <xf:action event="xforms-ready">

                    <xf:var
                        name="import-invalid-data"
                        value="xs:boolean((xxf:get-request-parameter('import-invalid-data')[. castable as xs:boolean], false())[1])"/>

                    <!-- Get precomputed list of invalid rows from session -->
                    <xf:var
                        name="invalid-rows"
                        as="xs:integer*"
                        value="
                            for $t in xxf:split(xxf:get-session-attribute('org.orbeon.fr.import.invalid-rows'))
                                return xs:integer($t)"/>

                    <!-- Remember original empty data -->
                    <xf:insert
                        ref="instance('fr-empty-data')"
                        origin="xxf:instance('fr-form-instance')"/>

                    <xf:var
                        name="non-empty-headers"
                        value="instance()/row[1]/c[normalize-space(.) and @r]"/>

                    <xf:setvalue
                        ref="instance('fr-import-stats')/total"
                        value="count(instance()/row) - 1"/>

                    <xf:action type="xpath">
                        xxf:set-session-attribute('org.orbeon.fr.import.total', xs:integer(instance('fr-import-stats')/total)),
                        xxf:set-session-attribute('org.orbeon.fr.import.processed', 0),
                        xxf:set-session-attribute('org.orbeon.fr.import.succeeded', 0)
                    </xf:action>

                    <!-- Iterate over all rows -->
                    <xf:action iterate="instance()/row[position() gt 1]">
                        <xf:var name="row" value="."/>
                        <xf:var name="p"   value="position()"/>

                        <!-- Check at each iteration whether to stop -->
                        <xf:action if="not(xxf:get-session-attribute('org.orbeon.fr.import.cancel'))">

                            <!-- Only consider valid rows unless importing invalid ones is explicitly allowed -->
                            <xf:action if="$import-invalid-data or empty(index-of($invalid-rows, $p))">
                                <!-- Start with empty data -->
                                <xf:var
                                    name="new"
                                    value="xxf:create-document()"/>

                                <xf:insert
                                    context="$new"
                                    origin="instance('fr-empty-data')"/>

                                <!-- Fill data -->
                                <xf:action iterate="$non-empty-headers">
                                    <xf:var name="r"          value="@r/string()"/>
                                    <xf:var name="raw-header" value="normalize-space(.)"/>
                                    <xf:var name="v"          value="($row/c[@r = $r]/string(), '')[1]"/>

                                    <xf:setvalue
                                        ref="$new//*[not(*) and name() = utils:makeNCName($raw-header, false())]"
                                        value="$v"/>
                                </xf:action>

                                <!-- Set filled data -->
                                <xf:insert ref="xxf:instance('fr-form-instance')" origin="$new"/>
<commit_msg>Fix #4408 "Excel import: exception when validating data"<commit_after><!--
  Copyright (C) 2011 Orbeon, Inc.

  This program is free software; you can redistribute it and/or modify it under the terms of the
  GNU Lesser General Public License as published by the Free Software Foundation; either version
  2.1 of the License, or (at your option) any later version.

  This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
  without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  See the GNU Lesser General Public License for more details.

  The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
  -->
<p:config xmlns:p="http://www.orbeon.com/oxf/pipeline"
          xmlns:oxf="http://www.orbeon.com/oxf/processors"
          xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
          xmlns:xs="http://www.w3.org/2001/XMLSchema"
          xmlns:xh="http://www.w3.org/1999/xhtml"
          xmlns:xf="http://www.w3.org/2002/xforms"
          xmlns:ev="http://www.w3.org/2001/xml-events"
          xmlns:xxf="http://orbeon.org/oxf/xml/xforms"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xmlns:saxon="http://saxon.sf.net/"
          xmlns:fr="http://orbeon.org/oxf/xml/form-runner"
          xmlns:utils="java:org.orbeon.oxf.xml.SaxonUtils">

    <p:param type="input" name="instance"/>
    <p:param type="output" name="data"/>

    <!-- Apply unzip/XForms model/zip/result -->
    <p:processor name="oxf:pipeline">
        <p:input name="instance" href="#instance"/>
        <p:input name="config" href="apply.xpl"/>
        <p:input name="xforms-model">
            <!-- XForms model that gets inserted into Form Runner to perform import upon initialization -->
            <xf:model
                id="fr-batch-validation-model"
                xxf:xpath-analysis="true"
                xxf:function-library="org.orbeon.oxf.fr.library.FormRunnerFunctionLibrary">

                <xf:instance src="input:data" id="fr-batch-data" xxf:readonly="true"/>
                <xf:instance id="fr-empty-data"><dummy/></xf:instance>
                <xf:instance id="fr-import-stats" xxf:exclude-result-prefixes="#all">
                    <stats>
                        <total>0</total>
                        <processed>0</processed>
                        <succeeded>0</succeeded>
                    </stats>
                </xf:instance>

                <xf:submission
                    id="fr-send-stats"
                    ref="instance('fr-import-stats')"
                    method="post"
                    action="echo:"
                    replace="all"/>

                <xf:action event="xforms-ready">

                    <xf:var
                        name="import-invalid-data"
                        value="xs:boolean((xxf:get-request-parameter('import-invalid-data')[. castable as xs:boolean], false())[1])"/>

                    <!-- Get precomputed list of invalid rows from session -->
                    <xf:var
                        name="invalid-rows"
                        as="xs:integer*"
                        value="
                            for $t in xxf:split(xxf:get-session-attribute('org.orbeon.fr.import.invalid-rows'))
                                return xs:integer($t)"/>

                    <!-- Remember original empty data -->
                    <xf:insert
                        ref="instance('fr-empty-data')"
                        origin="xxf:instance('fr-form-instance')"/>

                    <xf:var
                        name="non-empty-headers"
                        value="instance()/row[1]/c[normalize-space(.) and @r]"/>

                    <xf:setvalue
                        ref="instance('fr-import-stats')/total"
                        value="count(instance()/row) - 1"/>

                    <xf:action type="xpath">
                        xxf:set-session-attribute('org.orbeon.fr.import.total', xs:integer(instance('fr-import-stats')/total)),
                        xxf:set-session-attribute('org.orbeon.fr.import.processed', 0),
                        xxf:set-session-attribute('org.orbeon.fr.import.succeeded', 0)
                    </xf:action>

                    <!-- Iterate over all rows -->
                    <xf:action iterate="instance()/row[position() gt 1]">
                        <xf:var name="row" value="."/>
                        <xf:var name="p"   value="position()"/>

                        <!-- Check at each iteration whether to stop -->
                        <xf:action if="not(xxf:get-session-attribute('org.orbeon.fr.import.cancel'))">

                            <!-- Only consider valid rows unless importing invalid ones is explicitly allowed -->
                            <xf:action if="$import-invalid-data or empty(index-of($invalid-rows, $p))">
                                <!-- Start with empty data -->
                                <xf:var
                                    name="new"
                                    value="xxf:create-document()"/>

                                <xf:insert
                                    context="$new"
                                    origin="instance('fr-empty-data')"/>

                                <!-- Fill data -->
                                <xf:action iterate="$non-empty-headers">
                                    <xf:var name="r"          value="@r/string()"/>
                                    <xf:var name="raw-header" value="normalize-space(.)"/>
                                    <xf:var name="v"          value="($row/c[@r = $r]/string(), '')[1]"/>

                                    <xf:setvalue
                                        ref="$new//*[not(*) and name() = utils:makeNCName($raw-header, false())]"
                                        value="$v"/>
                                </xf:action>

                                <!-- Set filled data -->
                                <xf:insert ref="xxf:instance('fr-form-instance')" origin="$new"/>
                                <xf:rebuild model="fr-form-model"/>

{'fraction_non_alphanumeric': 0.13978319341429618, 'fraction_numerical': 0.007824598581791508, 'mean_word_length': 1.368268674001158, 'pattern_counts': {'":': 0, '<': 116, '<?xml version=': 0, '>': 116, 'https://': 0, 'lorem ipsum': 0, 'www.': 18, 'xml': 32}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7389898', 'n_tokens_mistral': 3384, 'n_tokens_neox': 3109, 'n_words': 603}