<commit_before>/*******************************************************************************
 * Copyright (c) 2008 CWI.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    <EMAIL> - initial API and implementation

 *******************************************************************************/

package io.usethesource.vallang.io;

import java.io.IOException;
import java.io.Reader;
import java.util.Objects;
import java.util.Set;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import io.usethesource.vallang.IListWriter;
import io.usethesource.vallang.IMapWriter;
import io.usethesource.vallang.ISetWriter;
import io.usethesource.vallang.IValue;
import io.usethesource.vallang.IValueFactory;
import io.usethesource.vallang.exceptions.FactParseError;
import io.usethesource.vallang.exceptions.FactTypeUseException;
import io.usethesource.vallang.exceptions.UnsupportedTypeException;
import io.usethesource.vallang.type.Type;
import io.usethesource.vallang.type.TypeFactory;
import io.usethesource.vallang.type.TypeStore;

/**
 * This IValueReader parses and validates certain forms of XML and deserializes
 * it as IValues. The forms of XML allowed are limited by a number of different
 * value types. In particular, it allows: <ul>
 *   <li> TreeSortTypes and TreeNodeTypes </li>
 *   <li> lists, sets, relations and maps, but not unless they are wrapped by a single
 *     ConstructorType. I.o.w. a container must be the only child of a tree node.
 *     Elements of containers are juxtapositioned as children of this node.</li>
 *   <li> tuples, but not nested ones. And tuples of containers are not allowed.
 *     elements of tuples are juxtapositioned in the xml files.</li>
 *   <li> basic types, such as str, int, double; with the same restriction as for
 *     container types, they must be the only child of a tree node.</li>
 *   <li> lists of tuples, sets of tuples and maps of tuples are allowed, but not
 *     lists of lists, tuples of tuples, lists in tuples, sets in tuples, etc.
 *     If such nesting is needed, it is required to use a wrapping tree node.</li>
 * </ul>
 * There is no support for NamedTypes yet, only TreeSortType and ConstructorType are 
 * allowed.
 *     
 * The limitations of this class are governed by wanting to avoid ambiguity
 * while validating XML using the pdb's type system and the inherent impedance
 * mismatch between the type system of pdb and the structure of XML.
 * 
 * Use this class to import many forms of XML data into PDB.
 * 
 */
public class XMLReader extends AbstractTextReader {
    private static final DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
    private static final TypeFactory TF = TypeFactory.getInstance();

    public IValue read(IValueFactory factory, TypeStore store, Type type, Reader stream)
            throws FactTypeUseException, IOException {

        try {
            Document doc = domFactory.newDocumentBuilder().parse(new InputSource(stream));
            return new Parser(factory, store).parse(doc.getDocumentElement(), type);
        } catch (SAXException se) {
            throw new IOException("Parsing of value failed because XML was invalid: " + se.getMessage());
        } catch (ParserConfigurationException pce) {
            throw new IOException("Parsing of value failed because XML configuration is wrong: " + pce.getMessage());
        } catch (DOMException de) {
            throw new IOException("Parsing of value failed because of a XML document failure: " + de.getMessage());
        } catch (NumberFormatException nfe) {
            throw new FactParseError("Expected a number, got something different", nfe);
        }
    }

    private static class Parser {
        private final IValueFactory vf;
        private final TypeStore ts;
        
        public Parser(IValueFactory vf, TypeStore ts) {
            this.vf = vf;
            this.ts = ts;
        }

        private IValue parse(@Nullable Node node, Type expected) {
            if (node == null) {
                throw new IllegalArgumentException();
            }

            if (expected.isAbstractData()) {
                Type sort = expected;
                String name = node.getNodeName();

                if (isListWrapper(name,  sort)) {
                    return parseList(node,  sort);
                }
                else if (isSetWrapper(name, sort)) {
                    return parseSet(node, sort);
                }
                else if (isRelationWrapper(name, sort)) {
                    return parseRelation(node, sort);
                }
                else if (isMapWrapper(name, sort)) {
                    return parseMap(node, sort);
                }
                else {
                    return parseTreeSort(node, sort);
                }
            }
            else if (expected.equivalent(TF.stringType())) {
                return parseString(node);
            }
            else if (expected.equivalent(TF.integerType())) {
                return parseInt(node);
            }
            else if (expected.equivalent(TF.realType())) {
                return parseDouble(node);
            }
            else if (expected.equivalent(TF.rationalType())) {
                return parseRational(node);
            }
            else if (expected.isExternalType()) {
                // external types default to string
                return parseString(node);
            }

            throw new UnsupportedTypeException(
                    "Outermost or nested tuples, lists, sets, relations or maps are not allowed.", expected);
        }

        private boolean isListWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isSubtypeOf(TF.listType(TF.valueType()));
            }

            return false;
        }

        private boolean isSetWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isSubtypeOf(TF.setType(TF.valueType()));
            }

            return false;
        }

        private boolean isRelationWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isSubtypeOf(TF.setType(TF.valueType()))
                        && nodeType.getFieldTypes().getFieldType(0).getElementType().isFixedWidth();
            }

            return false;
        }

        private boolean isMapWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isMap();
            }

            return false;
        }

        // TODO: implement this
        private IValue parseRational(Node node) {
            String contents = getSafeNodeValue(node).trim();
            String[] parts = contents.split("r");
            if (parts.length == 2) {
                return vf.rational(vf.integer(Integer.parseInt(parts[0])), vf.integer(Integer.parseInt(parts[0])));
            }
            throw new FactParseError(contents, 0);
        }

        private IValue parseDouble(Node node) {
            return vf.real(Double.parseDouble(getSafeNodeValue(node).trim()));
        }

        private IValue parseInt(Node node) {
            return vf.integer(Integer.parseInt(getSafeNodeValue(node).trim()));<commit_msg>removed TODO comment<commit_after>/*******************************************************************************
 * Copyright (c) 2008 CWI.
 * All rights reserved. This program and the accompanying materials
 * are made available under the terms of the Eclipse Public License v1.0
 * which accompanies this distribution, and is available at
 * http://www.eclipse.org/legal/epl-v10.html
 *
 * Contributors:
 *    <EMAIL> - initial API and implementation

 *******************************************************************************/

package io.usethesource.vallang.io;

import java.io.IOException;
import java.io.Reader;
import java.util.Objects;
import java.util.Set;

import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;

import org.checkerframework.checker.nullness.qual.NonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
import org.w3c.dom.DOMException;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;

import io.usethesource.vallang.IListWriter;
import io.usethesource.vallang.IMapWriter;
import io.usethesource.vallang.ISetWriter;
import io.usethesource.vallang.IValue;
import io.usethesource.vallang.IValueFactory;
import io.usethesource.vallang.exceptions.FactParseError;
import io.usethesource.vallang.exceptions.FactTypeUseException;
import io.usethesource.vallang.exceptions.UnsupportedTypeException;
import io.usethesource.vallang.type.Type;
import io.usethesource.vallang.type.TypeFactory;
import io.usethesource.vallang.type.TypeStore;

/**
 * This IValueReader parses and validates certain forms of XML and deserializes
 * it as IValues. The forms of XML allowed are limited by a number of different
 * value types. In particular, it allows: <ul>
 *   <li> TreeSortTypes and TreeNodeTypes </li>
 *   <li> lists, sets, relations and maps, but not unless they are wrapped by a single
 *     ConstructorType. I.o.w. a container must be the only child of a tree node.
 *     Elements of containers are juxtapositioned as children of this node.</li>
 *   <li> tuples, but not nested ones. And tuples of containers are not allowed.
 *     elements of tuples are juxtapositioned in the xml files.</li>
 *   <li> basic types, such as str, int, double; with the same restriction as for
 *     container types, they must be the only child of a tree node.</li>
 *   <li> lists of tuples, sets of tuples and maps of tuples are allowed, but not
 *     lists of lists, tuples of tuples, lists in tuples, sets in tuples, etc.
 *     If such nesting is needed, it is required to use a wrapping tree node.</li>
 * </ul>
 * There is no support for NamedTypes yet, only TreeSortType and ConstructorType are 
 * allowed.
 *     
 * The limitations of this class are governed by wanting to avoid ambiguity
 * while validating XML using the pdb's type system and the inherent impedance
 * mismatch between the type system of pdb and the structure of XML.
 * 
 * Use this class to import many forms of XML data into PDB.
 * 
 */
public class XMLReader extends AbstractTextReader {
    private static final DocumentBuilderFactory domFactory = DocumentBuilderFactory.newInstance();
    private static final TypeFactory TF = TypeFactory.getInstance();

    public IValue read(IValueFactory factory, TypeStore store, Type type, Reader stream)
            throws FactTypeUseException, IOException {

        try {
            Document doc = domFactory.newDocumentBuilder().parse(new InputSource(stream));
            return new Parser(factory, store).parse(doc.getDocumentElement(), type);
        } catch (SAXException se) {
            throw new IOException("Parsing of value failed because XML was invalid: " + se.getMessage());
        } catch (ParserConfigurationException pce) {
            throw new IOException("Parsing of value failed because XML configuration is wrong: " + pce.getMessage());
        } catch (DOMException de) {
            throw new IOException("Parsing of value failed because of a XML document failure: " + de.getMessage());
        } catch (NumberFormatException nfe) {
            throw new FactParseError("Expected a number, got something different", nfe);
        }
    }

    private static class Parser {
        private final IValueFactory vf;
        private final TypeStore ts;
        
        public Parser(IValueFactory vf, TypeStore ts) {
            this.vf = vf;
            this.ts = ts;
        }

        private IValue parse(@Nullable Node node, Type expected) {
            if (node == null) {
                throw new IllegalArgumentException();
            }

            if (expected.isAbstractData()) {
                Type sort = expected;
                String name = node.getNodeName();

                if (isListWrapper(name,  sort)) {
                    return parseList(node,  sort);
                }
                else if (isSetWrapper(name, sort)) {
                    return parseSet(node, sort);
                }
                else if (isRelationWrapper(name, sort)) {
                    return parseRelation(node, sort);
                }
                else if (isMapWrapper(name, sort)) {
                    return parseMap(node, sort);
                }
                else {
                    return parseTreeSort(node, sort);
                }
            }
            else if (expected.equivalent(TF.stringType())) {
                return parseString(node);
            }
            else if (expected.equivalent(TF.integerType())) {
                return parseInt(node);
            }
            else if (expected.equivalent(TF.realType())) {
                return parseDouble(node);
            }
            else if (expected.equivalent(TF.rationalType())) {
                return parseRational(node);
            }
            else if (expected.isExternalType()) {
                // external types default to string
                return parseString(node);
            }

            throw new UnsupportedTypeException(
                    "Outermost or nested tuples, lists, sets, relations or maps are not allowed.", expected);
        }

        private boolean isListWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isSubtypeOf(TF.listType(TF.valueType()));
            }

            return false;
        }

        private boolean isSetWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isSubtypeOf(TF.setType(TF.valueType()));
            }

            return false;
        }

        private boolean isRelationWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isSubtypeOf(TF.setType(TF.valueType()))
                        && nodeType.getFieldTypes().getFieldType(0).getElementType().isFixedWidth();
            }

            return false;
        }

        private boolean isMapWrapper(String name, Type expected) {
            Set<Type> nodeTypes = ts.lookupConstructor(expected, name);

            if (nodeTypes.size() > 0) {
                Type nodeType = nodeTypes.iterator().next();
                return nodeType.getArity() == 1
                        && nodeType.getFieldTypes().getFieldType(0).isMap();
            }

            return false;
        }

        private IValue parseRational(Node node) {
            String contents = getSafeNodeValue(node).trim();
            String[] parts = contents.split("r");
            if (parts.length == 2) {
                return vf.rational(vf.integer(Integer.parseInt(parts[0])), vf.integer(Integer.parseInt(parts[0])));
            }
            throw new FactParseError(contents, 0);
        }

        private IValue parseDouble(Node node) {
            return vf.real(Double.parseDouble(getSafeNodeValue(node).trim()));
        }

        private IValue parseInt(Node node) {
            return vf.integer(Integer.parseInt(getSafeNodeValue(node).trim()));
{'fraction_non_alphanumeric': 0.1100943665999428, 'fraction_numerical': 0.0033171289676865883, 'mean_word_length': 2.3620457604306866, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7656490', 'n_tokens_mistral': 4626, 'n_tokens_neox': 4426, 'n_words': 1320}