<commit_before>/*
 * $Id$ [Nov 5, 2004]
 *
 * Copyright (c) 2004, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 	o Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * 	o Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * 	o Neither the name of <NAME> nor the names of any other contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package net.fortuna.ical4j.data;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StreamTokenizer;
import java.net.URISyntaxException;
import java.text.ParseException;

import net.fortuna.ical4j.model.Calendar;
import net.fortuna.ical4j.model.Component;
import net.fortuna.ical4j.util.StringUtils;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class CalendarParserImpl implements CalendarParser {

    private static final int WORD_CHAR_START = 32;

    private static final int WORD_CHAR_END = 126;

    private static final int WHITESPACE_CHAR_START = 0;

    private static final int WHITESPACE_CHAR_END = 20;

    private static Log log = LogFactory.getLog(CalendarParserImpl.class);

    public final void parse(final InputStream in, final ContentHandler handler)
            throws IOException, ParserException {
        parse(new InputStreamReader(in), handler);
    }

    public final void parse(final Reader in, final ContentHandler handler)
            throws IOException, ParserException {

        try {
            StreamTokenizer tokeniser = new StreamTokenizer(in);
            tokeniser.resetSyntax();
            tokeniser.wordChars(WORD_CHAR_START, WORD_CHAR_END);
            tokeniser.whitespaceChars(WHITESPACE_CHAR_START,
                    WHITESPACE_CHAR_END);
            tokeniser.ordinaryChar(':');
            tokeniser.ordinaryChar(';');
            tokeniser.ordinaryChar('=');
            tokeniser.eolIsSignificant(true);
            tokeniser.whitespaceChars(0, 0);
            tokeniser.quoteChar('"');

            // BEGIN:VCALENDAR
            assertToken(tokeniser, Calendar.BEGIN);

            assertToken(tokeniser, ':');

            assertToken(tokeniser, Calendar.VCALENDAR);

            assertToken(tokeniser, StreamTokenizer.TT_EOL);

            handler.startCalendar();

            // parse calendar properties..
            parsePropertyList(tokeniser, handler);

            // parse components..
            parseComponentList(tokeniser, handler);

            // END:VCALENDAR
            //assertToken(tokeniser,Calendar.END);

            assertToken(tokeniser, ':');

            assertToken(tokeniser, Calendar.VCALENDAR);

            handler.endCalendar();
        } catch (Exception e) {

            if (e instanceof IOException) {
                throw (IOException) e;
            }
            if (e instanceof ParserException) {
                throw (ParserException) e;
            } else {
                throw new ParserException("An error ocurred during parsing", e);
            }
        }
    }

    /**
     * Parses an iCalendar property list from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a property list
     * @throws IOException
     * @throws ParseException
     * @throws URISyntaxException
     * @throws URISyntaxException
     * @throws ParserException
     */
    private void parsePropertyList(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParseException,
            URISyntaxException, ParserException {

        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        while (!Component.BEGIN.equals(tokeniser.sval)
                && !Component.END.equals(tokeniser.sval)) {

            parseProperty(tokeniser, handler);

            assertToken(tokeniser, StreamTokenizer.TT_WORD);
        }
    }

    /**
     * Parses an iCalendar property from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a property
     * @throws IOException
     * @throws ParserException
     * @throws URISyntaxException
     * @throws ParseException
     */
    private void parseProperty(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParserException,
            URISyntaxException, ParseException {

        String name = tokeniser.sval;

        // debugging..
        if (log.isDebugEnabled()) {
            log.debug("Property [" + name + "]");
        }

        handler.startProperty(name);

        parseParameterList(tokeniser, handler);

        // it appears that control tokens (ie. ':') are allowed
        // after the first instance on a line is used.. as such
        // we must continue appending to value until EOL is
        // reached..
        //assertToken(tokeniser, StreamTokenizer.TT_WORD);

        //String value = tokeniser.sval;
        StringBuffer value = new StringBuffer();

        //assertToken(tokeniser,StreamTokenizer.TT_EOL);

        while (tokeniser.nextToken() != StreamTokenizer.TT_EOL) {

            if (tokeniser.ttype == StreamTokenizer.TT_WORD) {
                value.append(tokeniser.sval);
            } else if (tokeniser.ttype == '"') {
                value.append((char) tokeniser.ttype);
                value.append(tokeniser.sval);
                //                value.append((char) tokeniser.ttype);
            } else {
                value.append((char) tokeniser.ttype);
            }
        }

        handler.propertyValue(StringUtils.unescape(value.toString()));
        handler.endProperty(name);
    }

    /**
     * Parses a list of iCalendar parameters by parsing the specified stream
     * tokeniser.
     * 
     * @param tokeniser
     * @return
     * @throws IOException
     * @throws ParserException
     * @throws URISyntaxException
     * @throws ParseException
     */
    private void parseParameterList(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParserException,
            URISyntaxException, ParseException {

        while (tokeniser.nextToken() == ';') {
            parseParameter(tokeniser, handler);
        }
    }

    private void parseParameter(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParserException,
            URISyntaxException, ParseException {

        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        String paramName = tokeniser.sval;

        // debugging..
        if (log.isDebugEnabled()) {
            log.debug("Parameter [" + paramName + "]");
        }

        assertToken(tokeniser, '=');

        StringBuffer paramValue = new StringBuffer();

        // preserve quote chars..
        if (tokeniser.nextToken() == '"') {
            paramValue.append('"');
            paramValue.append(tokeniser.sval);
            paramValue.append('"');
        } else {
            paramValue.append(tokeniser.sval);
        }

        handler.parameter(paramName, StringUtils
                .unescape(paramValue.toString()));
    }

    /**
     * Parses an iCalendar component list from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a component list
     * @throws IOException
     * @throws ParseException
     * @throws URISyntaxException
     * @throws ParserException
     */
    private void parseComponentList(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParseException,
            URISyntaxException, ParserException {

        while (Component.BEGIN.equals(tokeniser.sval)) {

            parseComponent(tokeniser, handler);

            assertToken(tokeniser, StreamTokenizer.TT_WORD);
        }
    }

    /**
     * Parses an iCalendar component from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a component
     * @throws IOException
     * @throws ParseException
     * @throws URISyntaxException
     * @throws ParserException
     */
    private void parseComponent(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParseException,
            URISyntaxException, ParserException {

        assertToken(tokeniser, ':');

        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        String name = tokeniser.sval;

        handler.startComponent(name);

        assertToken(tokeniser, StreamTokenizer.TT_EOL);

        parsePropertyList(tokeniser, handler);

        // a special case for VTIMEZONE component which contains
        // sub-components..
        if (Component.VTIMEZONE.equals(name)) {
            parseComponentList(tokeniser, handler);
        }
        // VEVENT/VTODO components may optionally have embedded VALARM
        // components..
        else if ((Component.VEVENT.equals(name) || Component.VTODO.equals(name))
                && Component.BEGIN.equals(tokeniser.sval)) {
            parseComponentList(tokeniser, handler);
        }

        assertToken(tokeniser, ':');

        assertToken(tokeniser, name);

        assertToken(tokeniser, StreamTokenizer.TT_EOL);

        handler.endComponent(name);
    }

    /**
     * Asserts that the next token in the stream matches the specified token.
     * 
     * @param tokeniser
     *            stream tokeniser to perform assertion on
     * @param token
     *            expected token
     * @throws IOException
     *             when unable to read from stream
     * @throws ParseException
     *             when next token in the stream does not match the expected
     *             token
     */
    private void assertToken(final StreamTokenizer tokeniser, final int token)
            throws IOException, ParserException, ParseException {

        if (tokeniser.nextToken() != token) {

            throw new ParserException("Expected [" + token + "], read ["
                    + tokeniser.ttype + "] at line " + tokeniser.lineno());
        }

        if (log.isDebugEnabled()) {
            log.debug("[" + token + "]");
        }
    }

    /**
     * Asserts that the next token in the stream matches the specified token.
     * 
     * @param tokeniser
     *            stream tokeniser to perform assertion on
     * @param token
     *            expected token
     * @throws IOException
     *             when unable to read from stream
     * @throws ParseException
     *             when next token in the stream does not match the expected
     *             token
     */
    private void assertToken(final StreamTokenizer tokeniser, final String token)
            throws IOException, ParseException, ParserException {

        // ensure next token is a word token..
        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        if (!token.equals(tokeniser.sval)) {

            throw new ParserException("Expected [" + token + "], read ["
                    + tokeniser.sval + "] at line " + tokeniser.lineno());
        }

        if (log.isDebugEnabled()) {
            log.debug("[" + token + "]");
        }
    }
}<commit_msg>Absorb extraneous whitespace between components (required for some files generated by KOrganizer)<commit_after>/*
 * $Id$ [Nov 5, 2004]
 *
 * Copyright (c) 2004, <NAME>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * 	o Redistributions of source code must retain the above copyright
 * notice, this list of conditions and the following disclaimer.
 *
 * 	o Redistributions in binary form must reproduce the above copyright
 * notice, this list of conditions and the following disclaimer in the
 * documentation and/or other materials provided with the distribution.
 *
 * 	o Neither the name of <NAME> nor the names of any other contributors
 * may be used to endorse or promote products derived from this software
 * without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 * A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
package net.fortuna.ical4j.data;

import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StreamTokenizer;
import java.net.URISyntaxException;
import java.text.ParseException;

import net.fortuna.ical4j.model.Calendar;
import net.fortuna.ical4j.model.Component;
import net.fortuna.ical4j.util.StringUtils;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class CalendarParserImpl implements CalendarParser {

    private static final int WORD_CHAR_START = 32;

    private static final int WORD_CHAR_END = 126;

    private static final int WHITESPACE_CHAR_START = 0;

    private static final int WHITESPACE_CHAR_END = 20;

    private static Log log = LogFactory.getLog(CalendarParserImpl.class);

    public final void parse(final InputStream in, final ContentHandler handler)
            throws IOException, ParserException {
        parse(new InputStreamReader(in), handler);
    }

    public final void parse(final Reader in, final ContentHandler handler)
            throws IOException, ParserException {

        try {
            StreamTokenizer tokeniser = new StreamTokenizer(in);
            tokeniser.resetSyntax();
            tokeniser.wordChars(WORD_CHAR_START, WORD_CHAR_END);
            tokeniser.whitespaceChars(WHITESPACE_CHAR_START,
                    WHITESPACE_CHAR_END);
            tokeniser.ordinaryChar(':');
            tokeniser.ordinaryChar(';');
            tokeniser.ordinaryChar('=');
            tokeniser.eolIsSignificant(true);
            tokeniser.whitespaceChars(0, 0);
            tokeniser.quoteChar('"');

            // BEGIN:VCALENDAR
            assertToken(tokeniser, Calendar.BEGIN);

            assertToken(tokeniser, ':');

            assertToken(tokeniser, Calendar.VCALENDAR);

            assertToken(tokeniser, StreamTokenizer.TT_EOL);

            handler.startCalendar();

            // parse calendar properties..
            parsePropertyList(tokeniser, handler);

            // parse components..
            parseComponentList(tokeniser, handler);

            // END:VCALENDAR
            //assertToken(tokeniser,Calendar.END);

            assertToken(tokeniser, ':');

            assertToken(tokeniser, Calendar.VCALENDAR);

            handler.endCalendar();
        } catch (Exception e) {

            if (e instanceof IOException) {
                throw (IOException) e;
            }
            if (e instanceof ParserException) {
                throw (ParserException) e;
            } else {
                throw new ParserException("An error ocurred during parsing", e);
            }
        }
    }

    /**
     * Parses an iCalendar property list from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a property list
     * @throws IOException
     * @throws ParseException
     * @throws URISyntaxException
     * @throws URISyntaxException
     * @throws ParserException
     */
    private void parsePropertyList(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParseException,
            URISyntaxException, ParserException {

        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        while (!Component.BEGIN.equals(tokeniser.sval)
                && !Component.END.equals(tokeniser.sval)) {

            parseProperty(tokeniser, handler);

            assertToken(tokeniser, StreamTokenizer.TT_WORD);
        }
    }

    /**
     * Parses an iCalendar property from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a property
     * @throws IOException
     * @throws ParserException
     * @throws URISyntaxException
     * @throws ParseException
     */
    private void parseProperty(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParserException,
            URISyntaxException, ParseException {

        String name = tokeniser.sval;

        // debugging..
        if (log.isDebugEnabled()) {
            log.debug("Property [" + name + "]");
        }

        handler.startProperty(name);

        parseParameterList(tokeniser, handler);

        // it appears that control tokens (ie. ':') are allowed
        // after the first instance on a line is used.. as such
        // we must continue appending to value until EOL is
        // reached..
        //assertToken(tokeniser, StreamTokenizer.TT_WORD);

        //String value = tokeniser.sval;
        StringBuffer value = new StringBuffer();

        //assertToken(tokeniser,StreamTokenizer.TT_EOL);

        while (tokeniser.nextToken() != StreamTokenizer.TT_EOL) {

            if (tokeniser.ttype == StreamTokenizer.TT_WORD) {
                value.append(tokeniser.sval);
            } else if (tokeniser.ttype == '"') {
                value.append((char) tokeniser.ttype);
                value.append(tokeniser.sval);
                //                value.append((char) tokeniser.ttype);
            } else {
                value.append((char) tokeniser.ttype);
            }
        }

        handler.propertyValue(StringUtils.unescape(value.toString()));
        handler.endProperty(name);
    }

    /**
     * Parses a list of iCalendar parameters by parsing the specified stream
     * tokeniser.
     * 
     * @param tokeniser
     * @return
     * @throws IOException
     * @throws ParserException
     * @throws URISyntaxException
     * @throws ParseException
     */
    private void parseParameterList(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParserException,
            URISyntaxException, ParseException {

        while (tokeniser.nextToken() == ';') {
            parseParameter(tokeniser, handler);
        }
    }

    private void parseParameter(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParserException,
            URISyntaxException, ParseException {

        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        String paramName = tokeniser.sval;

        // debugging..
        if (log.isDebugEnabled()) {
            log.debug("Parameter [" + paramName + "]");
        }

        assertToken(tokeniser, '=');

        StringBuffer paramValue = new StringBuffer();

        // preserve quote chars..
        if (tokeniser.nextToken() == '"') {
            paramValue.append('"');
            paramValue.append(tokeniser.sval);
            paramValue.append('"');
        } else {
            paramValue.append(tokeniser.sval);
        }

        handler.parameter(paramName, StringUtils
                .unescape(paramValue.toString()));
    }

    /**
     * Parses an iCalendar component list from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a component list
     * @throws IOException
     * @throws ParseException
     * @throws URISyntaxException
     * @throws ParserException
     */
    private void parseComponentList(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParseException,
            URISyntaxException, ParserException {

        while (Component.BEGIN.equals(tokeniser.sval)) {
            parseComponent(tokeniser, handler);
            
            // absorb extraneous whitespace between components..
            try {
                while (true) {
                    assertToken(tokeniser, StreamTokenizer.TT_EOL);
                }
            }
            catch (ParserException pe) {
            }
//            assertToken(tokeniser, StreamTokenizer.TT_WORD);
        }
    }

    /**
     * Parses an iCalendar component from the specified stream tokeniser.
     * 
     * @param tokeniser
     * @return a component
     * @throws IOException
     * @throws ParseException
     * @throws URISyntaxException
     * @throws ParserException
     */
    private void parseComponent(final StreamTokenizer tokeniser,
            final ContentHandler handler) throws IOException, ParseException,
            URISyntaxException, ParserException {

        assertToken(tokeniser, ':');

        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        String name = tokeniser.sval;

        handler.startComponent(name);

        assertToken(tokeniser, StreamTokenizer.TT_EOL);

        parsePropertyList(tokeniser, handler);

        // a special case for VTIMEZONE component which contains
        // sub-components..
        if (Component.VTIMEZONE.equals(name)) {
            parseComponentList(tokeniser, handler);
        }
        // VEVENT/VTODO components may optionally have embedded VALARM
        // components..
        else if ((Component.VEVENT.equals(name) || Component.VTODO.equals(name))
                && Component.BEGIN.equals(tokeniser.sval)) {
            parseComponentList(tokeniser, handler);
        }

        assertToken(tokeniser, ':');

        assertToken(tokeniser, name);

        assertToken(tokeniser, StreamTokenizer.TT_EOL);

        handler.endComponent(name);
    }

    /**
     * Asserts that the next token in the stream matches the specified token.
     * 
     * @param tokeniser
     *            stream tokeniser to perform assertion on
     * @param token
     *            expected token
     * @throws IOException
     *             when unable to read from stream
     * @throws ParseException
     *             when next token in the stream does not match the expected
     *             token
     */
    private void assertToken(final StreamTokenizer tokeniser, final int token)
            throws IOException, ParserException, ParseException {

        if (tokeniser.nextToken() != token) {
            throw new ParserException("Expected [" + token + "], read ["
                    + tokeniser.ttype + "] at line " + tokeniser.lineno());
        }

        if (log.isDebugEnabled()) {
            log.debug("[" + token + "]");
        }
    }

    /**
     * Asserts that the next token in the stream matches the specified token.
     * 
     * @param tokeniser
     *            stream tokeniser to perform assertion on
     * @param token
     *            expected token
     * @throws IOException
     *             when unable to read from stream
     * @throws ParseException
     *             when next token in the stream does not match the expected
     *             token
     */
    private void assertToken(final StreamTokenizer tokeniser, final String token)
            throws IOException, ParseException, ParserException {

        // ensure next token is a word token..
        assertToken(tokeniser, StreamTokenizer.TT_WORD);

        if (!token.equals(tokeniser.sval)) {
            throw new ParserException("Expected [" + token + "], read ["
                    + tokeniser.sval + "] at line " + tokeniser.lineno());
        }

        if (log.isDebugEnabled()) {
            log.debug("[" + token + "]");
        }
    }
}
{'fraction_non_alphanumeric': 0.08534537895746722, 'fraction_numerical': 0.0018388231531819635, 'mean_word_length': 2.3056289640591965, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1457906', 'n_tokens_mistral': 6602, 'n_tokens_neox': 6200, 'n_words': 1812}