<commit_before>/*
 * Copyright 2015 XXX
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package xxx.web.comments.debates.impl;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import xxx.web.comments.Utils;
import xxx.web.comments.createdebate.Argument;
import xxx.web.comments.createdebate.Debate;
import xxx.web.comments.debates.DebateParser;

import java.io.IOException;
import java.io.InputStream;

/**
 * @author <NAME>
 */
public class ProConOrgParser implements DebateParser
{
    @Override
    public Debate parseDebate(InputStream inputStream)
            throws IOException
    {
        Debate result = new Debate();

        Document doc = Jsoup.parse(inputStream, "UTF-8", "http://www.procon.org/");

        // Set the Url of the doc


        // title
        Element body = doc.body();
        Elements debateTitleElements = body.select("p[class=title]").select("p[style]");

        if (debateTitleElements.first() == null) {
            // not a debate
            return null;
        }

        String title = Utils.normalize(debateTitleElements.first().text());
        result.setTitle(title);

        Element trAnswers = body.select("tr > td > b:contains(PRO \\(yes\\))").parents().first().parents().first().nextElementSibling();

        // the PRO side
        Element proTd = trAnswers.select("td").get(0);
        Element conTd = trAnswers.select("td").get(1);

        System.out.println(proTd.select("blockquote").size());
        System.out.println(conTd.select("blockquote").size());

        Elements texts = proTd.select("blockquote > div[class=editortext]");
        for (Element text: texts) {
            Argument argument = new Argument();
            argument.setStance("pro");
            argument.setText(removeQuotes(text.text()));

            // TODO make sure we parse paragraphs as well

            result.getArgumentList().add(argument);
        }

        texts = conTd.select("blockquote > div[class=editortext]");
        for (Element text: texts) {
            Argument argument = new Argument();
            argument.setStance("con");
            argument.setText(removeQuotes(text.text()));

            result.getArgumentList().add(argument);
        }

        /*
        Element twoSidesAndScores = body
                .select("table[style=margin:0 auto;padding:0;border:0;text-align:center;width:98%;]")
                .first();

        if (twoSidesAndScores == null) {
            // this is not a two-side debate
            return null;
        }

        Elements twoSides = twoSidesAndScores.select("div[class=sideTitle]");
        if (twoSides.size() != 2) {
            // this is not a two-side debate
            return null;
        }

        // description
        StringBuilder debateDescriptionBuilder = new StringBuilder();
        Element description = body.select("#description").first();

        if (description != null) {
            Element descriptionText = description.select("div[class=centered debatelongDesc]")
                    .first();
            if (descriptionText.select("p").isEmpty()) {
                // just extract the text
                debateDescriptionBuilder.append(descriptionText.text());
            }
            else if ("".equals(descriptionText.select("p").first().text())) {
                // extract paragraphs
                for (Element p : descriptionText.select("p").select("span")) {
                    debateDescriptionBuilder.append(p.ownText());
                    debateDescriptionBuilder.append("\n");
                }
            }
            else {
                // extract paragraphs
                for (Element p : descriptionText.select("p")) {
                    debateDescriptionBuilder.append(p.text());
                    debateDescriptionBuilder.append("\n");
                }
            }
        }
        result.setDescription(Utils.normalize(debateDescriptionBuilder.toString()));
        */

        return result;
    }

    private static String removeQuotes(String s)
    {
        return s.replaceAll("[(^\")(\"$)]", "");
    }
}<commit_msg>Parsing procon.org<commit_after>/*
 * Copyright 2015 XXX
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package xxx.web.comments.debates.impl;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import xxx.web.comments.Utils;
import xxx.web.comments.createdebate.Argument;
import xxx.web.comments.createdebate.Debate;
import xxx.web.comments.debates.DebateParser;

import java.io.IOException;
import java.io.InputStream;

/**
 * @author <NAME>
 */
public class ProConOrgParser
        implements DebateParser
{
    @Override
    public Debate parseDebate(InputStream inputStream)
            throws IOException
    {
        Debate result = new Debate();

        Document doc = Jsoup.parse(inputStream, "UTF-8", "http://www.procon.org/");

        // Set the Url of the doc

        // title
        Element body = doc.body();
        Elements debateTitleElements = body.select("p[class=title]").select("p[style]");

        if (debateTitleElements.first() == null) {
            // not a debate
            return null;
        }

        String title = Utils.normalize(debateTitleElements.first().text());
        result.setTitle(title);

        Element trAnswers = body.select("tr > td > b:contains(PRO \\(yes\\))").parents().first()
                .parents().first().nextElementSibling();

        // the PRO side
        Element proTd = trAnswers.select("td").get(0);
        Element conTd = trAnswers.select("td").get(1);

        System.out.println(proTd.select("blockquote").size());
        System.out.println(conTd.select("blockquote").size());

        Elements texts = proTd.select("blockquote > div[class=editortext]");
        for (Element text : texts) {
            Argument argument = new Argument();
            argument.setStance("pro");
            argument.setText(extractPlainTextFromTextElement(text));

            result.getArgumentList().add(argument);
        }

        texts = conTd.select("blockquote > div[class=editortext]");
        for (Element text : texts) {
            Argument argument = new Argument();
            argument.setStance("con");
            argument.setText(extractPlainTextFromTextElement(text));

            result.getArgumentList().add(argument);
        }

        return result;
    }

    /**
     * Extracts the document of the quote
     *
     * @param textElement text quote element
     * @return plain string with paragraphs kept
     */
    protected static String extractPlainTextFromTextElement(Element textElement)
    {
        StringBuilder sb = new StringBuilder();

        for (Node childNode : textElement.childNodes()) {
            if (childNode instanceof Element) {
                Element childElement = (Element) childNode;

                String tagName = childElement.tagName();

                if ("p".equals(tagName) || "span".equals(tagName)) {
                    sb.append(childElement.text());
                    sb.append("\n");
                }
                else if ("br".equals(tagName)) {
                    // prevent double newlines
                    sb = new StringBuilder(sb.toString().trim());
                    sb.append("\n");
                }

            }
            else if (childNode instanceof TextNode) {
                TextNode textNode = (TextNode) childNode;

                sb.append(textNode.text());
            }
        }

        // remove leading + ending quotes
        return Utils.normalize(sb.toString()).replaceAll("[(^\")(\"$)]", "");
    }
}
{'fraction_non_alphanumeric': 0.1215726263312939, 'fraction_numerical': 0.0031724450487196918, 'mean_word_length': 2.2680488707885966, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '381420', 'n_tokens_mistral': 2413, 'n_tokens_neox': 2293, 'n_words': 568}