<commit_before>package edu.kit.ksri.paperfinder.scholar.parser;

import edu.kit.ksri.paperfinder.model.Article;
import org.jsoup.nodes.Element;

import java.util.List;
import java.util.stream.Collectors;

/**
 * This parser works with the 2014-11-28 Google Scholar result HTML format
 */
public class ResultsParser extends ParserBase {
    public ResultsParser(String html) {
        super(html);
    }

    @Override
    public List<Article> parse() {
        return getContent().getElementsByClass("gs_r")
                .stream()
                .filter(element -> element.children().hasClass("gs_ri"))
                .map(this::buildArticleFromElement)
                .collect(Collectors.toList());
    }

    private Article buildArticleFromElement(Element element) {
        Article article = new Article();

        Element title = element.select(".gs_rt > a").first();
        article.setTitle(title.text());

        Element metaInfo = element.getElementsByClass("gs_a").first();
        String metaInfoText = metaInfo.text();
        String[] metaInfoTextParts = metaInfoText.split(" - ");
        String author = metaInfoTextParts[0];
        article.setAuthor(author);

        String publicationAndYear = metaInfoTextParts[1];
        String year = publicationAndYear.substring(publicationAndYear.length()-4, publicationAndYear.length());
        article.setYearPublished(Integer.parseInt(year));

        if (publicationAndYear.length() > 6) {
            String publication = publicationAndYear.substring(0, publicationAndYear.length()-4);
            article.setPublication(publication);
        }

        String source = metaInfoTextParts[metaInfoTextParts.length-1];
        article.setSource(source);

        Element citationsElement = element.select(".gs_ri > .gs_fl > a").first();
        String citationText = citationsElement.text();
        article.setCitations(Integer.parseInt(citationText.replaceAll("[\\D]", "")));

        return article;
    }
}<commit_msg>Complete Refactor of ResultsParser Looks less nice, but should be bulletproof<commit_after>package edu.kit.ksri.paperfinder.scholar.parser;

import edu.kit.ksri.paperfinder.model.Article;
import org.jsoup.nodes.Element;

import java.util.List;
import java.util.stream.Collectors;

import static edu.kit.ksri.paperfinder.util.NumberUtils.parseInt;


/**
 * This parser works with the 2014-11-28 Google Scholar result HTML format
 */
public class ResultsParser extends ParserBase {
    public ResultsParser(String html) {
        super(html);
    }

    @Override
    public List<Article> parse() {
        return getContent().getElementsByClass("gs_r")
                .stream()
                .filter(element -> element.children().hasClass("gs_ri"))
                .map(this::buildArticleFromElement)
                .collect(Collectors.toList());
    }

    private Article buildArticleFromElement(Element element) {
        Article article = new Article();

        try {
            Element title = element.select(".gs_rt > a").first();
            article.setTitle(title.text());
        } catch (Exception e) {
            article.setTitle("n/a");
        }

        String[] metaInfoTextParts = new String[0];
        try {
            Element metaInfo = element.getElementsByClass("gs_a").first();
            String metaInfoText = metaInfo.text();
            metaInfoTextParts = metaInfoText.split(" - ");
            String author = metaInfoTextParts[0];
            article.setAuthor(author);
        } catch (Exception e) {
            article.setAuthor("n/a");
        }

        String publicationAndYear = null;
        try {
            publicationAndYear = metaInfoTextParts[1];
            String year = publicationAndYear.substring(publicationAndYear.length()-4, publicationAndYear.length());
            article.setYearPublished(parseInt(year, -1));
        } catch (Exception e) {
            article.setYearPublished(-1);
        }

        if (publicationAndYear.length() > 6) {
            try {
                String publication = publicationAndYear.substring(0, publicationAndYear.length()-4);
                article.setPublication(publication);
            } catch (Exception e) {
                article.setPublication("n/a");
            }
        }

        try {
            String source = metaInfoTextParts[metaInfoTextParts.length-1];
            article.setSource(source);
        } catch (Exception e) {
            article.setSource("n/a");
        }

        try {
            Element citationsElement = element.select(".gs_ri > .gs_fl > a").first();
            String citationText = citationsElement.text();
            article.setCitations(parseInt(citationText.replaceAll("[\\D]", ""),0));
        } catch (Exception e) {
            article.setCitations(-1);
        }

        try {
            Element abstractElement = element.select(".gs_ri > .gs_rs").first();
            String abstractText = abstractElement.text();
            article.setAbstractText(abstractText);
        } catch (Exception e) {
            article.setAbstractText("n/a");
        }

        try {
            Element pdfLinkElement = element.select(".gs_ttss > a").first();
            if (pdfLinkElement != null) {
                String pdfLink = pdfLinkElement.attr("href");
                article.setPdfLink(pdfLink);
            }
        } catch (Exception e) {
            article.setPdfLink("");
        }

        return article;
    }
}
{'fraction_non_alphanumeric': 0.11804538799414349, 'fraction_numerical': 0.006405563689604685, 'mean_word_length': 2.3384239462431275, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '420291', 'n_tokens_mistral': 1488, 'n_tokens_neox': 1426, 'n_words': 275}