<commit_before>package com.gurkensalat.osm.repository;

import com.gurkensalat.osm.entity.DitibPlace;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Tag;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
import static org.apache.commons.collections4.ListUtils.emptyIfNull;
import static org.apache.commons.io.IOUtils.closeQuietly;

public class DitibRepositoryImpl implements DitibRepository
{
    private static final Logger LOGGER = LoggerFactory.getLogger(DitibRepositoryImpl.class);

    private final String CSS_PATH = "td.body_text2";

    public List<DitibPlace> parse(File resourceFile)
    {
        List<DitibPlace> result = new ArrayList<DitibPlace>();

        try
        {
            Document document = Jsoup.parse(resourceFile, "UTF-8", "http://www.ditib.de/");

            Elements selection = document.select(CSS_PATH);
            LOGGER.debug("Found {} matching elements", selection.size());

            if (isNotEmpty(selection))
            {
                int hitNumber = 0;
                for (Element element : selection)
                {
                    LOGGER.debug("-------------------------------------------------------");
                    LOGGER.debug("Found: {}, {}", hitNumber++, element);

                    Elements rows = selection.select("tbody > tr");
                    if (isNotEmpty(rows))
                    {

                        int rowNumber = 0;
                        for (Element row : rows)
                        {
                            LOGGER.debug("  qw qw qw -------------------------------------------------------");
                            String foo = element.toString();
                            LOGGER.debug("  qw qw qw {}", rowNumber++, element);
                        }
                    }

                    DitibPlace place = new DitibPlace();
                    place = extractPlaceName(safeGetElement(rows, 0), place);
                    result.add(place);
                }
            }
        }
        catch (IOException ioe)
        {
            LOGGER.error("While parsing {}", resourceFile, ioe);
        }

        result = emptyIfNull(result);
        return result;
    }

    protected DitibPlace extractPlaceName(Element block, DitibPlace place)
    {
        LOGGER.debug("extractPlaceName()");

        // fsck css, just run indexOf :-(
        String data = block.toString();
        int from = data.lastIndexOf("<strong>");
        int to = data.lastIndexOf("</strong>");

        if (from != -1 && to != -1)
        {
            data = data.substring(from + 8, to);
            data = data.replaceAll("&nbsp;", "");
            place.setDitibCode(data);
        }

        return place;
    }

    protected Element safeGetElement(Elements e, int index)
    {
        Element result = new Element(Tag.valueOf("body"), "no-where");

        if (e != null)
        {
            if (e.size() > index)
            {
                result = e.get(index);
            }
        }

        return result;
    }

    public void prettify(File target, File resourceFile)<commit_msg>Slightly improved parser<commit_after>package com.gurkensalat.osm.repository;

import com.gurkensalat.osm.entity.DitibPlace;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Tag;
import org.jsoup.select.Elements;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

import static org.apache.commons.collections4.CollectionUtils.isNotEmpty;
import static org.apache.commons.collections4.ListUtils.emptyIfNull;
import static org.apache.commons.io.IOUtils.closeQuietly;

public class DitibRepositoryImpl implements DitibRepository
{
    private static final Logger LOGGER = LoggerFactory.getLogger(DitibRepositoryImpl.class);

    private final String CSS_PATH = "td.body_text2";

    public List<DitibPlace> parse(File resourceFile)
    {
        List<DitibPlace> result = new ArrayList<DitibPlace>();

        try
        {
            Document document = Jsoup.parse(resourceFile, "UTF-8", "http://www.ditib.de/");

            Elements selection = document.select(CSS_PATH);
            LOGGER.debug("Found {} matching elements", selection.size());

            if (isNotEmpty(selection))
            {
                int hitNumber = 0;
                for (Element element : selection)
                {
                    LOGGER.debug("-------------------------------------------------------");
                    LOGGER.debug("Found: {}, {}", hitNumber++, element);

                    Elements rows = selection.select("tbody > tr");
                    if (isNotEmpty(rows))
                    {
                        DitibPlace place = new DitibPlace();
                        place = extractPlaceCode(safeGetElement(rows, 0), place);
                        place = extractFoo1(safeGetElement(rows, 1), place);
                        place = extractFoo2(safeGetElement(rows, 2), place);
                        place = extractFoo3(safeGetElement(rows, 3), place);
                        place = extractFoo4(safeGetElement(rows, 4), place);
                        result.add(place);
                    }
                }
            }
        }
        catch (IOException ioe)
        {
            LOGGER.error("While parsing {}", resourceFile, ioe);
        }

        result = emptyIfNull(result);
        return result;
    }

    protected DitibPlace extractPlaceCode(Element block, DitibPlace place)
    {
        LOGGER.debug("extractPlaceCode()");
        LOGGER.debug("-------------------------------------------------------");
        LOGGER.debug("{}", block.toString());
        LOGGER.debug("-------------------------------------------------------");

        // fsck css, just run indexOf :-(
        String data = block.toString();
        int from = data.lastIndexOf("<strong>");
        int to = data.lastIndexOf("</strong>");

        if (from != -1 && to != -1)
        {
            data = data.substring(from + 8, to);
            data = data.replaceAll("&nbsp;", "");
            place.setDitibCode(data);
        }

        return place;
    }

    protected DitibPlace extractFoo1(Element block, DitibPlace place)
    {
        LOGGER.debug("extractFoo1()");
        LOGGER.debug("-------------------------------------------------------");
        LOGGER.debug("{}", block.toString());
        LOGGER.debug("-------------------------------------------------------");

        return place;
    }

    protected DitibPlace extractFoo2(Element block, DitibPlace place)
    {
        LOGGER.debug("-------------------------------------------------------");
        LOGGER.debug("extractFoo2()");
        LOGGER.debug("{}", block.toString());
        LOGGER.debug("-------------------------------------------------------");

        return place;
    }

    protected DitibPlace extractFoo3(Element block, DitibPlace place)
    {
        LOGGER.debug("extractFoo3()");
        LOGGER.debug("-------------------------------------------------------");
        LOGGER.debug("{}", block.toString());
        LOGGER.debug("-------------------------------------------------------");

        return place;
    }

    protected DitibPlace extractFoo4(Element block, DitibPlace place)
    {
        LOGGER.debug("extractFoo4()");
        LOGGER.debug("-------------------------------------------------------");
        LOGGER.debug("{}", block.toString());
        LOGGER.debug("-------------------------------------------------------");

        return place;
    }

    protected Element safeGetElement(Elements e, int index)
    {
        Element result = new Element(Tag.valueOf("body"), "no-where");

        if (e != null)
        {
            if (e.size() > index)
            {
                result = e.get(index);
            }
        }

        return result;
    }

    public void prettify(File target, File resourceFile)
{'fraction_non_alphanumeric': 0.19389822428792755, 'fraction_numerical': 0.004647836968180193, 'mean_word_length': 1.5050746268656716, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5220808', 'n_tokens_mistral': 2181, 'n_tokens_neox': 2158, 'n_words': 443}