<commit_before>package de.najidev.mensaupb.stw;

import android.content.*;
import android.text.*;

import org.slf4j.*;

import java.io.*;
import java.util.*;

import de.najidev.mensaupb.*;

public class StwParser {

    private static final Logger LOGGER = LoggerFactory.getLogger(StwParser.class.getSimpleName());

    public static List<ContentValues> parseInputStream(InputStream inputStream) {
        if(BuildConfig.DEBUG) LOGGER.debug("parseInputStream({})", inputStream);

        List<ContentValues> parsedMenus = new ArrayList<ContentValues>();

        Scanner sc = new Scanner(inputStream, "windows-1252");

        sc.nextLine(); // skip description line

        while (sc.hasNextLine()) {
            String[] parts = prepareNextLine(sc);
            if(skipThisLine(parts)) {
                continue;
            }else{
            ContentValues menu = parseLine(parts);
            parsedMenus.add(menu);
            }
        }

        if(BuildConfig.DEBUG) LOGGER.debug("parseInputStream({}) done", inputStream);
        return parsedMenus;
    }

    private static boolean skipThisLine(String[] parts) {
        if(BuildConfig.DEBUG) LOGGER.debug("skipThisLine(..): \"Mensa Hamm\" == {}",parts[1]);
        return TextUtils.equals("Mensa Hamm", parts[1]);
    }

    private static String[] prepareNextLine(Scanner sc) {
        if(BuildConfig.DEBUG) LOGGER.debug("prepareNextLine(..)");

        String line = sc.nextLine();

        if(BuildConfig.DEBUG) LOGGER.debug("nextLine -> ({})", line);

        line = line.replaceAll("\"", "");

        if(BuildConfig.DEBUG) LOGGER.debug("prepareNextLine(..) done");
        return line.split(";");
    }

    private static ContentValues parseLine(String[] parts) {
        if(BuildConfig.DEBUG) LOGGER.debug("parseLine({}) {}", parts, "");

        ContentValues cv = new ContentValues();<commit_msg>updates csv parser to skip blank lines<commit_after>package de.najidev.mensaupb.stw;

import android.content.*;
import android.text.*;

import org.slf4j.*;

import java.io.*;
import java.util.*;

import de.najidev.mensaupb.*;

public class StwParser {

    private static final Logger LOGGER = LoggerFactory.getLogger(StwParser.class.getSimpleName());

    public static List<ContentValues> parseInputStream(InputStream inputStream) {
        if(BuildConfig.DEBUG) LOGGER.debug("parseInputStream({})", inputStream);

        List<ContentValues> parsedMenus = new ArrayList<ContentValues>();

        Scanner sc = new Scanner(inputStream, "windows-1252");

        sc.nextLine(); // skip description line

        while (sc.hasNextLine()) {
            String[] parts = prepareNextLine(sc);
            if(skipThisLine(parts)) {
                continue;
            } else {
            ContentValues menu = parseLine(parts);
            parsedMenus.add(menu);
            }
        }

        if(BuildConfig.DEBUG) LOGGER.debug("parseInputStream({}) done", inputStream);
        return parsedMenus;
    }

    private static boolean skipThisLine(String[] parts) {
        if(BuildConfig.DEBUG) LOGGER.debug("skipThisLine(..): \"Mensa Hamm\" == {}",parts[1]);
        return parts.length < 9 || TextUtils.equals("Mensa Hamm", parts[1]);
    }

    private static String[] prepareNextLine(Scanner sc) {
        if(BuildConfig.DEBUG) LOGGER.debug("prepareNextLine(..)");

        String line = sc.nextLine();

        if(BuildConfig.DEBUG) LOGGER.debug("nextLine -> ({})", line);

        line = line.replaceAll("\"", "");

        if(BuildConfig.DEBUG) LOGGER.debug("prepareNextLine(..) done");
        return line.split(";");
    }

    private static ContentValues parseLine(String[] parts) {
        if(BuildConfig.DEBUG) LOGGER.debug("parseLine({}) {}", parts, "");

        ContentValues cv = new ContentValues();
{'fraction_non_alphanumeric': 0.1431212041193557, 'fraction_numerical': 0.003960918933192501, 'mean_word_length': 2.9623430962343096, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5552447', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1081, 'n_words': 223}