<commit_before>package bkromhout.fdl.parsers;

import bkromhout.fdl.Site;
import bkromhout.fdl.util.C;
import bkromhout.fdl.util.Sites;
import bkromhout.fdl.util.Util;

import java.io.File;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Parses the input file.
 */
public class InputFileParser extends FileParser {
    /**
     * Regex for extracting host name from a url.
     */
    private static Pattern hostRegex = Pattern.compile("^(http[s]?://)?([^:/\\s]+)(/.*)?$");
    /**
     * Regex for matching lines which point to local story directories.
     */
    private static Pattern localFicRegex = Pattern.compile(""); // TODO create regex to match "@fdl:dirbook=[name]"

    /**
     * Create a new {@link InputFileParser} to parse the given file.
     * <p>
     * This class should be used as a oneshot call since it parses the file upon creation and exposes no public
     * members.
     * @param inputFile Input file.
     */
    public InputFileParser(File inputFile) {
        super(FileType.INPUT, inputFile);
    }

    @Override
    protected void init() {
        // Do nothing.
    }

    /**
     * Processes a line from the input file, attempting to parse a story site url and assign it to one of the url lists.
     * Won't put any invalid or repeat lines in the url lists.
     * @param line Line from the input file.
     */
    @Override
    protected void processLine(String line) throws IllegalStateException {
        // TODO make this support local stories too!


        // Try to match this line to a url so that we can extract the host.
        Matcher hostMatcher = hostRegex.matcher(line);
        if (!hostMatcher.matches()) {
            if (!line.trim().isEmpty()) Util.loudf(C.PROCESS_LINE_FAILED, type, line);
            return;
        }
        String hostString = hostMatcher.group(2).toLowerCase();
        // Add story url to a Site's list, or log it if none of them could take it (malformed or unsupported site url).
        if (!tryAddUrlToSomeSite(hostString, line)) Util.logf(C.PROCESS_LINE_FAILED, type, line);
    }

    /**
     * Using the given host string, figure out which site we can add the given url string to and add it.
     * @param hostString String parsed from the line that should contain a substring which is equal to some {@link
     *                   bkromhout.fdl.Site#host} value.
     * @param url        The url to add to some {@link bkromhout.fdl.Site Site}'s {@link bkromhout.fdl.Site#urls url
     *                   list}.
     * @return True if a {@link bkromhout.fdl.Site} was found to add the url to, otherwise false. Returns false
     * immediately if either parameter is null or empty.
     */
    private boolean tryAddUrlToSomeSite(String hostString, String url) {
        if (hostString == null || hostString.isEmpty() || url == null || url.isEmpty()) return false;
        // Iterate through the list of sites to find one to add the url to.
        for (Site site : Sites.all()) {
            if (hostString.contains(site.getHost())) {
                // Found a site which we can add this URL to.
                site.getUrls().add(url);
                return true;
            }
        }
        // No site found to add the url to.
        return false;
    }
}<commit_msg>Finished modifying InputFileParser to parse local story lines.<commit_after>package bkromhout.fdl.parsers;

import bkromhout.fdl.Site;
import bkromhout.fdl.events.AddLocalStoryDirEvent;
import bkromhout.fdl.util.C;
import bkromhout.fdl.util.Sites;
import bkromhout.fdl.util.Util;

import java.io.File;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Parses the input file.
 */
public class InputFileParser extends FileParser {
    /**
     * Regex for extracting host name from a url.
     */
    private static Pattern hostRegex = Pattern.compile("^(http[s]?://)?([^:/\\s]+)(/.*)?$");
    /**
     * Regex for matching lines which point to local story directories.
     * <p>
     * Will match a line if, ignoring leading whitespace, it starts with "@fdl:ls=". (There can be whitespace before the
     * "=", but anything after it will be part of group 1).
     * <p>
     * If a line matches, group 1 will contain the rest of the line following "=". This text is intended to be used as
     * the name of a directory that is relative to the folder that the input file in in.
     */
    private static Pattern localStoryRegex = Pattern.compile("^\\s*@fdl:ls\\s*=(.*)$");

    /**
     * Create a new {@link InputFileParser} to parse the given file.
     * <p>
     * This class should be used as a oneshot call since it parses the file upon creation and exposes no public
     * members.
     * @param inputFile Input file.
     */
    public InputFileParser(File inputFile) {
        super(FileType.INPUT, inputFile);
    }

    @Override
    protected void init() {
        // Do nothing.
    }

    @Override
    protected void processLine(String line) throws IllegalStateException {
        // Check if this line specifies a local story.
        Matcher lsMatcher = localStoryRegex.matcher(line);
        if (lsMatcher.matches()) {
            // Get the directory name from the line, stripping any leading/trailing whitespace.
            String dirName = lsMatcher.group(1).trim();
            // Add the directory name to the list of local story directories in the local story processor if non-empty.
            if (!dirName.isEmpty()) C.getEventBus().post(new AddLocalStoryDirEvent(dirName));
            return;
        }

        // Try to match this line as an http(s) url so that we can extract the host.
        Matcher hostMatcher = hostRegex.matcher(line);
        if (hostMatcher.matches()) {
            String hostString = hostMatcher.group(2).toLowerCase();
            // Try to add the line (which we now know is a url) to a Site's list, and return immediately if there is
            // a site which can handle it.
            if (tryAddUrlToSomeSite(hostString, line)) return;
        }

        // Verbose log this line if we got here and it's non-empty, we couldn't process it.
        if (!line.trim().isEmpty()) Util.loudf(C.PROCESS_LINE_FAILED, type, line);
    }

    /**
     * Using the given host string, figure out which site we can add the given url string to and add it.
     * @param hostString String parsed from the line that should contain a substring which is equal to some {@link
     *                   bkromhout.fdl.Site#host} value.
     * @param url        The url to add to some {@link bkromhout.fdl.Site Site}'s {@link bkromhout.fdl.Site#urls url
     *                   list}.
     * @return True if a {@link bkromhout.fdl.Site} was found to add the url to, otherwise false. Returns false
     * immediately if either parameter is null or empty.
     */
    private boolean tryAddUrlToSomeSite(String hostString, String url) {
        if (hostString == null || hostString.isEmpty() || url == null || url.isEmpty()) return false;
        // Iterate through the list of sites to find one to add the url to.
        for (Site site : Sites.all()) {
            if (hostString.contains(site.getHost())) {
                // Found a site which we can add this URL to.
                site.getUrls().add(url);
                return true;
            }
        }
        // No site found to add the url to.
        return false;
    }
}
{'fraction_non_alphanumeric': 0.09909422738948222, 'fraction_numerical': 0.000675949709341625, 'mean_word_length': 2.371923427529626, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4044404', 'n_tokens_mistral': 2122, 'n_tokens_neox': 2068, 'n_words': 805}