<commit_before>package io.github.lonamiwebs.stringlate.classes.applications;

import android.util.Xml;

import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlSerializer;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.regex.Pattern;

public class ApplicationListParser {
    // We don't use namespaces
    private static final String ns = null;

    private static final String ID = "id";
    private static final String LAST_UPDATED = "lastupdated";
    private static final String NAME = "name";
    private static final String DESCRIPTION = "summary";
    private static final String ICON = "icon";
    private static final String SOURCE_URL = "source";

    // We will only parse https GitHub urls
    private static final Pattern GITHUB_PATTERN =
            Pattern.compile("^https?://github.com/\\w+/\\w+(?:/.*)?$");

    //region Xml -> ApplicationsList

    public static ArrayList<Application> parseFromXml(InputStream in, HashSet<String> installedPackages)
            throws XmlPullParserException, IOException {

        try {
            XmlPullParser parser = Xml.newPullParser();
            parser.setFeature(XmlPullParser.FEATURE_PROCESS_NAMESPACES, false);
            parser.setInput(in, null);
            parser.nextTag();

            ArrayList<Application> apps = readFdroid(parser);
            // Now set which applications are installed on the device
            for (Application app : apps) {
                if (installedPackages.contains(app.getPackageName()))
                    app.setInstalled(true);
            }

            return apps;
        } finally {
            try {
                in.close();
            } catch (IOException e) { }
        }
    }

    // Reads the <fdroid> tag and returns a list of its <application> tags
    private static ArrayList<Application> readFdroid(XmlPullParser parser)
            throws XmlPullParserException, IOException {

        ArrayList<Application> apps = new ArrayList<>();

        parser.require(XmlPullParser.START_TAG, ns, "fdroid");
        while (parser.next() != XmlPullParser.END_TAG) {
            if (parser.getEventType() != XmlPullParser.START_TAG)
                continue;

            String name = parser.getName();
            if (name.equals("application")) {
                Application app = readApplication(parser);
                if (GITHUB_PATTERN.matcher(app.getSourceCodeUrl()).matches()) {
                    apps.add(app);
                }
            }
            else
                skip(parser);
        }
        return apps;
    }

    // Reads a <application id="...">...</application> tag from the xml
    private static Application readApplication(XmlPullParser parser)
            throws XmlPullParserException, IOException {

        String packageName, lastUpdated, name, description, iconName, sourceCodeUrl;
        packageName = lastUpdated = name = description = iconName = sourceCodeUrl = "";

        parser.require(XmlPullParser.START_TAG, ns, "application");
        while (parser.next() != XmlPullParser.END_TAG) {
            if (parser.getEventType() != XmlPullParser.START_TAG)
                continue;

            switch (parser.getName()) {
                case ID:
                    packageName = readText(parser);
                    break;<commit_msg>Fixed application parser skipping a lot of applications<commit_after>package io.github.lonamiwebs.stringlate.classes.applications;

import android.util.Log;
import android.util.Xml;

import org.xmlpull.v1.XmlPullParser;
import org.xmlpull.v1.XmlPullParserException;
import org.xmlpull.v1.XmlSerializer;

import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.regex.Pattern;

public class ApplicationListParser {
    // We don't use namespaces
    private static final String ns = null;

    private static final String ID = "id";
    private static final String LAST_UPDATED = "lastupdated";
    private static final String NAME = "name";
    private static final String DESCRIPTION = "summary";
    private static final String ICON = "icon";
    private static final String SOURCE_URL = "source";

    // We will only parse https GitHub urls
    private static final Pattern GITHUB_PATTERN =
            Pattern.compile("^https?://github.com/([\\w-]+)/([\\w-]+)(?:/.*)?$");

    //region Xml -> ApplicationsList

    public static ArrayList<Application> parseFromXml(InputStream in, HashSet<String> installedPackages)
            throws XmlPullParserException, IOException {

        try {
            XmlPullParser parser = Xml.newPullParser();
            parser.setFeature(XmlPullParser.FEATURE_PROCESS_NAMESPACES, false);
            parser.setInput(in, null);
            parser.nextTag();

            ArrayList<Application> apps = readFdroid(parser);
            // Now set which applications are installed on the device
            for (Application app : apps) {
                if (installedPackages.contains(app.getPackageName()))
                    app.setInstalled(true);
            }

            return apps;
        } finally {
            try {
                in.close();
            } catch (IOException e) { }
        }
    }

    // Reads the <fdroid> tag and returns a list of its <application> tags
    private static ArrayList<Application> readFdroid(XmlPullParser parser)
            throws XmlPullParserException, IOException {

        ArrayList<Application> apps = new ArrayList<>();

        parser.require(XmlPullParser.START_TAG, ns, "fdroid");
        while (parser.next() != XmlPullParser.END_TAG) {
            if (parser.getEventType() != XmlPullParser.START_TAG)
                continue;

            String name = parser.getName();
            if (name.equals("application")) {
                Application app = readApplication(parser);
                if (GITHUB_PATTERN.matcher(app.getSourceCodeUrl()).matches()) {
                    apps.add(app);
                } else if (app.getSourceCodeUrl().contains("github")) {
                    Log.w("LONAMIWEBS", "Why didn't this pass?: "+app.getSourceCodeUrl());
                }
            }
            else
                skip(parser);
        }
        return apps;
    }

    // Reads a <application id="...">...</application> tag from the xml
    private static Application readApplication(XmlPullParser parser)
            throws XmlPullParserException, IOException {

        String packageName, lastUpdated, name, description, iconName, sourceCodeUrl;
        packageName = lastUpdated = name = description = iconName = sourceCodeUrl = "";

        parser.require(XmlPullParser.START_TAG, ns, "application");
        while (parser.next() != XmlPullParser.END_TAG) {
            if (parser.getEventType() != XmlPullParser.START_TAG)
                continue;

            switch (parser.getName()) {
                case ID:
                    packageName = readText(parser);
                    break;
{'fraction_non_alphanumeric': 0.10078059659882911, 'fraction_numerical': 0.0008363534987454698, 'mean_word_length': 2.3232978230662344, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2775655', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1798, 'n_words': 473}