<commit_before><!--
    @(#)userwatchlist.jsp 0.02 31/08/2017
    Copyright (C) 2015 - 20xx MER-C

    This is free software: you are free to change and redistribute it under the
    Affero GNU GPL version 3 or later, see <https://www.gnu.org/licenses/agpl.html>
    for details. There is NO WARRANTY, to the extent permitted by law.
-->

<%
    request.setAttribute("toolname", "User watchlist");
    request.setAttribute("earliest_default", LocalDate.now(ZoneOffset.UTC).minusDays(30));

    String inputpage = request.getParameter("page");
    String inputpage_url = "";
    String inputpage_attribute = "";
    if (inputpage != null)
    {
        inputpage_url = ServletUtils.sanitizeForURL(inputpage);
        inputpage_attribute = ServletUtils.sanitizeForAttribute(inputpage);
    }

    String temp = request.getParameter("skip");
    int skip = (temp == null) ? 0 : Integer.parseInt(temp);
    skip = Math.max(skip, 0);

    Wiki enWiki = Wiki.createInstance("en.wikipedia.org");
    enWiki.setMaxLag(-1);
    enWiki.setQueryLimit(30000); // 60 network requests
    Users userUtils = Users.of(enWiki);
    Revisions revisionUtils = Revisions.of(enWiki);
%>

<%@ include file="datevalidate.jspf" %>
<%@ include file="header.jsp" %>

<p>
This tool retrieves contributions of a list of users. There is a limit of 50
users per request, though the list may be of indefinite length.

<p>
Syntax: one user per line, reason after # . Example:

<pre>
Example user # Copyright violations
// This is a comment
Someone # Spam
</pre>

<form action="./userwatchlist.jsp" method=GET>
<table>
<tr>
    <td>Input page or category:
    <td>
        <input type=text size=50 name=page required value="<%= inputpage_attribute %>">
        <%
        if (inputpage != null)
        {
        %>
        (<a href="<%= enWiki.getPageUrl(inputpage) %>">visit</a> |
        <a href="<%= enWiki.getIndexPhpUrl() + "?action=edit&title=" + inputpage_url %>">edit</a>)
        <%
        }
        %>

<tr><td>Show changes from:
    <td><input type=date name=earliest value="<%= earliest %>"> to
        <input type=date name=latest value="<%= latest %>"> (inclusive)
<tr><td>Skip:
    <td><input type=number size=50 name=skip value="<%= skip %>">
</table>
<input type=submit value="Submit">
</form>

<%
    if (inputpage == null)
    {
        %>
<%@ include file="footer.jsp" %>
        <%
    }

    Map<String, String> input = new LinkedHashMap<>();
    if (enWiki.namespace(inputpage) == Wiki.CATEGORY_NAMESPACE)
    {
        String[] catmembers = enWiki.getCategoryMembers(inputpage);
        for (String member : catmembers)
            input.put(enWiki.removeNamespace(member), "");
    }
    else if (inputpage.matches("^User:.+/.+\\.(cs|j)s$"))
    {
        String us = inputpage.substring(5, inputpage.indexOf('/'));
        Wiki.User us2 = enWiki.getUser(us);
        if (us2 == null || !us2.isA("sysop"))
        {
            request.setAttribute("error", "TESTING WOOP WOOP WOOP!");
%>
<%@ include file="footer.jsp" %>
<%
        }
        String text = enWiki.getPageText(inputpage);
        if (text == null)
        {
            request.setAttribute("error", "ERROR: page &quot;" + ServletUtils.sanitizeForHTML(inputpage) + "&quot; does not exist!");
%>
<%@ include file="footer.jsp" %>
<%
        }
        // parse input
        String[] lines = text.split("\n");

        for (String user : lines)
        {
            // remove comments, parse reasons
            user = user.trim();
            if (user.contains("//"))
                user = user.substring(0, user.indexOf("//")).trim();
            int boundary = user.indexOf("#");
            String reason = "";
            if (boundary >= 0)
            {
                reason = user.substring(boundary + 1).trim();
                user = user.substring(0, boundary).trim();
            }
            if (user.isEmpty())
                continue;
            input.put(user, reason);
        }
    }
    else
    {
        request.setAttribute("error", "TESTING WOOP WOOP WOOP!");
%>
<%@ include file="footer.jsp" %>
<%
    }

    if (input.isEmpty())
    {
        request.setAttribute("error", "ERROR: no users found!");
%>
<%@ include file="footer.jsp" %>
<%
    }

    // top pagination
    String requesturl = "./userwatchlist.jsp?page=" +  inputpage_url + "&earliest=" + earliest
        + "&latest=" + latest + "&skip=";
    out.println("<hr>");
    if (skip > 0)
        out.print("<a href=\"" + requesturl + Math.max(0, skip - 50) + "\">Previous 50</a> | ");
    else
        out.print("Previous 50 | ");
    if (input.size() - skip > 50)
        out.println("<a href=\"" + requesturl + (skip + 50) + "\">Next 50</a>");
    else
        out.println("Next 50");

    // fetch contributions
    Wiki.RequestHelper rh = enWiki.new RequestHelper()
        .withinDateRange(earliest_odt, latest_odt);
    List<String> users = new ArrayList<>(input.keySet());
    List<String> userstofetch = users.subList(skip, Math.min(skip + 50, users.size() - 1));
    List<List<Wiki.Revision>> contribs = enWiki.contribs(userstofetch, null, rh);

    for (int i = 0; i < userstofetch.size(); i++)
    {
        String user = userstofetch.get(i);
        String reason = ServletUtils.sanitizeForHTML(input.get(user));
        // user links
        %><commit_msg>userwatchlist.jsp: add new page only filter.<commit_after><!--
    @(#)userwatchlist.jsp 0.02 31/08/2017
    Copyright (C) 2015 - 20xx MER-C

    This is free software: you are free to change and redistribute it under the
    Affero GNU GPL version 3 or later, see <https://www.gnu.org/licenses/agpl.html>
    for details. There is NO WARRANTY, to the extent permitted by law.
-->

<%
    request.setAttribute("toolname", "User watchlist");
    request.setAttribute("earliest_default", LocalDate.now(ZoneOffset.UTC).minusDays(30));

    String inputpage = request.getParameter("page");
    String inputpage_url = "";
    String inputpage_attribute = "";
    if (inputpage != null)
    {
        inputpage_url = ServletUtils.sanitizeForURL(inputpage);
        inputpage_attribute = ServletUtils.sanitizeForAttribute(inputpage);
    }

    String temp = request.getParameter("skip");
    int skip = (temp == null) ? 0 : Integer.parseInt(temp);
    skip = Math.max(skip, 0);
    boolean newonly = (request.getParameter("newonly") != null);

    Wiki enWiki = Wiki.createInstance("en.wikipedia.org");
    enWiki.setMaxLag(-1);
    enWiki.setQueryLimit(30000); // 60 network requests
    Users userUtils = Users.of(enWiki);
    Revisions revisionUtils = Revisions.of(enWiki);
%>

<%@ include file="datevalidate.jspf" %>
<%@ include file="header.jsp" %>

<p>
This tool retrieves contributions of a list of users. There is a limit of 50
users per request, though the list may be of indefinite length.

<p>
Syntax: one user per line, reason after # . Example:

<pre>
Example user # Copyright violations
// This is a comment
Someone # Spam
</pre>

<form action="./userwatchlist.jsp" method=GET>
<table>
<tr>
    <td>Input page or category:
    <td>
        <input type=text size=50 name=page required value="<%= inputpage_attribute %>">
        <%
        if (inputpage != null)
        {
        %>
        (<a href="<%= enWiki.getPageUrl(inputpage) %>">visit</a> |
        <a href="<%= enWiki.getIndexPhpUrl() + "?action=edit&title=" + inputpage_url %>">edit</a>)
        <%
        }
        %>

<tr><td>Show changes from:
    <td><input type=date name=earliest value="<%= earliest %>"> to
        <input type=date name=latest value="<%= latest %>"> (inclusive)
<tr><td>Show:
    <td><input type=checkbox name=newonly value=1<%= newonly ? " checked" : 
        "" %>>New pages only
<tr><td>Skip:
    <td><input type=number size=50 name=skip value="<%= skip %>">
</table>
<input type=submit value="Submit">
</form>

<%
    if (inputpage == null)
    {
        %>
<%@ include file="footer.jsp" %>
        <%
    }

    Map<String, String> input = new LinkedHashMap<>();
    if (enWiki.namespace(inputpage) == Wiki.CATEGORY_NAMESPACE)
    {
        String[] catmembers = enWiki.getCategoryMembers(inputpage);
        for (String member : catmembers)
            input.put(enWiki.removeNamespace(member), "");
    }
    else if (inputpage.matches("^User:.+/.+\\.(cs|j)s$"))
    {
        String us = inputpage.substring(5, inputpage.indexOf('/'));
        Wiki.User us2 = enWiki.getUser(us);
        if (us2 == null || !us2.isA("sysop"))
        {
            request.setAttribute("error", "TESTING WOOP WOOP WOOP!");
%>
<%@ include file="footer.jsp" %>
<%
        }
        String text = enWiki.getPageText(inputpage);
        if (text == null)
        {
            request.setAttribute("error", "ERROR: page &quot;" + ServletUtils.sanitizeForHTML(inputpage) + "&quot; does not exist!");
%>
<%@ include file="footer.jsp" %>
<%
        }
        // parse input
        String[] lines = text.split("\n");

        for (String user : lines)
        {
            // remove comments, parse reasons
            user = user.trim();
            if (user.contains("//"))
                user = user.substring(0, user.indexOf("//")).trim();
            int boundary = user.indexOf("#");
            String reason = "";
            if (boundary >= 0)
            {
                reason = user.substring(boundary + 1).trim();
                user = user.substring(0, boundary).trim();
            }
            if (user.isEmpty())
                continue;
            input.put(user, reason);
        }
    }
    else
    {
        request.setAttribute("error", "TESTING WOOP WOOP WOOP!");
%>
<%@ include file="footer.jsp" %>
<%
    }

    if (input.isEmpty())
    {
        request.setAttribute("error", "ERROR: no users found!");
%>
<%@ include file="footer.jsp" %>
<%
    }

    // top pagination
    String requesturl = "./userwatchlist.jsp?page=" +  inputpage_url + "&earliest=" + earliest
        + "&latest=" + latest + "&skip=";
    out.println("<hr>");
    if (skip > 0)
        out.print("<a href=\"" + requesturl + Math.max(0, skip - 50) + "\">Previous 50</a> | ");
    else
        out.print("Previous 50 | ");
    if (input.size() - skip > 50)
        out.println("<a href=\"" + requesturl + (skip + 50) + "\">Next 50</a>");
    else
        out.println("Next 50");

    // fetch contributions
    Wiki.RequestHelper rh = enWiki.new RequestHelper()
        .withinDateRange(earliest_odt, latest_odt);
    if (newonly)
    {
        Map<String, Boolean> tempmap = new HashMap<>();
        tempmap.put("new", Boolean.TRUE);
        rh = rh.filterBy(tempmap);
    }
    List<String> users = new ArrayList<>(input.keySet());
    List<String> userstofetch = users.subList(skip, Math.min(skip + 50, users.size()));
    List<List<Wiki.Revision>> contribs = enWiki.contribs(userstofetch, null, rh);

    for (int i = 0; i < userstofetch.size(); i++)
    {
        String user = userstofetch.get(i);
        String reason = ServletUtils.sanitizeForHTML(input.get(user));
        // user links
        %>
{'fraction_non_alphanumeric': 0.16369912790697674, 'fraction_numerical': 0.011627906976744186, 'mean_word_length': 2.5755115297174407, 'pattern_counts': {'":': 0, '<': 136, '<?xml version=': 0, '>': 140, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2969895', 'n_tokens_mistral': 3663, 'n_tokens_neox': 3450, 'n_words': 813}