<commit_before>/**
 *  Copyright 2007-2011 Zuse Institute Berlin
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */
package de.zib.scalaris.examples.wikipedia.bliki;

import info.bliki.api.Connector;
import info.bliki.api.User;

import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.servlet.RequestDispatcher;
import javax.servlet.Servlet;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;

import com.skjegstad.utils.BloomFilter;

import de.zib.scalaris.examples.wikipedia.CircularByteArrayOutputStream;
import de.zib.scalaris.examples.wikipedia.NamespaceUtils;
import de.zib.scalaris.examples.wikipedia.Options;
import de.zib.scalaris.examples.wikipedia.Options.STORE_CONTRIB_TYPE;
import de.zib.scalaris.examples.wikipedia.PageHistoryResult;
import de.zib.scalaris.examples.wikipedia.RevisionResult;
import de.zib.scalaris.examples.wikipedia.SavePageResult;
import de.zib.scalaris.examples.wikipedia.ValueResult;
import de.zib.scalaris.examples.wikipedia.WikiServletContext;
import de.zib.scalaris.examples.wikipedia.bliki.WikiPageListBean.FormType;
import de.zib.scalaris.examples.wikipedia.data.Contributor;
import de.zib.scalaris.examples.wikipedia.data.Page;
import de.zib.scalaris.examples.wikipedia.data.Revision;
import de.zib.scalaris.examples.wikipedia.data.SiteInfo;
import de.zib.scalaris.examples.wikipedia.data.xml.WikiDump;
import de.zib.scalaris.examples.wikipedia.plugin.PluginClassLoader;
import de.zib.scalaris.examples.wikipedia.plugin.WikiEventHandler;
import de.zib.scalaris.examples.wikipedia.plugin.WikiPlugin;

/**
 * Servlet for handling wiki page display and editing.
 * 
 * @param <Connection> connection to a DB
 * 
 * @author <NAME>, <EMAIL>
 */
public abstract class WikiServlet<Connection> extends HttpServlet implements
        Servlet, WikiServletContext, WikiServletDataHandler<Connection> {
    protected static final String MAIN_PAGE = "Main Page";
    protected static final int IMPORT_REDIRECT_EVERY = 5; // seconds

    private static final long serialVersionUID = 1L;
    
    /**
     * Version of the "Wikipedia on Scalaris" example implementation.
     */
    public static final String version = "0.4.1+svn";

    protected SiteInfo siteinfo = null;
    protected MyNamespace namespace = null;
    
    protected boolean initialized = false;

    /**
     * Name under which the servlet is available.
     */
    public static final String wikiBaseURL = "wiki";
    /**
     * URL for page links
     */
    public static final String linkBaseURL = wikiBaseURL + "?title=${title}";
    /**
     * URL for image links
     */
    public static final String imageBaseURL = wikiBaseURL + "?get_image=${image}";

    protected static final Pattern MATCH_WIKI_IMPORT_FILE = Pattern.compile(".*((\\.xml(\\.gz|\\.bz2)?)|\\.db)$");
    protected static final Pattern MATCH_WIKI_IMAGE_PX = Pattern.compile("^[0-9]*px-");
    protected static final Pattern MATCH_WIKI_IMAGE_SVG_PNG = Pattern.compile("\\.svg\\.png$");
    /*
     * http://simple.wiktionary.org/wiki/Main_Page
     * http://bar.wikipedia.org/wiki/Hauptseitn
     * https://secure.wikimedia.org/wikipedia/en/wiki/Main_Page
     */
    protected static final Pattern MATCH_WIKI_SITE_BASE = Pattern.compile("^(http[s]?://.+)(/wiki/.*)$");
    
    protected String currentImport = "";

    protected static CircularByteArrayOutputStream importLog = null;
    protected WikiDump importHandler = null;
    
    protected List<WikiEventHandler> eventHandlers = new LinkedList<WikiEventHandler>();
    
    protected BloomFilter<String> existingPages = new BloomFilter<String>(MyWikiModel.existingPagesFPR, 100);

    /**
     * Creates the servlet. 
     */
    public WikiServlet() {
        super();
    }

    /**
     * Servlet initialisation: creates the connection to the erlang node and
     * imports site information.
     */
    @Override
    public void init(ServletConfig config) throws ServletException {
        super.init(config);
        final String WIKI_USE_NEW_SCALARIS_OPS = config.getInitParameter("WIKI_USE_NEW_SCALARIS_OPS");
        if (WIKI_USE_NEW_SCALARIS_OPS != null) {
            Options.WIKI_USE_NEW_SCALARIS_OPS = Boolean.parseBoolean(WIKI_USE_NEW_SCALARIS_OPS);
        }
        final String WIKI_USE_BACKLINKS = config.getInitParameter("WIKI_USE_BACKLINKS");
        if (WIKI_USE_BACKLINKS != null) {
            Options.WIKI_USE_BACKLINKS = Boolean.parseBoolean(WIKI_USE_BACKLINKS);
        }
        final String WIKI_SAVEPAGE_RETRIES = config.getInitParameter("WIKI_SAVEPAGE_RETRIES");
        if (WIKI_SAVEPAGE_RETRIES != null) {
            Options.WIKI_SAVEPAGE_RETRIES = parseInt(WIKI_SAVEPAGE_RETRIES, Options.WIKI_SAVEPAGE_RETRIES);
        }
        final String WIKI_SAVEPAGE_RETRY_DELAY = config.getInitParameter("WIKI_SAVEPAGE_RETRY_DELAY");
        if (WIKI_SAVEPAGE_RETRY_DELAY != null) {
            Options.WIKI_SAVEPAGE_RETRY_DELAY = parseInt(WIKI_SAVEPAGE_RETRY_DELAY, Options.WIKI_SAVEPAGE_RETRY_DELAY);
        }
        final String WIKI_REBUILD_PAGES_CACHE = config.getInitParameter("WIKI_REBUILD_PAGES_CACHE");
        if (WIKI_REBUILD_PAGES_CACHE != null) {
            Options.WIKI_REBUILD_PAGES_CACHE = parseInt(WIKI_REBUILD_PAGES_CACHE, Options.WIKI_REBUILD_PAGES_CACHE);
        }
        final String WIKI_STORE_CONTRIBUTIONS = config.getInitParameter("WIKI_STORE_CONTRIBUTIONS");
        if (WIKI_STORE_CONTRIBUTIONS != null) {
            try {
                Options.WIKI_STORE_CONTRIBUTIONS = STORE_CONTRIB_TYPE.fromString(WIKI_SAVEPAGE_RETRY_DELAY);
            } catch (IllegalArgumentException e) {
            }
        }
    }
    
    /**
     * Loads the siteinfo object.<commit_msg>- Wiki on Scalaris: fixed WIKI_STORE_CONTRIBUTIONS option retrieval<commit_after>/**
 *  Copyright 2007-2011 Zuse Institute Berlin
 *
 *   Licensed under the Apache License, Version 2.0 (the "License");
 *   you may not use this file except in compliance with the License.
 *   You may obtain a copy of the License at
 *
 *       http://www.apache.org/licenses/LICENSE-2.0
 *
 *   Unless required by applicable law or agreed to in writing, software
 *   distributed under the License is distributed on an "AS IS" BASIS,
 *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *   See the License for the specific language governing permissions and
 *   limitations under the License.
 */
package de.zib.scalaris.examples.wikipedia.bliki;

import info.bliki.api.Connector;
import info.bliki.api.User;

import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.math.BigInteger;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Enumeration;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Random;
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.servlet.RequestDispatcher;
import javax.servlet.Servlet;
import javax.servlet.ServletConfig;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;

import com.skjegstad.utils.BloomFilter;

import de.zib.scalaris.examples.wikipedia.CircularByteArrayOutputStream;
import de.zib.scalaris.examples.wikipedia.NamespaceUtils;
import de.zib.scalaris.examples.wikipedia.Options;
import de.zib.scalaris.examples.wikipedia.Options.STORE_CONTRIB_TYPE;
import de.zib.scalaris.examples.wikipedia.PageHistoryResult;
import de.zib.scalaris.examples.wikipedia.RevisionResult;
import de.zib.scalaris.examples.wikipedia.SavePageResult;
import de.zib.scalaris.examples.wikipedia.ValueResult;
import de.zib.scalaris.examples.wikipedia.WikiServletContext;
import de.zib.scalaris.examples.wikipedia.bliki.WikiPageListBean.FormType;
import de.zib.scalaris.examples.wikipedia.data.Contributor;
import de.zib.scalaris.examples.wikipedia.data.Page;
import de.zib.scalaris.examples.wikipedia.data.Revision;
import de.zib.scalaris.examples.wikipedia.data.SiteInfo;
import de.zib.scalaris.examples.wikipedia.data.xml.WikiDump;
import de.zib.scalaris.examples.wikipedia.plugin.PluginClassLoader;
import de.zib.scalaris.examples.wikipedia.plugin.WikiEventHandler;
import de.zib.scalaris.examples.wikipedia.plugin.WikiPlugin;

/**
 * Servlet for handling wiki page display and editing.
 * 
 * @param <Connection> connection to a DB
 * 
 * @author <NAME>, <EMAIL>
 */
public abstract class WikiServlet<Connection> extends HttpServlet implements
        Servlet, WikiServletContext, WikiServletDataHandler<Connection> {
    protected static final String MAIN_PAGE = "Main Page";
    protected static final int IMPORT_REDIRECT_EVERY = 5; // seconds

    private static final long serialVersionUID = 1L;
    
    /**
     * Version of the "Wikipedia on Scalaris" example implementation.
     */
    public static final String version = "0.4.1+svn";

    protected SiteInfo siteinfo = null;
    protected MyNamespace namespace = null;
    
    protected boolean initialized = false;

    /**
     * Name under which the servlet is available.
     */
    public static final String wikiBaseURL = "wiki";
    /**
     * URL for page links
     */
    public static final String linkBaseURL = wikiBaseURL + "?title=${title}";
    /**
     * URL for image links
     */
    public static final String imageBaseURL = wikiBaseURL + "?get_image=${image}";

    protected static final Pattern MATCH_WIKI_IMPORT_FILE = Pattern.compile(".*((\\.xml(\\.gz|\\.bz2)?)|\\.db)$");
    protected static final Pattern MATCH_WIKI_IMAGE_PX = Pattern.compile("^[0-9]*px-");
    protected static final Pattern MATCH_WIKI_IMAGE_SVG_PNG = Pattern.compile("\\.svg\\.png$");
    /*
     * http://simple.wiktionary.org/wiki/Main_Page
     * http://bar.wikipedia.org/wiki/Hauptseitn
     * https://secure.wikimedia.org/wikipedia/en/wiki/Main_Page
     */
    protected static final Pattern MATCH_WIKI_SITE_BASE = Pattern.compile("^(http[s]?://.+)(/wiki/.*)$");
    
    protected String currentImport = "";

    protected static CircularByteArrayOutputStream importLog = null;
    protected WikiDump importHandler = null;
    
    protected List<WikiEventHandler> eventHandlers = new LinkedList<WikiEventHandler>();
    
    protected BloomFilter<String> existingPages = new BloomFilter<String>(MyWikiModel.existingPagesFPR, 100);

    /**
     * Creates the servlet. 
     */
    public WikiServlet() {
        super();
    }

    /**
     * Servlet initialisation: creates the connection to the erlang node and
     * imports site information.
     */
    @Override
    public void init(ServletConfig config) throws ServletException {
        super.init(config);
        final String WIKI_USE_NEW_SCALARIS_OPS = config.getInitParameter("WIKI_USE_NEW_SCALARIS_OPS");
        if (WIKI_USE_NEW_SCALARIS_OPS != null) {
            Options.WIKI_USE_NEW_SCALARIS_OPS = Boolean.parseBoolean(WIKI_USE_NEW_SCALARIS_OPS);
        }
        final String WIKI_USE_BACKLINKS = config.getInitParameter("WIKI_USE_BACKLINKS");
        if (WIKI_USE_BACKLINKS != null) {
            Options.WIKI_USE_BACKLINKS = Boolean.parseBoolean(WIKI_USE_BACKLINKS);
        }
        final String WIKI_SAVEPAGE_RETRIES = config.getInitParameter("WIKI_SAVEPAGE_RETRIES");
        if (WIKI_SAVEPAGE_RETRIES != null) {
            Options.WIKI_SAVEPAGE_RETRIES = parseInt(WIKI_SAVEPAGE_RETRIES, Options.WIKI_SAVEPAGE_RETRIES);
        }
        final String WIKI_SAVEPAGE_RETRY_DELAY = config.getInitParameter("WIKI_SAVEPAGE_RETRY_DELAY");
        if (WIKI_SAVEPAGE_RETRY_DELAY != null) {
            Options.WIKI_SAVEPAGE_RETRY_DELAY = parseInt(WIKI_SAVEPAGE_RETRY_DELAY, Options.WIKI_SAVEPAGE_RETRY_DELAY);
        }
        final String WIKI_REBUILD_PAGES_CACHE = config.getInitParameter("WIKI_REBUILD_PAGES_CACHE");
        if (WIKI_REBUILD_PAGES_CACHE != null) {
            Options.WIKI_REBUILD_PAGES_CACHE = parseInt(WIKI_REBUILD_PAGES_CACHE, Options.WIKI_REBUILD_PAGES_CACHE);
        }
        final String WIKI_STORE_CONTRIBUTIONS = config.getInitParameter("WIKI_STORE_CONTRIBUTIONS");
        if (WIKI_STORE_CONTRIBUTIONS != null) {
            try {
                Options.WIKI_STORE_CONTRIBUTIONS = STORE_CONTRIB_TYPE.fromString(WIKI_STORE_CONTRIBUTIONS);
            } catch (IllegalArgumentException e) {
            }
        }
    }
    
    /**
     * Loads the siteinfo object.
{'fraction_non_alphanumeric': 0.09752332842208658, 'fraction_numerical': 0.0032273907247597, 'mean_word_length': 3.9202623403520884, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3231484', 'n_tokens_mistral': 4554, 'n_tokens_neox': 4399, 'n_words': 890}