<commit_before>package com.github.wovnio.wovnjava;

import java.io.IOException;
import java.io.PrintWriter;

import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class WovnServletFilter implements Filter {
    private Settings settings;
    private final HtmlChecker htmlChecker = new HtmlChecker();

    public static final String VERSION = Settings.VERSION;  // for backward compatibility

    @Override
    public void init(FilterConfig config) throws ServletException {
        this.settings = new Settings(config);
    }

    @Override
    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws ServletException, IOException
    {
        RequestOptions requestOptions = new RequestOptions(this.settings, request);

        if (requestOptions.getDisableMode()) {
            chain.doFilter(request, response);
        } else {
            ((HttpServletResponse)response).setHeader("X-Wovn-Handler", "wovnjava_" + Settings.VERSION);

            Headers headers = new Headers((HttpServletRequest)request, this.settings);
            String lang = headers.getPathLang();
            boolean hasShorterPath = settings.urlPattern.equals("path") && lang.length() > 0 && lang.equals(settings.defaultLang);
            if (hasShorterPath) {
                ((HttpServletResponse) response).sendRedirect(headers.redirectLocation(settings.defaultLang));
            } else if (headers.isValidPath() && htmlChecker.canTranslatePath(headers.pathName)) {
                tryTranslate(headers, requestOptions, (HttpServletRequest)request, (HttpServletResponse)response, chain);
            } else {
                WovnHttpServletRequest wovnRequest = new WovnHttpServletRequest((HttpServletRequest)request, headers);
                chain.doFilter(wovnRequest, response);
            }
        }
    }

    @Override
    public void destroy() {
    }

    private void tryTranslate(Headers headers, RequestOptions requestOptions, HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
        WovnHttpServletRequest wovnRequest = new WovnHttpServletRequest(request, headers);
        WovnHttpServletResponse wovnResponse = new WovnHttpServletResponse(response, headers);

        ResponseHeaders responseHeaders = new ResponseHeaders(response);
        responseHeaders.setApiStatus("Unused");

        if (settings.urlPattern.equals("path") && headers.getPathLang().length() > 0) {
            wovnRequest.getRequestDispatcher(headers.pathNameKeepTrailingSlash).forward(wovnRequest, wovnResponse);
        } else {
            chain.doFilter(wovnRequest, wovnResponse);
        }

        String originalBody = wovnResponse.toString();
        if (originalBody != null) {
            // text
            String body = null;
            if (htmlChecker.canTranslate(response.getContentType(), headers.pathName, originalBody)) {
                // html
                Api api = new Api(settings, headers, requestOptions, responseHeaders);
                Interceptor interceptor = new Interceptor(headers, settings, api, responseHeaders);
                body = interceptor.translate(originalBody);
            } else {
                // css, javascript or others
                body = originalBody;
            }
            wovnResponse.setContentLength(body.getBytes().length);
            wovnResponse.setCharacterEncoding("utf-8");
            PrintWriter out = response.getWriter();
            out.write(body);
            out.close();
        } else {
            // binary
            ServletOutputStream out = response.getOutputStream();
            out.write(wovnResponse.getData());
            out.close();
        }
    }
}
<commit_msg>Strip language code and X-Wovn-Handler also for wovnDisable mode (#97)<commit_after>package com.github.wovnio.wovnjava;

import java.io.IOException;
import java.io.PrintWriter;

import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

public class WovnServletFilter implements Filter {
    private Settings settings;
    private final HtmlChecker htmlChecker = new HtmlChecker();

    public static final String VERSION = Settings.VERSION;  // for backward compatibility

    @Override
    public void init(FilterConfig config) throws ServletException {
        this.settings = new Settings(config);
    }

    @Override
    public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws ServletException, IOException
    {
        ((HttpServletResponse)response).setHeader("X-Wovn-Handler", "wovnjava_" + Settings.VERSION);

        RequestOptions requestOptions = new RequestOptions(this.settings, request);
        Headers headers = new Headers((HttpServletRequest)request, this.settings);

        String lang = headers.getPathLang();
        boolean isRequestWithDefaultLanguageCode = settings.urlPattern.equals("path") && lang.length() > 0 && lang.equals(settings.defaultLang);
        boolean canProcessRequest = !requestOptions.getDisableMode() && headers.isValidPath() && htmlChecker.canTranslatePath(headers.pathName);

        if (isRequestWithDefaultLanguageCode) {
            /* Send 302 redirect to equivalent URL without default language code */
            ((HttpServletResponse) response).sendRedirect(headers.redirectLocation(settings.defaultLang));
        } else if (canProcessRequest) {
            /* Process the request */
            tryTranslate(headers, requestOptions, (HttpServletRequest)request, (HttpServletResponse)response, chain);
        } else {
            /* Strip language code and pass through the request and response untouched */
            WovnHttpServletRequest wovnRequest = new WovnHttpServletRequest((HttpServletRequest)request, headers);
            chain.doFilter(wovnRequest, response);
        }
    }

    @Override
    public void destroy() {
    }

    private void tryTranslate(Headers headers, RequestOptions requestOptions, HttpServletRequest request, HttpServletResponse response, FilterChain chain) throws IOException, ServletException {
        WovnHttpServletRequest wovnRequest = new WovnHttpServletRequest(request, headers);
        WovnHttpServletResponse wovnResponse = new WovnHttpServletResponse(response, headers);

        ResponseHeaders responseHeaders = new ResponseHeaders(response);
        responseHeaders.setApiStatus("Unused");

        if (settings.urlPattern.equals("path") && headers.getPathLang().length() > 0) {
            wovnRequest.getRequestDispatcher(headers.pathNameKeepTrailingSlash).forward(wovnRequest, wovnResponse);
        } else {
            chain.doFilter(wovnRequest, wovnResponse);
        }

        String originalBody = wovnResponse.toString();
        if (originalBody != null) {
            // text
            String body = null;
            if (htmlChecker.canTranslate(response.getContentType(), headers.pathName, originalBody)) {
                // html
                Api api = new Api(settings, headers, requestOptions, responseHeaders);
                Interceptor interceptor = new Interceptor(headers, settings, api, responseHeaders);
                body = interceptor.translate(originalBody);
            } else {
                // css, javascript or others
                body = originalBody;
            }
            wovnResponse.setContentLength(body.getBytes().length);
            wovnResponse.setCharacterEncoding("utf-8");
            PrintWriter out = response.getWriter();
            out.write(body);
            out.close();
        } else {
            // binary
            ServletOutputStream out = response.getOutputStream();
            out.write(wovnResponse.getData());
            out.close();
        }
    }
}

{'fraction_non_alphanumeric': 0.09049188044245705, 'fraction_numerical': 0.0012944222169922334, 'mean_word_length': 3.088023088023088, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2275596', 'n_tokens_mistral': 2095, 'n_tokens_neox': 2061, 'n_words': 464}