<commit_before>/**
 * AbstractPushServlet
 * Copyright 27.07.2015 by <NAME>, @zyzo
 * <p/>
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * <p/>
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * <p/>
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program in the file lgpl21.txt
 * If not, see <http://www.gnu.org/licenses/>.
 */

package org.loklak.api.server.push;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.github.fge.jsonschema.core.report.ProcessingReport;

import org.loklak.api.client.ClientConnection;
import org.loklak.api.server.RemoteAccess;
import org.loklak.data.DAO;
import org.loklak.data.MessageEntry;
import org.loklak.geo.LocationSource;
import org.loklak.geo.PlaceContext;
import org.loklak.harvester.JsonFieldConverter;
import org.loklak.harvester.JsonValidator;
import org.loklak.harvester.SourceType;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

public abstract class AbstractPushServlet extends HttpServlet {

    private JsonValidator validator;
    private JsonFieldConverter converter;

    @Override
    public void init() throws ServletException {
        try {
            validator = new JsonValidator(this.getValidatorSchema());
        } catch (IOException e) {
            DAO.log("Unable to initialize push servlet validator : " + e.getMessage());
            e.printStackTrace();
        }
        try {
            converter = new JsonFieldConverter(this.getConversionSchema());
        } catch (IOException e) {
            DAO.log("Unable to initialize push servlet field converter : " + e.getMessage());
            e.printStackTrace();
        }
    }

    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        this.doGet(request, response);
    }

    @SuppressWarnings("unchecked")
    @Override
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        RemoteAccess.Post post = RemoteAccess.evaluate(request);
        String remoteHash = Integer.toHexString(Math.abs(post.getClientHost().hashCode()));

        // manage DoS
        if (post.isDoS_blackout()) {
            response.sendError(503, "your request frequency is too high");
            return;
        }

        String url = post.get("url", "");
        if (url == null || url.length() == 0) {
            response.sendError(400, "your request does not contain an url to your data object");
            return;
        }
        String screen_name = post.get("screen_name", "");
        if (screen_name == null || screen_name.length() == 0) {
            response.sendError(400, "your request does not contain required screen_name parameter");
            return;
        }

        Map<String, Object> map;
        byte[] jsonText;
        try {
            jsonText = ClientConnection.download(url);
            map = DAO.jsonMapper.readValue(jsonText, DAO.jsonTypeRef);
        } catch (Exception e) {
            response.sendError(400, "error reading json file from url");
            return;
        }

        // validation phase
        ProcessingReport report = this.validator.validate(new String(jsonText));
        if (!report.isSuccess()) {
            response.sendError(400, "json does not conform to schema : " + this.getValidatorSchema().name() + "\n" + report);
            return;
        }

        // conversion phase
        Object extractResults = extractMessages(map);
        List<Map<String, Object>> messages;
        if (extractResults instanceof List) {
            messages = (List<Map<String, Object>>) extractResults;
        } else if (extractResults instanceof Map) {
            messages = new ArrayList<>();
            messages.add((Map<String, Object>) extractResults);
        } else {
            throw new IOException("extractMessages must return either a List or a Map");
        }
        List<Map<String, Object>> convertedMessages = this.converter.convert(messages);

        PushReport nodePushReport = new PushReport();
        ObjectWriter ow = new ObjectMapper().writerWithDefaultPrettyPrinter();
        // custom treatment for each message
        for (int i = 0; i < convertedMessages.size(); i++) {<commit_msg>Fix IndexOutOfBoundException when performing push  request<commit_after>/**
 * AbstractPushServlet
 * Copyright 27.07.2015 by <NAME>, @zyzo
 * <p/>
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 * <p/>
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 * <p/>
 * You should have received a copy of the GNU Lesser General Public License
 * along with this program in the file lgpl21.txt
 * If not, see <http://www.gnu.org/licenses/>.
 */

package org.loklak.api.server.push;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.github.fge.jsonschema.core.report.ProcessingReport;

import org.loklak.api.client.ClientConnection;
import org.loklak.api.server.RemoteAccess;
import org.loklak.data.DAO;
import org.loklak.data.MessageEntry;
import org.loklak.geo.LocationSource;
import org.loklak.geo.PlaceContext;
import org.loklak.harvester.JsonFieldConverter;
import org.loklak.harvester.JsonValidator;
import org.loklak.harvester.SourceType;

import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;

public abstract class AbstractPushServlet extends HttpServlet {

    private JsonValidator validator;
    private JsonFieldConverter converter;

    @Override
    public void init() throws ServletException {
        try {
            validator = new JsonValidator(this.getValidatorSchema());
        } catch (IOException e) {
            DAO.log("Unable to initialize push servlet validator : " + e.getMessage());
            e.printStackTrace();
        }
        try {
            converter = new JsonFieldConverter(this.getConversionSchema());
        } catch (IOException e) {
            DAO.log("Unable to initialize push servlet field converter : " + e.getMessage());
            e.printStackTrace();
        }
    }

    @Override
    protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        this.doGet(request, response);
    }

    @SuppressWarnings("unchecked")
    @Override
    protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
        RemoteAccess.Post post = RemoteAccess.evaluate(request);
        String remoteHash = Integer.toHexString(Math.abs(post.getClientHost().hashCode()));

        // manage DoS
        if (post.isDoS_blackout()) {
            response.sendError(503, "your request frequency is too high");
            return;
        }

        String url = post.get("url", "");
        if (url == null || url.length() == 0) {
            response.sendError(400, "your request does not contain an url to your data object");
            return;
        }
        String screen_name = post.get("screen_name", "");
        if (screen_name == null || screen_name.length() == 0) {
            response.sendError(400, "your request does not contain required screen_name parameter");
            return;
        }

        Map<String, Object> map;
        byte[] jsonText;
        try {
            jsonText = ClientConnection.download(url);
            map = DAO.jsonMapper.readValue(jsonText, DAO.jsonTypeRef);
        } catch (Exception e) {
            response.sendError(400, "error reading json file from url");
            return;
        }

        // validation phase
        ProcessingReport report = this.validator.validate(new String(jsonText));
        if (!report.isSuccess()) {
            response.sendError(400, "json does not conform to schema : " + this.getValidatorSchema().name() + "\n" + report);
            return;
        }

        // conversion phase
        Object extractResults = extractMessages(map);
        List<Map<String, Object>> messages;
        if (extractResults instanceof List) {
            messages = (List<Map<String, Object>>) extractResults;
        } else if (extractResults instanceof Map) {
            messages = new ArrayList<>();
            messages.add((Map<String, Object>) extractResults);
        } else {
            throw new IOException("extractMessages must return either a List or a Map. Get " + (extractResults == null ? "null" : extractResults.getClass().getCanonicalName()) + " instead");
        }
        List<Map<String, Object>> convertedMessages = this.converter.convert(messages);

        PushReport nodePushReport = new PushReport();
        ObjectWriter ow = new ObjectMapper().writerWithDefaultPrettyPrinter();
        // custom treatment for each message
        for (int i = 0; i < messages.size(); i++) {
{'fraction_non_alphanumeric': 0.09986252945797329, 'fraction_numerical': 0.00589159465828751, 'mean_word_length': 2.9707602339181287, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6076466', 'n_tokens_mistral': 2719, 'n_tokens_neox': 2641, 'n_words': 797}