<commit_before>/**
 * Copyright 2014 Telefonica Investigación y Desarrollo, S.A.U
 *
 * This file is part of fiware-connectors (FI-WARE project).
 *
 * fiware-connectors is free software: you can redistribute it and/or modify it under the terms of the GNU Affero
 * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 * fiware-connectors is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
 * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
 * for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with fiware-connectors. If not, see
 * http://www.gnu.org/licenses/.
 *
 * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es
 */

package es.tid.fiware.fiwareconnectors.cygnus.interceptors;

import com.google.gson.Gson;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequest;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequest.ContextElement;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequest.ContextElementResponse;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequestSAXHandler;
import es.tid.fiware.fiwareconnectors.cygnus.utils.Constants;
import es.tid.fiware.fiwareconnectors.cygnus.utils.Utils;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.apache.log4j.Logger;
import org.xml.sax.InputSource;

/**
 * Custom interceptor in charge of extracting the destination where the data must be persisted. This destination is
 * added as a 'destination' header.
 * 
 * @author frb
 */
public class DestinationExtractor implements Interceptor {
    
    private Logger logger;
    private String matchingTableFile;
    private ArrayList<MatchingRule> matchingTable;
    
    /**
     * Constructor.
     */
    public DestinationExtractor(String matchingTableFile) {
        logger = Logger.getLogger(DestinationExtractor.class);
        this.matchingTableFile = matchingTableFile;
    } // DestinationExtractor
    
    /**
     * Gets the matching table. This is protected since it is only going to be used in the tests.
     * @return
     */
    protected ArrayList<MatchingRule> getMatchingTable() {
        return matchingTable;
    } // getMatchingTable
    
    @Override
    public void initialize() {
        // load the matching table from the file where it is described
        matchingTable = new ArrayList<MatchingRule>();
        BufferedReader reader = null;
        
        try {
            reader = new BufferedReader(new FileReader(matchingTableFile));
        } catch (FileNotFoundException e) {
            logger.error("Runtime error (File not found. Details=" + e.getMessage() + ")");
            return;
        } // try catch
        
        String line;
        
        try {
            while ((line = reader.readLine()) != null) {
                if (line.startsWith("#") || line.length() == 0) {
                    continue;
                } // if
                
                String[] tokens = line.split("\\|");
                int id = new Integer(tokens[0]).intValue();
                ArrayList<String> fields = new ArrayList<String>(Arrays.asList(tokens[1].split(",")));
                matchingTable.add(new MatchingRule(id, fields, tokens[2], tokens[3], tokens[4]));
            } // while
        } catch (IOException e) {
            logger.error("Runtime error (I/O exception. Details=" + e.getMessage() + ")");
        } // try catch
    } // initialize
 
    @Override
    public Event intercept(Event event) {
        // get the original headers and body
        Map<String, String> headers = event.getHeaders();
        String body = new String(event.getBody());
        
        // get some original header values
        String fiwareServicePath = headers.get(Constants.HEADER_SERVICE_PATH);
        
        // parse the original body
        NotifyContextRequest notification = null;

        if (headers.get(Constants.HEADER_CONTENT_TYPE).contains("application/json")) {
            Gson gson = new Gson();

            try {
                notification = gson.fromJson(body, NotifyContextRequest.class);<commit_msg>Discard invalid matching rules at DetinationExtractor<commit_after>/**
 * Copyright 2014 Telefonica Investigación y Desarrollo, S.A.U
 *
 * This file is part of fiware-connectors (FI-WARE project).
 *
 * fiware-connectors is free software: you can redistribute it and/or modify it under the terms of the GNU Affero
 * General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your
 * option) any later version.
 * fiware-connectors is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the
 * implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License
 * for more details.
 *
 * You should have received a copy of the GNU Affero General Public License along with fiware-connectors. If not, see
 * http://www.gnu.org/licenses/.
 *
 * For those usages not covered by the GNU Affero General Public License please contact with iot_support at tid dot es
 */

package es.tid.fiware.fiwareconnectors.cygnus.interceptors;

import com.google.gson.Gson;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequest;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequest.ContextElement;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequest.ContextElementResponse;
import es.tid.fiware.fiwareconnectors.cygnus.containers.NotifyContextRequestSAXHandler;
import es.tid.fiware.fiwareconnectors.cygnus.utils.Constants;
import es.tid.fiware.fiwareconnectors.cygnus.utils.Utils;
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.parsers.SAXParser;
import javax.xml.parsers.SAXParserFactory;
import org.apache.flume.Context;
import org.apache.flume.Event;
import org.apache.flume.interceptor.Interceptor;
import org.apache.log4j.Logger;
import org.xml.sax.InputSource;

/**
 * Custom interceptor in charge of extracting the destination where the data must be persisted. This destination is
 * added as a 'destination' header.
 * 
 * @author frb
 */
public class DestinationExtractor implements Interceptor {
    
    private Logger logger;
    private String matchingTableFile;
    private ArrayList<MatchingRule> matchingTable;
    
    /**
     * Constructor.
     */
    public DestinationExtractor(String matchingTableFile) {
        logger = Logger.getLogger(DestinationExtractor.class);
        this.matchingTableFile = matchingTableFile;
    } // DestinationExtractor
    
    /**
     * Gets the matching table. This is protected since it is only going to be used in the tests.
     * @return
     */
    protected ArrayList<MatchingRule> getMatchingTable() {
        return matchingTable;
    } // getMatchingTable
    
    @Override
    public void initialize() {
        // load the matching table from the file where it is described
        matchingTable = new ArrayList<MatchingRule>();
        BufferedReader reader = null;
        
        try {
            reader = new BufferedReader(new FileReader(matchingTableFile));
        } catch (FileNotFoundException e) {
            logger.error("Runtime error (File not found. Details=" + e.getMessage() + ")");
            return;
        } // try catch
        
        String line;
        
        try {
            while ((line = reader.readLine()) != null) {
                if (line.startsWith("#") || line.length() == 0) {
                    continue;
                } // if
                
                String[] tokens = line.split("\\|");
                
                if (tokens.length < 5 || tokens[0].length() == 0 || tokens[1].length() == 0 || tokens[2].length() == 0
                        || tokens[3].length() == 0 || tokens[4].length() == 0) {
                    logger.warn("Malformed matching rule, it will be discarded. Details=" + line);
                    continue;
                } // if
                
                int id = new Integer(tokens[0]).intValue();
                ArrayList<String> fields = new ArrayList<String>(Arrays.asList(tokens[1].split(",")));
                matchingTable.add(new MatchingRule(id, fields, tokens[2], tokens[3], tokens[4]));
            } // while
        } catch (IOException e) {
            logger.error("Runtime error (I/O exception. Details=" + e.getMessage() + ")");
        } // try catch
    } // initialize
 
    @Override
    public Event intercept(Event event) {
        // get the original headers and body
        Map<String, String> headers = event.getHeaders();
        String body = new String(event.getBody());
        
        // get some original header values
        String fiwareServicePath = headers.get(Constants.HEADER_SERVICE_PATH);
        
        // parse the original body
        NotifyContextRequest notification = null;

        if (headers.get(Constants.HEADER_CONTENT_TYPE).contains("application/json")) {
            Gson gson = new Gson();

            try {
                notification = gson.fromJson(body, NotifyContextRequest.class);
{'fraction_non_alphanumeric': 0.09665647499753427, 'fraction_numerical': 0.0034520169641976527, 'mean_word_length': 2.8192090395480225, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1094716', 'n_tokens_mistral': 2732, 'n_tokens_neox': 2681, 'n_words': 798}