<commit_before>/*******************************************************************************
 * Copyright 2014 Barcelona Supercomputing Center (BSC)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package com.servioticy.dispatcher;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.servioticy.datamodel.serviceobject.SO010;
import com.servioticy.datamodel.serviceobject.SOChannel;
import com.servioticy.datamodel.serviceobject.SOStream;
import com.servioticy.datamodel.serviceobject.SOStream010;
import com.servioticy.datamodel.sensorupdate.SUChannel;
import com.servioticy.datamodel.sensorupdate.SensorUpdate;
import com.servioticy.dispatcher.jsonprocessors.AliasReplacer;
import com.servioticy.dispatcher.jsonprocessors.JsonPathReplacer;
import org.elasticsearch.common.geo.GeoPoint;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;

/**
 * @author <NAME> <<EMAIL>>
 */
public class SOProcessor010 extends SOProcessor{

    private ObjectMapper mapper;
    AliasReplacer aliases;
    LinkedHashMap<String, PSOStream> streams;
    LinkedHashMap<String, Object> queries;
    SO010 so;
    Map<String, HashSet<String>> streamsByDocId;
    Map<String, HashSet<String>> docIdsByStream;

    public SOProcessor010(SO010 so, ObjectMapper mapper) throws JsonParseException, JsonMappingException, IOException {

        this.mapper = mapper;
        this.streamsByDocId = new HashMap<String, HashSet<String>>();
        this.docIdsByStream = new HashMap<String, HashSet<String>>();

        this.mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);

        this.so = so;
        aliases = new AliasReplacer(this.so.getAliases());

        // Streams
        this.streams = new LinkedHashMap<String, PSOStream>();

    }

    private void addStreamByDocIds(String stream, Set<String> docIds) {
        for (String docId : docIds) {
            if (!this.streamsByDocId.containsKey(docId)) {
                this.streamsByDocId.put(docId, new HashSet<String>());
            }
            this.streamsByDocId.get(docId).add(stream);
        }
    }

    public String replaceAliases() throws JsonGenerationException, JsonMappingException, IOException {
        for (Map.Entry<String, SOStream> streamEntry : this.so.getStreams(this.mapper).entrySet()) {
            SOStream010 stream = (SOStream010) streamEntry.getValue();
            for (Map.Entry<String, SOChannel> channelEntry : stream.getChannels().entrySet()) {
                SOChannel channel = channelEntry.getValue();

                channel.setCurrentValue(aliases.replace(channel.getCurrentValue()));
                channel.setUnit(aliases.replace(channel.getUnit() == null ? "" : channel.getUnit()));
                channel.setType(aliases.replace(channel.getType()));
            }

            stream.setPreFilter(stream.getPreFilter() == null ? "true" : (aliases.replace(stream.getPreFilter())));
            stream.setPostFilter(stream.getPostFilter() == null ? "true" : aliases.replace(stream.getPostFilter()));
        }
        return this.mapper.writeValueAsString(this.so);
    }

    public void compileJSONPaths() {
        for (Map.Entry<String, SOStream> streamEntry : this.so.getStreams(this.mapper).entrySet()) {
            PSOStream pstream = new PSOStream();
            SOStream010 stream = (SOStream010) streamEntry.getValue();
            String streamId = streamEntry.getKey();

            this.docIdsByStream.put(streamId, new HashSet<String>());

            // Channels
            pstream.channels = new LinkedHashMap<String, PSOChannel>();
            for (Map.Entry<String, SOChannel> channelEntry : stream.getChannels().entrySet()) {
                PSOChannel pchannel = new PSOChannel();
                SOChannel channel = channelEntry.getValue();
                pchannel.currentValue = null;
                if (channel.getCurrentValue() != null) {
                    pchannel.currentValue = new JsonPathReplacer(channel.getCurrentValue(), mapper);
                    // Set the objective streams for each group of SUs
                    Set<String> docIds = pchannel.currentValue.getJsonPathIds();
                    addStreamByDocIds(streamEntry.getKey(), docIds);
                    this.docIdsByStream.get(streamId).addAll(docIds);
                }

                pchannel.type = channel.getType();

                pstream.channels.put(channelEntry.getKey(), pchannel);
            }
            pstream.preFilter = null;
            if (stream.getPreFilter() != null) {
                pstream.preFilter = new JsonPathReplacer(stream.getPreFilter(), mapper);
                this.docIdsByStream.get(streamId).addAll(pstream.preFilter.getJsonPathIds());
                addStreamByDocIds(streamId, pstream.preFilter.getJsonPathIds());
            }


            pstream.postFilter = null;
            if (stream.getPreFilter() != null) {<commit_msg>Update SOProcessor010.java<commit_after>/*******************************************************************************
 * Copyright 2014 Barcelona Supercomputing Center (BSC)
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ******************************************************************************/
package com.servioticy.dispatcher;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.JsonMappingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.servioticy.datamodel.serviceobject.SO010;
import com.servioticy.datamodel.serviceobject.SOChannel;
import com.servioticy.datamodel.serviceobject.SOStream;
import com.servioticy.datamodel.serviceobject.SOStream010;
import com.servioticy.datamodel.sensorupdate.SUChannel;
import com.servioticy.datamodel.sensorupdate.SensorUpdate;
import com.servioticy.dispatcher.jsonprocessors.AliasReplacer;
import com.servioticy.dispatcher.jsonprocessors.JsonPathReplacer;
import org.elasticsearch.common.geo.GeoPoint;

import javax.script.ScriptEngine;
import javax.script.ScriptEngineManager;
import javax.script.ScriptException;
import java.io.IOException;
import java.util.*;
import java.util.Map.Entry;

/**
 * @author <NAME> <<EMAIL>>
 */
public class SOProcessor010 extends SOProcessor{

    private ObjectMapper mapper;
    AliasReplacer aliases;
    LinkedHashMap<String, PSOStream> streams;
    LinkedHashMap<String, Object> queries;
    SO010 so;
    Map<String, HashSet<String>> streamsByDocId;
    Map<String, HashSet<String>> docIdsByStream;

    public SOProcessor010(SO010 so, ObjectMapper mapper) throws JsonParseException, JsonMappingException, IOException {

        this.mapper = mapper;
        this.streamsByDocId = new HashMap<String, HashSet<String>>();
        this.docIdsByStream = new HashMap<String, HashSet<String>>();

        this.mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);

        this.so = so;
        aliases = new AliasReplacer(this.so.getAliases());

        // Streams
        this.streams = new LinkedHashMap<String, PSOStream>();

    }

    private void addStreamByDocIds(String stream, Set<String> docIds) {
        for (String docId : docIds) {
            if (!this.streamsByDocId.containsKey(docId)) {
                this.streamsByDocId.put(docId, new HashSet<String>());
            }
            this.streamsByDocId.get(docId).add(stream);
        }
    }

    public String replaceAliases() throws JsonGenerationException, JsonMappingException, IOException {
        for (Map.Entry<String, SOStream> streamEntry : this.so.getStreams(this.mapper).entrySet()) {
            SOStream010 stream = (SOStream010) streamEntry.getValue();
            for (Map.Entry<String, SOChannel> channelEntry : stream.getChannels().entrySet()) {
                SOChannel channel = channelEntry.getValue();

                channel.setCurrentValue(aliases.replace(channel.getCurrentValue()));
                channel.setUnit(aliases.replace(channel.getUnit() == null ? "" : channel.getUnit()));
                channel.setType(aliases.replace(channel.getType()));
            }

            stream.setPreFilter(stream.getPreFilter() == null ? "true" : (aliases.replace(stream.getPreFilter())));
            stream.setPostFilter(stream.getPostFilter() == null ? "true" : aliases.replace(stream.getPostFilter()));
        }
        return this.mapper.writeValueAsString(this.so);
    }

    public void compileJSONPaths() {
        for (Map.Entry<String, SOStream> streamEntry : this.so.getStreams(this.mapper).entrySet()) {
            PSOStream pstream = new PSOStream();
            SOStream010 stream = (SOStream010) streamEntry.getValue();
            String streamId = streamEntry.getKey();

            this.docIdsByStream.put(streamId, new HashSet<String>());

            // Channels
            pstream.channels = new LinkedHashMap<String, PSOChannel>();
            for (Map.Entry<String, SOChannel> channelEntry : stream.getChannels().entrySet()) {
                PSOChannel pchannel = new PSOChannel();
                SOChannel channel = channelEntry.getValue();
                pchannel.currentValue = null;
                if (channel.getCurrentValue() != null) {
                    pchannel.currentValue = new JsonPathReplacer(channel.getCurrentValue(), mapper);
                    // Set the objective streams for each group of SUs
                    Set<String> docIds = pchannel.currentValue.getJsonPathIds();
                    addStreamByDocIds(streamEntry.getKey(), docIds);
                    this.docIdsByStream.get(streamId).addAll(docIds);
                }

                pchannel.type = channel.getType();

                pstream.channels.put(channelEntry.getKey(), pchannel);
            }
            pstream.preFilter = null;
            if (stream.getPreFilter() != null) {
                pstream.preFilter = new JsonPathReplacer(stream.getPreFilter(), mapper);
                this.docIdsByStream.get(streamId).addAll(pstream.preFilter.getJsonPathIds());
                addStreamByDocIds(streamId, pstream.preFilter.getJsonPathIds());
            }


            pstream.postFilter = null;
            if (stream.getPostFilter() != null) {
{'fraction_non_alphanumeric': 0.12837609723160026, 'fraction_numerical': 0.006667792032410533, 'mean_word_length': 3.248476156328433, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3433693', 'n_tokens_mistral': 3298, 'n_tokens_neox': 3123, 'n_words': 653}