<commit_before>package de.ids_mannheim.korap.query.serialize;

import static org.junit.Assert.*;

import java.io.IOException;
import java.util.ArrayList;

import org.junit.Test;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;

import de.ids_mannheim.korap.query.serialize.QuerySerializer;

/**
 * Tests for JSON-LD serialization of PoliqarpPlus queries.
 * 
 * @author <NAME> (<EMAIL>)
 * @version 1.0
 */
public class PoliqarpPlusQueryProcessorTest {

    String query;
    ArrayList<JsonNode> operands;

    QuerySerializer qs = new QuerySerializer();
    ObjectMapper mapper = new ObjectMapper();
    JsonNode res;


    @Test
    public void testContext () throws JsonProcessingException, IOException {
        query = "foo";
        String contextString = "http://korap.ids-mannheim.de/ns/koral/0.3/context.jsonld";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals(contextString, res.get("@context").asText());
    }


    @Test
    public void testSingleTokens () throws JsonProcessingException, IOException {
        query = "[base=Mann]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("Mann", res.at("/query/wrap/key").asText());
        assertEquals("lemma", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[orth!=Frau]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("Frau", res.at("/query/wrap/key").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:ne", res.at("/query/wrap/match").asText());

        query = "[p!=NN]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("NN", res.at("/query/wrap/key").asText());
        assertEquals("p", res.at("/query/wrap/layer").asText());
        assertEquals("match:ne", res.at("/query/wrap/match").asText());

        query = "[!p!=NN]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("NN", res.at("/query/wrap/key").asText());
        assertEquals("p", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[base=schland/x]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals(".*?schland.*?", res.at("/query/wrap/key").asText());
        assertEquals("lemma", res.at("/query/wrap/layer").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
    }


    @Test
    public void testNegatedTokens () throws JsonProcessingException,
            IOException {
        query = "![p!=NN]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("NN", res.at("/query/wrap/key").asText());
        assertEquals("p", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
    }


    @Test
    public void testValue () throws JsonProcessingException, IOException {
        query = "[mate/m=temp:pres]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals("temp", res.at("/query/wrap/key").asText());
        assertEquals("pres", res.at("/query/wrap/value").asText());
        assertEquals("m", res.at("/query/wrap/layer").asText());
        assertEquals("mate", res.at("/query/wrap/foundry").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
    }


    @Test
    public void testRegex () throws JsonProcessingException, IOException {
        query = "[orth=\"M(a|ä)nn(er)?\"]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals("M(a|ä)nn(er)?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[orth=\"M(a|ä)nn(er)?\"/x]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".*?M(a|ä)nn(er)?.*?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "\".*?Mann.*?\"";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".*?Mann.*?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "z.B./x";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        System.out.println("QUERY IS  "+ res);
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".*?z\\.B\\..*?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        // Escape regex symbols
        query = "\"a\\.\"";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        System.out.println("QUERY IS  "+ res);
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
        assertEquals("a\\.", res.at("/query/wrap/key").asText());
    }


    @Test
    public void testPunct () throws JsonProcessingException, IOException {
        query = "[punct=.]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".", res.at("/query/wrap/key").asText());
        assertEquals("type:punct", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[punct=\".\"]";
        qs.setQuery(query, "poliqarpplus");<commit_msg>tests hot fix<commit_after>package de.ids_mannheim.korap.query.serialize;

import static org.junit.Assert.*;

import java.io.IOException;
import java.util.ArrayList;

import org.junit.Test;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Lists;

import de.ids_mannheim.korap.query.serialize.QuerySerializer;

/**
 * Tests for JSON-LD serialization of PoliqarpPlus queries.
 * 
 * @author <NAME> (<EMAIL>)
 * @version 1.0
 */
public class PoliqarpPlusQueryProcessorTest {

    String query;
    ArrayList<JsonNode> operands;

    QuerySerializer qs = new QuerySerializer();
    ObjectMapper mapper = new ObjectMapper();
    JsonNode res;


    @Test
    public void testContext () throws JsonProcessingException, IOException {
        query = "foo";
        String contextString = "http://korap.ids-mannheim.de/ns/koral/0.3/context.jsonld";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals(contextString, res.get("@context").asText());
    }


    @Test
    public void testSingleTokens () throws JsonProcessingException, IOException {
        query = "[base=Mann]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("Mann", res.at("/query/wrap/key").asText());
        assertEquals("lemma", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[orth!=Frau]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("Frau", res.at("/query/wrap/key").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:ne", res.at("/query/wrap/match").asText());

        query = "[p!=NN]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("NN", res.at("/query/wrap/key").asText());
        assertEquals("p", res.at("/query/wrap/layer").asText());
        assertEquals("match:ne", res.at("/query/wrap/match").asText());

        query = "[!p!=NN]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("NN", res.at("/query/wrap/key").asText());
        assertEquals("p", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[base=schland/x]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals(".*?schland.*?", res.at("/query/wrap/key").asText());
        assertEquals("lemma", res.at("/query/wrap/layer").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
    }


    @Test
    public void testNegatedTokens () throws JsonProcessingException,
            IOException {
        query = "![p!=NN]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("NN", res.at("/query/wrap/key").asText());
        assertEquals("p", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
    }


    @Test
    public void testValue () throws JsonProcessingException, IOException {
        query = "[mate/m=temp:pres]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals("temp", res.at("/query/wrap/key").asText());
        assertEquals("pres", res.at("/query/wrap/value").asText());
        assertEquals("m", res.at("/query/wrap/layer").asText());
        assertEquals("mate", res.at("/query/wrap/foundry").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
    }


    // todo:
    @Test
    public void testRegex () throws JsonProcessingException, IOException {
        query = "[orth=\"M(a|ä)nn(er)?\"]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals("M(a|ä)nn(er)?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[orth=\"M(a|ä)nn(er)?\"/x]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".*?M(a|ä)nn(er)?.*?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "\".*?Mann.*?\"";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".*?Mann.*?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "z.B./x";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        //System.out.println("QUERY IS  "+ res);
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".*?z\\.B\\..*?", res.at("/query/wrap/key").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        // Escape regex symbols
        query = "\"a\\.\"";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        //System.out.println("QUERY IS  "+ res);
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals("type:regex", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());
        assertEquals("a\\.", res.at("/query/wrap/key").asText());
    }


    @Test
    public void testPunct () throws JsonProcessingException, IOException {
        query = "[punct=.]";
        qs.setQuery(query, "poliqarpplus");
        res = mapper.readTree(qs.toJSON());
        assertEquals("koral:token", res.at("/query/@type").asText());
        assertEquals("koral:term", res.at("/query/wrap/@type").asText());
        assertEquals(".", res.at("/query/wrap/key").asText());
        assertEquals("type:punct", res.at("/query/wrap/type").asText());
        assertEquals("orth", res.at("/query/wrap/layer").asText());
        assertEquals("match:eq", res.at("/query/wrap/match").asText());

        query = "[punct=\".\"]";
        qs.setQuery(query, "poliqarpplus");
{'fraction_non_alphanumeric': 0.21717940907706365, 'fraction_numerical': 0.00048735912275357904, 'mean_word_length': 3.2363870967741937, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4234596', 'n_tokens_mistral': 5118, 'n_tokens_neox': 5019, 'n_words': 662}