<commit_before>package org.elinker.core.spotter;

import eu.freme.common.exception.InternalServerErrorException;
import lombok.RequiredArgsConstructor;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Collection;
import java.util.stream.Stream;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.StandardOpenOption.APPEND;

@Component
@RequiredArgsConstructor
public class FremeSpotter implements ApplicationListener<ContextRefreshedEvent> {

    @Value("${freme.ner.dictionary:}")
    private String dictionary = "";

    private Boolean hasBuilt = false;

    private Trie.TrieBuilder builder;

    private Trie trie;

    private void buildDictionary() {

        if (dictionary == null || dictionary.isEmpty()) {
            trie = builder.build();
            return;
        }


        builder = Trie.builder().removeOverlaps().caseInsensitive();

        try (Stream<String> stream = Files.lines(Paths.get(dictionary))) {
            stream.forEach(s -> {
                builder.addKeyword(s);
            });
        } catch (IOException e) {
            throw new InternalServerErrorException(String.format("It was not possible to read the spotter dictionary." +
                    " Please check if the path is correct (%s)", dictionary));
        }
        trie = builder.build();
    }

    public void addKey(String key) {

        if (dictionary == null || dictionary.isEmpty()) {
            return;
        }

        try {
            builder.addKeyword(key);
            trie = builder.build();
            Files.write(Paths.get(dictionary), key.concat(System.lineSeparator()).getBytes(UTF_8), APPEND);
        } catch (IOException e) {
            throw new InternalServerErrorException("It was not possible to add a key into the spotter dictionary.");
        }

    }

    public Collection<Emit> parseText(String text) {
        return trie.parseText(text);
    }

    @Override
    public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
        if (!hasBuilt) {
            buildDictionary();
        }

    }

}
<commit_msg>Checking for trie initialization<commit_after>package org.elinker.core.spotter;

import eu.freme.common.exception.InternalServerErrorException;
import lombok.RequiredArgsConstructor;
import org.ahocorasick.trie.Emit;
import org.ahocorasick.trie.Trie;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.stereotype.Component;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.stream.Stream;

import static java.nio.charset.StandardCharsets.UTF_8;
import static java.nio.file.StandardOpenOption.APPEND;

@Component
@RequiredArgsConstructor
public class FremeSpotter implements ApplicationListener<ContextRefreshedEvent> {

    @Value("${freme.ner.dictionary:}")
    private String dictionary = "";

    private Boolean hasBuilt = false;

    private Trie.TrieBuilder builder;

    private Trie trie;

    private void buildDictionary() {

        if (dictionary == null || dictionary.isEmpty()) {
            return;
        }


        builder = Trie.builder().removeOverlaps().caseInsensitive();

        try (Stream<String> stream = Files.lines(Paths.get(dictionary))) {
            stream.forEach(s -> {
                builder.addKeyword(s);
            });
        } catch (IOException e) {
            throw new InternalServerErrorException(String.format("It was not possible to read the spotter dictionary." +
                    " Please check if the path is correct (%s)", dictionary));
        }
        trie = builder.build();
    }

    public void addKey(String key) {

        if (dictionary == null || dictionary.isEmpty()) {
            return;
        }

        try {
            builder.addKeyword(key);
            trie = builder.build();
            Files.write(Paths.get(dictionary), key.concat(System.lineSeparator()).getBytes(UTF_8), APPEND);
        } catch (IOException e) {
            throw new InternalServerErrorException("It was not possible to add a key into the spotter dictionary.");
        }

    }

    public Collection<Emit> parseText(String text) {
        if (trie != null) {
            return trie.parseText(text);
        }
        return new ArrayList<>();
    }

    @Override
    public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
        if (!hasBuilt) {
            buildDictionary();
        }

    }

}

{'fraction_non_alphanumeric': 0.10382076415283056, 'fraction_numerical': 0.0008001600320064013, 'mean_word_length': 2.897116134060795, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4552058', 'n_tokens_mistral': 1368, 'n_tokens_neox': 1325, 'n_words': 299}