<commit_before>package me.rkfg.xmpp.bot.plugins;

import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import me.rkfg.xmpp.bot.domain.Markov;
import me.rkfg.xmpp.bot.domain.MarkovFirstWord;
import me.rkfg.xmpp.bot.domain.MarkovFirstWordCount;

import org.apache.commons.lang3.StringUtils;
import org.hibernate.NonUniqueResultException;
import org.hibernate.Session;
import org.jivesoftware.smack.packet.Message;

import ru.ppsrk.gwt.client.ClientAuthException;
import ru.ppsrk.gwt.client.LogicException;
import ru.ppsrk.gwt.server.HibernateCallback;
import ru.ppsrk.gwt.server.HibernateUtil;

public class MarkovCollectorPlugin extends MessagePluginImpl {

    private static int SEGMENT_WORDS = 4; // words per segment

    @Override
    public Pattern getPattern() {
        return Pattern.compile("(.+)", Pattern.DOTALL);
    }

    @Override
    public String process(Message message, final Matcher matcher) {
        if (!isMessageFromUser(message)) {
            // don't store system messages
            return null;
        }
        final String text = matcher.group(1).replace('\n', ' ');
        processLine(text);
        return null;
    }

    protected synchronized void processLine(final String text) {
        try {
            HibernateUtil.exec(new HibernateCallback<Void>() {

                @Override
                public Void run(Session session) throws LogicException, ClientAuthException {
                    String words[] = text.split("\\s+");
                    int c = 0;
                    int pos = 0;
                    String segment = null;
                    String firstWord = null;
                    String lastWord = null;
                    do {
                        if (c + SEGMENT_WORDS < words.length) {
                            segment = StringUtils.join(Arrays.copyOfRange(words, c + 1, c + SEGMENT_WORDS), " ");
                            firstWord = purify(words[c]);
                            lastWord = purify(words[c + SEGMENT_WORDS - 1]);
                        } else {
                            if (c + 1 < words.length) {
                                segment = StringUtils.join(Arrays.copyOfRange(words, c + 1, words.length), " ");
                            } else {
                                segment = "";
                            }
                            firstWord = purify(words[c]);
                            lastWord = purify(words[words.length - 1]);
                        }
                        if (segment.length() < 256 && firstWord.length() < 256 && lastWord.length() < 256 && !firstWord.isEmpty()
                                && !lastWord.isEmpty()) {
                            Markov markov = (Markov) session.merge(new Markov(segment, pos, purify(firstWord), purify(lastWord)));
                            try {
                                MarkovFirstWordCount count = (MarkovFirstWordCount) session
                                        .createQuery("from MarkovFirstWordCount where word = :fw").setString("fw", firstWord)
                                        .uniqueResult();
                                if (count == null) {
                                    count = (MarkovFirstWordCount) session.merge(new MarkovFirstWordCount(firstWord));
                                }
                                count.incCount();
                                session.merge(new MarkovFirstWord(count, count.getCount() - 1, markov));
                            } catch (NonUniqueResultException e) {
                                log.warn("Non-unique word found: {}", firstWord);
                            }
                            pos++;
                        }
                    } while (c++ < words.length - SEGMENT_WORDS);
                    return null;
                }
            });
        } catch (ClientAuthException e) {
            e.printStackTrace();
        } catch (LogicException e) {
            e.printStackTrace();
        }
    }

    public static String purify(String str) {
        return str.toLowerCase().trim().replaceAll("[,.?!;:\"'`«»()\\[\\]{}]", "");
    }
}
<commit_msg>Skip empty message in Markov Collector.<commit_after>package me.rkfg.xmpp.bot.plugins;

import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import me.rkfg.xmpp.bot.domain.Markov;
import me.rkfg.xmpp.bot.domain.MarkovFirstWord;
import me.rkfg.xmpp.bot.domain.MarkovFirstWordCount;

import org.apache.commons.lang3.StringUtils;
import org.hibernate.NonUniqueResultException;
import org.hibernate.Session;
import org.jivesoftware.smack.packet.Message;

import ru.ppsrk.gwt.client.ClientAuthException;
import ru.ppsrk.gwt.client.LogicException;
import ru.ppsrk.gwt.server.HibernateCallback;
import ru.ppsrk.gwt.server.HibernateUtil;

public class MarkovCollectorPlugin extends MessagePluginImpl {

    private static int SEGMENT_WORDS = 4; // words per segment

    @Override
    public Pattern getPattern() {
        return Pattern.compile("(.+)", Pattern.DOTALL);
    }

    @Override
    public String process(Message message, final Matcher matcher) {
        if (!isMessageFromUser(message)) {
            // don't store system messages
            return null;
        }
        final String text = matcher.group(1).replace('\n', ' ');
        processLine(text);
        return null;
    }

    protected synchronized void processLine(final String text) {
        try {
            HibernateUtil.exec(new HibernateCallback<Void>() {

                @Override
                public Void run(Session session) throws LogicException, ClientAuthException {
                    String words[] = text.split("\\s+");
                    if (words.length == 0) {
                        return null;
                    }
                    int c = 0;
                    int pos = 0;
                    String segment = null;
                    String firstWord = null;
                    String lastWord = null;
                    do {
                        if (c + SEGMENT_WORDS < words.length) {
                            segment = StringUtils.join(Arrays.copyOfRange(words, c + 1, c + SEGMENT_WORDS), " ");
                            firstWord = purify(words[c]);
                            lastWord = purify(words[c + SEGMENT_WORDS - 1]);
                        } else {
                            if (c + 1 < words.length) {
                                segment = StringUtils.join(Arrays.copyOfRange(words, c + 1, words.length), " ");
                            } else {
                                segment = "";
                            }
                            firstWord = purify(words[c]);
                            lastWord = purify(words[words.length - 1]);
                        }
                        if (segment.length() < 256 && firstWord.length() < 256 && lastWord.length() < 256 && !firstWord.isEmpty()
                                && !lastWord.isEmpty()) {
                            Markov markov = (Markov) session.merge(new Markov(segment, pos, purify(firstWord), purify(lastWord)));
                            try {
                                MarkovFirstWordCount count = (MarkovFirstWordCount) session
                                        .createQuery("from MarkovFirstWordCount where word = :fw").setString("fw", firstWord)
                                        .uniqueResult();
                                if (count == null) {
                                    count = (MarkovFirstWordCount) session.merge(new MarkovFirstWordCount(firstWord));
                                }
                                count.incCount();
                                session.merge(new MarkovFirstWord(count, count.getCount() - 1, markov));
                            } catch (NonUniqueResultException e) {
                                log.warn("Non-unique word found: {}", firstWord);
                            }
                            pos++;
                        }
                    } while (c++ < words.length - SEGMENT_WORDS);
                    return null;
                }
            });
        } catch (ClientAuthException e) {
            e.printStackTrace();
        } catch (LogicException e) {
            e.printStackTrace();
        }
    }

    public static String purify(String str) {
        return str.toLowerCase().trim().replaceAll("[,.?!;:\"'`«»()\\[\\]{}]", "");
    }
}

{'fraction_non_alphanumeric': 0.10538063909774437, 'fraction_numerical': 0.00481672932330827, 'mean_word_length': 1.3872686483454852, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2276323', 'n_tokens_mistral': 2229, 'n_tokens_neox': 2101, 'n_words': 440}