<commit_before>package com.gmail.inverseconduit.utils;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class PrintUtils {

    private static final String  messageTokenRegex =
                                                           "(\\[[^]]++\\]\\(https?+:\\/\\/[^\\s\"]++\\h*\\\"[^\"]++\\\"\\)|([-*_]{1,3})?+\\[(meta\\-)?tag:[^]]++\\]\\2|\\`.*?\\`|[\\w\\.]*)";

    private static final Pattern markdownTokenizer = Pattern.compile(messageTokenRegex);

    public static String FixedFont(String msg) {
        if (msg.isEmpty())
            return msg;
        StringBuilder builder = new StringBuilder();
        for (String segment : msg.split("\n"))
            builder.append("    ").append(segment);
        return builder.toString();
    }

    public static String truncate(String message) {
        if (message.lastIndexOf(".") < 500 && message.lastIndexOf(".") != -1) { return message.substring(0, message.lastIndexOf(".")); }
        message = message.substring(0, 495);
        //this one can cut quite a lot of stuff..
        if (message.lastIndexOf(".") < 500 && message.lastIndexOf(".") != -1) { return message.substring(0, message.lastIndexOf(".")); }
        return message + "...";
    }

    public static List<String> splitUsefully(String message) {
        Matcher m = markdownTokenizer.matcher(message);
        m.find();
        int groups = m.groupCount();
        List<String> tokens = new ArrayList<>();
        for (int i = 1; i <= groups; i++ ) {
            tokens.add(m.group(i));
        }
        return tokens;
    }
}
<commit_msg>Stopped returning null elements from the splitUsefully method<commit_after>package com.gmail.inverseconduit.utils;

import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class PrintUtils {

    private static final String  messageTokenRegex =
                                                           "(\\[[^]]++\\]\\(https?+:\\/\\/[^\\s\"]++\\h*\\\"[^\"]++\\\"\\)|([-*_]{1,3})?+\\[(meta\\-)?tag:[^]]++\\]\\2|\\`.*?\\`|[\\w\\.]*)";

    private static final Pattern markdownTokenizer = Pattern.compile(messageTokenRegex);

    public static String FixedFont(String msg) {
        if (msg.isEmpty())
            return msg;
        StringBuilder builder = new StringBuilder();
        for (String segment : msg.split("\n"))
            builder.append("    ").append(segment);
        return builder.toString();
    }

    public static String truncate(String message) {
        if (message.lastIndexOf(".") < 500 && message.lastIndexOf(".") != -1) { return message.substring(0, message.lastIndexOf(".")); }
        message = message.substring(0, 495);
        //this one can cut quite a lot of stuff..
        if (message.lastIndexOf(".") < 500 && message.lastIndexOf(".") != -1) { return message.substring(0, message.lastIndexOf(".")); }
        return message + "...";
    }

    public static List<String> splitUsefully(String message) {
        Matcher m = markdownTokenizer.matcher(message);
        m.find();
        int groups = m.groupCount();
        List<String> tokens = new ArrayList<>();
        for (int i = 1; i <= groups; i++ ) {
            String matchedGroup = m.group(i);
            if (null != matchedGroup) {
                tokens.add(matchedGroup);
            }
        }
        return tokens;
    }
}

{'fraction_non_alphanumeric': 0.18245206275421266, 'fraction_numerical': 0.010459035444509006, 'mean_word_length': 2.2481132075471697, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4152073', 'n_tokens_mistral': 1026, 'n_tokens_neox': 984, 'n_words': 227}