<commit_before>package com.commerzinfo.categorize;

import com.commerzinfo.data.DataRow;
import com.commerzinfo.util.PredicateUtil;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.Collections2;
import com.google.common.collect.Maps;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class CategoryCollection {
    private static Logger logger = LoggerFactory.getLogger(CategoryCollection.class);
    private static final Collection<Category> allCategories = new ArrayList<Category>();
    private static final Matcher whitespaceMatcher = Pattern.compile("\\s+", Pattern.CASE_INSENSITIVE).matcher("");

    public static final String CATCHALL = "catchall";
    public static final Function<Category, String> CATEGORY_STRING_FUNCTION = new Function<Category, String>() {
        @Override
        public String apply(@Nullable Category input) {
            return (input != null) ? input.getCategoryName() : "";
        }
    };
    private static final LinkedHashMap<String, Matcher> categoryMap = Maps.newLinkedHashMap();

    @SuppressWarnings("unchecked")
    public static Collection<Category> createCategories(File configFile) {
        allCategories.clear();

        if (configFile != null && configFile.isFile()) {
            try {
                LinkedProperties properties = new LinkedProperties();
                properties.load(new FileReader(configFile));

                for (String propName : properties.stringPropertyNames()) {
                    String value = properties.getProperty(propName);

                    whitespaceMatcher.reset(value);
                    if (whitespaceMatcher.find()) {
                        String oldValue = value;
                        value = whitespaceMatcher.replaceAll("\\\\s+");
                        logger.info("replacing whitespaces " + oldValue + "=>" + value);
                    }

                    Matcher matcher = Pattern.compile(value, Pattern.CASE_INSENSITIVE).matcher("");
                    categoryMap.put(propName, matcher);
                    addCat(allCategories, propName, PredicateUtil.findPatternPredicate(value));
                }
            } catch (Exception e) {
                throw new RuntimeException("configFile=" + configFile.getAbsolutePath(), e);
            }
        }

        addCat(allCategories, CATCHALL, Predicates.<DataRow>alwaysTrue());
        categoryMap.put(CATCHALL, Pattern.compile(".*", Pattern.CASE_INSENSITIVE).matcher(""));
        return allCategories;
    }

    private static void addCat(Collection<Category> categoryCollection, String categoryName, Predicate<DataRow> predicate) {
        categoryCollection.add(new Category(categoryName, predicate));
    }

    public static Collection<Category> getAllCategories() {
        return allCategories;
    }

    public static Collection<String> getAllCategoryNames() {
        return Collections2.transform(allCategories, CATEGORY_STRING_FUNCTION);
    }

    public static LinkedHashMap<String, Matcher> getCategoryMap() {
        return categoryMap;
    }
}<commit_msg>retrieve properties in given order from the file<commit_after>package com.commerzinfo.categorize;

import com.commerzinfo.data.DataRow;
import com.commerzinfo.util.PredicateUtil;
import com.google.common.base.Function;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.collect.Collections2;
import com.google.common.collect.Maps;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class CategoryCollection {
    public static final String CATCHALL = "catchall";
    public static final Function<Category, String> CATEGORY_STRING_FUNCTION = new Function<Category, String>() {
        @Override
        public String apply(@Nullable Category input) {
            return (input != null) ? input.getCategoryName() : "";
        }
    };
    private static final Collection<Category> allCategories = new ArrayList<Category>();
    private static final Matcher whitespaceMatcher = Pattern.compile("\\s+", Pattern.CASE_INSENSITIVE).matcher("");
    private static final LinkedHashMap<String, Matcher> categoryMap = Maps.newLinkedHashMap();
    private static Logger logger = LoggerFactory.getLogger(CategoryCollection.class);

    @SuppressWarnings("unchecked")
    public static Collection<Category> createCategories(File configFile) {
        allCategories.clear();

        if (configFile != null && configFile.isFile()) {
            try {
                for (Map.Entry<String, String> entry : getPropertyMap(configFile).entrySet()) {
                    String catName = entry.getKey();
                    String regex = entry.getValue();

                    whitespaceMatcher.reset(regex);
                    if (whitespaceMatcher.find()) {
                        String oldValue = regex;
                        regex = whitespaceMatcher.replaceAll("\\\\s+");
                        logger.info("replacing whitespaces " + oldValue + "=>" + regex);
                    }

                    Matcher matcher = Pattern.compile(regex, Pattern.CASE_INSENSITIVE).matcher("");
                    categoryMap.put(catName, matcher);
                    addCat(allCategories, catName, PredicateUtil.findPatternPredicate(regex));
                }
            } catch (Exception e) {
                throw new RuntimeException("configFile=" + configFile.getAbsolutePath(), e);
            }
        }

        addCat(allCategories, CATCHALL, Predicates.<DataRow>alwaysTrue());
        categoryMap.put(CATCHALL, Pattern.compile(".*", Pattern.CASE_INSENSITIVE).matcher(""));
        return allCategories;
    }

    private static Map<String, String> getPropertyMap(File configFile) throws IOException {
        Map<String, String> propertyMap = new LinkedHashMap<String, String>();
        for (String line : FileUtils.readLines(configFile, "UTF-8")) {
            if (!line.contains("="))
                continue;
            final String[] split = line.split("=");
            propertyMap.put(split[0], split[1]);
        }
        return propertyMap;
    }

    private static void addCat(Collection<Category> categoryCollection, String categoryName, Predicate<DataRow> predicate) {
        categoryCollection.add(new Category(categoryName, predicate));
    }

    public static Collection<Category> getAllCategories() {
        return allCategories;
    }

    public static Collection<String> getAllCategoryNames() {
        return Collections2.transform(allCategories, CATEGORY_STRING_FUNCTION);
    }

    public static LinkedHashMap<String, Matcher> getCategoryMap() {
        return categoryMap;
    }
}
{'fraction_non_alphanumeric': 0.10482985729967069, 'fraction_numerical': 0.0015093304061470911, 'mean_word_length': 2.9700435729847494, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7320647', 'n_tokens_mistral': 1908, 'n_tokens_neox': 1848, 'n_words': 407}