<commit_before>package org.insightcentre.richcontext;

import net.recommenders.rival.core.DataModelIF;
import net.recommenders.rival.core.DataModelUtils;
import net.recommenders.rival.split.splitter.CrossValidationSplitter;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;

/**
 * Created by fpena on 05/09/2017.
 */
public class RichContextDataInitializer {


    private final int numFolds;
    private final long seed;
    private final RichContextResultsProcessor.Strategy strategy;
    private final boolean coldStart;


    private Map<String, Review> reviewsMap;
    private String ratingsFolderPath;
    private String jsonRatingsFile;


    public RichContextDataInitializer(
            String cacheFolder, String propertiesFile)
            throws IOException {

        Properties properties = Properties.loadProperties(propertiesFile);
        numFolds = properties.getCrossValidationNumFolds();
        seed = properties.getSeed();
        strategy = RichContextResultsProcessor.Strategy.valueOf(
                (properties.getStrategy().toUpperCase(Locale.ENGLISH)));
        RichContextResultsProcessor.ContextFormat contextFormat = RichContextResultsProcessor.ContextFormat.valueOf(
                properties.getContextFormat().toUpperCase(Locale.ENGLISH));
        RichContextResultsProcessor.Dataset dataset = RichContextResultsProcessor.Dataset.valueOf(
                properties.getDataset().toUpperCase(Locale.ENGLISH));
        int numTopics = properties.getNumTopics();
        coldStart = properties.getEvaluateColdStart();

        jsonRatingsFile = cacheFolder + dataset.toString().toLowerCase() +
                "_recsys_formatted_context_records_ensemble_" +
                "numtopics-" + numTopics + "_iterations-100_passes-10_targetreview-specific_" +
                "normalized_contextformat-" + contextFormat.toString().toLowerCase()
                + "_lang-en_bow-NN_document_level-review_targettype-context_" +
                "min_item_reviews-10.json";

        this.ratingsFolderPath = Utils.getRatingsFolderPath(jsonRatingsFile);
    }


    /**
     * Creates the cross-validation splits
     */
    private void prepareSplits() throws IOException {

        String dataFile = jsonRatingsFile;
        boolean perUser = false;
        JsonParser parser = new JsonParser();

        DataModelIF<Long, Long> data = parser.parseData(new File(dataFile));

        // Build reviews map
        this.reviewsMap = new HashMap<>();
        Set<Long> itemsSet = new HashSet<>();
        Set<Long> usersSet = new HashSet<>();
        for (Review review : parser.getReviews()) {
            reviewsMap.put(review.getUser_item_key(), review);
            itemsSet.add(review.getItemId());
            usersSet.add(review.getUserId());
        }
        int numItems = itemsSet.size();
        int numUsers = usersSet.size();
        System.err.println("Num items: " + numItems);
        System.err.println("Num users: " + numUsers);


        DataModelIF<Long, Long>[] splits =
                new CrossValidationSplitter<Long, Long>(
                        numFolds, perUser, seed).split(data);
        File dir = new File(ratingsFolderPath);
        if (!dir.exists()) {
            if (!dir.mkdir()) {
                System.err.println("Directory " + dir + " could not be created");
                return;
            }
        }
        for (int i = 0; i < splits.length / 2; i++) {

            String foldPath = ratingsFolderPath + "fold_" + i + "/";
            File foldDir = new File(foldPath);
            if (!foldDir.exists()) {
                if (!foldDir.mkdir()) {
                    System.err.println("Directory " + foldDir + " could not be created");
                    return;
                }
            }

            DataModelIF<Long, Long> training = splits[2 * i];
            DataModelIF<Long, Long> test = splits[2 * i + 1];

            String trainingFile = foldPath + "train.csv";
            String testFile = foldPath + "test.csv";
            boolean overwrite = true;
            DataModelUtils.saveDataModel(training, trainingFile, overwrite, "\t");
            DataModelUtils.saveDataModel(test, testFile, overwrite, "\t");
        }
    }


    /**
     * Transform the split given by {@code fold} into the format required by
     * LibFM
     *
     * @param fold the cross-validation fold
     */
    private void transformSplitToLibfm(int fold) throws IOException {

        System.err.println("Transform split " + fold + " to LibFM");

        String foldPath = ratingsFolderPath + "fold_" + fold + "/";
        String trainFile = foldPath + "train.csv";
        String testFile = foldPath + "test.csv";
        String predictionsFile = foldPath + "predictions.csv";
        List<Review> incompleteTrainReviews = ReviewCsvDao.readCsvFile(trainFile);
        List<Review> incompleteTestReviews = ReviewCsvDao.readCsvFile(testFile);

        List<Review> completeTrainReviews = new ArrayList<>();
        List<Review> completeTestReviews = new ArrayList<>();

        for (Review review : incompleteTrainReviews) {
            Review completeReview = reviewsMap.get(review.getUser_item_key());
            completeTrainReviews.add(completeReview);
        }
        for (Review review : incompleteTestReviews) {
            Review completeReview = reviewsMap.get(review.getUser_item_key());
            completeTestReviews.add(completeReview);
        }

        String libfmTrainFile = foldPath + "libfm_train.libfm";
//        String libfmTestFile = foldPath + "libfm_test.libfm";
        String libfmPredictionsFile = foldPath + "libfm_predictions_" +
                strategy.getPredictionType() + ".libfm";

        Map<String, Integer> oneHotIdMap =
                LibfmExporter.getOneHot(reviewsMap.values());

        LibfmExporter.exportRecommendations(
                completeTrainReviews, libfmTrainFile, oneHotIdMap);

        switch (strategy) {
            case TEST_ITEMS:
            case USER_TEST:
                LibfmExporter.exportRecommendations(
                        completeTestReviews, libfmPredictionsFile, oneHotIdMap);
                break;
            case REL_PLUS_N: {
                LibfmExporter.exportRankingPredictionsFile(
                        completeTrainReviews, completeTestReviews,
                        libfmPredictionsFile, oneHotIdMap, predictionsFile
                );<commit_msg>Added cache support for the RichContextDataInitalizer class<commit_after>package org.insightcentre.richcontext;

import net.recommenders.rival.core.DataModelIF;
import net.recommenders.rival.core.DataModelUtils;
import net.recommenders.rival.split.splitter.CrossValidationSplitter;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Random;
import java.util.Set;

/**
 * Created by fpena on 05/09/2017.
 */
public class RichContextDataInitializer {


    private final int numFolds;
    private final long seed;
    private final RichContextResultsProcessor.Strategy strategy;
    private final boolean coldStart;


    private Map<String, Review> reviewsMap;
    private String ratingsFolderPath;
    private String jsonRatingsFile;


    public RichContextDataInitializer(
            String cacheFolder, String propertiesFile)
            throws IOException {

        Properties properties = Properties.loadProperties(propertiesFile);
        numFolds = properties.getCrossValidationNumFolds();
        seed = properties.getSeed();
        strategy = RichContextResultsProcessor.Strategy.valueOf(
                (properties.getStrategy().toUpperCase(Locale.ENGLISH)));
        RichContextResultsProcessor.ContextFormat contextFormat = RichContextResultsProcessor.ContextFormat.valueOf(
                properties.getContextFormat().toUpperCase(Locale.ENGLISH));
        RichContextResultsProcessor.Dataset dataset = RichContextResultsProcessor.Dataset.valueOf(
                properties.getDataset().toUpperCase(Locale.ENGLISH));
        int numTopics = properties.getNumTopics();
        coldStart = properties.getEvaluateColdStart();

        jsonRatingsFile = cacheFolder + dataset.toString().toLowerCase() +
                "_recsys_formatted_context_records_ensemble_" +
                "numtopics-" + numTopics + "_iterations-100_passes-10_targetreview-specific_" +
                "normalized_contextformat-" + contextFormat.toString().toLowerCase()
                + "_lang-en_bow-NN_document_level-review_targettype-context_" +
                "min_item_reviews-10.json";

        this.ratingsFolderPath = Utils.getRatingsFolderPath(jsonRatingsFile);
    }


    /**
     * Creates the cross-validation splits
     */
    private void prepareSplits() throws IOException {

        String dataFile = jsonRatingsFile;
        boolean perUser = false;
        JsonParser parser = new JsonParser();

        DataModelIF<Long, Long> data = parser.parseData(new File(dataFile));

        // Build reviews map
        this.reviewsMap = new HashMap<>();
        Set<Long> itemsSet = new HashSet<>();
        Set<Long> usersSet = new HashSet<>();
        for (Review review : parser.getReviews()) {
            reviewsMap.put(review.getUser_item_key(), review);
            itemsSet.add(review.getItemId());
            usersSet.add(review.getUserId());
        }
        int numItems = itemsSet.size();
        int numUsers = usersSet.size();
        System.err.println("Num items: " + numItems);
        System.err.println("Num users: " + numUsers);


        DataModelIF<Long, Long>[] splits =
                new CrossValidationSplitter<Long, Long>(
                        numFolds, perUser, seed).split(data);
        File dir = new File(ratingsFolderPath);
        if (!dir.exists()) {
            if (!dir.mkdir()) {
                System.err.println("Directory " + dir + " could not be created");
                return;
            }
        }
        for (int i = 0; i < splits.length / 2; i++) {

            String foldPath = ratingsFolderPath + "fold_" + i + "/";
            File foldDir = new File(foldPath);
            if (!foldDir.exists()) {
                if (!foldDir.mkdir()) {
                    System.err.println("Directory " + foldDir + " could not be created");
                    return;
                }
            }

            DataModelIF<Long, Long> training = splits[2 * i];
            DataModelIF<Long, Long> test = splits[2 * i + 1];

            String trainingFile = foldPath + "train.csv";
            String testFile = foldPath + "test.csv";
            boolean overwrite = true;
            DataModelUtils.saveDataModel(training, trainingFile, overwrite, "\t");
            DataModelUtils.saveDataModel(test, testFile, overwrite, "\t");
        }
    }


    /**
     * Transform the split given by {@code fold} into the format required by
     * LibFM
     *
     * @param fold the cross-validation fold
     */
    private void transformSplitToLibfm(int fold) throws IOException {

        System.err.println("Transform split " + fold + " to LibFM");
        boolean overwrite = false;

        String foldPath = ratingsFolderPath + "fold_" + fold + "/";
        String libfmTrainFile = foldPath + "libfm_train.libfm";
//        String libfmTestFile = foldPath + "libfm_test.libfm";
        String libfmPredictionsFile = foldPath + "libfm_predictions_" +
                strategy.getPredictionType() + ".libfm";

        if (new File(libfmTrainFile).exists() &&
                new File(libfmPredictionsFile).exists() && !overwrite) {
            System.err.println(
                    "LibFM files for split " + fold + " already exist");
            return;
        }

        String trainFile = foldPath + "train.csv";
        String testFile = foldPath + "test.csv";
        String predictionsFile = foldPath + "predictions.csv";
        List<Review> incompleteTrainReviews = ReviewCsvDao.readCsvFile(trainFile);
        List<Review> incompleteTestReviews = ReviewCsvDao.readCsvFile(testFile);

        List<Review> completeTrainReviews = new ArrayList<>();
        List<Review> completeTestReviews = new ArrayList<>();

        for (Review review : incompleteTrainReviews) {
            Review completeReview = reviewsMap.get(review.getUser_item_key());
            completeTrainReviews.add(completeReview);
        }
        for (Review review : incompleteTestReviews) {
            Review completeReview = reviewsMap.get(review.getUser_item_key());
            completeTestReviews.add(completeReview);
        }

        Map<String, Integer> oneHotIdMap =
                LibfmExporter.getOneHot(reviewsMap.values());

        LibfmExporter.exportRecommendations(
                completeTrainReviews, libfmTrainFile, oneHotIdMap);

        switch (strategy) {
            case TEST_ITEMS:
            case USER_TEST:
                LibfmExporter.exportRecommendations(
                        completeTestReviews, libfmPredictionsFile, oneHotIdMap);
                break;
            case REL_PLUS_N: {
                LibfmExporter.exportRankingPredictionsFile(
                        completeTrainReviews, completeTestReviews,
                        libfmPredictionsFile, oneHotIdMap, predictionsFile
                );
{'fraction_non_alphanumeric': 0.09821756033154845, 'fraction_numerical': 0.002934057067409961, 'mean_word_length': 2.445539550164266, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6208049', 'n_tokens_mistral': 3725, 'n_tokens_neox': 3523, 'n_words': 754}