// Path: online/src/net/myrrix/online/RescorerProvider.java
// public interface RescorerProvider {
// 
//   /**
//    * @param userIDs user(s) for which recommendations are being made, which may be needed in the rescoring logic.
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link IDRescorer} to use with {@link ServerRecommender#recommend(long, int, IDRescorer)}
//    *  or {@code null} if none should be used. The resulting {@link IDRescorer} will be passed each candidate
//    *  item ID to {@link IDRescorer#isFiltered(long)}, and each non-filtered candidate with its original score
//    *  to {@link IDRescorer#rescore(long, double)}
//    */
//   IDRescorer getRecommendRescorer(long[] userIDs, MyrrixRecommender recommender, String... args);
// 
//   /**
//    * @param itemIDs items that the anonymous user is associated to
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link IDRescorer} to use with {@link ServerRecommender#recommendToAnonymous(long[], int, IDRescorer)}
//    *  or {@code null} if none should be used. The resulting {@link IDRescorer} will be passed each candidate
//    *  item ID to {@link IDRescorer#isFiltered(long)}, and each non-filtered candidate with its original score
//    *  to {@link IDRescorer#rescore(long, double)}
//    */
//   IDRescorer getRecommendToAnonymousRescorer(long[] itemIDs, MyrrixRecommender recommender, String... args);
// 
//   /**
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link IDRescorer} to use with {@link ServerRecommender#mostPopularItems(int, IDRescorer)}
//    *  or {@code null} if none should be used.
//    */
//   IDRescorer getMostPopularItemsRescorer(MyrrixRecommender recommender, String... args);
// 
//   /**
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link Rescorer} to use with {@link ServerRecommender#mostSimilarItems(long[], int, Rescorer)}
//    *  or {@code null} if none should be used. The {@link Rescorer} will be passed, to its
//    *  {@link Rescorer#isFiltered(Object)} method, a {@link LongPair} containing the candidate item ID
//    *  as its first element, and the item ID passed in the user query as its second element.
//    *  Each non-filtered {@link LongPair} is passed with its original score to
//    *  {@link Rescorer#rescore(Object, double)}
//    */
//   Rescorer<LongPair> getMostSimilarItemsRescorer(MyrrixRecommender recommender, String... args);
// 
// }


import java.io.File;
import java.util.Collection;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Multimap;
import org.apache.commons.math3.random.RandomGenerator;
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.myrrix.common.MyrrixRecommender;
import net.myrrix.common.collection.FastByIDMap;
import net.myrrix.common.collection.FastIDSet;
import net.myrrix.common.parallel.Paralleler;
import net.myrrix.common.parallel.Processor;
import net.myrrix.common.random.RandomManager;
import net.myrrix.common.random.RandomUtils;
import net.myrrix.online.RescorerProvider;
/*
 * Copyright Myrrix Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.myrrix.online.eval;




/**
 * <p>This implementation calculates Area under curve (AUC), which may be understood as the probability
 * that a random "good" recommendation is ranked higher than a random "bad" recommendation.</p>
 * 
 * <p>This class can be run as a Java program; the single argument is a directory containing test data.
 * The {@link EvaluationResult} is printed to standard out.</p>
 *
 * @author Sean Owen
 * @since 1.0
 */
public final class AUCEvaluator extends AbstractEvaluator {

  private static final Logger log = LoggerFactory.getLogger(AUCEvaluator.class);

  @Override
  protected boolean isSplitTestByPrefValue() {
    return true;
  }

  @Override
  public EvaluationResult evaluate(MyrrixRecommender recommender,
// Path: online/src/net/myrrix/online/RescorerProvider.java
// public interface RescorerProvider {
// 
//   /**
//    * @param userIDs user(s) for which recommendations are being made, which may be needed in the rescoring logic.
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link IDRescorer} to use with {@link ServerRecommender#recommend(long, int, IDRescorer)}
//    *  or {@code null} if none should be used. The resulting {@link IDRescorer} will be passed each candidate
//    *  item ID to {@link IDRescorer#isFiltered(long)}, and each non-filtered candidate with its original score
//    *  to {@link IDRescorer#rescore(long, double)}
//    */
//   IDRescorer getRecommendRescorer(long[] userIDs, MyrrixRecommender recommender, String... args);
// 
//   /**
//    * @param itemIDs items that the anonymous user is associated to
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link IDRescorer} to use with {@link ServerRecommender#recommendToAnonymous(long[], int, IDRescorer)}
//    *  or {@code null} if none should be used. The resulting {@link IDRescorer} will be passed each candidate
//    *  item ID to {@link IDRescorer#isFiltered(long)}, and each non-filtered candidate with its original score
//    *  to {@link IDRescorer#rescore(long, double)}
//    */
//   IDRescorer getRecommendToAnonymousRescorer(long[] itemIDs, MyrrixRecommender recommender, String... args);
// 
//   /**
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link IDRescorer} to use with {@link ServerRecommender#mostPopularItems(int, IDRescorer)}
//    *  or {@code null} if none should be used.
//    */
//   IDRescorer getMostPopularItemsRescorer(MyrrixRecommender recommender, String... args);
// 
//   /**
//    * @param recommender the recommender instance that is rescoring results
//    * @param args arguments, if any, that should be used when making the {@link IDRescorer}. This is additional
//    *  information from the request that may be necessary to its logic, like current location. What it means
//    *  is up to the implementation.
//    * @return {@link Rescorer} to use with {@link ServerRecommender#mostSimilarItems(long[], int, Rescorer)}
//    *  or {@code null} if none should be used. The {@link Rescorer} will be passed, to its
//    *  {@link Rescorer#isFiltered(Object)} method, a {@link LongPair} containing the candidate item ID
//    *  as its first element, and the item ID passed in the user query as its second element.
//    *  Each non-filtered {@link LongPair} is passed with its original score to
//    *  {@link Rescorer#rescore(Object, double)}
//    */
//   Rescorer<LongPair> getMostSimilarItemsRescorer(MyrrixRecommender recommender, String... args);
// 
// }


// Path: online/src/net/myrrix/online/eval/AUCEvaluator.java
import java.io.File;
import java.util.Collection;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.Multimap;
import org.apache.commons.math3.random.RandomGenerator;
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.recommender.RecommendedItem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import net.myrrix.common.MyrrixRecommender;
import net.myrrix.common.collection.FastByIDMap;
import net.myrrix.common.collection.FastIDSet;
import net.myrrix.common.parallel.Paralleler;
import net.myrrix.common.parallel.Processor;
import net.myrrix.common.random.RandomManager;
import net.myrrix.common.random.RandomUtils;
import net.myrrix.online.RescorerProvider;

/*
 * Copyright Myrrix Ltd
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package net.myrrix.online.eval;




/**
 * <p>This implementation calculates Area under curve (AUC), which may be understood as the probability
 * that a random "good" recommendation is ranked higher than a random "bad" recommendation.</p>
 * 
 * <p>This class can be run as a Java program; the single argument is a directory containing test data.
 * The {@link EvaluationResult} is printed to standard out.</p>
 *
 * @author Sean Owen
 * @since 1.0
 */
public final class AUCEvaluator extends AbstractEvaluator {

  private static final Logger log = LoggerFactory.getLogger(AUCEvaluator.class);

  @Override
  protected boolean isSplitTestByPrefValue() {
    return true;
  }

  @Override
  public EvaluationResult evaluate(MyrrixRecommender recommender,
                                   RescorerProvider provider, // ignored