<commit_before>package services.nlp.recommendation;

import java.math.RoundingMode;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

import play.libs.Json;
import services.nlp.microserviceutil.NLPResultUtil;
import services.nlp.tfidf.TFIDFResult;
import services.util.Sorter;

public class DeckRecommendation {


	public static JsonNode calculateLuceneValuesForTfidfMapAndReturnAsJsonNode(Map<String,Double> map, int maxEntriesToUse, String keynameForSolr){
		
		ObjectNode result = Json.newObject();
		
		if(map.isEmpty()){
			return result;
		}
		
		List<Entry<String, Double>> sorted = Sorter.sortByValueAndReturnAsList(map, true);
		StringBuilder sbLuceneQuery = new StringBuilder();
		Double highestValue = sorted.get(0).getValue();
		
		DecimalFormat df = new DecimalFormat("#.####");
		df.setRoundingMode(RoundingMode.HALF_UP);

		ArrayNode arrayNode = Json.newArray();
		int counter = 0;
		
		for (Entry<String, Double> entry : sorted) {
			
			counter++;
			if(maxEntriesToUse>=0 && counter > maxEntriesToUse){
				break;
			}
			String key = entry.getKey();
			Double value = entry.getValue();
			Double valueNormalized = value / highestValue;
			String valueRounded = df.format(valueNormalized).replace(',', '.');
			
			String valueForLuceneQuery = valueRounded;
			// remove leading zero (normalized can be only start with 0 or 1
			if(valueRounded.startsWith("0")){
				valueForLuceneQuery = valueRounded.substring(1);
			}
			
			sbLuceneQuery.append(" " + keynameForSolr + ":");
			sbLuceneQuery.append(key);
			sbLuceneQuery.append("^");
			sbLuceneQuery.append(valueForLuceneQuery);
			
			ObjectNode entryAsJson = Json.newObject();
			entryAsJson.put("key", key);
			entryAsJson.put("value", value);
			entryAsJson.put("valueNormalized", valueNormalized);
			entryAsJson.put("valueRounded", valueRounded);
			entryAsJson.put("valueForLuceneQuery", valueForLuceneQuery);

			arrayNode.add(entryAsJson);
		}
		String luceneQuery = sbLuceneQuery.toString();
		
		
		result.put("luceneQuery", luceneQuery);
		result.set("values", arrayNode);
		
		return result;
	}
	
	
	public static Map<String, Double> recreateMapFromJson(JsonNode jsonNode, String valueNameToUse){
		
		Map<String, Double> map = new HashMap<String, Double>();
		ArrayNode arrayNode = (ArrayNode) jsonNode.get("values");
		Iterator<JsonNode> iterator = arrayNode.iterator();
		while (iterator.hasNext()){
			JsonNode node = iterator.next();
			String entryKey = node.get("key").asText();
			Double value = node.get(valueNameToUse).asDouble();
			map.put(entryKey, value);
		}
		
		return map;
	}
	
	public static ObjectNode createDeckRecommendationBackgroundInfoNodeIncludingLuceneQueryFromTFIDFResult(TFIDFResult tfidfResult, int maxTermsToConsider){
		
		ObjectNode result = Json.newObject();
		
		result.put("language", tfidfResult.getLanguage());
		result.put("numberOfDecksWithGivenLanguage", tfidfResult.getNumberOfDecksInPlatformWithGivenLanguage());
		result.put("numberOfDecksOverall", tfidfResult.getNumberOfDecksInPlatformOverall());
		result.put("tfidfValuesWereCalculatedLanguageDependent", tfidfResult.isTfidfValuesWereCalculatedLanguageDependent());
		
		Map<String,Map<String,Double>> tfidfmap = tfidfResult.getTfidfMap();
		Set<String> providers= tfidfmap.keySet();

		StringBuilder sb = new StringBuilder();
		ArrayNode providersArrayNode = Json.newArray();
		for (String provider : providers) {
			
			Map<String,Double> tfidfMap = tfidfmap.get(provider);
			String keynameForSolr = NLPResultUtil.getSolrNameForProviderName(provider);<commit_msg>bug fix: DeckRecommendation.createDeckRecommendationBackgroundInfoNodeIncludingLuceneQueryFromTFIDFResult: if no entries for provider available (tfidf map is empty): skip this<commit_after>package services.nlp.recommendation;

import java.math.RoundingMode;
import java.text.DecimalFormat;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Map.Entry;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;

import play.libs.Json;
import services.nlp.microserviceutil.NLPResultUtil;
import services.nlp.tfidf.TFIDFResult;
import services.util.Sorter;

public class DeckRecommendation {


	public static JsonNode calculateLuceneValuesForTfidfMapAndReturnAsJsonNode(Map<String,Double> map, int maxEntriesToUse, String keynameForSolr){
		
		ObjectNode result = Json.newObject();
		
		if(map.isEmpty()){
			return result;
		}
		
		List<Entry<String, Double>> sorted = Sorter.sortByValueAndReturnAsList(map, true);
		StringBuilder sbLuceneQuery = new StringBuilder();
		Double highestValue = sorted.get(0).getValue();
		
		DecimalFormat df = new DecimalFormat("#.####");
		df.setRoundingMode(RoundingMode.HALF_UP);

		ArrayNode arrayNode = Json.newArray();
		int counter = 0;
		
		for (Entry<String, Double> entry : sorted) {
			
			counter++;
			if(maxEntriesToUse>=0 && counter > maxEntriesToUse){
				break;
			}
			String key = entry.getKey();
			Double value = entry.getValue();
			Double valueNormalized = value / highestValue;
			String valueRounded = df.format(valueNormalized).replace(',', '.');
			
			String valueForLuceneQuery = valueRounded;
			// remove leading zero (normalized can be only start with 0 or 1
			if(valueRounded.startsWith("0")){
				valueForLuceneQuery = valueRounded.substring(1);
			}
			
			sbLuceneQuery.append(" " + keynameForSolr + ":");
			sbLuceneQuery.append(key);
			sbLuceneQuery.append("^");
			sbLuceneQuery.append(valueForLuceneQuery);
			
			ObjectNode entryAsJson = Json.newObject();
			entryAsJson.put("key", key);
			entryAsJson.put("value", value);
			entryAsJson.put("valueNormalized", valueNormalized);
			entryAsJson.put("valueRounded", valueRounded);
			entryAsJson.put("valueForLuceneQuery", valueForLuceneQuery);

			arrayNode.add(entryAsJson);
		}
		String luceneQuery = sbLuceneQuery.toString();
		
		
		result.put("luceneQuery", luceneQuery);
		result.set("values", arrayNode);
		
		return result;
	}
	
	
	public static Map<String, Double> recreateMapFromJson(JsonNode jsonNode, String valueNameToUse){
		
		Map<String, Double> map = new HashMap<String, Double>();
		ArrayNode arrayNode = (ArrayNode) jsonNode.get("values");
		Iterator<JsonNode> iterator = arrayNode.iterator();
		while (iterator.hasNext()){
			JsonNode node = iterator.next();
			String entryKey = node.get("key").asText();
			Double value = node.get(valueNameToUse).asDouble();
			map.put(entryKey, value);
		}
		
		return map;
	}
	
	public static ObjectNode createDeckRecommendationBackgroundInfoNodeIncludingLuceneQueryFromTFIDFResult(TFIDFResult tfidfResult, int maxTermsToConsider){
		
		ObjectNode result = Json.newObject();
		
		result.put("language", tfidfResult.getLanguage());
		result.put("numberOfDecksWithGivenLanguage", tfidfResult.getNumberOfDecksInPlatformWithGivenLanguage());
		result.put("numberOfDecksOverall", tfidfResult.getNumberOfDecksInPlatformOverall());
		result.put("tfidfValuesWereCalculatedLanguageDependent", tfidfResult.isTfidfValuesWereCalculatedLanguageDependent());
		
		Map<String,Map<String,Double>> tfidfmap = tfidfResult.getTfidfMap();
		Set<String> providers= tfidfmap.keySet();

		StringBuilder sb = new StringBuilder();
		ArrayNode providersArrayNode = Json.newArray();
		for (String provider : providers) {
			
			Map<String,Double> tfidfMap = tfidfmap.get(provider);
			if(tfidfMap.isEmpty()){
				continue;
			}
			String keynameForSolr = NLPResultUtil.getSolrNameForProviderName(provider);
{'fraction_non_alphanumeric': 0.11669203450025367, 'fraction_numerical': 0.0017757483510908167, 'mean_word_length': 5.3794498381877025, 'pattern_counts': {'":': 2, '<': 27, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4935947', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2417, 'n_words': 453}