<commit_before>package com.google.sps.servlets;

import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EmbeddedEntity;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.PreparedQuery;
import java.io.PrintWriter;
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.LinkedHashMap;
import java.util.PriorityQueue;
import java.util.Collections;
import com.google.gson.JsonObject;
import com.google.gson.Gson;
import com.google.sps.servlets.ServletHelper;
import com.google.sps.Objects.TFIDFStringHelper;
import com.google.sps.Objects.Tag;
import com.google.sps.Objects.Comment;
import error.ErrorHandler;

/**
 * This servlet generates groups "tags" for all groups - calculated using TF-IDF.
 */
@WebServlet("/tags-tfidf")
public class TagsTFIDFServlet extends HttpServlet {

  private DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();

  @Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    List<Long> groupIds = getGroupIds();

    LinkedHashMap<Long, LinkedHashMap<String, Integer>> groupMap = getGroupMap(groupIds, response);
    LinkedHashMap<String, Integer> occurenceMap = getTotalOccurences(groupMap);

    calculateTFIDF(groupMap, occurenceMap, response);
  }

  /** 
   * Queries the database to return a list of all groupIds. 
   */
  private List<Long> getGroupIds() {
    // Preparing query instance to retrieve all Groups
    Query query = new Query("Group");
    PreparedQuery results = datastore.prepare(query);

    List<Long> groupIds = new ArrayList<>();
    for (Entity entity : results.asIterable()) {
      long groupId = (long) entity.getKey().getId();
      groupIds.add(groupId);
    }

    return groupIds;
  }

  /** 
   * Returns a mapping of all groupIds to their n-grams (and occurences). 
   */
  private LinkedHashMap<Long, LinkedHashMap<String, Integer>> getGroupMap(
      List<Long> groupIds, HttpServletResponse response) throws IOException {

    LinkedHashMap<Long, LinkedHashMap<String, Integer>> groupMap = 
        new LinkedHashMap<Long, LinkedHashMap<String, Integer>>();
        
    for (long groupId : groupIds) {
      Entity groupEntity = ServletHelper.getEntityFromId(response, groupId, datastore, "Group");
      List<Long> postIds = (ArrayList<Long>) groupEntity.getProperty("posts");
      List<String> textData = getGroupPostsText(postIds, response);
      addChallengeText(groupEntity, textData, response);

      // TODO: Parallelize this process to occur for each string concurrently.
      ArrayList<LinkedHashMap<String, Integer>> ngramsList = new ArrayList<>();
      for (String text : textData) {
        ngramsList.add(TFIDFStringHelper.ngramTokenizer(text));
      }
      groupMap.put(groupId, TFIDFStringHelper.combineMaps(ngramsList));
    }

    return groupMap;
  }

  /** 
   * Returns a list of Post content Strings, given a list of postIds. 
   */
  private List<String> getGroupPostsText(List<Long> postIds, HttpServletResponse response) 
      throws IOException {

    List<String> textData = new ArrayList<>();
    for (long postId : postIds) {
      Entity postEntity = ServletHelper.getEntityFromId(response, postId, datastore, "Post");
      String postText = (String) postEntity.getProperty("postText");
      textData.add(postText);
      addCommentsText(postEntity, textData);
    }

    return textData;
  }

  /**
   * Given a post, adds that post's comments to a list of group text data.
   */
  private void addCommentsText(Entity postEntity, List<String> textData) {
    
    ArrayList<EmbeddedEntity> comments = 
        (ArrayList<EmbeddedEntity>) postEntity.getProperty("comments");

    if (comments != null) {
      for (EmbeddedEntity commentEntity : comments) {
        String commentText = (String) commentEntity.getProperty("commentText");
        textData.add(commentText);
      }
    }
  }

  /**
   * Given a post, adds that post's comments to a list of group text data.
   */
  private void addChallengeText(Entity groupEntity, List<String> textData,
      HttpServletResponse response) throws IOException {
    List<Long> challengeIds = (ArrayList<Long>) groupEntity.getProperty("challenges");<commit_msg>Add challenges to tfidf calc - comment<commit_after>package com.google.sps.servlets;

import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EmbeddedEntity;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.PreparedQuery;
import java.io.PrintWriter;
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
import java.util.Map;
import java.util.LinkedHashMap;
import java.util.PriorityQueue;
import java.util.Collections;
import com.google.gson.JsonObject;
import com.google.gson.Gson;
import com.google.sps.servlets.ServletHelper;
import com.google.sps.Objects.TFIDFStringHelper;
import com.google.sps.Objects.Tag;
import com.google.sps.Objects.Comment;
import error.ErrorHandler;

/**
 * This servlet generates groups "tags" for all groups - calculated using TF-IDF.
 */
@WebServlet("/tags-tfidf")
public class TagsTFIDFServlet extends HttpServlet {

  private DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();

  @Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    List<Long> groupIds = getGroupIds();

    LinkedHashMap<Long, LinkedHashMap<String, Integer>> groupMap = getGroupMap(groupIds, response);
    LinkedHashMap<String, Integer> occurenceMap = getTotalOccurences(groupMap);

    calculateTFIDF(groupMap, occurenceMap, response);
  }

  /** 
   * Queries the database to return a list of all groupIds. 
   */
  private List<Long> getGroupIds() {
    // Preparing query instance to retrieve all Groups
    Query query = new Query("Group");
    PreparedQuery results = datastore.prepare(query);

    List<Long> groupIds = new ArrayList<>();
    for (Entity entity : results.asIterable()) {
      long groupId = (long) entity.getKey().getId();
      groupIds.add(groupId);
    }

    return groupIds;
  }

  /** 
   * Returns a mapping of all groupIds to their n-grams (and occurences). 
   */
  private LinkedHashMap<Long, LinkedHashMap<String, Integer>> getGroupMap(
      List<Long> groupIds, HttpServletResponse response) throws IOException {

    LinkedHashMap<Long, LinkedHashMap<String, Integer>> groupMap = 
        new LinkedHashMap<Long, LinkedHashMap<String, Integer>>();
        
    for (long groupId : groupIds) {
      Entity groupEntity = ServletHelper.getEntityFromId(response, groupId, datastore, "Group");
      List<Long> postIds = (ArrayList<Long>) groupEntity.getProperty("posts");
      List<String> textData = getGroupPostsText(postIds, response);
      addChallengeText(groupEntity, textData, response);

      // TODO: Parallelize this process to occur for each string concurrently.
      ArrayList<LinkedHashMap<String, Integer>> ngramsList = new ArrayList<>();
      for (String text : textData) {
        ngramsList.add(TFIDFStringHelper.ngramTokenizer(text));
      }
      groupMap.put(groupId, TFIDFStringHelper.combineMaps(ngramsList));
    }

    return groupMap;
  }

  /** 
   * Returns a list of Post content Strings, given a list of postIds. 
   */
  private List<String> getGroupPostsText(List<Long> postIds, HttpServletResponse response) 
      throws IOException {

    List<String> textData = new ArrayList<>();
    for (long postId : postIds) {
      Entity postEntity = ServletHelper.getEntityFromId(response, postId, datastore, "Post");
      String postText = (String) postEntity.getProperty("postText");
      textData.add(postText);
      addCommentsText(postEntity, textData);
    }

    return textData;
  }

  /**
   * Given a post, adds that post's comments to a list of group text data.
   */
  private void addCommentsText(Entity postEntity, List<String> textData) {
    
    ArrayList<EmbeddedEntity> comments = 
        (ArrayList<EmbeddedEntity>) postEntity.getProperty("comments");

    if (comments != null) {
      for (EmbeddedEntity commentEntity : comments) {
        String commentText = (String) commentEntity.getProperty("commentText");
        textData.add(commentText);
      }
    }
  }

  /**
   * Add's a group's challenge names into a list of group text data.
   */
  private void addChallengeText(Entity groupEntity, List<String> textData,
      HttpServletResponse response) throws IOException {
    List<Long> challengeIds = (ArrayList<Long>) groupEntity.getProperty("challenges");
{'fraction_non_alphanumeric': 0.10429447852760736, 'fraction_numerical': 0.0, 'mean_word_length': 4.237880496054115, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3751280', 'n_tokens_mistral': 2722, 'n_tokens_neox': 2666, 'n_words': 646}