<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.sps.servlets;

import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.Filter;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Query.FilterPredicate;
import com.google.maps.model.LatLng;
import com.google.sps.Utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

@WebServlet("/search")
public class SearchServlet extends HttpServlet {

  private static final Logger LOGGER = Logger.getLogger(SearchServlet.class.getName());
  // Conversion rate from mi to km
  private static final double MI_TO_KM = 1.609;
  // Number of keywords to produce
  private static final int NUM_KEYWORDS = 5;
  // Minimum number of instances to be considered a keyword
  private static final int MIN_INSTANCES = 2;
  // List of words to ignore
  // This list is incomplete
  private static final List<String> IRRELEVANT_WORDS =
      new ArrayList<String>(Arrays.asList("the", "is", "for", "in", "of", "so", "to"));

  // Keywords retrieved from the frontend in the doGet method
  private static List<String> searchKeywords;
  // Tags retrieved from the frontend in the doGet method
  private static List<String> searchTags;

  /** Comparator used to compare the relevance of two events to the user and their search tags. */
  private static final Comparator<Entity> TAGS_SEARCH_RELEVANCE =
      new Comparator<Entity>() {
        @Override
        public int compare(Entity o1, Entity o2) {
          int tagResult = compareUsingTags(o1, o2);
          if (tagResult != 0) {
            return tagResult;
          } else {
            return compareDistances(o1, o2);
          }
        }
      };

  /**
   * Comparator used to compare the relevance of two events to the user and their search keywords.
   */
  private static final Comparator<Entity> KEYWORD_SEARCH_RELEVANCE =
      new Comparator<Entity>() {
        @Override
        public int compare(Entity o1, Entity o2) {
          int keywordResult = compareUsingKeywords(o1, o2);
          if (keywordResult != 0) {
            return keywordResult;
          } else {
            return compareDistances(o1, o2);
          }
        }
      };

  /**
   * Comparator used to compare the relevance of two events to the user and their search
   * keywords/tags.
   */
  public static final Comparator<Entity> COMBINE_SEARCH_RELEVANCE =
      new Comparator<Entity>() {
        @Override
        public int compare(Entity o1, Entity o2) {
          int tagResult = compareUsingTags(o1, o2);
          if (tagResult != 0) {
            return tagResult;
          }

          int keywordResult = compareUsingKeywords(o1, o2);
          if (keywordResult != 0) {
            return keywordResult;
          }

          return compareDistances(o1, o2);
        }
      };

  @Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    final long startTime = System.nanoTime();
    // String search query input
    String searchQueryKeywords = request.getParameter("searchKeywords");
    // List of all the tags we are searching for
    String searchQueryTags = request.getParameter("tags");
    int tagCount = 0;
    if (searchQueryTags != null) {
      searchTags = new ArrayList<String>(Arrays.asList(searchQueryTags.split(",")));
      if (!searchTags.get(0).equals("")) {
        tagCount = searchTags.size();
      }
    }

    boolean hasTags = true;
    boolean hasKeywords = true;
    if (searchQueryTags == null || searchTags.get(0).equals("")) {
      hasTags = false;
    }
    if (searchQueryKeywords == null || searchQueryKeywords.equals("")) {
      hasKeywords = false;
    }

    Query query = null;
    int keywordCount = 0;
    if (hasKeywords) {
      // List of keywords we're using to search
      searchKeywords = new ArrayList<String>(getSeparateWords(searchQueryKeywords));
      for (int i = 0; i < searchKeywords.size(); i++) {
        searchKeywords.set(i, searchKeywords.get(i).toLowerCase());
      }
      searchKeywords.removeAll(IRRELEVANT_WORDS);
      keywordCount = searchKeywords.size();
    } else {
      searchKeywords = new ArrayList<String>();
    }

    if (!hasTags && hasKeywords) {
      // Filter to check if the event has any of the keywords we're searching for
      Filter keywordsFilter = new FilterPredicate("keywords", FilterOperator.IN, searchKeywords);
      query = new Query("Event").setFilter(keywordsFilter);
    } else if (hasTags) {
      Filter tagsFilter = new FilterPredicate("tags", FilterOperator.IN, searchTags);
      query = new Query("Event").setFilter(tagsFilter);
    } else {
      query = new Query("Event");
    }

    DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
    PreparedQuery results = datastore.prepare(query);
    final long entityCount = results.countEntities(FetchOptions.Builder.withDefaults());
    List<Entity> events =
        new ArrayList<Entity>(results.asList(FetchOptions.Builder.withDefaults()));

    // Get location of user
    String location = request.getParameter("location");
    LatLng userLocation = Utils.getLatLng(location);

    // Get distance between user and the location of all the events
    for (Entity event : events) {
      LatLng eventLocation = Utils.getLatLng(event.getProperty("address").toString());

      int distance = Utils.getDistance(userLocation, eventLocation);
      event.setProperty("distance", distance);
    }

    // Get location cutoff converted from mi to km
    int cutoff =
        Math.toIntExact(
            Math.round(
                Integer.parseInt(Utils.getParameter(request, "searchDistance", "")) * MI_TO_KM));
    // Remove from the list any events outside the cutoff or that aren't drivable
    events.removeIf(
        e -> ((int) e.getProperty("distance")) > cutoff || ((int) e.getProperty("distance")) < 0);<commit_msg>Begin addition of GeoPt system for Datastore filter<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.sps.servlets;

import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.Filter;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Query.FilterPredicate;
import com.google.maps.model.LatLng;
import com.google.sps.Utils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import com.google.appengine.api.datastore.Query.GeoRegion.Circle;
import com.google.appengine.api.datastore.GeoPt;
import com.google.appengine.api.datastore.Query.StContainsFilter;

@WebServlet("/search")
public class SearchServlet extends HttpServlet {

  private static final Logger LOGGER = Logger.getLogger(SearchServlet.class.getName());
  // Conversion rate from mi to km
  private static final double MI_TO_KM = 1.609;
  // Number of keywords to produce
  private static final int NUM_KEYWORDS = 5;
  // Minimum number of instances to be considered a keyword
  private static final int MIN_INSTANCES = 2;
  // List of words to ignore
  // This list is incomplete
  private static final List<String> IRRELEVANT_WORDS =
      new ArrayList<String>(Arrays.asList("the", "is", "for", "in", "of", "so", "to"));

  // Keywords retrieved from the frontend in the doGet method
  private static List<String> searchKeywords;
  // Tags retrieved from the frontend in the doGet method
  private static List<String> searchTags;

  /** Comparator used to compare the relevance of two events to the user and their search tags. */
  private static final Comparator<Entity> TAGS_SEARCH_RELEVANCE =
      new Comparator<Entity>() {
        @Override
        public int compare(Entity o1, Entity o2) {
          int tagResult = compareUsingTags(o1, o2);
          if (tagResult != 0) {
            return tagResult;
          } else {
            return compareDistances(o1, o2);
          }
        }
      };

  /**
   * Comparator used to compare the relevance of two events to the user and their search keywords.
   */
  private static final Comparator<Entity> KEYWORD_SEARCH_RELEVANCE =
      new Comparator<Entity>() {
        @Override
        public int compare(Entity o1, Entity o2) {
          int keywordResult = compareUsingKeywords(o1, o2);
          if (keywordResult != 0) {
            return keywordResult;
          } else {
            return compareDistances(o1, o2);
          }
        }
      };

  /**
   * Comparator used to compare the relevance of two events to the user and their search
   * keywords/tags.
   */
  public static final Comparator<Entity> COMBINE_SEARCH_RELEVANCE =
      new Comparator<Entity>() {
        @Override
        public int compare(Entity o1, Entity o2) {
          int tagResult = compareUsingTags(o1, o2);
          if (tagResult != 0) {
            return tagResult;
          }

          int keywordResult = compareUsingKeywords(o1, o2);
          if (keywordResult != 0) {
            return keywordResult;
          }

          return compareDistances(o1, o2);
        }
      };

  @Override
  public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
    final long startTime = System.nanoTime();
    // String search query input
    String searchQueryKeywords = request.getParameter("searchKeywords");
    // List of all the tags we are searching for
    String searchQueryTags = request.getParameter("tags");
    int tagCount = 0;
    if (searchQueryTags != null) {
      searchTags = new ArrayList<String>(Arrays.asList(searchQueryTags.split(",")));
      if (!searchTags.get(0).equals("")) {
        tagCount = searchTags.size();
      }
    }

    boolean hasTags = true;
    boolean hasKeywords = true;
    if (searchQueryTags == null || searchTags.get(0).equals("")) {
      hasTags = false;
    }
    if (searchQueryKeywords == null || searchQueryKeywords.equals("")) {
      hasKeywords = false;
    }

    Query query = null;
    int keywordCount = 0;
    if (hasKeywords) {
      // List of keywords we're using to search
      searchKeywords = new ArrayList<String>(getSeparateWords(searchQueryKeywords));
      for (int i = 0; i < searchKeywords.size(); i++) {
        searchKeywords.set(i, searchKeywords.get(i).toLowerCase());
      }
      searchKeywords.removeAll(IRRELEVANT_WORDS);
      keywordCount = searchKeywords.size();
    } else {
      searchKeywords = new ArrayList<String>();
    }

    // Get location of user
    String location = request.getParameter("location");
    LatLng userLocation = Utils.getLatLng(location);

    // Get location cutoff converted from mi to km
    int cutoff =
        Math.toIntExact(
            Math.round(
                Integer.parseInt(Utils.getParameter(request, "searchDistance", "")) * MI_TO_KM));

    GeoPt userPt = new GeoPt(userLocation.lat(), userLocation.lng());
    Circle region = new Circle(userPt, cutoff * 1000);
    Filter distanceFilter = new StContainsFilter("geoPt", region);
    query = new Query("Event").setFilter(distanceFilter);

    DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
    PreparedQuery results = datastore.prepare(query);
    final long entityCount = results.countEntities(FetchOptions.Builder.withDefaults());
    List<Entity> events =
        new ArrayList<Entity>(results.asList(FetchOptions.Builder.withDefaults()));

    // Get distance between user and the location of all the events
    for (Entity event : events) {
      LatLng eventLocation = Utils.getLatLng(event.getProperty("address").toString());

      int distance = Utils.getDistance(userLocation, eventLocation);
      event.setProperty("distance", distance);
    }

    // Remove from the list any events outside the cutoff or that aren't drivable
    // Should be deprecated and unneeded now
    events.removeIf(
        e -> ((int) e.getProperty("distance")) > cutoff || ((int) e.getProperty("distance")) < 0);
{'fraction_non_alphanumeric': 0.10194005980112648, 'fraction_numerical': 0.006397329810166191, 'mean_word_length': 3.2324896998234256, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '370356', 'n_tokens_mistral': 4081, 'n_tokens_neox': 3980, 'n_words': 1177}