<commit_before>/*
 * CodeSearchOracle.java
 *
 * Copyright (C) 2009-12 by RStudio, Inc.
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */
package org.rstudio.studio.client.workbench.codesearch;

import java.util.ArrayList;
import java.util.Comparator;

import org.rstudio.core.client.CodeNavigationTarget;
import org.rstudio.core.client.DuplicateHelper;
import org.rstudio.core.client.FilePosition;
import org.rstudio.core.client.Invalidation;
import org.rstudio.core.client.StringUtil;
import org.rstudio.core.client.TimeBufferedCommand;
import org.rstudio.core.client.files.FileSystemItem;
import org.rstudio.core.client.regex.Match;
import org.rstudio.core.client.regex.Pattern;
import org.rstudio.studio.client.common.SimpleRequestCallback;
import org.rstudio.studio.client.workbench.WorkbenchContext;
import org.rstudio.studio.client.workbench.codesearch.model.CodeSearchResults;
import org.rstudio.studio.client.workbench.codesearch.model.FileItem;
import org.rstudio.studio.client.workbench.codesearch.model.SourceItem;
import org.rstudio.studio.client.workbench.codesearch.model.CodeSearchServerOperations;

import com.google.gwt.user.client.ui.SuggestOracle;
import com.google.inject.Inject;

public class CodeSearchOracle extends SuggestOracle
{
   @Inject
   public CodeSearchOracle(CodeSearchServerOperations server,
                           WorkbenchContext workbenchContext)
   {
      server_ = server;
      workbenchContext_ = workbenchContext;
   }
   
   private static int scoreMatch(CodeSearchSuggestion suggestion, String query)
   {
      return scoreMatch(suggestion.getMatchedString(), query, suggestion.isFileTarget());
   }
   
   // NOTE: When modifying this function, you should ensure that the associated
   // code on the server side is modified to include the same logic as well!
   // (see: SessionCodeSearch.cpp)
   public static int scoreMatch(String suggestion, String query, boolean isFile)
   {
      String suggestionLower = suggestion.toLowerCase();
      String queryLower = query.toLowerCase();
      
      // No penalty for identical results
      if (suggestion == query)
         return 0;
      
      int query_n = query.length();
      
      int totalPenalty = 0;
      
      // Get query matches in string (ordered)
      // Note: we have already guaranteed this to be a subsequence so
      // this will succeed
      int[] matches = StringUtil.subsequenceIndices(suggestionLower, queryLower);
      
      // Loop over the matches and assign a score
      for (int j = 0; j < query_n; j++)
      {
         int matchPos = matches[j];
         
         // The initial penalty is equal to the match position
         int penalty = matchPos;
         
         // Less penalty if character follows special delim
         if (matchPos >= 1)
         {
            char prevChar = suggestionLower.charAt(matchPos - 1);
            if (prevChar == '_' || prevChar == '-' ||
                  (!isFile && prevChar == '.'))
            {
               penalty = j;
            }
         }
         
         // Less penalty for case-sensitive matches
         if (suggestion.charAt(matchPos) == query.charAt(j))
            penalty--;
         
         // More penalty for 'uninteresting' files (e.g. .Rd)
         String extension = StringUtil.getExtension(suggestionLower);
         if (extension.toLowerCase() == "rd")
            penalty += 3;
         
         totalPenalty += penalty;
      }
      
      // Penalize file targets
      if (isFile)
         totalPenalty++;
      
      return totalPenalty;
   }
   
   @Override
   public void requestSuggestions(final Request request, 
                                  final Callback callback)
   {    
      // invalidate any outstanding search
      searchInvalidation_.invalidate();
      
      // first see if we can serve the request from the cache
      for (int i=resultCache_.size() - 1; i >= 0; i--)
      {
         // get the previous result
         SearchResult res = resultCache_.get(i);
         
         // exact match of previous query
         if (request.getQuery().equals(res.getQuery()))
         {
            callback.onSuggestionsReady(request, 
                                        new Response(res.getSuggestions()));
            return;
         }
         
         // if this query is a further refinement of a non-overflowed 
         // previous query then satisfy it by filtering the previous results
         if (!res.getMoreAvailable() && 
             request.getQuery().startsWith(res.getQuery()))
         {
            Pattern pattern = null;
            String query = request.getQuery();
            String queryLower = query.toLowerCase();
            
            if (queryLower.indexOf('*') != -1)
               pattern = patternForTerm(queryLower);
            
            ArrayList<CodeSearchSuggestion> suggestions =
                                       new ArrayList<CodeSearchSuggestion>();
            for (int s=0; s<res.getSuggestions().size(); s++)
            {
               CodeSearchSuggestion sugg = res.getSuggestions().get(s);
               
               String name = sugg.getMatchedString().toLowerCase();
               if (pattern != null)
               {
                  Match match = pattern.match(name, 0);
                  if (match != null && match.getIndex() == 0)
                     suggestions.add(sugg);
               }
               else
               {
                  int colonIndex = query.indexOf(":");
                  if (colonIndex == -1)
                     colonIndex = query.length();
                  
                  if (StringUtil.isSubsequence(name, query.substring(0, colonIndex)))
                     suggestions.add(sugg);
               }
            }
            
            // process and cache suggestions. note that this adds an item to
            // the end of the resultCache_ (which we are currently iterating
            // over) no biggie because we are about to return from the loop
            suggestions = processSuggestions(request, suggestions, false);
            
            // sort suggestions
            sortSuggestions(suggestions, query);
            
            // return suggestions
            callback.onSuggestionsReady(request, new Response(suggestions));
            
            return;
         } 
      }
      
      // failed to short-circuit via the cache, hit the server
      codeSearch_.enqueRequest(request, callback); 
   }
     
   public CodeNavigationTarget navigationTarget(String query,
                                                Suggestion suggestion)<commit_msg>make subsequence check case-insensitive<commit_after>/*
 * CodeSearchOracle.java
 *
 * Copyright (C) 2009-12 by RStudio, Inc.
 *
 * Unless you have received this program directly from RStudio pursuant
 * to the terms of a commercial license agreement with RStudio, then
 * this program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */
package org.rstudio.studio.client.workbench.codesearch;

import java.util.ArrayList;
import java.util.Comparator;

import org.rstudio.core.client.CodeNavigationTarget;
import org.rstudio.core.client.DuplicateHelper;
import org.rstudio.core.client.FilePosition;
import org.rstudio.core.client.Invalidation;
import org.rstudio.core.client.StringUtil;
import org.rstudio.core.client.TimeBufferedCommand;
import org.rstudio.core.client.files.FileSystemItem;
import org.rstudio.core.client.regex.Match;
import org.rstudio.core.client.regex.Pattern;
import org.rstudio.studio.client.common.SimpleRequestCallback;
import org.rstudio.studio.client.workbench.WorkbenchContext;
import org.rstudio.studio.client.workbench.codesearch.model.CodeSearchResults;
import org.rstudio.studio.client.workbench.codesearch.model.FileItem;
import org.rstudio.studio.client.workbench.codesearch.model.SourceItem;
import org.rstudio.studio.client.workbench.codesearch.model.CodeSearchServerOperations;

import com.google.gwt.user.client.ui.SuggestOracle;
import com.google.inject.Inject;

public class CodeSearchOracle extends SuggestOracle
{
   @Inject
   public CodeSearchOracle(CodeSearchServerOperations server,
                           WorkbenchContext workbenchContext)
   {
      server_ = server;
      workbenchContext_ = workbenchContext;
   }
   
   private static int scoreMatch(CodeSearchSuggestion suggestion, String query)
   {
      return scoreMatch(suggestion.getMatchedString(), query, suggestion.isFileTarget());
   }
   
   // NOTE: When modifying this function, you should ensure that the associated
   // code on the server side is modified to include the same logic as well!
   // (see: SessionCodeSearch.cpp)
   public static int scoreMatch(String suggestion, String query, boolean isFile)
   {
      String suggestionLower = suggestion.toLowerCase();
      String queryLower = query.toLowerCase();
      
      // No penalty for identical results
      if (suggestion == query)
         return 0;
      
      int query_n = query.length();
      
      int totalPenalty = 0;
      
      // Get query matches in string (ordered)
      // Note: we have already guaranteed this to be a subsequence so
      // this will succeed
      int[] matches = StringUtil.subsequenceIndices(suggestionLower, queryLower);
      
      // Loop over the matches and assign a score
      for (int j = 0; j < query_n; j++)
      {
         int matchPos = matches[j];
         
         // The initial penalty is equal to the match position
         int penalty = matchPos;
         
         // Less penalty if character follows special delim
         if (matchPos >= 1)
         {
            char prevChar = suggestionLower.charAt(matchPos - 1);
            if (prevChar == '_' || prevChar == '-' ||
                  (!isFile && prevChar == '.'))
            {
               penalty = j;
            }
         }
         
         // Less penalty for case-sensitive matches
         if (suggestion.charAt(matchPos) == query.charAt(j))
            penalty--;
         
         // More penalty for 'uninteresting' files (e.g. .Rd)
         String extension = StringUtil.getExtension(suggestionLower);
         if (extension.toLowerCase() == "rd")
            penalty += 3;
         
         totalPenalty += penalty;
      }
      
      // Penalize file targets
      if (isFile)
         totalPenalty++;
      
      return totalPenalty;
   }
   
   @Override
   public void requestSuggestions(final Request request, 
                                  final Callback callback)
   {    
      // invalidate any outstanding search
      searchInvalidation_.invalidate();
      
      // first see if we can serve the request from the cache
      for (int i=resultCache_.size() - 1; i >= 0; i--)
      {
         // get the previous result
         SearchResult res = resultCache_.get(i);
         
         // exact match of previous query
         if (request.getQuery().equals(res.getQuery()))
         {
            callback.onSuggestionsReady(request, 
                                        new Response(res.getSuggestions()));
            return;
         }
         
         // if this query is a further refinement of a non-overflowed 
         // previous query then satisfy it by filtering the previous results
         if (!res.getMoreAvailable() && 
             request.getQuery().startsWith(res.getQuery()))
         {
            Pattern pattern = null;
            String query = request.getQuery();
            String queryLower = query.toLowerCase();
            
            if (queryLower.indexOf('*') != -1)
               pattern = patternForTerm(queryLower);
            
            ArrayList<CodeSearchSuggestion> suggestions =
                                       new ArrayList<CodeSearchSuggestion>();
            for (int s=0; s<res.getSuggestions().size(); s++)
            {
               CodeSearchSuggestion sugg = res.getSuggestions().get(s);
               
               String name = sugg.getMatchedString().toLowerCase();
               if (pattern != null)
               {
                  Match match = pattern.match(name, 0);
                  if (match != null && match.getIndex() == 0)
                     suggestions.add(sugg);
               }
               else
               {
                  int colonIndex = query.indexOf(":");
                  if (colonIndex == -1)
                     colonIndex = query.length();
                  
                  if (StringUtil.isSubsequence(name, query.substring(0, colonIndex), true))
                     suggestions.add(sugg);
               }
            }
            
            // process and cache suggestions. note that this adds an item to
            // the end of the resultCache_ (which we are currently iterating
            // over) no biggie because we are about to return from the loop
            suggestions = processSuggestions(request, suggestions, false);
            
            // sort suggestions
            sortSuggestions(suggestions, query);
            
            // return suggestions
            callback.onSuggestionsReady(request, new Response(suggestions));
            
            return;
         } 
      }
      
      // failed to short-circuit via the cache, hit the server
      codeSearch_.enqueRequest(request, callback); 
   }
     
   public CodeNavigationTarget navigationTarget(String query,
                                                Suggestion suggestion)
{'fraction_non_alphanumeric': 0.08693213684800898, 'fraction_numerical': 0.003224901850813236, 'mean_word_length': 1.9546396023198012, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6892609', 'n_tokens_mistral': 3688, 'n_tokens_neox': 3582, 'n_words': 1011}