<commit_before>/*
 * Copyright 2012, Red Hat, Inc. and individual contributors as indicated by the
 * @author tags. See the copyright.txt file in the distribution for a full
 * listing of individual contributors.
 * 
 * This is free software; you can redistribute it and/or modify it under the
 * terms of the GNU Lesser General Public License as published by the Free
 * Software Foundation; either version 2.1 of the License, or (at your option)
 * any later version.
 * 
 * This software is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
 * details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this software; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF
 * site: http://www.fsf.org.
 */
package org.zanata.webtrans.server.rpc;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import org.jboss.seam.ScopeType;
import org.jboss.seam.annotations.In;
import org.jboss.seam.annotations.Logger;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.annotations.Scope;
import org.jboss.seam.log.Log;
import org.zanata.exception.ZanataServiceException;
import org.zanata.model.HLocale;
import org.zanata.model.HTextFlow;
import org.zanata.search.FilterConstraints;
import org.zanata.security.ZanataIdentity;
import org.zanata.service.LocaleService;
import org.zanata.service.TextFlowSearchService;
import org.zanata.webtrans.server.ActionHandlerFor;
import org.zanata.webtrans.shared.model.TransUnit;
import org.zanata.webtrans.shared.rpc.GetProjectTransUnitLists;
import org.zanata.webtrans.shared.rpc.GetProjectTransUnitListsResult;
import com.google.common.base.Strings;
import com.ibm.icu.lang.UCharacter;

import lombok.extern.slf4j.Slf4j;
import net.customware.gwt.dispatch.server.ExecutionContext;
import net.customware.gwt.dispatch.shared.ActionException;

/**
 * @see GetProjectTransUnitLists
 * @author <NAME>, <EMAIL>
 */
@Name("webtrans.gwt.GetProjectTransUnitListsHandler")
@Scope(ScopeType.STATELESS)
@ActionHandlerFor(GetProjectTransUnitLists.class)
@Slf4j
public class GetProjectTransUnitListsHandler extends AbstractActionHandler<GetProjectTransUnitLists, GetProjectTransUnitListsResult>
{
   @In
   private LocaleService localeServiceImpl;

   @In
   private TextFlowSearchService textFlowSearchServiceImpl;

   @In
   private TransUnitTransformer transUnitTransformer;

   @In
   private ZanataIdentity identity;

   @Override
   public GetProjectTransUnitListsResult execute(GetProjectTransUnitLists action, ExecutionContext context) throws ActionException
   {
      identity.checkLoggedIn();
      log.info("Searching all targets for workspace {}", action.getWorkspaceId());

      HashMap<Long, List<TransUnit>> matchingTUs = new HashMap<Long, List<TransUnit>>();
      HashMap<Long, String> docPaths = new HashMap<Long, String>();
      if (Strings.isNullOrEmpty(action.getSearchString()))
      {
         // TODO empty searches shouldn't be requested, consider replacing this
         // with an error, or making behaviour return all targets for the
         // project (consider performance).
         return new GetProjectTransUnitListsResult(action, docPaths, matchingTUs);
      }

      FilterConstraints filterConstraints = FilterConstraints.filterBy(action.getSearchString()).excludeNew().caseSensitive(action.isCaseSensitive());
      if (!action.isSearchInSource())
      {
         filterConstraints.ignoreSource();
      }
      if (!action.isSearchInTarget())
      {
         filterConstraints.ignoreTarget();
      }
      // TODO handle exception thrown by search service
      List<HTextFlow> matchingFlows = textFlowSearchServiceImpl.findTextFlows(action.getWorkspaceId(), action.getDocumentPaths(), filterConstraints);
      log.info("Returned {} results for search", matchingFlows.size());

      HLocale hLocale;
      try
      {
         hLocale = localeServiceImpl.validateLocaleByProjectIteration(action.getWorkspaceId().getLocaleId(), action.getWorkspaceId().getProjectIterationId().getProjectSlug(), action.getWorkspaceId().getProjectIterationId().getIterationSlug());
      }
      catch (ZanataServiceException e)
      {
         throw new ActionException(e);
      }

      //FIXME remove when analyzer handles leading & trailing whitespace
      boolean needsWhitespaceCheck = !action.getSearchString().equals(action.getSearchString().trim());
      String searchString = action.getSearchString();
      if (!action.isCaseSensitive())
      {
         searchString = foldCase(searchString);
      }


      for (HTextFlow textFlow : matchingFlows)
      {
         // FIXME temporary check for leading and trailing whitespace to compensate
         // for NGramAnalyzer trimming strings before tokenization. This should
         // be removed when updating to a lucene version with the whitespace
         // issue resolved.
         if (needsWhitespaceCheck)
         {
            boolean whitespaceMatch = false;
            if (action.isSearchInSource())
            {
               for (String content : textFlow.getContents())
               {
                  String contentStr = content;
                  if (!action.isCaseSensitive())
                  {
                     contentStr = foldCase(contentStr);
                  }
                  if (contentStr.contains(searchString))
                  {
                     whitespaceMatch = true;
                     break;
                  }
               }
            }
            if (!whitespaceMatch && action.isSearchInTarget())
            {
               // FIXME hibernate n + 1 select happening here
               List<String> targetContents = textFlow.getTargets().get(hLocale.getId()).getContents();
               for (String content : targetContents)
               {
                  String contentStr = content;
                  if (!action.isCaseSensitive())
                  {
                     contentStr = foldCase(contentStr);
                  }
                  if (contentStr.contains(searchString))
                  {
                     whitespaceMatch = true;
                     break;
                  }
               }
            }
            if (!whitespaceMatch)
            {
               continue;
            }
         }

         List<TransUnit> listForDoc = matchingTUs.get(textFlow.getDocument().getId());
         if (listForDoc == null)
         {
            listForDoc = new ArrayList<TransUnit>();
         }

         TransUnit transUnit = transUnitTransformer.transform(textFlow, hLocale);
         listForDoc.add(transUnit);
         matchingTUs.put(textFlow.getDocument().getId(), listForDoc);
         docPaths.put(textFlow.getDocument().getId(), textFlow.getDocument().getDocId());
      }

      return new GetProjectTransUnitListsResult(action, docPaths, matchingTUs);
   }

   private String foldCase(String original)
   {
      char[] buffer = original.toCharArray();
      for (int i=0; i<buffer.length; i++)
      {
         buffer[i] = (char) UCharacter.foldCase(buffer[i], true);
      }
      return new String(buffer);
   }

   @Override
   public void rollback(GetProjectTransUnitLists action, GetProjectTransUnitListsResult result, ExecutionContext context) throws ActionException
   {
   }

}<commit_msg>refactor GetProjectTransUnitListsHandler<commit_after>/*
 * Copyright 2012, Red Hat, Inc. and individual contributors as indicated by the
 * @author tags. See the copyright.txt file in the distribution for a full
 * listing of individual contributors.
 * 
 * This is free software; you can redistribute it and/or modify it under the
 * terms of the GNU Lesser General Public License as published by the Free
 * Software Foundation; either version 2.1 of the License, or (at your option)
 * any later version.
 * 
 * This software is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more
 * details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with this software; if not, write to the Free Software Foundation,
 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA, or see the FSF
 * site: http://www.fsf.org.
 */
package org.zanata.webtrans.server.rpc;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import org.jboss.seam.ScopeType;
import org.jboss.seam.annotations.In;
import org.jboss.seam.annotations.Name;
import org.jboss.seam.annotations.Scope;
import org.zanata.exception.ZanataServiceException;
import org.zanata.model.HLocale;
import org.zanata.model.HTextFlow;
import org.zanata.search.FilterConstraints;
import org.zanata.security.ZanataIdentity;
import org.zanata.service.LocaleService;
import org.zanata.service.TextFlowSearchService;
import org.zanata.webtrans.server.ActionHandlerFor;
import org.zanata.webtrans.shared.model.TransUnit;
import org.zanata.webtrans.shared.rpc.GetProjectTransUnitLists;
import org.zanata.webtrans.shared.rpc.GetProjectTransUnitListsResult;
import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;
import com.ibm.icu.lang.UCharacter;

import lombok.extern.slf4j.Slf4j;
import net.customware.gwt.dispatch.server.ExecutionContext;
import net.customware.gwt.dispatch.shared.ActionException;

/**
 * @see GetProjectTransUnitLists
 * @author <NAME>, <EMAIL>
 */
@Name("webtrans.gwt.GetProjectTransUnitListsHandler")
@Scope(ScopeType.STATELESS)
@ActionHandlerFor(GetProjectTransUnitLists.class)
@Slf4j
public class GetProjectTransUnitListsHandler extends AbstractActionHandler<GetProjectTransUnitLists, GetProjectTransUnitListsResult>
{
   @In
   private LocaleService localeServiceImpl;

   @In
   private TextFlowSearchService textFlowSearchServiceImpl;

   @In
   private TransUnitTransformer transUnitTransformer;

   @In
   private ZanataIdentity identity;

   @Override
   public GetProjectTransUnitListsResult execute(GetProjectTransUnitLists action, ExecutionContext context) throws ActionException
   {
      identity.checkLoggedIn();
      log.info("Searching all targets for workspace {}", action.getWorkspaceId());
      HLocale hLocale;
      try
      {
         hLocale = localeServiceImpl.validateLocaleByProjectIteration(action.getWorkspaceId().getLocaleId(), action.getWorkspaceId().getProjectIterationId().getProjectSlug(), action.getWorkspaceId().getProjectIterationId().getIterationSlug());
      }
      catch (ZanataServiceException e)
      {
         throw new ActionException(e);
      }

      HashMap<Long, List<TransUnit>> matchingTUs = new HashMap<Long, List<TransUnit>>();
      HashMap<Long, String> docPaths = new HashMap<Long, String>();
      if (Strings.isNullOrEmpty(action.getSearchString()))
      {
         // TODO empty searches shouldn't be requested, consider replacing this
         // with an error, or making behaviour return all targets for the
         // project (consider performance).
         return new GetProjectTransUnitListsResult(action, docPaths, matchingTUs);
      }

      FilterConstraints filterConstraints = FilterConstraints.filterBy(action.getSearchString()).excludeNew().caseSensitive(action.isCaseSensitive());
      if (!action.isSearchInSource())
      {
         filterConstraints.ignoreSource();
      }
      if (!action.isSearchInTarget())
      {
         filterConstraints.ignoreTarget();
      }
      // TODO handle exception thrown by search service
      List<HTextFlow> matchingFlows = textFlowSearchServiceImpl.findTextFlows(action.getWorkspaceId(), action.getDocumentPaths(), filterConstraints);
      log.info("Returned {} results for search", matchingFlows.size());

      //FIXME remove when analyzer handles leading & trailing whitespace
      boolean needsWhitespaceCheck = !action.getSearchString().equals(action.getSearchString().trim());
      Iterable<HTextFlow> result = matchingFlows;
      if (needsWhitespaceCheck)
      {
         // FIXME temporary check for leading and trailing whitespace to compensate
         // for NGramAnalyzer trimming strings before tokenization. This should
         // be removed when updating to a lucene version with the whitespace
         // issue resolved.
         result = Iterables.filter(matchingFlows, new WhitespaceMatchPredicate(action, hLocale.getId()));
      }

      for (HTextFlow textFlow : result)
      {
         List<TransUnit> listForDoc = matchingTUs.get(textFlow.getDocument().getId());
         if (listForDoc == null)
         {
            listForDoc = new ArrayList<TransUnit>();
         }

         TransUnit transUnit = transUnitTransformer.transform(textFlow, hLocale);
         listForDoc.add(transUnit);
         matchingTUs.put(textFlow.getDocument().getId(), listForDoc);
         docPaths.put(textFlow.getDocument().getId(), textFlow.getDocument().getDocId());
      }
      return new GetProjectTransUnitListsResult(action, docPaths, matchingTUs);
   }

   private static String foldCase(String original)
   {
      char[] buffer = original.toCharArray();
      for (int i=0; i<buffer.length; i++)
      {
         buffer[i] = (char) UCharacter.foldCase(buffer[i], true);
      }
      return new String(buffer);
   }

   @Override
   public void rollback(GetProjectTransUnitLists action, GetProjectTransUnitListsResult result, ExecutionContext context) throws ActionException
   {
   }

   private static class WhitespaceMatchPredicate implements Predicate<HTextFlow>
   {

      private final GetProjectTransUnitLists action;
      private final Long localeId;
      private ContainSearchTermPredicate containSearchTermPredicate;

      private WhitespaceMatchPredicate(GetProjectTransUnitLists action, Long localeId)
      {
         this.action = action;
         this.localeId = localeId;
         containSearchTermPredicate = new ContainSearchTermPredicate(action);
      }

      @Override
      public boolean apply(HTextFlow textFlow)
      {
         if (action.isSearchInSource())
         {
            Optional<String> optional = Iterables.tryFind(textFlow.getContents(), containSearchTermPredicate);
            if (optional.isPresent())
            {
               return true;
            }
         }

         if (action.isSearchInTarget())
         {
            // FIXME hibernate n + 1 select happening here
            List<String> targetContents = textFlow.getTargets().get(localeId).getContents();
            Optional<String> optional = Iterables.tryFind(targetContents, containSearchTermPredicate);
            return optional.isPresent();
         }
         return false;
      }

   }

   private static class ContainSearchTermPredicate implements Predicate<String>
   {
      private final String searchTerm;
      private final boolean caseSensitive;

      private ContainSearchTermPredicate(GetProjectTransUnitLists action)
      {
         caseSensitive = action.isCaseSensitive();
         searchTerm = !caseSensitive ? foldCase(action.getSearchString()) : action.getSearchString();
      }

      @Override
      public boolean apply(String input)
      {
         String contentStr = !caseSensitive ? foldCase(input) : input;
         return contentStr.contains(searchTerm);
      }
   }
}
{'fraction_non_alphanumeric': 0.08884059823597086, 'fraction_numerical': 0.0028122203758149046, 'mean_word_length': 3.1328578975171686, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7254730', 'n_tokens_mistral': 4220, 'n_tokens_neox': 4079, 'n_words': 1033}