<commit_before>/*
 * Copyright 2012, Red Hat, Inc. and individual contributors
 * as indicated by the @author tags. See the copyright.txt file in the
 * distribution for a full listing of individual contributors.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package org.zanata.client.commands.push;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.codec.binary.Hex;
import org.jboss.resteasy.client.ClientResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.zanata.client.commands.PushPullCommand;
import org.zanata.client.commands.PushPullType;
import org.zanata.client.commands.push.RawPushStrategy.TranslationFilesVisitor;
import org.zanata.client.config.LocaleMapping;
import org.zanata.client.exceptions.ConfigException;
import org.zanata.client.util.ConsoleUtils;
import org.zanata.rest.DocumentFileUploadForm;
import org.zanata.rest.StringSet;
import org.zanata.rest.client.IFileResource;
import org.zanata.rest.client.ISourceDocResource;
import org.zanata.rest.client.ITranslatedDocResource;
import org.zanata.rest.client.ZanataProxyFactory;
import org.zanata.rest.dto.ChunkUploadResponse;

/**
 * Command to send files directly to the server without parsing on the client.
 * 
 * @author <NAME>, <a href="mailto:<EMAIL>"><EMAIL></a>
 *
 */
public class RawPushCommand extends PushPullCommand<PushOptions>
{
   private static final Logger log = LoggerFactory.getLogger(PushCommand.class);

   protected final IFileResource fileResource;

   public RawPushCommand(PushOptions opts)
   {
      super(opts);
      this.fileResource = getRequestFactory().getFileResource();
   }

   public RawPushCommand(PushOptions opts, ZanataProxyFactory factory, ISourceDocResource sourceDocResource, ITranslatedDocResource translationResources, URI uri)
   {
      super(opts, factory, sourceDocResource, translationResources, uri);
      this.fileResource = factory.getFileResource();
   }

   @Override
   public void run() throws Exception
   {
      PushCommand.logOptions(log, getOpts());
      log.warn("Using EXPERIMENTAL project type 'raw'.");

      // only supporting single module for now

      File sourceDir = getOpts().getSrcDir();
      if (!sourceDir.exists())
      {
         boolean enableModules = getOpts().getEnableModules();
         // TODO remove when modules implemented
         if (enableModules)
         {
            log.warn("enableModules=true but multi-modules not yet supported for this command. Using single module push.");
            enableModules = false;
         }

         if (enableModules)
         {
            log.info("source directory '" + sourceDir + "' not found; skipping docs push for module " + getOpts().getCurrentModule());
            return;
         }
         else
         {
            throw new RuntimeException("directory '" + sourceDir + "' does not exist - check srcDir option");
         }
      }

      RawPushStrategy strat = new RawPushStrategy();
      strat.setPushOptions(getOpts());

      List<String> types = new ArrayList<String>();

      ClientResponse<String> response = fileResource.acceptedFileTypes();
      StringSet serverAcceptedTypes = new StringSet(response.getEntity());
      for (String type : getOpts().getFileTypes())
      {
         if (serverAcceptedTypes.contains(type))
         {
            types.add(type);
         }
         else
         {
            log.warn("Requested type '{}' is not supported by the target server and will be ignored.", type);
         }
      }

      if (types.isEmpty())
      {
         log.info("no valid types specified; nothing to do");
         return;
      }

      String[] srcFiles = strat.getSrcFiles(sourceDir, getOpts().getIncludes(), getOpts().getExcludes(), types, true, getOpts().getCaseSensitive());

      // TODO handle obsolete document deletion
      log.warn("Obsolete document removal is not yet implemented, no documents will be removed from the server.");

      if (srcFiles.length == 0)
      {
         log.info("no documents in module: {}; nothing to do", getOpts().getCurrentModule());
         return;<commit_msg>prevent null pointer exception when file types are not specified for raw push command<commit_after>/*
 * Copyright 2012, Red Hat, Inc. and individual contributors
 * as indicated by the @author tags. See the copyright.txt file in the
 * distribution for a full listing of individual contributors.
 *
 * This is free software; you can redistribute it and/or modify it
 * under the terms of the GNU Lesser General Public License as
 * published by the Free Software Foundation; either version 2.1 of
 * the License, or (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this software; if not, write to the Free
 * Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
 * 02110-1301 USA, or see the FSF site: http://www.fsf.org.
 */
package org.zanata.client.commands.push;

import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;
import java.security.DigestInputStream;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.codec.binary.Hex;
import org.jboss.resteasy.client.ClientResponse;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.zanata.client.commands.PushPullCommand;
import org.zanata.client.commands.PushPullType;
import org.zanata.client.commands.push.RawPushStrategy.TranslationFilesVisitor;
import org.zanata.client.config.LocaleMapping;
import org.zanata.client.exceptions.ConfigException;
import org.zanata.client.util.ConsoleUtils;
import org.zanata.rest.DocumentFileUploadForm;
import org.zanata.rest.StringSet;
import org.zanata.rest.client.IFileResource;
import org.zanata.rest.client.ISourceDocResource;
import org.zanata.rest.client.ITranslatedDocResource;
import org.zanata.rest.client.ZanataProxyFactory;
import org.zanata.rest.dto.ChunkUploadResponse;

/**
 * Command to send files directly to the server without parsing on the client.
 * 
 * @author <NAME>, <a href="mailto:<EMAIL>"><EMAIL></a>
 *
 */
public class RawPushCommand extends PushPullCommand<PushOptions>
{
   private static final Logger log = LoggerFactory.getLogger(PushCommand.class);

   protected final IFileResource fileResource;

   public RawPushCommand(PushOptions opts)
   {
      super(opts);
      this.fileResource = getRequestFactory().getFileResource();
   }

   public RawPushCommand(PushOptions opts, ZanataProxyFactory factory, ISourceDocResource sourceDocResource, ITranslatedDocResource translationResources, URI uri)
   {
      super(opts, factory, sourceDocResource, translationResources, uri);
      this.fileResource = factory.getFileResource();
   }

   @Override
   public void run() throws Exception
   {
      PushCommand.logOptions(log, getOpts());
      log.warn("Using EXPERIMENTAL project type 'raw'.");

      // only supporting single module for now

      File sourceDir = getOpts().getSrcDir();
      if (!sourceDir.exists())
      {
         boolean enableModules = getOpts().getEnableModules();
         // TODO remove when modules implemented
         if (enableModules)
         {
            log.warn("enableModules=true but multi-modules not yet supported for this command. Using single module push.");
            enableModules = false;
         }

         if (enableModules)
         {
            log.info("source directory '" + sourceDir + "' not found; skipping docs push for module " + getOpts().getCurrentModule());
            return;
         }
         else
         {
            throw new RuntimeException("directory '" + sourceDir + "' does not exist - check srcDir option");
         }
      }

      RawPushStrategy strat = new RawPushStrategy();
      strat.setPushOptions(getOpts());

      List<String> types = new ArrayList<String>();

      ClientResponse<String> response = fileResource.acceptedFileTypes();
      StringSet serverAcceptedTypes = new StringSet(response.getEntity());
      if (getOpts().getFileTypes() != null)
      {
         for (String type : getOpts().getFileTypes())
         {
            if (serverAcceptedTypes.contains(type))
            {
               types.add(type);
            }
            else
            {
               log.warn("Requested type '{}' is not supported by the target server and will be ignored.", type);
            }
         }
      }

      if (types.isEmpty())
      {
         log.info("no valid types specified; nothing to do");
         return;
      }

      String[] srcFiles = strat.getSrcFiles(sourceDir, getOpts().getIncludes(), getOpts().getExcludes(), types, true, getOpts().getCaseSensitive());

      // TODO handle obsolete document deletion
      log.warn("Obsolete document removal is not yet implemented, no documents will be removed from the server.");

      if (srcFiles.length == 0)
      {
         log.info("no documents in module: {}; nothing to do", getOpts().getCurrentModule());
         return;
{'fraction_non_alphanumeric': 0.0958179930142547, 'fraction_numerical': 0.0037760785424336826, 'mean_word_length': 3.366859027205276, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '144534', 'n_tokens_mistral': 2870, 'n_tokens_neox': 2785, 'n_words': 875}