<commit_before>/*
 * Copyright © 2017 <NAME>, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.wrangler.service.explorer;

import co.cask.cdap.api.annotation.TransactionControl;
import co.cask.cdap.api.annotation.TransactionPolicy;
import co.cask.cdap.api.annotation.UseDataSet;
import co.cask.cdap.api.data.schema.Schema;
import co.cask.cdap.api.dataset.Dataset;
import co.cask.cdap.api.service.http.AbstractHttpServiceHandler;
import co.cask.cdap.api.service.http.HttpServiceContext;
import co.cask.cdap.api.service.http.HttpServiceRequest;
import co.cask.cdap.api.service.http.HttpServiceResponder;
import co.cask.cdap.internal.io.SchemaTypeAdapter;
import co.cask.wrangler.PropertyIds;
import co.cask.wrangler.RequestExtractor;
import co.cask.wrangler.SamplingMethod;
import co.cask.wrangler.ServiceUtils;
import co.cask.wrangler.api.Row;
import co.cask.wrangler.dataset.workspace.DataType;
import co.cask.wrangler.dataset.workspace.WorkspaceDataset;
import co.cask.wrangler.sampling.Bernoulli;
import co.cask.wrangler.sampling.Poisson;
import co.cask.wrangler.sampling.Reservoir;
import co.cask.wrangler.service.connections.ConnectionType;
import co.cask.wrangler.utils.ObjectSerDe;
import com.google.common.base.Charsets;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.apache.twill.filesystem.Location;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.security.Security;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.QueryParam;

import static co.cask.wrangler.ServiceUtils.error;
import static co.cask.wrangler.ServiceUtils.sendJson;
import static co.cask.wrangler.service.directive.DirectivesService.WORKSPACE_DATASET;

/**
 * A {@link FilesystemExplorer} is a HTTP Service handler for exploring the filesystem.
 * It provides capabilities for listing file(s) and directories. It also provides metadata.
 */
public class FilesystemExplorer extends AbstractHttpServiceHandler {
  private static final Logger LOG = LoggerFactory.getLogger(FilesystemExplorer.class);
  private static final Gson gson =
    new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create();
  private Explorer explorer;
  private static final String COLUMN_NAME = "body";
  private static final int FILE_SIZE = 10 * 1024 * 1024;

  @UseDataSet(WORKSPACE_DATASET)
  private WorkspaceDataset table;

  /**
   * Lists the content of the path specified using the {@Location}.
   *
   * @param request HTTP Request Handler
   * @param responder HTTP Response Handler
   * @param path to the location in the filesystem
   * @throws Exception
   */
  @TransactionPolicy(value = TransactionControl.EXPLICIT)
  @Path("explorer/fs")
  @GET
  public void list(HttpServiceRequest request, HttpServiceResponder responder,
                   @QueryParam("path") String path,
                   @QueryParam("hidden") boolean hidden) throws Exception {

    try {
      Map<String, Object> listing = explorer.browse(path, hidden);
      sendJson(responder, HttpURLConnection.HTTP_OK, gson.toJson(listing));
    } catch (ExplorerException e) {
      error(responder, e.getMessage());
    }
  }

  /**
   * Given a path, reads the file into the workspace.
   *
   * @param request HTTP request handler.
   * @param responder HTTP response handler.
   * @param path to the location in the filesystem.
   * @param lines number of lines to extracted from file if it's a text/plain.
   * @param sampler sampling method to be used.
   */
  @Path("explorer/fs/read")
  @GET
  public void read(HttpServiceRequest request, HttpServiceResponder responder,
                   @QueryParam("path") String path, @QueryParam("lines") int lines,
                   @QueryParam("sampler") String sampler, @QueryParam("fraction") double fraction) {
    RequestExtractor extractor = new RequestExtractor(request);
    String header = extractor.getHeader(RequestExtractor.CONTENT_TYPE_HEADER, null);

    if (header == null) {
      error(responder, "Content-Type header not specified.");
      return;
    }

    if (header.equalsIgnoreCase("text/plain")) {
      loadSamplableFile(responder, path, lines, fraction, sampler);
    } else if (header.equalsIgnoreCase("application/xml")) {
      loadFile(responder, path, DataType.RECORDS);
    } else if (header.equalsIgnoreCase("application/json")) {
      loadFile(responder, path, DataType.TEXT);
    } else if (header.equalsIgnoreCase("application/avro")
      || header.equalsIgnoreCase("application/protobuf")
      || header.equalsIgnoreCase("application/excel")
      || header.contains("image/")) {<commit_msg>Fixing the check for making files wrangleable when mime type is text/<commit_after>/*
 * Copyright © 2017 Cask Data, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

package co.cask.wrangler.service.explorer;

import co.cask.cdap.api.annotation.TransactionControl;
import co.cask.cdap.api.annotation.TransactionPolicy;
import co.cask.cdap.api.annotation.UseDataSet;
import co.cask.cdap.api.data.schema.Schema;
import co.cask.cdap.api.dataset.Dataset;
import co.cask.cdap.api.service.http.AbstractHttpServiceHandler;
import co.cask.cdap.api.service.http.HttpServiceContext;
import co.cask.cdap.api.service.http.HttpServiceRequest;
import co.cask.cdap.api.service.http.HttpServiceResponder;
import co.cask.cdap.internal.io.SchemaTypeAdapter;
import co.cask.wrangler.PropertyIds;
import co.cask.wrangler.RequestExtractor;
import co.cask.wrangler.SamplingMethod;
import co.cask.wrangler.ServiceUtils;
import co.cask.wrangler.api.Row;
import co.cask.wrangler.dataset.workspace.DataType;
import co.cask.wrangler.dataset.workspace.WorkspaceDataset;
import co.cask.wrangler.sampling.Bernoulli;
import co.cask.wrangler.sampling.Poisson;
import co.cask.wrangler.sampling.Reservoir;
import co.cask.wrangler.service.connections.ConnectionType;
import co.cask.wrangler.utils.ObjectSerDe;
import com.google.common.base.Charsets;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import org.apache.twill.filesystem.Location;
import org.bouncycastle.jce.provider.BouncyCastleProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.BufferedInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.security.Security;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.QueryParam;

import static co.cask.wrangler.ServiceUtils.error;
import static co.cask.wrangler.ServiceUtils.sendJson;
import static co.cask.wrangler.service.directive.DirectivesService.WORKSPACE_DATASET;

/**
 * A {@link FilesystemExplorer} is a HTTP Service handler for exploring the filesystem.
 * It provides capabilities for listing file(s) and directories. It also provides metadata.
 */
public class FilesystemExplorer extends AbstractHttpServiceHandler {
  private static final Logger LOG = LoggerFactory.getLogger(FilesystemExplorer.class);
  private static final Gson gson =
    new GsonBuilder().registerTypeAdapter(Schema.class, new SchemaTypeAdapter()).create();
  private Explorer explorer;
  private static final String COLUMN_NAME = "body";
  private static final int FILE_SIZE = 10 * 1024 * 1024;

  @UseDataSet(WORKSPACE_DATASET)
  private WorkspaceDataset table;

  /**
   * Lists the content of the path specified using the {@Location}.
   *
   * @param request HTTP Request Handler
   * @param responder HTTP Response Handler
   * @param path to the location in the filesystem
   * @throws Exception
   */
  @TransactionPolicy(value = TransactionControl.EXPLICIT)
  @Path("explorer/fs")
  @GET
  public void list(HttpServiceRequest request, HttpServiceResponder responder,
                   @QueryParam("path") String path,
                   @QueryParam("hidden") boolean hidden) throws Exception {

    try {
      Map<String, Object> listing = explorer.browse(path, hidden);
      sendJson(responder, HttpURLConnection.HTTP_OK, gson.toJson(listing));
    } catch (ExplorerException e) {
      error(responder, e.getMessage());
    }
  }

  /**
   * Given a path, reads the file into the workspace.
   *
   * @param request HTTP request handler.
   * @param responder HTTP response handler.
   * @param path to the location in the filesystem.
   * @param lines number of lines to extracted from file if it's a text/plain.
   * @param sampler sampling method to be used.
   */
  @Path("explorer/fs/read")
  @GET
  public void read(HttpServiceRequest request, HttpServiceResponder responder,
                   @QueryParam("path") String path, @QueryParam("lines") int lines,
                   @QueryParam("sampler") String sampler, @QueryParam("fraction") double fraction) {
    RequestExtractor extractor = new RequestExtractor(request);
    String header = extractor.getHeader(RequestExtractor.CONTENT_TYPE_HEADER, null);

    if (header == null) {
      error(responder, "Content-Type header not specified.");
      return;
    }

    if (header.equalsIgnoreCase("text/plain") || header.contains("text/")) {
      loadSamplableFile(responder, path, lines, fraction, sampler);
    } else if (header.equalsIgnoreCase("application/xml")) {
      loadFile(responder, path, DataType.RECORDS);
    } else if (header.equalsIgnoreCase("application/json")) {
      loadFile(responder, path, DataType.TEXT);
    } else if (header.equalsIgnoreCase("application/avro")
      || header.equalsIgnoreCase("application/protobuf")
      || header.equalsIgnoreCase("application/excel")
      || header.contains("image/")) {
{'fraction_non_alphanumeric': 0.10338600451467268, 'fraction_numerical': 0.0036117381489841984, 'mean_word_length': 4.38715953307393, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '834778', 'n_tokens_mistral': 3346, 'n_tokens_neox': 3273, 'n_words': 847}