<commit_before>/*
 * Copyright 2013 Cloudera Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.kitesdk.cli.commands;

import com.beust.jcommander.internal.Lists;
import com.beust.jcommander.internal.Sets;
import com.google.common.io.Files;
import java.io.BufferedWriter;
import java.io.File;
import java.util.Set;
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecordBuilder;
import org.apache.hadoop.conf.Configuration;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.kitesdk.cli.TestUtil;
import org.kitesdk.data.DatasetRepositories;
import org.kitesdk.data.DatasetRepository;
import org.kitesdk.data.MiniDFSTest;
import org.kitesdk.data.spi.filesystem.DatasetTestUtilities;
import org.slf4j.Logger;

import static org.mockito.Mockito.mock;

public class TestCopyCommandCluster extends MiniDFSTest {

  private static final String source = "users_source";
  private static final String dest = "users_dest";
  private static final String avsc = "target/user.avsc";
  private static String repoUri;

  @BeforeClass
  public static void createSourceDataset() throws Exception {
    repoUri = "hdfs://" + getDFS().getUri().getAuthority() + "/tmp/data";
    TestUtil.run("delete", source, "-r", repoUri, "-d", "target/data");

    String csv = "target/users.csv";
    BufferedWriter writer = Files.newWriter(
        new File(csv), CSVSchemaCommand.SCHEMA_CHARSET);
    writer.append("id,username,email\n");
    writer.append("1,test,<EMAIL>\n");
    writer.append("2,user,<EMAIL>\n");
    writer.close();

    TestUtil.run("-v", "csv-schema", csv, "-o", avsc, "--class", "User");
    TestUtil.run("create", source, "-s", avsc,
        "-r", repoUri, "-d", "target/data");
    TestUtil.run("csv-import", csv, source, "-r", repoUri, "-d", "target/data");
  }

  @AfterClass
  public static void deleteSourceDataset() throws Exception {
    TestUtil.run("delete", source, "-r", repoUri, "-d", "target/data");
  }

  private Logger console;
  private CopyCommand command;

  @Before
  public void createDestination() throws Exception {
    TestUtil.run("delete", dest, "-r", repoUri, "-d", "target/data");
    TestUtil.run("create", dest, "-s", avsc, "-r", repoUri, "-d", "target/data");
    this.console = mock(Logger.class);
    this.command = new CopyCommand(console);
    command.setConf(new Configuration());
  }

  @After
  public void deleteDestination() throws Exception {
    TestUtil.run("delete", dest, "-r", repoUri, "-d", "target/data");
  }

  @Test
  public void testBasicCopy() throws Exception {
    command.repoURI = repoUri;
    command.datasets = Lists.newArrayList(source, dest);

    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);

    DatasetRepository repo = DatasetRepositories.open("repo:" + repoUri);
    int size = DatasetTestUtilities.datasetSize(repo.load(source));
    Assert.assertEquals("Should contain copied records", 2, size);
  }
}<commit_msg>CDK-488: Add tests for --no-compaction and --num-writers.<commit_after>/*
 * Copyright 2013 Cloudera Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.kitesdk.cli.commands;

import com.beust.jcommander.internal.Lists;
import com.beust.jcommander.internal.Sets;
import com.google.common.collect.Iterators;
import com.google.common.io.Files;
import java.io.BufferedWriter;
import java.io.File;
import java.net.URI;
import java.util.Set;
import org.apache.avro.Schema;
import org.apache.avro.SchemaBuilder;
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecordBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.kitesdk.cli.TestUtil;
import org.kitesdk.data.Dataset;
import org.kitesdk.data.DatasetDescriptor;
import org.kitesdk.data.DatasetRepositories;
import org.kitesdk.data.DatasetRepository;
import org.kitesdk.data.Datasets;
import org.kitesdk.data.MiniDFSTest;
import org.kitesdk.data.PartitionStrategy;
import org.kitesdk.data.spi.URIBuilder;
import org.kitesdk.data.spi.filesystem.DatasetTestUtilities;
import org.kitesdk.data.spi.filesystem.FileSystemDataset;
import org.slf4j.Logger;

import static org.mockito.Matchers.contains;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoMoreInteractions;

public class TestCopyCommandCluster extends MiniDFSTest {

  private static final String source = "users_source";
  private static final String dest = "users_dest";
  private static final String avsc = "target/user.avsc";
  private static String repoUri;

  @BeforeClass
  public static void createSourceDataset() throws Exception {
    repoUri = "hdfs://" + getDFS().getUri().getAuthority() + "/tmp/data";
    TestUtil.run("delete", source, "-r", repoUri, "-d", "target/data");

    String csv = "target/users.csv";
    BufferedWriter writer = Files.newWriter(
        new File(csv), CSVSchemaCommand.SCHEMA_CHARSET);
    writer.append("id,username,email\n");
    writer.append("1,test,<EMAIL>\n");
    writer.append("2,user,<EMAIL>\n");
    writer.append("3,user3,<EMAIL>\n");
    writer.append("4,user4,<EMAIL>\n");
    writer.append("5,user5,<EMAIL>\n");
    writer.append("6,user6,<EMAIL>\n");
    writer.close();

    TestUtil.run("-v", "csv-schema", csv, "-o", avsc, "--class", "User");
    TestUtil.run("create", source, "-s", avsc,
        "-r", repoUri, "-d", "target/data");
    TestUtil.run("csv-import", csv, source, "-r", repoUri, "-d", "target/data");
  }

  @AfterClass
  public static void deleteSourceDataset() throws Exception {
    TestUtil.run("delete", source, "-r", repoUri, "-d", "target/data");
  }

  private Logger console;
  private CopyCommand command;

  @Before
  public void createDestination() throws Exception {
    TestUtil.run("delete", dest, "-r", repoUri, "-d", "target/data");
    TestUtil.run("create", dest, "-s", avsc, "-r", repoUri, "-d", "target/data");
    this.console = mock(Logger.class);
    this.command = new CopyCommand(console);
    command.setConf(new Configuration());
  }

  @After
  public void deleteDestination() throws Exception {
    TestUtil.run("delete", dest, "-r", repoUri, "-d", "target/data");
  }

  @Test
  public void testBasicCopy() throws Exception {
    command.repoURI = repoUri;
    command.datasets = Lists.newArrayList(source, dest);

    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);

    DatasetRepository repo = DatasetRepositories.open("repo:" + repoUri);
    int size = DatasetTestUtilities.datasetSize(repo.load(dest));
    Assert.assertEquals("Should contain copied records", 6, size);

    verify(console).info("Added {} records to \"{}\"", 6l, dest);
    verifyNoMoreInteractions(console);
  }

  @Test
  public void testCopyWithoutCompaction() throws Exception {
    command.repoURI = repoUri;
    command.noCompaction = true;
    command.datasets = Lists.newArrayList(source, dest);

    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);

    DatasetRepository repo = DatasetRepositories.open("repo:" + repoUri);
    int size = DatasetTestUtilities.datasetSize(repo.load(dest));
    Assert.assertEquals("Should contain copied records", 6, size);

    verify(console).info("Added {} records to \"{}\"", 6l, dest);
    verifyNoMoreInteractions(console);
  }

  @Test
  @SuppressWarnings("unchecked")
  public void testCopyWithNumWriters() throws Exception {
    command.repoURI = repoUri;
    command.numWriters = 3;
    command.datasets = Lists.newArrayList(source, dest);

    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);

    DatasetRepository repo = DatasetRepositories.open("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load(dest);
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);

    verify(console).info("Added {} records to \"{}\"", 6l, dest);
    verifyNoMoreInteractions(console);
  }

  @Test
  @SuppressWarnings("unchecked")
  public void testPartitionedCopyWithNumWriters() throws Exception {
    command.repoURI = repoUri;
    command.numWriters = 3;
    command.datasets = Lists.newArrayList(source, "dest_partitioned");
    URI dsUri = new URIBuilder("repo:" + repoUri, "dest_partitioned").build();
    Datasets.create(dsUri, new DatasetDescriptor.Builder()
        .partitionStrategy(new PartitionStrategy.Builder()
            .hash("id", 2)
            .build())
        .schema(SchemaBuilder.record("User").fields()
            .requiredString("id")
            .optionalString("username")
            .optionalString("email")
            .endRecord())
        .build());

    int rc = command.run();
    Assert.assertEquals("Should return success", 0, rc);

    DatasetRepository repo = DatasetRepositories.open("repo:" + repoUri);
    FileSystemDataset<GenericData.Record> ds =
        (FileSystemDataset<GenericData.Record>) repo.<GenericData.Record>
            load("dest_partitioned");
    int size = DatasetTestUtilities.datasetSize(ds);
    Assert.assertEquals("Should contain copied records", 6, size);

    Assert.assertEquals("Should produce 2 partitions",
        2, Iterators.size(ds.pathIterator()));

    verify(console).info("Added {} records to \"{}\"", 6l, "dest_partitioned");
    verifyNoMoreInteractions(console);
  }
}
{'fraction_non_alphanumeric': 0.1367789570835256, 'fraction_numerical': 0.00479926165205353, 'mean_word_length': 4.189655172413793, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '378637', 'n_tokens_mistral': 3308, 'n_tokens_neox': 3262, 'n_words': 808}