<commit_before>package metadata.etl.dataset.hbase;

import metadata.etl.dataset.hdfs.HdfsMetadataEtl;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;

/**
 * Created by lake on 17-5-25.
 */
public class HBaseMetadataEtlTest {
    HBaseMetadataEtl ds;

    @BeforeTest
    public void setUp()
            throws Exception {
        ds = new HBaseMetadataEtl(2, 0L);
    }

    @Test(groups = {"needConfig"})
    public void testRun()
            throws Exception {
        ds.run();
    }

    @Test(groups = {"needConfig"})
    public void testExtract()
            throws Exception {
        ds.extract();
        //TODO check it copy back the files
    }

    @Test(groups = {"needConfig"})
    public void testTransform()
            throws Exception {
        ds.transform();
        //TODO check it generate the final csv file
    }

    @Test(groups = {"needConfig"})
    public void testLoad()
            throws Exception {
        ds.load();
    }
}
<commit_msg>hbase etl test<commit_after>package metadata.etl.dataset.hbase;

import metadata.etl.dataset.hdfs.HdfsMetadataEtl;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;

import java.io.File;

/**
 * Created by lake on 17-5-25.
 */
public class HBaseMetadataEtlTest {
    HBaseMetadataEtl ds;

    @BeforeTest
    public void setUp()
            throws Exception {
        ds = new HBaseMetadataEtl(2, 0L);
    }

    @Test
    public void testRun()
            throws Exception {
        ds.run();
        File file = new File("/usr/tmp/hbase-data/hbase_meta");
        assert file.exists() == true;
    }

    @Test(groups = {"needConfig"})
    public void testExtract()
            throws Exception {
        ds.extract();
        //TODO check it copy back the files
    }

    @Test(groups = {"needConfig"})
    public void testTransform()
            throws Exception {
        ds.transform();
        //TODO check it generate the final csv file
    }

    @Test(groups = {"needConfig"})
    public void testLoad()
            throws Exception {
        ds.load();
    }
}

{'fraction_non_alphanumeric': 0.11084681255946717, 'fraction_numerical': 0.006660323501427212, 'mean_word_length': 2.2009132420091326, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5834761', 'n_tokens_mistral': 642, 'n_tokens_neox': 616, 'n_words': 160}