<commit_before>//
// Treasure Data Bulk-Import Tool in Java
//
// Copyright (C) 2012 - 2013 <NAME>
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//
package com.treasure_data.td_import;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.logging.Logger;

import com.treasure_data.client.TreasureDataClient;
import com.treasure_data.client.bulkimport.BulkImportClient;
import com.treasure_data.model.bulkimport.SessionSummary;
import com.treasure_data.td_import.TaskResult;
import com.treasure_data.td_import.prepare.MultiThreadPrepareProcessor;
import com.treasure_data.td_import.prepare.PrepareConfiguration;
import com.treasure_data.td_import.source.LocalFileSource;
import com.treasure_data.td_import.source.Source;
import com.treasure_data.td_import.upload.MultiThreadUploadProcessor;
import com.treasure_data.td_import.upload.UploadConfiguration;
import com.treasure_data.td_import.upload.UploadProcessor;

public class BulkImport extends Import {
    private static final Logger LOG = Logger.getLogger(BulkImport.class.getName());

    public BulkImport(Properties props) {
        super(props);
    }

    public List<com.treasure_data.td_import.TaskResult<?>> doPrepare(final String[] args)
            throws Exception {
        // create prepare configuration
        PrepareConfiguration conf = createPrepareConf(args);

        // extract and get source names from command-line arguments
        Source[] sources = getSources(conf, 1);

        MultiThreadPrepareProcessor proc =
                createAndStartPrepareProcessor(conf);

        // create prepare tasks
        com.treasure_data.td_import.prepare.Task[] tasks =
                createPrepareTasks(conf, sources);

        // start prepare tasks
        startPrepareTasks(conf, tasks);

        return stopPrepareProcessor(proc);
    }

    public List<com.treasure_data.td_import.TaskResult<?>> doUpload(final String[] args)
            throws Exception {
        // create configuration for 'upload' processing
        UploadConfiguration uploadConf = createUploadConf(args);

        // create TreasureDataClient and BulkImportClient objects
        TreasureDataClient tdClient = new TreasureDataClient(uploadConf.getProperties());
        BulkImportClient biClient = new BulkImportClient(tdClient);

        // configure session name
        TaskResult<?> r = null;
        String sessionName;
        int filePos;
        if (uploadConf.autoCreate()) { // 'auto-create-session'
            // create session automatically<commit_msg>minor change of BulkImport class<commit_after>//
// Treasure Data Bulk-Import Tool in Java
//
// Copyright (C) 2012 - 2013 <NAME>
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
//
package com.treasure_data.td_import;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.logging.Logger;

import com.treasure_data.client.TreasureDataClient;
import com.treasure_data.client.bulkimport.BulkImportClient;
import com.treasure_data.model.bulkimport.SessionSummary;
import com.treasure_data.td_import.TaskResult;
import com.treasure_data.td_import.prepare.MultiThreadPrepareProcessor;
import com.treasure_data.td_import.prepare.PrepareConfiguration;
import com.treasure_data.td_import.source.LocalFileSource;
import com.treasure_data.td_import.source.Source;
import com.treasure_data.td_import.upload.MultiThreadUploadProcessor;
import com.treasure_data.td_import.upload.UploadConfiguration;
import com.treasure_data.td_import.upload.UploadProcessor;

public class BulkImport extends Import {
    private static final Logger LOG = Logger.getLogger(BulkImport.class.getName());

    public BulkImport(Properties props) {
        super(props);
    }

    public List<com.treasure_data.td_import.TaskResult<?>> doPrepare(final String[] args)
            throws Exception {
        // create prepare configuration
        PrepareConfiguration prepareConf = createPrepareConf(args);

        // extract and get source names from command-line arguments
        Source[] srcs = getSources(prepareConf, 1);

        MultiThreadPrepareProcessor proc =
                createAndStartPrepareProcessor(prepareConf);

        // create prepare tasks
        com.treasure_data.td_import.prepare.Task[] tasks =
                createPrepareTasks(prepareConf, srcs);

        // start prepare tasks
        startPrepareTasks(prepareConf, tasks);

        return stopPrepareProcessor(proc);
    }

    public List<com.treasure_data.td_import.TaskResult<?>> doUpload(final String[] args)
            throws Exception {
        // create configuration for 'upload' processing
        UploadConfiguration uploadConf = createUploadConf(args);

        // create TreasureDataClient and BulkImportClient objects
        TreasureDataClient tdClient = new TreasureDataClient(uploadConf.getProperties());
        BulkImportClient biClient = new BulkImportClient(tdClient);

        // configure session name
        TaskResult<?> r = null;
        String sessionName;
        int filePos;
        if (uploadConf.autoCreate()) { // 'auto-create-session'
            // create session automatically
{'fraction_non_alphanumeric': 0.0855614973262032, 'fraction_numerical': 0.004089336269267065, 'mean_word_length': 3.4437456324248776, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6269203', 'n_tokens_mistral': 1709, 'n_tokens_neox': 1661, 'n_words': 457}