<commit_before>/** s3_multipart_cmd.cc
    <NAME>, 29th May 2013
    Copyright (c) 2013 Datacratic.  All rights reserved.
    
    
*/

#include "soa/service/s3.h"
#include "jml/utils/filter_streams.h"
#include <boost/program_options/cmdline.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/positional_options.hpp> 
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>

#include <poll.h>

namespace po = boost::program_options;

using namespace std;
using namespace Datacratic;
using namespace ML;

int main(int argc, char* argv[])
{
    string outputFile ="s3://tests.datacratic.com/logger/2013-05-30/router-2013-05-30-23:00:00.log.gz";
    string localFile;
    string s3KeyId = "<KEY>";
    string s3Key = "<KEY>";
    bool cancel = false; //whether the multipart upload should be cancelled
    
    po::options_description desc("Main options");
    desc.add_options()
        ("output-uri,o", po::value<string>(&outputFile), "Output files/uris (can have multiple file/s3://bucket/object)")
        ("s3-key-id,I", po::value<string>(&s3KeyId), "S3 key id")
        ("s3-key,K", po::value<string>(&s3Key), "S3 key")
        ("cancel,c", po::value<bool>(&cancel), "whether or not to cancel the upload")<commit_msg>removed obsolete and private infos<commit_after>/** s3_multipart_cmd.cc
    <NAME>, 29th May 2013
    Copyright (c) 2013 Datacratic.  All rights reserved.
    
    
*/

#include "soa/service/s3.h"
#include "jml/utils/filter_streams.h"
#include <boost/program_options/cmdline.hpp>
#include <boost/program_options/options_description.hpp>
#include <boost/program_options/positional_options.hpp> 
#include <boost/program_options/parsers.hpp>
#include <boost/program_options/variables_map.hpp>

#include <poll.h>

namespace po = boost::program_options;

using namespace std;
using namespace Datacratic;
using namespace ML;

int main(int argc, char* argv[])
{
    string outputFile;
    string localFile;
    string s3KeyId;
    string s3Key;
    bool cancel = false; //whether the multipart upload should be cancelled
    
    po::options_description desc("Main options");
    desc.add_options()
        ("output-uri,o", po::value<string>(&outputFile), "Output files/uris (can have multiple file/s3://bucket/object)")
        ("s3-key-id,I", po::value<string>(&s3KeyId), "S3 key id")
        ("s3-key,K", po::value<string>(&s3Key), "S3 key")
        ("cancel,c", po::value<bool>(&cancel), "whether or not to cancel the upload")
{'fraction_non_alphanumeric': 0.1468725257323832, 'fraction_numerical': 0.02573238321456849, 'mean_word_length': 4.29769392033543, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '331184', 'n_tokens_mistral': 887, 'n_tokens_neox': 849, 'n_words': 194}