<commit_before>/*
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

var fs = require('fs');
var path = require('path');
var AWS = require('aws-sdk');

module.exports = function(grunt) {
    grunt.registerMultiTask('upload-s3-bucket', 'Upload files to Amazon S3 bucket', function() {
        // Merge task-specific and/or target-specific options with these defaults.
        var options = this.options({
            bucket: '',
            overwrite: 'no',
            files: '.'
        });

        console.log(options);

        if(fs.existsSync('./awsconfig.json'))
            AWS.config.loadFromPath('./awsconfig.json');

        console.log(AWS.config.credentials);

        var s3 = new AWS.S3();

        var done = this.async();
        var taskCount = 0;
        var allTasksOK = true;

        var addTask = function(){
            ++taskCount;
        };

        var completeTask = function(allOK){
            if(!allOK)
                allTasksOK = false;

            --taskCount;

            if(taskCount == 0){
                console.log('All done '+allTasksOK);
                done(allTasksOK);
            }
        };

        var processFiles = function(base, files){
            files.forEach(function(element, index, array){
                if(element.substr(0, 1) == '.'){
                    //skip hidden files
                }else{
                    var fullPath = path.join(base, element);
                    var stat = fs.statSync(fullPath);
                    if(stat.isDirectory()){
                        addTask();
                        fs.readdir(fullPath, function(err, files){
                            if(err){
                                console.log(err, err.stack);
                                completeTask(false);
                            }else{
                                processFiles(fullPath, files);
                            }
                        });
                    }else{
                        //console.log(fullPath);
                        uploadFile(fullPath);
                    }
                }
            });

            completeTask(true);
        };

        var uploadFile = function(fullPath){
            addTask();
            var params = {
                Bucket: options.bucket, // required
                Key: fullPath, // required
//                ACL: 'public-read | bucket-owner-full-control',//'private | public-read | public-read-write | authenticated-read | bucket-owner-read | bucket-owner-full-control',
//                CacheControl: 'STRING_VALUE',
//                ContentDisposition: 'STRING_VALUE',
//                ContentEncoding: 'STRING_VALUE',
//                ContentLanguage: 'STRING_VALUE',
//                ContentType: 'STRING_VALUE',
//                Expires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
//                GrantFullControl: 'STRING_VALUE',
//                GrantRead: 'STRING_VALUE',
//                GrantReadACP: 'STRING_VALUE',
//                GrantWriteACP: 'STRING_VALUE',
//                Metadata: {
//                    someKey: 'STRING_VALUE',
//                    // anotherKey: ...
//                },
//                SSECustomerAlgorithm: 'STRING_VALUE',
//                SSECustomerKey: 'STRING_VALUE',
//                SSECustomerKeyMD5: 'STRING_VALUE',
//                ServerSideEncryption: 'AES256',<commit_msg>Upload respect overwrite flag<commit_after>/*
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

var fs = require('fs');
var path = require('path');
var AWS = require('aws-sdk');

module.exports = function(grunt) {
    grunt.registerMultiTask('upload-s3-bucket', 'Upload files to Amazon S3 bucket', function() {
        // Merge task-specific and/or target-specific options with these defaults.
        var options = this.options({
            bucket: '',
            overwrite: 'no',
            files: '.'
        });

        console.log(options);

        if(fs.existsSync('./awsconfig.json'))
            AWS.config.loadFromPath('./awsconfig.json');

        console.log(AWS.config.credentials);

        var s3 = new AWS.S3();

        var done = this.async();
        var taskCount = 0;
        var allTasksOK = true;

        var addTask = function(){
            ++taskCount;
        };

        var completeTask = function(allOK){
            if(!allOK)
                allTasksOK = false;

            --taskCount;

            if(taskCount == 0){
                console.log('All done '+allTasksOK);
                done(allTasksOK);
            }
        };

        var processFiles = function(base, files){
            files.forEach(function(element, index, array){
                if(element.substr(0, 1) == '.'){
                    //skip hidden files
                }else{
                    var fullPath = path.join(base, element);
                    var stat = fs.statSync(fullPath);
                    if(stat.isDirectory()){
                        addTask();
                        fs.readdir(fullPath, function(err, files){
                            if(err){
                                console.log(err, err.stack);
                                completeTask(false);
                            }else{
                                processFiles(fullPath, files);
                            }
                        });
                    }else{
                        //console.log(fullPath);
                        if(options.overwrite != 'yes'){
                            addTask();
                            var params = {
                                Bucket: options.bucket, // required
                                Key: fullPath // required
                            };
                            s3.headObject(params, function(err, data) {
                                if(err){
                                    if(err.statusCode == 404){
                                        uploadFile(fullPath);
                                        completeTask(true);
                                    }else{
                                        console.log('Skipping '+fullPath);
                                        completeTask(true);
                                    }
                                }else{
                                    //object exists
                                    console.log('Skipping '+fullPath);
                                    completeTask(true);
                                }
                            });
                        }else{
                            uploadFile(fullPath);
                        }
                    }
                }
            });

            completeTask(true);
        };

        var uploadFile = function(fullPath){
            addTask();

            var params = {
                Bucket: options.bucket, // required
                Key: fullPath, // required
//                ACL: 'public-read | bucket-owner-full-control',//'private | public-read | public-read-write | authenticated-read | bucket-owner-read | bucket-owner-full-control',
//                CacheControl: 'STRING_VALUE',
//                ContentDisposition: 'STRING_VALUE',
//                ContentEncoding: 'STRING_VALUE',
//                ContentLanguage: 'STRING_VALUE',
//                ContentType: 'STRING_VALUE',
//                Expires: new Date || 'Wed Dec 31 1969 16:00:00 GMT-0800 (PST)' || 123456789,
//                GrantFullControl: 'STRING_VALUE',
//                GrantRead: 'STRING_VALUE',
//                GrantReadACP: 'STRING_VALUE',
//                GrantWriteACP: 'STRING_VALUE',
//                Metadata: {
//                    someKey: 'STRING_VALUE',
//                    // anotherKey: ...
//                },
//                SSECustomerAlgorithm: 'STRING_VALUE',
//                SSECustomerKey: 'STRING_VALUE',
//                SSECustomerKeyMD5: 'STRING_VALUE',
//                ServerSideEncryption: 'AES256',
{'fraction_non_alphanumeric': 0.11435282482383484, 'fraction_numerical': 0.010631722091729509, 'mean_word_length': 1.093143596377749, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2235027', 'n_tokens_mistral': 2064, 'n_tokens_neox': 1866, 'n_words': 386}