<commit_before>var fs = require("fs");
var path = require("path");
var util = require("util");
var async = require("async");

var servlet = require("./lib/servlet");

module.exports = function (grunt) {
    grunt.registerMultiTask("sling-content", function () {
        var options = this.options({
            host: "localhost",
            port: 8080,
            user: "admin",
            pass: "<PASSWORD>"
        });

        var done = this.async();

        var postServlet = new servlet.Post(options);

        this.files.forEach(postRoots);

        function postRoots(file) {
            file.src.forEach(function (root) {
                if (grunt.file.exists(root) === false) {
                    grunt.fail.fatal("The directory " + root + " does not exist");
                }

                if (grunt.file.isDir(root) === false) {
                    grunt.file.fatal("The path " + root + " is not a directory");
                }
            });

            var tasks = file.src.map(function (directory) {
                return createRootDirectoryTask(directory, "/");
            });

            async.parallel(tasks, done);
        }

        function createRootDirectoryTask(root, resource) {
            return function (done) {
                postRecursively(root, resource, done);
            };
        }

        function postRecursively(root, resource, done) {

            // Read content of the current directory

            var children = fs.readdirSync(root);

            // Filter out files

            var files = children.filter(function (child) {
                return grunt.file.isFile(root, child) && path.extname(child) !== ".json";
            });

            // Filter out directories

            var directories = children.filter(function (child) {
                return grunt.file.isDir(root, child);
            });

            // Filter out descriptors

            var descriptors = children.filter(function (child) {
                return grunt.file.isFile(root, child) && path.extname(child) === ".json";
            });

            // Create a descriptor map (base name -> json content)

            function toDescriptorMap(map, descriptor) {
                var base = path.basename(descriptor, ".json");
                var json = grunt.file.readJSON(path.join(root, descriptor));

                map[base] = json;

                return map;
            }

            var descriptorMap = descriptors.reduce(toDescriptorMap, {});

            // Return an object with properties for the selected file

            function propertiesFor(name, initial) {
                var base = path.basename(name, path.extname(name));

                var descriptor = descriptorMap[base] || {};

                function toProperties(map, name) {
                    map[name] = descriptor[name];
                    return map;
                }

                return Object.keys(descriptor).reduce(toProperties, initial || {});
            }

            // Creates a task to create/update a file

            function toFileTasks(file) {
                return function (done) {
                    grunt.log.writeln("File: " + path.join(root, file));
                    postServlet.createFile(resource, path.join(root, file), propertiesFor(file), done);
                };
            }

            var fileTasks = files.map(toFileTasks);

            // Creates a task to create/update a folder

            function toDirectoryTask(directory) {
                return function (done) {
                    grunt.log.writeln("Dir : " + path.join(root, directory));
                    postServlet.create(resource + directory, propertiesFor(directory), done);
                };
            }

            var directoryTasks = directories.map(toDirectoryTask);

            // Creates a task to create/update generic nodes

            function unusedDescriptor(name) {
                var usedForDirectories = directories.some(function (directory) {
                    return directory === name;
                });

                var usedForFiles = files.some(function (file) {
                    return path.basename(file, path.extname(file)) === name;
                });

                return !usedForFiles && !usedForDirectories;
            }

            function toNodeTask(name) {
                return function (done) {
                    grunt.log.writeln("Node: " + path.join(root, name + ".json"));
                    postServlet.create(resource + name, propertiesFor(name), done);<commit_msg>Use sling:Folder as primary type for folders - closes #1<commit_after>var fs = require("fs");
var path = require("path");
var util = require("util");
var async = require("async");

var servlet = require("./lib/servlet");

module.exports = function (grunt) {
    grunt.registerMultiTask("sling-content", function () {
        var options = this.options({
            host: "localhost",
            port: 8080,
            user: "admin",
            pass: "<PASSWORD>"
        });

        var done = this.async();

        var postServlet = new servlet.Post(options);

        this.files.forEach(postRoots);

        function postRoots(file) {
            file.src.forEach(function (root) {
                if (grunt.file.exists(root) === false) {
                    grunt.fail.fatal("The directory " + root + " does not exist");
                }

                if (grunt.file.isDir(root) === false) {
                    grunt.file.fatal("The path " + root + " is not a directory");
                }
            });

            var tasks = file.src.map(function (directory) {
                return createRootDirectoryTask(directory, "/");
            });

            async.parallel(tasks, done);
        }

        function createRootDirectoryTask(root, resource) {
            return function (done) {
                postRecursively(root, resource, done);
            };
        }

        function postRecursively(root, resource, done) {

            // Read content of the current directory

            var children = fs.readdirSync(root);

            // Filter out files

            var files = children.filter(function (child) {
                return grunt.file.isFile(root, child) && path.extname(child) !== ".json";
            });

            // Filter out directories

            var directories = children.filter(function (child) {
                return grunt.file.isDir(root, child);
            });

            // Filter out descriptors

            var descriptors = children.filter(function (child) {
                return grunt.file.isFile(root, child) && path.extname(child) === ".json";
            });

            // Create a descriptor map (base name -> json content)

            function toDescriptorMap(map, descriptor) {
                var base = path.basename(descriptor, ".json");
                var json = grunt.file.readJSON(path.join(root, descriptor));

                map[base] = json;

                return map;
            }

            var descriptorMap = descriptors.reduce(toDescriptorMap, {});

            // Return an object with properties for the selected file

            function propertiesFor(name, initial) {
                var base = path.basename(name, path.extname(name));

                var descriptor = descriptorMap[base] || {};

                function toProperties(map, name) {
                    map[name] = descriptor[name];
                    return map;
                }

                return Object.keys(descriptor).reduce(toProperties, initial || {});
            }

            // Creates a task to create/update a file

            function toFileTasks(file) {
                return function (done) {
                    grunt.log.writeln("File: " + path.join(root, file));
                    postServlet.createFile(resource, path.join(root, file), propertiesFor(file), done);
                };
            }

            var fileTasks = files.map(toFileTasks);

            // Creates a task to create/update a folder

            function toDirectoryTask(directory) {
                return function (done) {
                    grunt.log.writeln("Dir : " + path.join(root, directory));

                    var properties = propertiesFor(directory, {
                        "jcr:primaryType": "sling:Folder"
                    });

                    postServlet.create(resource + directory, properties, done);
                };
            }

            var directoryTasks = directories.map(toDirectoryTask);

            // Creates a task to create/update generic nodes

            function unusedDescriptor(name) {
                var usedForDirectories = directories.some(function (directory) {
                    return directory === name;
                });

                var usedForFiles = files.some(function (file) {
                    return path.basename(file, path.extname(file)) === name;
                });

                return !usedForFiles && !usedForDirectories;
            }

            function toNodeTask(name) {
                return function (done) {
                    grunt.log.writeln("Node: " + path.join(root, name + ".json"));
                    postServlet.create(resource + name, propertiesFor(name), done);
{'fraction_non_alphanumeric': 0.11208650037469221, 'fraction_numerical': 0.0009634942725618242, 'mean_word_length': 1.5700137551581843, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2094969', 'n_tokens_mistral': 2379, 'n_tokens_neox': 2191, 'n_words': 544}