<commit_before>/*!
GPII Data Source

Copyright 2012 OCAD University

Licensed under the New BSD license. You may not use this file except in
compliance with this License.

You may obtain a copy of the License at
https://github.com/gpii/universal/LICENSE.txt
*/

(function () {

    "use strict";

    var fluid = require("infusion"),
        gpii = fluid.registerNamespace("gpii"),
        fs = require("fs"),
        http = require("http"),
        url = require("url"),
        when = require("when"),
        eUC = "encodeURIComponent:";

    /*
        A data source component, also used as a grade. That is used as an abstraction for
        a data resouce. Its methods include:
            * get - to get the data from data resource
            * set - to set the data (only if writable options is set as true)
    */
    fluid.defaults("gpii.dataSource", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        components: {
            errback: {
                type: "gpii.dataSource.errback"
            },
            modelParser: {
                type: "gpii.dataSource.modelParser"
            }
        },
        invokers: {
            get: "gpii.dataSource.get"
        },
        nickName: "dataSource", // framework bug FLUID-4636 - this is not resolved
        writable: false,
        preInitFunction: "gpii.dataSource.preInit"
    });

    /*
        Model parser component that does both parsing and stringification of json
        payloads for a dataSource.
    */
    fluid.defaults("gpii.dataSource.modelParser", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        invokers: {
            stringify: "gpii.dataSource.modelParser.stringify",
            parse: "gpii.dataSource.modelParser.parse"
        }
    });

    fluid.demands("gpii.dataSource.modelParser.parse", "gpii.dataSource.modelParser", {
        args: [{
            responseParser: "{dataSource}.options.responseParser",
            errback: "{errback}"
        }, "{arguments}.0"]
    });

    /*
        Parses the payload from the source of data.
        Arguments:
            * options {Object} - contains responseParses (a function or a name of a
            global function) that parses the raw response data; errback - a component
            whose "onError" event if fired if there's something wrong with the 
            payload (e.g. the returned data is an error).
            * data {Object|String} - raw data from the source.
    */
    gpii.dataSource.modelParser.parse = function (options, data) {
        var responseParser = options.responseParser;
        data = typeof data === "string" ? JSON.parse(data) : data;
        if (data && data.isError) {
            options.errback.events.onError.fire(data);
        }
        if (responseParser) {
            data = typeof responseParser === "string" ?
                fluid.invokeGlobalFunction(responseParser, [data]) :
                responseParser(data);
        }
        return data;
    };

    fluid.demands("gpii.dataSource.modelParser.stringify", "gpii.dataSource.modelParser", {
        args: "{arguments}.0"
    });

    /*
        Stringifies the data to be sent to the source (in no data does nothing).
        Arguments:
            * model {Object|String} - raw data to be sent to the source.
    */
    gpii.dataSource.modelParser.stringify = function (model) {
        if (!model) {
            return;
        }
        return typeof model === "string" ? model : JSON.stringify(model);
    };

    /*
        Errback component that handles errors when interacting with dataSource by
        completing client request (through firing request's onError event).
    */
    fluid.defaults("gpii.dataSource.errback", {
        gradeNames: ["autoInit", "fluid.eventedComponent"],
        events: {
            onError: null
        },
        listeners: {
            onError: "{that}.handleError"
        },
        preInitFunction: "gpii.dataSource.errback.preInit",
        invokers: {
            handleError: "gpii.dataSource.errback.handleError"
        }
    });

    gpii.dataSource.errback.preInit = function (that) {
        that.handleError = function (data) {
            that.handleError(data);
        };
    };

    fluid.demands("gpii.dataSource.errback.handleError", "gpii.dataSource.errback", {
        args: [
            "{callbackWrapper}",
            "{requestProxy}",
            "{arguments}.0"
        ]
    });

    /*
        Fires "onError" event of requestsProxy - a lightweight wrapper for
        requests component.
        Arguments:
            * callbackWrapper - component that wraps a function in context of
            current http request.
            * requestProxy - a wrapper for requests component.
    */
    gpii.dataSource.errback.handleError = function (callbackWrapper, requestProxy, data) {
        var fireOnError = callbackWrapper.wrap(requestProxy.events.onError.fire);
        fireOnError(data);
    };

    /*
        UrlResolver component that is responsible for handling and expansion of urls or
        paths that point to the source of data.
    */
    fluid.defaults("gpii.dataSource.urlResolver", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        components: {
            urlExpander: {
                type: "gpii.urlExpander"
            }
        },
        invokers: {
            resolve: "gpii.dataSource.urlResolver.resolve"
        }
    });

    fluid.demands("gpii.dataSource.urlResolver.resolve", "gpii.dataSource.urlResolver", {
        args: [
            "{urlExpander}.expand",
            "{dataSource}.options.url",
            "{dataSource}.options.termMap",
            "{arguments}.0"
        ]
    });

    /*
        Resolves (expands) a url or path in the current context.
        Arguments:
            * expand - a method of urlExpander that expands values within the url template.
            * url - an actual url or path to expand.
            * termMap - a set of configuration for urlExpander's expand method.
            * directModel - a set of data used to expand the url template.
    */
    gpii.dataSource.urlResolver.resolve = function (expand, url, termMap, directModel) {
        var map = fluid.copy(termMap);
        map = fluid.transform(map, function (entry) {
            var encode = false;
            if (entry.indexOf(eUC) === 0) {
                encode = true;
                entry = entry.substring(eUC.length);
            }
            if (entry.charAt(0) === "%") {
                entry = fluid.get(directModel, entry.substring(1));
            }
            if (encode) {
                entry = encodeURIComponent(entry);
            }
            return entry;
        });
        var replaced = fluid.stringTemplate(url, map);
        replaced = expand(replaced);
        return replaced;
    };

    /*
        A type of a data source specific to communicating with a URL resource.
    */
    fluid.defaults("gpii.dataSource.URL", {
        gradeNames: ["gpii.dataSource", "autoInit"],
        components: {
            urlResolver: {
                type: "gpii.dataSource.urlResolver"
            }
        },
        termMap: {},
        writeMethod: "POST"
    });

    /*
        A type of the URL data source specific to communicating with Couch DB.
    */
    fluid.defaults("gpii.dataSource.CouchDB", {
        gradeNames: ["gpii.dataSource.URL", "autoInit"],
        writeMethod: "PUT",
        components: {
            gradeContext: {
                type: "fluid.typeFount",
                options: {
                    targetTypeName: "gpii.dataSource.URL"
                }
            }
        },
        finalInitFunction: "gpii.dataSource.CouchDB.finalInit"
    });

    /*
        Cleans up couch specific fields from returned document.
    */
    gpii.dataSource.CouchDB.wrapCallback = function (callback) {
        return function (resp) {
            var value = resp.value;
            if (resp.error) {
                value = {
                    isError: true,
                    message: resp.error + ": " + resp.reason
                };
            }
            callback(value);
        };
    };

    gpii.dataSource.CouchDB.finalInit = function (that) {
        var originalGet = that.get, originalSet;

        that.get = function (directModel, callback) {
            var couchDBCallback = gpii.dataSource.CouchDB.wrapCallback(callback);
            originalGet.apply(null, [directModel, couchDBCallback]);
        };

        if (!that.options.writable) {
            return;
        }
        originalSet = that.set;
        that.set = function (directModel, model, callback) {
            model = {value: model};
            var couchDBCallback = gpii.dataSource.CouchDB.wrapCallback(callback);
            originalGet(directModel, function (resp) {
                if (resp.ok) {
                    model.id = resp._id;
                    model.rev = resp._rev;
                }
                originalSet.apply(null, [directModel, model, couchDBCallback]);
            });
        };
    };

    gpii.dataSource.preInit = function (that) {
        that.nickName = "dataSource"; // work around FLUID-4636
        // Only enable set if writable is true.
        if (that.options.writable) {
            that.options.invokers.set = "gpii.dataSource.set";
        }
    };

    fluid.demands("gpii.dataSource.get", "gpii.dataSource.URL", {
        funcName: "gpii.dataSource.URL.handle",
        args: [{
            urlResolver: "{urlResolver}",
            errback: "{errback}",
            modelParser: "{modelParser}",
            callbackWrapper: "{callbackWrapper}"
        }, "{arguments}.0", "{arguments}.1"]
    });

    fluid.demands("gpii.dataSource.set", "gpii.dataSource.URL", {
        funcName: "gpii.dataSource.URL.handle",
        args: [{
            urlResolver: "{urlResolver}",
            errback: "{errback}",
            modelParser: "{modelParser}",
            callbackWrapper: "{callbackWrapper}",
            writeMethod: "{dataSource}.options.writeMethod"
        }, "{arguments}.0", "{arguments}.2", "{arguments}.1"]
    });

    /*
        Handles calls to data source's (URL and CouchDB) get and set.
        Arguments:
            * options {Object}- an options block that contains urlResolver,
            errback, modelParser and writeMethod ("POST" or "PUT").
            * directModel (Object) - a collection of variables used to expand a url.
            * callback {Function} - that is called when data is sent/received from
            the URL.
            * model {String|Object} - a payload returned from the set/get operation.
    */
    gpii.dataSource.URL.handle = function (options, directModel, callback, model) {
            // Expand URL.
        var url = options.urlResolver.resolve(directModel),
            // Test whether the URL is to a local file.
            isFileProtocol = gpii.dataSource.isFileProtocol(url),
            func = gpii.dataSource.URL.handle.url;
        if (isFileProtocol) {
            url = url.substring(7);
            func = gpii.dataSource.URL.handle.fs;
        }
        // If model exists and is an object - stringify it.
        model = options.modelParser.stringify(model);
        func.apply(null, [url, options, callback, model]);
    };

    gpii.dataSource.URL.handle.url = function (path, options, callback, model) {
        var urlObj = url.parse(path, true),
            opts = {
                host: urlObj.hostname,
                port: parseInt(urlObj.port, 10),
                path: urlObj.path,
                method: "GET"
            };
        if (model) {
            opts.headers = {
                "Content-Type": "application/json",
                "Content-Length": model.length
            };
            opts.method = options.writeMethod;
        }
        var req = http.request(opts, function (res) {
            var data = "";
            res.setEncoding("utf8");
            res.on("data", function (chunk) {
                data += chunk;
            });
            res.on("end", function () {
                data = options.modelParser.parse(data);
                callback(data);
            });
        });
        req.on("error", options.callbackWrapper.wrap(function (error) {
            options.errback.events.onError.fire({
                isError: true,
                message: error.message
            });
        }));
        req.end(model);
        return req;
    };

    gpii.dataSource.URL.handle.fs = function (fileName, options, callback, model) {
        var method = "readFile",
            args = [fileName, "utf8", options.callbackWrapper.wrap(function (error, data) {
                if (error) {
                    options.errback.events.onError.fire({
                        isError: true,
                        message: error.message
                    });
                    return;
                }
                data = options.modelParser.parse(data || model);
                callback(data);
            })];
        if (model) {
            method = "writeFile";
            args.splice(1, 0, model);
        }
        fs[method].apply(null, args);
    };

    /*
        Determines whether the URL is for local file system.
            * uri {String} - a uri to be tested.
    */
    gpii.dataSource.isFileProtocol = function(uri) {
        return /^file:\/\//.test(uri);
    };

    fluid.defaults("gpii.callbackWrappingPromiseData", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        components: {
            callbackWrapper: "{callbackWrapper}",
            rawSource: {
                type: "gpii.dataSource"
            }
        },<commit_msg>GPII-43: Improvements to data source. Improved error handling.<commit_after>/*!
GPII Data Source

Copyright 2012 OCAD University

Licensed under the New BSD license. You may not use this file except in
compliance with this License.

You may obtain a copy of the License at
https://github.com/gpii/universal/LICENSE.txt
*/

(function () {

    "use strict";

    var fluid = require("infusion"),
        gpii = fluid.registerNamespace("gpii"),
        fs = require("fs"),
        http = require("http"),
        url = require("url"),
        when = require("when"),
        eUC = "encodeURIComponent:";

    /*
        A data source component, also used as a grade. That is used as an abstraction for
        a data resouce. Its methods include:
            * get - to get the data from data resource
            * set - to set the data (only if writable options is set as true)
    */
    fluid.defaults("gpii.dataSource", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        components: {
            errback: {
                type: "gpii.dataSource.errback"
            },
            modelParser: {
                type: "gpii.dataSource.modelParser"
            }
        },
        invokers: {
            get: "gpii.dataSource.get"
        },
        nickName: "dataSource", // framework bug FLUID-4636 - this is not resolved
        writable: false,
        preInitFunction: "gpii.dataSource.preInit"
    });

    /*
        Model parser component that does both parsing and stringification of json
        payloads for a dataSource.
    */
    fluid.defaults("gpii.dataSource.modelParser", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        invokers: {
            stringify: "gpii.dataSource.modelParser.stringify",
            parse: "gpii.dataSource.modelParser.parse"
        }
    });

    fluid.demands("gpii.dataSource.modelParser.parse", "gpii.dataSource.modelParser", {
        args: ["{dataSource}.options.responseParser", "{arguments}.0"]
    });

    /*
        Parses the payload from the source of data.
        Arguments:
            * responseParser {Object} - a function or a name of a
              global function that parses the raw response data.
            * data {Object|String} - raw data from the source.
    */
    gpii.dataSource.modelParser.parse = function (responseParser, data) {
        data = typeof data === "string" ? JSON.parse(data) : data;
        if (data && data.isError) {
            return data;
        }
        if (responseParser) {
            data = typeof responseParser === "string" ?
                fluid.invokeGlobalFunction(responseParser, [data]) :
                responseParser(data);
        }
        return data;
    };

    fluid.demands("gpii.dataSource.modelParser.stringify", "gpii.dataSource.modelParser", {
        args: "{arguments}.0"
    });

    /*
        Stringifies the data to be sent to the source (in no data does nothing).
        Arguments:
            * model {Object|String} - raw data to be sent to the source.
    */
    gpii.dataSource.modelParser.stringify = function (model) {
        if (!model) {
            return;
        }
        return typeof model === "string" ? model : JSON.stringify(model);
    };

    /*
        Errback component that handles errors when interacting with dataSource by
        completing client request (through firing request's onError event).
    */
    fluid.defaults("gpii.dataSource.errback", {
        gradeNames: ["autoInit", "fluid.eventedComponent"],
        events: {
            onError: null
        },
        listeners: {
            onError: "{that}.handleError"
        },
        preInitFunction: "gpii.dataSource.errback.preInit",
        invokers: {
            handleError: "gpii.dataSource.errback.handleError"
        }
    });

    gpii.dataSource.errback.preInit = function (that) {
        that.handleError = function (data) {
            that.handleError(data);
        };
    };

    fluid.demands("gpii.dataSource.errback.handleError", "gpii.dataSource.errback", {
        args: [
            "{callbackWrapper}",
            "{requestProxy}",
            "{arguments}.0"
        ]
    });

    /*
        Fires "onError" event of requestsProxy - a lightweight wrapper for
        requests component.
        Arguments:
            * callbackWrapper - component that wraps a function in context of
            current http request.
            * requestProxy - a wrapper for requests component.
    */
    gpii.dataSource.errback.handleError = function (callbackWrapper, requestProxy, data) {
        var fireOnError = callbackWrapper.wrap(requestProxy.events.onError.fire);
        fireOnError(data);
    };

    /*
        UrlResolver component that is responsible for handling and expansion of urls or
        paths that point to the source of data.
    */
    fluid.defaults("gpii.dataSource.urlResolver", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        components: {
            urlExpander: {
                type: "gpii.urlExpander"
            }
        },
        invokers: {
            resolve: "gpii.dataSource.urlResolver.resolve"
        }
    });

    fluid.demands("gpii.dataSource.urlResolver.resolve", "gpii.dataSource.urlResolver", {
        args: [
            "{urlExpander}.expand",
            "{dataSource}.options.url",
            "{dataSource}.options.termMap",
            "{arguments}.0"
        ]
    });

    /*
        Resolves (expands) a url or path in the current context.
        Arguments:
            * expand - a method of urlExpander that expands values within the url template.
            * url - an actual url or path to expand.
            * termMap - a set of configuration for urlExpander's expand method.
            * directModel - a set of data used to expand the url template.
    */
    gpii.dataSource.urlResolver.resolve = function (expand, url, termMap, directModel) {
        var map = fluid.copy(termMap);
        map = fluid.transform(map, function (entry) {
            var encode = false;
            if (entry.indexOf(eUC) === 0) {
                encode = true;
                entry = entry.substring(eUC.length);
            }
            if (entry.charAt(0) === "%") {
                entry = fluid.get(directModel, entry.substring(1));
            }
            if (encode) {
                entry = encodeURIComponent(entry);
            }
            return entry;
        });
        var replaced = fluid.stringTemplate(url, map);
        replaced = expand(replaced);
        return replaced;
    };

    /*
        A type of a data source specific to communicating with a URL resource.
    */
    fluid.defaults("gpii.dataSource.URL", {
        gradeNames: ["gpii.dataSource", "autoInit"],
        components: {
            urlResolver: {
                type: "gpii.dataSource.urlResolver"
            }
        },
        termMap: {},
        writeMethod: "POST"
    });

    /*
        A type of the URL data source specific to communicating with Couch DB.
    */
    fluid.defaults("gpii.dataSource.CouchDB", {
        gradeNames: ["gpii.dataSource.URL", "autoInit"],
        writeMethod: "PUT",
        components: {
            gradeContext: {
                type: "fluid.typeFount",
                options: {
                    targetTypeName: "gpii.dataSource.URL"
                }
            }
        },
        finalInitFunction: "gpii.dataSource.CouchDB.finalInit"
    });

    /*
        Cleans up couch specific fields from returned document.
    */
    gpii.dataSource.CouchDB.wrapCallback = function (callback) {
        return function (resp) {
            var value = resp.value;
            if (resp.error) {
                value = {
                    isError: true,
                    message: resp.error + ": " + resp.reason
                };
            }
            callback(value);
        };
    };

    gpii.dataSource.CouchDB.finalInit = function (that) {
        var originalGet = that.get, originalSet;

        that.get = function (directModel, callback) {
            var couchDBCallback = gpii.dataSource.CouchDB.wrapCallback(callback);
            originalGet.apply(null, [directModel, couchDBCallback]);
        };

        if (!that.options.writable) {
            return;
        }
        originalSet = that.set;
        that.set = function (directModel, model, callback) {
            model = {value: model};
            var couchDBCallback = gpii.dataSource.CouchDB.wrapCallback(callback);
            originalGet(directModel, function (resp) {
                if (resp.ok) {
                    model.id = resp._id;
                    model.rev = resp._rev;
                }
                originalSet.apply(null, [directModel, model, couchDBCallback]);
            });
        };
    };

    gpii.dataSource.preInit = function (that) {
        that.nickName = "dataSource"; // work around FLUID-4636
        // Only enable set if writable is true.
        if (that.options.writable) {
            that.options.invokers.set = "gpii.dataSource.set";
        }
    };

    fluid.demands("gpii.dataSource.get", "gpii.dataSource.URL", {
        funcName: "gpii.dataSource.URL.handle",
        args: [{
            urlResolver: "{urlResolver}",
            errback: "{errback}",
            modelParser: "{modelParser}",
            callbackWrapper: "{callbackWrapper}"
        }, "{arguments}.0", "{arguments}.1"]
    });

    fluid.demands("gpii.dataSource.set", "gpii.dataSource.URL", {
        funcName: "gpii.dataSource.URL.handle",
        args: [{
            urlResolver: "{urlResolver}",
            errback: "{errback}",
            modelParser: "{modelParser}",
            callbackWrapper: "{callbackWrapper}",
            writeMethod: "{dataSource}.options.writeMethod"
        }, "{arguments}.0", "{arguments}.2", "{arguments}.1"]
    });

    /*
        Handles calls to data source's (URL and CouchDB) get and set.
        Arguments:
            * options {Object}- an options block that contains urlResolver,
            errback, modelParser and writeMethod ("POST" or "PUT").
            * directModel (Object) - a collection of variables used to expand a url.
            * callback {Function} - that is called when data is sent/received from
            the URL.
            * model {String|Object} - a payload returned from the set/get operation.
    */
    gpii.dataSource.URL.handle = function (options, directModel, callback, model) {
            // Expand URL.
        var url = options.urlResolver.resolve(directModel),
            // Test whether the URL is to a local file.
            isFileProtocol = gpii.dataSource.isFileProtocol(url),
            func = gpii.dataSource.URL.handle.url,
            oldCallback = callback;
        if (isFileProtocol) {
            url = url.substring(7);
            func = gpii.dataSource.URL.handle.fs;
        }
        options.fireOnError = options.errback.events.onError.fire;
        if (options.callbackWrapper) {
            options.fireOnError = options.callbackWrapper.wrap(options.fireOnError);
        }
        callback = function (data) {
            data = options.modelParser.parse(data);
            if (data && data.isError) {
                options.fireOnError(data);
                return;
            }
            oldCallback(data);
        };
        // If model exists and is an object - stringify it.
        model = options.modelParser.stringify(model);
        func.apply(null, [url, options, callback, model]);
    };

    gpii.dataSource.URL.handle.url = function (path, options, callback, model) {
        var urlObj = url.parse(path, true),
            opts = {
                host: urlObj.hostname,
                port: parseInt(urlObj.port, 10),
                path: urlObj.path,
                method: "GET"
            };
        if (model) {
            opts.headers = {
                "Content-Type": "application/json",
                "Content-Length": model.length
            };
            opts.method = options.writeMethod;
        }
        var req = http.request(opts, function (res) {
            var data = "";
            res.setEncoding("utf8");
            res.on("data", function (chunk) {
                data += chunk;
            });
            res.on("end", function () {
                callback(data);
            });
        });
        req.on("error", function (error) {
            options.fireOnError({
                isError: true,
                message: error.message
            });
        });
        req.end(model);
        return req;
    };

    gpii.dataSource.URL.handle.fs = function (fileName, options, callback, model) {
        var method = "readFile",
            args = [fileName, "utf8", function (error, data) {
                if (error) {
                    options.fireOnError({
                        isError: true,
                        message: error.message
                    });
                    return;
                }
                callback(data || model);
            }];
        if (model) {
            method = "writeFile";
            args.splice(1, 0, model);
        }
        fs[method].apply(null, args);
    };

    /*
        Determines whether the URL is for local file system.
            * uri {String} - a uri to be tested.
    */
    gpii.dataSource.isFileProtocol = function(uri) {
        return /^file:\/\//.test(uri);
    };

    fluid.defaults("gpii.callbackWrappingPromiseData", {
        gradeNames: ["autoInit", "fluid.littleComponent"],
        components: {
            callbackWrapper: "{callbackWrapper}",
            rawSource: {
                type: "gpii.dataSource"
            }
        },
{'fraction_non_alphanumeric': 0.11372991876434374, 'fraction_numerical': 0.0023314269061236385, 'mean_word_length': 1.7169437846397466, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7580271', 'n_tokens_mistral': 7553, 'n_tokens_neox': 7349, 'n_words': 1799}