<commit_before>var Knex = global.Packages.Knex,
    _ = global.Packages.lodash;


var _knex;

function initialize(params) {

    _knex = Knex.initialize({
        client: "mysql",
        connection: {
            host: params.host,
            user: params.uid,
            password: <PASSWORD>,
            database: params.db,
            charset: (params.charset) ? params.charset : "UTF8_GENERAL_CI"
        }
    });

    function deleteAllPropertiesWhichAreUndefined(object) {
        for (var k in object) {
            if (object[k] === undefined) {
                delete object[k];
            }
        }

        return object;
    }

    function zeroPad(number, length) {
        number = number.toString();
        while (number.length < length) {
            number = "0" + number;
        }

        return number;
    }

    function dateToString(date) {
        var dt = new Date(date);
        dt.setTime(dt.getTime() + (dt.getTimezoneOffset() * 60000));

        var year = dt.getFullYear();
        var month = zeroPad(dt.getMonth() + 1, 2);
        var day = zeroPad(dt.getDate(), 2);
        var hour = zeroPad(dt.getHours(), 2);
        var minute = zeroPad(dt.getMinutes(), 2);
        var second = zeroPad(dt.getSeconds(), 2);
        var millisecond = zeroPad(dt.getMilliseconds(), 3);

        return year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second + "." + millisecond;
    }



    function escape(val) {
        if (val === undefined || val === null) {
            return "NULL";
        }

        switch (typeof val) {
        case "boolean":
            return (val) ? "true" : "false";
        case "number":
            return val + "";
        }

        if (val instanceof Date) {
            val = dateToString(val);
        }


        val = val.replace(/[\0\n\r\b\t\\\'\"\x1a]/g, function(s) {
            switch (s) {
            case "\0":
                return "\\0";
            case "\n":
                return "\\n";
            case "\r":
                return "\\r";
            case "\b":
                return "\\b";
            case "\t":
                return "\\t";
            case "\x1a":
                return "\\Z";
            default:
                return "\\" + s;
            }
        });

        return val;
    }

    function getInsertSQL(ctx, object) {


        var isFirst = true;
        var dup = "";
        var sql = "INSERT INTO " + ctx.tableName + " SET ";

        for (var prop in object) {

            if (object[prop] || object[prop] === false) {

                if (isFirst) {
                    isFirst = false;
                }
                else {
                    sql += ", ";
                    dup += ", ";
                }
                sql += prop + " = '" + escape(object[prop]) + "'";
                dup += prop + " = values(" + prop + ")";
            }
        }


        sql += " ON DUPLICATE KEY UPDATE " + dup + ";";
        return sql;
    }

    function extend(attributes) {
        if (!attributes.tableName) {
            throw new Error("tableName must be defined");
        }

        function Model(obj) {

            //if model creation is not with new keyword then return it with new 
            if (!(this instanceof Model)) {
                return new Model(obj);
            }

            this.tableName = attributes.tableName;

            this.input = obj;
        }

        Model.prototype.update = function update(where_clause, set_fields) {
            var self = this;

            return _knex(self.tableName).where(where_clause).update(set_fields);

        };

        Model.prototype.save = function save() {
            var self = this;
            return _knex.raw(getInsertSQL(self, self.input));
        };

        Model.prototype.saveInBatch = function saveInBatch() {
            var self = this;
            var batch_statement = "";

            for (var index = 0; index < self.input.length; index++) {
                batch_statement += getInsertSQL(self, self.input[index]);
            }

            return _knex.raw(batch_statement);

        };



        Model.prototype.fetch = function fetch() {
            var self = this;
            if (arguments.length > 0) {

                return _knex(self.tableName).where(self.input).select(_.toArray(arguments)).then(function(obj) {
                    if (obj.length > 0) {
                        return self.getObjectFromDBObject(obj.shift());
                    }else{
                        return null;
                    }
                });
            }
            else {
                return _knex(self.tableName).where(self.input).select().then(function(obj) {
                    if (obj.length > 0) {
                        return self.getObjectFromDBObject(obj.shift());
                    }else{
                        return null;
                    }
                });
            }

        };

        Model.prototype.getInBatch = function getInBatch() {
            var self = this;
            var result = [];
            return _knex(self.tableName).whereIn(((self.idAttribute) ? self.idAttribute : "id"), self.input).then(function(output) {

                while (output.length > 0) {
                    result.push(self.getObjectFromDBObject(output.shift()));
                }
<commit_msg>mysql fix getDBObject<commit_after>var Knex = global.Packages.Knex,
    _ = global.Packages.lodash;


var _knex;

function initialize(params) {

    _knex = Knex.initialize({
        client: "mysql",
        connection: {
            host: params.host,
            user: params.uid,
            password: <PASSWORD>,
            database: params.db,
            charset: (params.charset) ? params.charset : "UTF8_GENERAL_CI"
        }
    });

    function deleteAllPropertiesWhichAreUndefined(object) {
        for (var k in object) {
            if (object[k] === undefined) {
                delete object[k];
            }
        }

        return object;
    }

    function zeroPad(number, length) {
        number = number.toString();
        while (number.length < length) {
            number = "0" + number;
        }

        return number;
    }

    function dateToString(date) {
        var dt = new Date(date);
        dt.setTime(dt.getTime() + (dt.getTimezoneOffset() * 60000));

        var year = dt.getFullYear();
        var month = zeroPad(dt.getMonth() + 1, 2);
        var day = zeroPad(dt.getDate(), 2);
        var hour = zeroPad(dt.getHours(), 2);
        var minute = zeroPad(dt.getMinutes(), 2);
        var second = zeroPad(dt.getSeconds(), 2);
        var millisecond = zeroPad(dt.getMilliseconds(), 3);

        return year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second + "." + millisecond;
    }



    function escape(val) {
        if (val === undefined || val === null) {
            return "NULL";
        }

        switch (typeof val) {
        case "boolean":
            return (val) ? "true" : "false";
        case "number":
            return val + "";
        }

        if (val instanceof Date) {
            val = dateToString(val);
        }


        val = val.replace(/[\0\n\r\b\t\\\'\"\x1a]/g, function(s) {
            switch (s) {
            case "\0":
                return "\\0";
            case "\n":
                return "\\n";
            case "\r":
                return "\\r";
            case "\b":
                return "\\b";
            case "\t":
                return "\\t";
            case "\x1a":
                return "\\Z";
            default:
                return "\\" + s;
            }
        });

        return val;
    }

    function getInsertSQL(ctx, object) {


        var isFirst = true;
        var dup = "";
        var sql = "INSERT INTO " + ctx.tableName + " SET ";

        for (var prop in object) {

            if (object[prop] || object[prop] === false) {

                if (isFirst) {
                    isFirst = false;
                }
                else {
                    sql += ", ";
                    dup += ", ";
                }
                sql += prop + " = '" + escape(object[prop]) + "'";
                dup += prop + " = values(" + prop + ")";
            }
        }


        sql += " ON DUPLICATE KEY UPDATE " + dup + ";";
        return sql;
    }

    function extend(attributes) {
        if (!attributes.tableName) {
            throw new Error("tableName must be defined");
        }

        function Model(obj) {

            //if model creation is not with new keyword then return it with new 
            if (!(this instanceof Model)) {
                return new Model(obj);
            }

            this.tableName = attributes.tableName;

            this.input = obj;
        }

        Model.prototype.update = function update(where_clause, set_fields) {
            var self = this;

            return _knex(self.tableName).where(where_clause).update(set_fields);

        };

        Model.prototype.save = function save() {
            var self = this;
            return _knex.raw(getInsertSQL(self, self.getDBObject(self.input)));
        };

        Model.prototype.saveInBatch = function saveInBatch() {
            var self = this;
            var batch_statement = "";

            for (var index = 0; index < self.input.length; index++) {
                batch_statement += getInsertSQL(self, self.getDBObject(self.input[index]));
            }

            return _knex.raw(batch_statement);

        };



        Model.prototype.fetch = function fetch() {
            var self = this;
            if (arguments.length > 0) {

                return _knex(self.tableName).where(self.getDBObject(self.input)).select(_.toArray(arguments)).then(function(obj) {
                    if (obj.length > 0) {
                        return self.getObjectFromDBObject(obj.shift());
                    }else{
                        return null;
                    }
                });
            }
            else {
                return _knex(self.tableName).where(self.getDBObject(self.input)).select().then(function(obj) {
                    if (obj.length > 0) {
                        return self.getObjectFromDBObject(obj.shift());
                    }else{
                        return null;
                    }
                });
            }

        };

        Model.prototype.getInBatch = function getInBatch() {
            var self = this;
            var result = [];
            return _knex(self.tableName).whereIn(((self.idAttribute) ? self.idAttribute : "id"), self.getDBObject(self.input)).then(function(output) {

                while (output.length > 0) {
                    result.push(self.getObjectFromDBObject(output.shift()));
                }

{'fraction_non_alphanumeric': 0.12966039128829826, 'fraction_numerical': 0.004429678848283499, 'mean_word_length': 1.3155982905982906, 'pattern_counts': {'":': 20, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2442167', 'n_tokens_mistral': 3083, 'n_tokens_neox': 2856, 'n_words': 652}