<commit_before>function Validator(options) {

    if (!(this instanceof Validator)) {
        return new Validator(options);
    }

    this.file = options.file ? options.file : null;

    this.log = [];
    this.line = 0;
    this.progress = 0;
    this._readBytes = 0;
    this._events = {};
    this.numLines = 0;

    this.linesToRead = 2000;
}

Validator.prototype = {
    init: function () {
        this.file = null;
        this.log = [];
        this.line = 0;
        this.numLines = 0;
        this.progress = 0;
        this._readBytes = 0;
    },
    stop: function (cb) {
        if (this._navigator != null) {
            this._navigator._stop = true;
        }
    },

    validate: function () {
        var me = this;

        /*Check if file is \r or \n , \r\n */
        this._detectCRSeparator(this.file, function (res) {
            var lastReadBytes = null;
            if (res) {
                me._navigator = new FileNavigator(me.file, undefined, {
                    newLineCode: '\r'.charCodeAt(0),
                    splitPattern: /\r/
                });
            } else {
                me._navigator = new FileNavigator(me.file);
            }

            me._totalBytes = me.file.size;
            var indexToStartWith = 0;

            // me._navigator.readSomeLines(indexToStartWith, function linesReadHandler(err, index, lines, eof, progress) {
            me._navigator.readLines(indexToStartWith, me.linesToRead, function linesReadHandler(err, index, lines, eof, progress) {
                if (err) {
                    me._emit("err");
                    return;
                }
                // console.log(lines.length);
                console.log(progress);
                for (var i = 0; i < lines.length; i++) {
                    var line = lines[i];
                    me.line++;

                    me.numLines++;
                    // me._emit("lines", [me.numLines]);
                    me._readBytes += line.length;
                    me.progress = (me._readBytes / me._totalBytes) * 100;
                    me.validateLine(line);
                }
                // me._emit("progress", [me.progress]);

                if (lastReadBytes == me._readBytes) {
                    me._emit("progress", 100);
                    me._emit("end");
                    me._validateEnd();
                    return;
                }

                if (me._navigator._stop != true) {
                    // me._navigator.readSomeLines(index + lines.length, linesReadHandler);
                    me._navigator.readLines(index + lines.length, me.linesToRead, linesReadHandler);
                } else {
                    me._emit("stop");
                    console.log("STOP!!!!!");
                }

            })

        });

    },
    validateLine: function (line) {
        return true;
    },
    _validateEnd: function () {
        this.validateEnd();
    },
    validateEnd: function () {<commit_msg>Solved error in validator.js<commit_after>function Validator(options) {

    if (!(this instanceof Validator)) {
        return new Validator(options);
    }

    this.file = options.file ? options.file : null;

    this.log = [];
    this.line = 0;
    this.progress = 0;
    this._readBytes = 0;
    this._events = {};
    this.numLines = 0;

    this.linesToRead = 2000;
}

Validator.prototype = {
    init: function () {
        this.file = null;
        this.log = [];
        this.line = 0;
        this.numLines = 0;
        this.progress = 0;
        this._readBytes = 0;
    },
    stop: function (cb) {
        if (this._navigator != null) {
            this._navigator._stop = true;
        }
    },

    validate: function () {
        var me = this;

        /*Check if file is \r or \n , \r\n */
        this._detectCRSeparator(this.file, function (res) {
            var lastReadBytes = null;
            if (res) {
                me._navigator = new FileNavigator(me.file, undefined, {
                    newLineCode: '\r'.charCodeAt(0),
                    splitPattern: /\r/
                });
            } else {
                me._navigator = new FileNavigator(me.file);
            }

            me._totalBytes = me.file.size;
            var indexToStartWith = 0;

            // me._navigator.readSomeLines(indexToStartWith, function linesReadHandler(err, index, lines, eof, progress) {
            me._navigator.readLines(indexToStartWith, me.linesToRead, function linesReadHandler(err, index, lines, eof, progress) {
                if (err) {
                    me._emit("err");
                    return;
                }
                // console.log(lines.length);
                console.log(progress);
                for (var i = 0; i < lines.length; i++) {
                    var line = lines[i];
                    me.line++;

                    me.numLines++;
                    // me._emit("lines", [me.numLines]);
                    me._readBytes += line.length;
                    me.progress = (me._readBytes / me._totalBytes) * 100;
                    me.validateLine(line);
                }
                // me._emit("progress", [me.progress]);
                if (lastReadBytes == me._readBytes) {
                    me._emit("progress", 100);
                    me._emit("end");
                    me._validateEnd();
                    return;
                }
                lastReadBytes= me._readBytes;

                if (me._navigator._stop != true) {
                    // me._navigator.readSomeLines(index + lines.length, linesReadHandler);
                    me._navigator.readLines(index + lines.length, me.linesToRead, linesReadHandler);
                } else {
                    me._emit("stop");
                    console.log("STOP!!!!!");
                }

            })

        });

    },
    validateLine: function (line) {
        return true;
    },
    _validateEnd: function () {
        this.validateEnd();
    },
    validateEnd: function () {
{'fraction_non_alphanumeric': 0.12437810945273632, 'fraction_numerical': 0.006965174129353234, 'mean_word_length': 1.3063097514340345, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4714932', 'n_tokens_mistral': 1731, 'n_tokens_neox': 1592, 'n_words': 325}