<commit_before>/*
 * self.js
 * Copyright (C) 2015 <NAME> <kovid at kovidgoyal.net>
 *
 * Distributed under terms of the BSD license.
 */
"use strict;";

var path = require('path');
var crypto = require('crypto');
var fs = require('fs');
var RapydScript = require('./compiler');

module.exports = function compile_self(base_path, src_path, lib_path, start_time) {
    var output_options = {'beautify': true, 'private_scope': false, 'omit_baselib': true, 'write_name': false};
	var baselib = RapydScript.parse_baselib(src_path, true);

    function timed(name, cont) {
        var t1 = new Date().getTime();
        console.log('Compiling', name, '...');
        var ret = cont();
        console.log('Compiled in', (new Date().getTime() - t1)/1000, 'seconds\n');
        return ret;
    }

	function parse_file(code, file) {
		return RapydScript.parse(code, {
			filename: file,
			basedir: path.dirname(file),
			auto_bind: false,
			libdir: path.join(src_path, 'lib'),
		});
	}


    var saved_hashes = {}, hashes = {}, compiled = {};
    var compiler_changed = false, sha1sum;
    var signatures = path.join(lib_path, 'signatures.json');
    try {
        saved_hashes = JSON.parse(fs.readFileSync(signatures, 'utf-8'));
    } catch (e) {
        if (e.code != 'ENOENT') throw (e);
    }

    sha1sum = crypto.createHash('sha1');
    RapydScript.FILES.concat([module.filename, path.join(base_path, 'tools', 'compiler.js')]).forEach(function (fpath) {
        sha1sum.update(fs.readFileSync(fpath));
    });
    hashes['#compiler#'] = sha1sum.digest('hex');
    source_hash = crypto.createHash('sha1');
    sources = {};
    RapydScript.FILENAMES.forEach(function (fname) {
        var src = path.join(src_path, fname + '.pyj');
        var h = crypto.createHash('sha1');
        var raw = fs.readFileSync(src, 'utf-8');
        // Remove copyright comment as it is unreadable in the output JS anyway
        var idx = raw.indexOf('"use strict');
        if (idx > -1) {
            var cmt = raw.substr(0, idx);
            var nlines = cmt.split('\n').length;
            raw = (new Array(nlines)).join('\n') + raw.substr(idx);
        }
        sources[src] = raw;
        source_hash.update(raw);
        h.update(raw);
        hashes[fname] = h.digest('hex');
    });
    source_hash = source_hash.digest('hex');
    compiler_changed = (hashes['#compiler#'] != saved_hashes['#compiler#']) ? true : false;
    function changed(name) {
        return compiler_changed || hashes[name] != saved_hashes[name];
    }

    function generate_baselib() {
        var output = '';
        Object.keys(baselib).forEach(function(key) {
            output += String(baselib[key]) + '\n\n';
        });
        return output;
    }

    if (changed('baselib')) compiled.baselib = timed('baselib', generate_baselib);
    RapydScript.FILENAMES.slice(1).forEach(function (fname) {
        if (changed(fname)) {<commit_msg>Revert "Preserve linenumbers when compiling self"<commit_after>/*
 * self.js
 * Copyright (C) 2015 <NAME> <kovid at kovidgoyal.net>
 *
 * Distributed under terms of the BSD license.
 */
"use strict;";

var path = require('path');
var crypto = require('crypto');
var fs = require('fs');
var RapydScript = require('./compiler');

module.exports = function compile_self(base_path, src_path, lib_path, start_time) {
    var output_options = {'beautify': true, 'private_scope': false, 'omit_baselib': true, 'write_name': false};
	var baselib = RapydScript.parse_baselib(src_path, true);

    function timed(name, cont) {
        var t1 = new Date().getTime();
        console.log('Compiling', name, '...');
        var ret = cont();
        console.log('Compiled in', (new Date().getTime() - t1)/1000, 'seconds\n');
        return ret;
    }

	function parse_file(code, file) {
		return RapydScript.parse(code, {
			filename: file,
			basedir: path.dirname(file),
			auto_bind: false,
			libdir: path.join(src_path, 'lib'),
		});
	}


    var saved_hashes = {}, hashes = {}, compiled = {};
    var compiler_changed = false, sha1sum;
    var signatures = path.join(lib_path, 'signatures.json');
    try {
        saved_hashes = JSON.parse(fs.readFileSync(signatures, 'utf-8'));
    } catch (e) {
        if (e.code != 'ENOENT') throw (e);
    }

    sha1sum = crypto.createHash('sha1');
    RapydScript.FILES.concat([module.filename, path.join(base_path, 'tools', 'compiler.js')]).forEach(function (fpath) {
        sha1sum.update(fs.readFileSync(fpath));
    });
    hashes['#compiler#'] = sha1sum.digest('hex');
    source_hash = crypto.createHash('sha1');
    sources = {};
    RapydScript.FILENAMES.forEach(function (fname) {
        var src = path.join(src_path, fname + '.pyj');
        var h = crypto.createHash('sha1');
        var raw = fs.readFileSync(src, 'utf-8');
        // Remove copyright comment as it is unreadable in the output JS anyway
        var idx = raw.indexOf('"use strict');
        if (idx > -1) raw = raw.substr(idx);
        sources[src] = raw;
        source_hash.update(raw);
        h.update(raw);
        hashes[fname] = h.digest('hex');
    });
    source_hash = source_hash.digest('hex');
    compiler_changed = (hashes['#compiler#'] != saved_hashes['#compiler#']) ? true : false;
    function changed(name) {
        return compiler_changed || hashes[name] != saved_hashes[name];
    }

    function generate_baselib() {
        var output = '';
        Object.keys(baselib).forEach(function(key) {
            output += String(baselib[key]) + '\n\n';
        });
        return output;
    }

    if (changed('baselib')) compiled.baselib = timed('baselib', generate_baselib);
    RapydScript.FILENAMES.slice(1).forEach(function (fname) {
        if (changed(fname)) {
{'fraction_non_alphanumeric': 0.1638655462184874, 'fraction_numerical': 0.007528011204481793, 'mean_word_length': 3.0633001422475106, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299925', 'n_tokens_mistral': 1932, 'n_tokens_neox': 1865, 'n_words': 400}