<commit_before>/*! resol-vbus | Copyright (c) 2013-2014, <NAME> | MIT license */
'use strict';



var crypto = require('crypto');
var fs = require('fs');
var path = require('path');


var _ = require('lodash');
var moment = require('moment');
var Q = require('q');
var request = require('request');


var HeaderSet = require('./header-set');
var utils = require('./utils');
var VBusRecordingConverter = require('./vbus-recording-converter');

var Recorder = require('./recorder');



var optionKeys = [
    'path',
];



var FileSystemRecorder = Recorder.extend({

    path: '.',

    constructor: function(options) {
        Recorder.call(this, options);

        _.extend(this, _.pick(options, optionKeys));
    },

    _getOptions: function() {
        var options = Recorder.prototype._getOptions.call(this);
        return _.extend(options, _.pick(this, optionKeys));
    },

    _getAbsoluteFilename: function(filename) {
        return path.resolve(this.path, this.getHash(this.id), filename);
    },

    _getCurrentSyncStateFilename: function(options) {
        return this._getAbsoluteFilename(this.getHash(options.id) + '_SyncState.json');
    },

    _getCurrentSyncState: function(options) {
        var filename = this._getCurrentSyncStateFilename(options);

        return Q.fcall(function() {
            return Q.npost(fs, 'readFile', [ filename ]);
        }).fail(function(err) {
            if (err.code === 'ENOENT') {
                return '{}';
            } else {
                throw err;
            }
        }).then(function(data) {
            return JSON.parse(data);
        });
    },

    _setCurrentSyncState: function(syncState, options) {
        var filename = this._getCurrentSyncStateFilename(options);

        return Q.fcall(function() {
            return JSON.stringify(syncState, null, '    ');
        }).then(function(data) {
            return Q.npost(fs, 'writeFile', [ filename, data ]);
        });
    },

    _recordSyncJob: function(recorder, syncJob) {
        var _this = this;

        var syncState = this._getSyncState(syncJob, 'destination', 'FileSystemRecorder');

        var syncVersion = syncState.version || 0;
        if (syncVersion === 0) {
            syncVersion = 1;
            syncState.infoListByInterval = {};
        }
        syncState.version = syncVersion;

        if (!syncState.infoListByInterval [syncJob.interval]) {
            syncState.infoListByInterval [syncJob.interval] = [];
        }

        var infoList = syncState.infoListByInterval [syncJob.interval];

        var lastTimestamp = null;

        var currentInfo = null;

        var outFile = null;

        var outConverter = null;

        var inConverter = new VBusRecordingConverter();

        inConverter.on('headerSet', function(headerSet) {
            var timestamp = headerSet.timestamp;

            if (lastTimestamp && (timestamp < lastTimestamp)) {
                // headersets are assumed to be played back in a chronological order,
                // so discard any headersets that are not...
                console.log('SKIPPING unlinear header sets');
                return;
            }
            lastTimestamp = timestamp;

            var timestampText = moment.utc(timestamp).format('YYYYMMDDHHmmssSSS');
<commit_msg>Refactor getting own sync state out into helper method.<commit_after>/*! resol-vbus | Copyright (c) 2013-2014, <NAME> | MIT license */
'use strict';



var crypto = require('crypto');
var fs = require('fs');
var path = require('path');


var _ = require('lodash');
var moment = require('moment');
var Q = require('q');
var request = require('request');


var HeaderSet = require('./header-set');
var utils = require('./utils');
var VBusRecordingConverter = require('./vbus-recording-converter');

var Recorder = require('./recorder');



var optionKeys = [
    'path',
];



var FileSystemRecorder = Recorder.extend({

    path: '.',

    constructor: function(options) {
        Recorder.call(this, options);

        _.extend(this, _.pick(options, optionKeys));
    },

    _getOptions: function() {
        var options = Recorder.prototype._getOptions.call(this);
        return _.extend(options, _.pick(this, optionKeys));
    },

    _getAbsoluteFilename: function(filename) {
        return path.resolve(this.path, this.getHash(this.id), filename);
    },

    _getCurrentSyncStateFilename: function(options) {
        return this._getAbsoluteFilename(this.getHash(options.id) + '_SyncState.json');
    },

    _getCurrentSyncState: function(options) {
        var filename = this._getCurrentSyncStateFilename(options);

        return Q.fcall(function() {
            return Q.npost(fs, 'readFile', [ filename ]);
        }).fail(function(err) {
            if (err.code === 'ENOENT') {
                return '{}';
            } else {
                throw err;
            }
        }).then(function(data) {
            return JSON.parse(data);
        });
    },

    _setCurrentSyncState: function(syncState, options) {
        var filename = this._getCurrentSyncStateFilename(options);

        return Q.fcall(function() {
            return JSON.stringify(syncState, null, '    ');
        }).then(function(data) {
            return Q.npost(fs, 'writeFile', [ filename, data ]);
        });
    },

    _getOwnSyncState: function(syncState, options) {
        var dstSyncState = this._getSyncState(syncState, 'destination', 'FileSystemRecorder');

        var syncVersion = syncState.version || 0;
        if (syncVersion === 0) {
            syncVersion = 1;
            syncState.infoListByInterval = {};
        }
        syncState.version = syncVersion;

        if (!syncState.infoListByInterval [options.interval]) {
            syncState.infoListByInterval [options.interval] = [];
        }

        var infoList = syncState.infoListByInterval [options.interval];

        return infoList;
    },

    _recordSyncJob: function(recorder, syncJob) {
        var _this = this;

        var infoList = _this._getOwnSyncState(syncJob, syncJob);

        var lastTimestamp = null;

        var currentInfo = null;

        var outFile = null;

        var outConverter = null;

        var inConverter = new VBusRecordingConverter();

        inConverter.on('headerSet', function(headerSet) {
            var timestamp = headerSet.timestamp;

            if (lastTimestamp && (timestamp < lastTimestamp)) {
                // headersets are assumed to be played back in a chronological order,
                // so discard any headersets that are not...
                console.log('SKIPPING unlinear header sets');
                return;
            }
            lastTimestamp = timestamp;

            var timestampText = moment.utc(timestamp).format('YYYYMMDDHHmmssSSS');

{'fraction_non_alphanumeric': 0.12682280159080866, 'fraction_numerical': 0.00324053616143762, 'mean_word_length': 2.413775766716943, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5903342', 'n_tokens_mistral': 1917, 'n_tokens_neox': 1818, 'n_words': 417}