<commit_before>#!/usr/bin/env node

'use strict';

var exec = require('child_process').exec;

var cmdDriveWwid = 'ls -l /dev/disk/by-id';
var cmdVdInfo = 'ls -l /dev/disk/by-path';
var cmdScsiId = 'lsscsi';
var options = {
    timeout: 10000 //10 seconds
};

/**
 * Parse the Drive WWID output
 * @param {String} idList - stdout for command 'ls -l /dev/disk/by-id'
 * @return {Object} include device name, Linux and ESXi WWIDs for each disks
 */
function parseDriveWwid(idList) {
    //idList example
    //"lrwxrwxrwx. 1 root root  9 Nov 18 20:30 ata-SATADOM-SV_3SE_20150522AA9992050085 -> ../../sdb"
    //returned string example
    // "ata-SATADOM-SV_3SE_20150522AA9992050085->../../sdb"
    var lines = idList.split('\n').map(function(line) {
        var split = line.split(/\s+/);
        return [split[8],split[10]].join('->');
    });

    //According to SCSI-3 spec, vendor specified logic unit name string is 60
	//Only IDE and SCSI disk will be retrieved
    var scsiLines = [], sataLines = [], wwnLines = [], usbLines = [], requiredStrLen = 60;
    lines.forEach(function(line){
        if ( line && !(line.match('part')) && line.match(/sd[a-z]$|hd[a-z]$/i)){
            var nameIndex = line.lastIndexOf('/'), idIndex = line.lastIndexOf('->');
            if (line.indexOf('scsi') === 0) {
                scsiLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
            else if (line.indexOf('ata') === 0) {
                sataLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
            else if (line.indexOf('wwn') === 0) {
                wwnLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
            else if (line.indexOf('usb') === 0) {
                usbLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
        }
    });

    //wwnLine example: wwn-0x5000cca23de9e287 -> ../../sdb
    //esxiWwn example: ["sdb", "naa.5000cca23de9e287"]
    var esxiWwn = wwnLines.map(function(wwnLine) {
        var line = wwnLine[1];
        var split = line.split(/-/);
        return [wwnLine[0], 'naa.' + split[1].slice(2)];
    });

    //ESXi SATA WWID should be ('t10.ATA_____' + VendorInfo + Sub + SerialNumber)
    //VendorInfo + Sub + SerialNumber should be 60 characters.
    //Sub is made of N(N = 60 - VendorInfo - SerialNumber) underline '_' symbols
    //ESXi SATA WWID should finally replace remaing dashs '-' with '2D', '3D' ...
    //esxiLine example:
    //["sda", "ata-32GB_SATA_Flash_Drive_B061430580090000000F"]
    //esxiSata example
    //["sda", "t10.ATA_____32GB_SATA_Flash_Drive___________________B061430580090000000F"]
    //Todo: confrim above analysis is correct for all SATA disks.
    var esxiSata = sataLines.map(function(esxiLine) {
        var line = esxiLine[1];
        var headIndex = line.indexOf('-'), snIndex = line.lastIndexOf('_');
        var headStr = ['t10.', line.slice(0, headIndex).toUpperCase(), '_____'].join(''),
            vendorStr = line.slice(headIndex + 1, snIndex + 1),
            snStr = line.slice(snIndex + 1),
            dashStr = '';
        for (var i = 0; i< requiredStrLen - vendorStr.length - snStr.length; i += 1){
            dashStr += '_';
        }
        var strLine = [headStr, vendorStr, dashStr, snStr].join('');
        //return strLine.replace('-', '2D');
        //ESXi driveid replaces Nth '-' with 'ND' like 2D, 3D
        var strArray = strLine.split('-');
        if (strArray.length !== 1) {
            strLine = strArray[0];
            for (i = 0; i < strArray.length -1; i += 1){
                strLine = [strLine, (i + 2).toString(), 'D', strArray[i + 1]].join('');
            }
        }
        return [esxiLine[0], strLine];
    });

    //If one device Name is mapped to both WWN and SATA array, use WWN element instead
    esxiWwn.forEach(function(line) {
        esxiSata.forEach(function(subline){
            if (line[0] === subline[0]){
                subline [1] = line[1];
            }
        });
    });

    //esxiLine example: ["sda", "scsi-35000c500725f45d7"]
    //esxiScsi example: ["sda", "naa.5000c500725f45d7"]
    var esxiScsi = scsiLines.map(function(esxiLine) {
        var line = esxiLine[1];
        var split = line.split(/-|_/);
        return [esxiLine[0], 'naa.' + split[1].slice(1)];
    });

    var esxiUsb = usbLines.map(function(esxiLine) {
        return [esxiLine[0], 'mpx.'];
    });

    //linuxLine example: ["sda", "scsi-35000c500725f45d7"]
    //linuxParsed example: ["sda", "/dev/disk/by-id/scsi-35000c500725f45d7"]
    var linuxParsed = sataLines.concat(scsiLines, usbLines).map(function(linuxLine) {
        return [linuxLine[0], '/dev/disk/by-id/' + linuxLine[1]];
    });

    return {esxiDriveIds: esxiSata.concat(esxiScsi, esxiUsb), linuxDriveIds: linuxParsed};
}


/**
 * Parse Virtual Drive output via by-path output
 * @param {String} pathList - stdout for command 'ls -l /dev/disk/by-path'
 * @return {Array} include device name, virtual disk name and scsi ID for each disk
 */
function parseVdInfo(pathList) {
    var lines = pathList.split('\n').map(function(line) {
        var split = line.split(/\s+/);
        return [split[8],split[10]].join('->');
    });
    var pciLines = [];
    lines.forEach(function(line){
        if ( line && !(line.match('part'))){
            var nameIndex = line.lastIndexOf('/'), idIndex = line.lastIndexOf('->');
            if (line.indexOf('pci') === 0) {
                pciLines.push([line.slice(nameIndex + 1), line.slice(0,idIndex)]);
            }
        }
    });

    return pciLines.map(function(line) {
        var scsiId = line[1].slice(line[1].lastIndexOf('-') + 1), vdStr;
        if (scsiId.split(':').length === 4){
            var scsiIdArray = scsiId.split(':');
            //Suppose controller id = 0 stands for JBOD
            if (scsiIdArray[1] === "0") {
                vdStr = '';
            } else {
                vdStr = ['/c', scsiIdArray[0], '/v', scsiIdArray[2]].join('');
            }
        }
        return [line[0], vdStr, scsiId];
    });<commit_msg>Update comments to mark JBOS should have no virtual disk info<commit_after>#!/usr/bin/env node

'use strict';

var exec = require('child_process').exec;

var cmdDriveWwid = 'ls -l /dev/disk/by-id';
var cmdVdInfo = 'ls -l /dev/disk/by-path';
var cmdScsiId = 'lsscsi';
var options = {
    timeout: 10000 //10 seconds
};

/**
 * Parse the Drive WWID output
 * @param {String} idList - stdout for command 'ls -l /dev/disk/by-id'
 * @return {Object} include device name, Linux and ESXi WWIDs for each disks
 */
function parseDriveWwid(idList) {
    //idList example
    //"lrwxrwxrwx. 1 root root  9 Nov 18 20:30 ata-SATADOM-SV_3SE_20150522AA9992050085 -> ../../sdb"
    //returned string example
    // "ata-SATADOM-SV_3SE_20150522AA9992050085->../../sdb"
    var lines = idList.split('\n').map(function(line) {
        var split = line.split(/\s+/);
        return [split[8],split[10]].join('->');
    });

    //According to SCSI-3 spec, vendor specified logic unit name string is 60
	//Only IDE and SCSI disk will be retrieved
    var scsiLines = [], sataLines = [], wwnLines = [], usbLines = [], requiredStrLen = 60;
    lines.forEach(function(line){
        if ( line && !(line.match('part')) && line.match(/sd[a-z]$|hd[a-z]$/i)){
            var nameIndex = line.lastIndexOf('/'), idIndex = line.lastIndexOf('->');
            if (line.indexOf('scsi') === 0) {
                scsiLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
            else if (line.indexOf('ata') === 0) {
                sataLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
            else if (line.indexOf('wwn') === 0) {
                wwnLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
            else if (line.indexOf('usb') === 0) {
                usbLines.push([line.slice(nameIndex + 1), line.slice(0, idIndex)]);
            }
        }
    });

    //wwnLine example: wwn-0x5000cca23de9e287 -> ../../sdb
    //esxiWwn example: ["sdb", "naa.5000cca23de9e287"]
    var esxiWwn = wwnLines.map(function(wwnLine) {
        var line = wwnLine[1];
        var split = line.split(/-/);
        return [wwnLine[0], 'naa.' + split[1].slice(2)];
    });

    //ESXi SATA WWID should be ('t10.ATA_____' + VendorInfo + Sub + SerialNumber)
    //VendorInfo + Sub + SerialNumber should be 60 characters.
    //Sub is made of N(N = 60 - VendorInfo - SerialNumber) underline '_' symbols
    //ESXi SATA WWID should finally replace remaing dashs '-' with '2D', '3D' ...
    //esxiLine example:
    //["sda", "ata-32GB_SATA_Flash_Drive_B061430580090000000F"]
    //esxiSata example
    //["sda", "t10.ATA_____32GB_SATA_Flash_Drive___________________B061430580090000000F"]
    //Todo: confrim above analysis is correct for all SATA disks.
    var esxiSata = sataLines.map(function(esxiLine) {
        var line = esxiLine[1];
        var headIndex = line.indexOf('-'), snIndex = line.lastIndexOf('_');
        var headStr = ['t10.', line.slice(0, headIndex).toUpperCase(), '_____'].join(''),
            vendorStr = line.slice(headIndex + 1, snIndex + 1),
            snStr = line.slice(snIndex + 1),
            dashStr = '';
        for (var i = 0; i< requiredStrLen - vendorStr.length - snStr.length; i += 1){
            dashStr += '_';
        }
        var strLine = [headStr, vendorStr, dashStr, snStr].join('');
        //return strLine.replace('-', '2D');
        //ESXi driveid replaces Nth '-' with 'ND' like 2D, 3D
        var strArray = strLine.split('-');
        if (strArray.length !== 1) {
            strLine = strArray[0];
            for (i = 0; i < strArray.length -1; i += 1){
                strLine = [strLine, (i + 2).toString(), 'D', strArray[i + 1]].join('');
            }
        }
        return [esxiLine[0], strLine];
    });

    //If one device Name is mapped to both WWN and SATA array, use WWN element instead
    esxiWwn.forEach(function(line) {
        esxiSata.forEach(function(subline){
            if (line[0] === subline[0]){
                subline [1] = line[1];
            }
        });
    });

    //esxiLine example: ["sda", "scsi-35000c500725f45d7"]
    //esxiScsi example: ["sda", "naa.5000c500725f45d7"]
    var esxiScsi = scsiLines.map(function(esxiLine) {
        var line = esxiLine[1];
        var split = line.split(/-|_/);
        return [esxiLine[0], 'naa.' + split[1].slice(1)];
    });

    var esxiUsb = usbLines.map(function(esxiLine) {
        return [esxiLine[0], 'mpx.'];
    });

    //linuxLine example: ["sda", "scsi-35000c500725f45d7"]
    //linuxParsed example: ["sda", "/dev/disk/by-id/scsi-35000c500725f45d7"]
    var linuxParsed = sataLines.concat(scsiLines, usbLines).map(function(linuxLine) {
        return [linuxLine[0], '/dev/disk/by-id/' + linuxLine[1]];
    });

    return {esxiDriveIds: esxiSata.concat(esxiScsi, esxiUsb), linuxDriveIds: linuxParsed};
}


/**
 * Parse Virtual Drive output via by-path output
 * @param {String} pathList - stdout for command 'ls -l /dev/disk/by-path'
 * @return {Array} include device name, virtual disk name and scsi ID for each disk
 */
function parseVdInfo(pathList) {
    var lines = pathList.split('\n').map(function(line) {
        var split = line.split(/\s+/);
        return [split[8],split[10]].join('->');
    });
    var pciLines = [];
    lines.forEach(function(line){
        if ( line && !(line.match('part'))){
            var nameIndex = line.lastIndexOf('/'), idIndex = line.lastIndexOf('->');
            if (line.indexOf('pci') === 0) {
                pciLines.push([line.slice(nameIndex + 1), line.slice(0,idIndex)]);
            }
        }
    });

    return pciLines.map(function(line) {
        var scsiId = line[1].slice(line[1].lastIndexOf('-') + 1), vdStr;
        if (scsiId.split(':').length === 4){
            var scsiIdArray = scsiId.split(':');
            //Suppose controller id = 0 stands for JBOD
            if (scsiIdArray[1] === "0") {
                //JBOD should have no virtual disk attribute
                vdStr = '';
            } else {
                vdStr = ['/c', scsiIdArray[0], '/v', scsiIdArray[2]].join('');
            }
        }
        return [line[0], vdStr, scsiId];
    });
{'fraction_non_alphanumeric': 0.17073764505396413, 'fraction_numerical': 0.04057453542156942, 'mean_word_length': 2.3543821448013067, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3924645', 'n_tokens_mistral': 4572, 'n_tokens_neox': 4179, 'n_words': 951}