<commit_before>var gulp = require("gulp");
var wdFrp = require("wdfrp");
var fs = require("fs-extra");
var path = require("path");

gulp.task("convert", function (done) {
    var convertFiles = require("./convertFiles");

    var convertMultiIndicesToSingleIndice = require("../convertIndices/convertMultiIndicesToSingleIndice");

    var filterPrimitiveDataByIndices = require("../filterPrimitiveData/filterPrimitiveDataByIndices");

    var compressToBinary = require("../compressToBinary/compressToBinary");

    var removeAttributeData = require("../removeAttributeData/removeAttributeData");

    var commandUtils = require("../../../../build/gulp_task/common/commandUtils");

    var Converter = require("../../dist/converter/Converter");


    var sourceDir = commandUtils.parseOption("--sourceDir") || "./source/",
        destDir = commandUtils.parseOption("--destDir") || "./dest/",
        isRemoveNullData = commandUtils.isDefinOption("--removeNullData") || true,
        isRemoveNormalData = commandUtils.isDefinOption("--removeNormalData") || false,
        isRemoveColorData = commandUtils.isDefinOption("--removeColorData") || false,
        isEmbedded = commandUtils.isDefinOption("--isEmbedded") || false;
    var converter = Converter.create();

    return convertFiles(isEmbedded, sourceDir, destDir)
        .flatMap(function(relativeDestWDFilePath){
            return wdFrp.fromNodeCallback(fs.readFile)(relativeDestWDFilePath)
                .map(function(fileBuffer){
                    return [fileBuffer, relativeDestWDFilePath];
                })
        })
        .filter(function(data){
            return !_isConvertedFromGLTF(data[0]);
        })
        .map(function(data){
            console.log("remove attribute data");
            return [removeAttributeData(isRemoveNormalData, isRemoveColorData, data[0]), data[1]];
        })
        .map(function(data){
            console.log("begin convertMultiIndicesToSingleIndice...");
            return [convertMultiIndicesToSingleIndice(isRemoveNullData, data[0]), data[1]];
        })
        .map(function(data){
            console.log("begin filterPrimitiveDataByIndices...");
            return [filterPrimitiveDataByIndices(isRemoveNullData, data[0]), data[1]];
        })
        .map(function(data){
            console.log("begin compressToBinary...");
            return [compressToBinary(isEmbedded, _getAbsoluteDir(sourceDir), data[0], data[1]), data[1]];
        })
        .map(function(data){
            console.log("write data to file");
            fs.writeFileSync(data[1], JSON.stringify(data[0]));
        })
        .subscribe(function (dataArr) {
        }, function (e) {
            console.log("error:", e.stack);
            done();
        }, function () {
            console.log("completed");
            done();
        });
});

function _isConvertedFromGLTF(fileBuffer){
    var json = JSON.parse(fileBuffer.toString());

    if(json.asset
        && json.asset.generator){
        return json.asset.generator.indexOf("GLTF") > -1;
    }

    return false;
}

function _getAbsoluteDir(dirPath) {
    return path.join(__dirname, "../../", dirPath);
}

<commit_msg>feat: if convert gltf file with embedded, warn "that gltf should use embedded version"<commit_after>var gulp = require("gulp");
var wdFrp = require("wdfrp");
var fs = require("fs-extra");
var path = require("path");

gulp.task("convert", function (done) {
    var convertFiles = require("./convertFiles");

    var convertMultiIndicesToSingleIndice = require("../convertIndices/convertMultiIndicesToSingleIndice");

    var filterPrimitiveDataByIndices = require("../filterPrimitiveData/filterPrimitiveDataByIndices");

    var compressToBinary = require("../compressToBinary/compressToBinary");

    var removeAttributeData = require("../removeAttributeData/removeAttributeData");

    var commandUtils = require("../../../../build/gulp_task/common/commandUtils");

    var Converter = require("../../dist/converter/Converter");


    var Log = require("../../../ts/Log");


    var sourceDir = commandUtils.parseOption("--sourceDir") || "./source/",
        destDir = commandUtils.parseOption("--destDir") || "./dest/",
        isRemoveNullData = commandUtils.isDefinOption("--removeNullData") || true,
        isRemoveNormalData = commandUtils.isDefinOption("--removeNormalData") || false,
        isRemoveColorData = commandUtils.isDefinOption("--removeColorData") || false,
        isEmbedded = commandUtils.isDefinOption("--isEmbedded") || false;
    var converter = Converter.create();

    return convertFiles(isEmbedded, sourceDir, destDir)
        .flatMap(function(relativeDestWDFilePath){
            return wdFrp.fromNodeCallback(fs.readFile)(relativeDestWDFilePath)
                .map(function(fileBuffer){
                    return [fileBuffer, relativeDestWDFilePath];
                })
        })
        .filter(function(data){
            if(_isConvertedFromGLTF(data[0])){
                if(isEmbedded){
                    Log.warn("GLTF should use the embedded version");
                }

                return false;
            }

            return true;
        })
        .map(function(data){
            console.log("remove attribute data");
            return [removeAttributeData(isRemoveNormalData, isRemoveColorData, data[0]), data[1]];
        })
        .map(function(data){
            console.log("begin convertMultiIndicesToSingleIndice...");
            return [convertMultiIndicesToSingleIndice(isRemoveNullData, data[0]), data[1]];
        })
        .map(function(data){
            console.log("begin filterPrimitiveDataByIndices...");
            return [filterPrimitiveDataByIndices(isRemoveNullData, data[0]), data[1]];
        })
        .map(function(data){
            console.log("begin compressToBinary...");
            return [compressToBinary(isEmbedded, _getAbsoluteDir(sourceDir), data[0], data[1]), data[1]];
        })
        .map(function(data){
            console.log("write data to file");
            fs.writeFileSync(data[1], JSON.stringify(data[0]));
        })
        .subscribe(function (dataArr) {
        }, function (e) {
            console.log("error:", e.stack);
            done();
        }, function () {
            console.log("completed");
            done();
        });
});

function _isConvertedFromGLTF(fileBuffer){
    var json = JSON.parse(fileBuffer.toString());

    if(json.asset
        && json.asset.generator){
        return json.asset.generator.indexOf("GLTF") > -1;
    }

    return false;
}

function _getAbsoluteDir(dirPath) {
    return path.join(__dirname, "../../", dirPath);
}


{'fraction_non_alphanumeric': 0.1459832134292566, 'fraction_numerical': 0.0038968824940047962, 'mean_word_length': 2.746771476698484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5926443', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1805, 'n_words': 302}