<commit_before>#!/usr/bin/env node

let firebase = require("firebase-admin"),
    glob = require("glob"),
    yamljs = require("yamljs"),
    fs = require("fs-extra"),
    crypto = require('crypto'),
    metaMarked = require("meta-marked"),
    path = require("path"),
    moment = require("moment");

const { getCompilationQuarterValue, getInfoFromPath } = require('./deploy-helper')

let argv = require("optimist").usage("Compile & deploy video - DON'T USE IF YOU DON'T KNOW WHAT IT DOES\n" +
    "Usage: $0 -b [string]")
    .alias({"b": "branch"})
    .describe({
        "b": "branch",
        "l": "language",
        "q": "quarter",
        "m": "mode"
    })
    .demand(["b"])
    .argv;

let branch = argv.b,
    compile_language = argv.l || "*",
    compile_quarter = argv.q || getCompilationQuarterValue(),
    mode = argv.m || "sync";

let API_HOST = "https://sabbath-school.adventech.io/api/",
    MEDIA_HOST = "https://sabbath-school-media.adventech.io/",
    API_VERSION = "v1",
    SOURCE_DIR = "src/",
    SOURCE_VIDEO_FILE = "video.yml",
    SOURCE_COVER_FILE = "cover.png",
    DIST_DIR = "dist/api/" + API_VERSION + "/",
    FIREBASE_DATABASE_VIDEO = "/api/" + API_VERSION + "/video";

let db
if (branch.toLowerCase() === "master") {
    API_HOST = "https://sabbath-school.adventech.io/api/";
    MEDIA_HOST = "https://sabbath-school-media.adventech.io/";
    firebase.initializeApp({
        databaseURL: "https://blistering-inferno-8720.firebaseio.com",
        credential: firebase.credential.cert(require('./deploy-creds.json')),
        databaseAuthVariableOverride: {
            uid: "deploy"
        }
    });
    db = firebase.database();
} else if (branch.toLowerCase() === "stage") {
    API_HOST = "https://sabbath-school-stage.adventech.io/api/";
    MEDIA_HOST = "https://sabbath-school-media-stage.adventech.io/";
    firebase.initializeApp({
        databaseURL: "https://sabbath-school-stage.firebaseio.com",
        credential: firebase.credential.cert(require('./deploy-creds-stage.json')),
        databaseAuthVariableOverride: {
            uid: "deploy"
        }
    });
    db = firebase.database();
} else {
    firebase = {
        app: function () {
            return {
                delete: function () {}
            }
        }
    };
    db = {
        ref: function () {
            return {
                set: function () {},
                child: function () {
                    return {
                        set: function (a) {},
                        once: function () {}
                    }
                }
            }
        },
        goOffline: function () {}
    }
}

let videoAPI = async function (mode) {
    const allowedVideoItemKeys = ['title', 'src', 'thumbnail', 'target', 'duration']
    console.log('Deploying video API');

    let videos = glob.sync(`${SOURCE_DIR}/${compile_language}/${compile_quarter}/${SOURCE_VIDEO_FILE}`);
    let videoAPIJson = []

    let curlConfig = ""

    for (let video of videos) {
        let videoSource = yamljs.load(`${video}`),
            info = getInfoFromPath(video);

        for (let artist of videoSource.video) {
            let videoInfo = {
                artist: artist.artist,
                clips: []
            }

            if (artist.thumbnail) {
                videoInfo.thumbnail = artist.thumbnail
            }

            for (let clip of artist.clips) {
                if (!clip['target'] || !clip['src']) { continue }

                let videoItem = {
                    artist: videoInfo.artist
                }

                videoItem.id = crypto.createHash('sha256').update(artist.artist + clip['target'] + clip['src']).digest('hex');

                for (let k of Object.keys(clip)) {
                    if (allowedVideoItemKeys.indexOf(k) >= 0) {
                        videoItem[k] = clip[k]
                    }
                }

                let extname = path.extname(videoItem.src)

                if (!extname.length || extname.length <= 1) {
                    extname = ".mp4"
                }

                videoItem.src = `${MEDIA_HOST}video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}`

                videoItem.targetIndex = videoItem.target.replace(/\//g, '-')

                if (!videoItem.thumbnail) {
                    if (artist.thumbnail && !/^http/.test(artist.thumbnail.trim())) {
                        artist.thumbnail = `${API_HOST}${API_VERSION}/images/${artist.thumbnail}`
                    }
                    videoItem.thumbnail = artist.thumbnail || `${API_HOST}${API_VERSION}/${info.language}/quarterlies/${info.quarterly}/${SOURCE_COVER_FILE}`
                }

                if (videoItem.duration) {
                    if (typeof videoItem.duration === 'number') {
                        videoItem.duration = moment("2015-01-01").startOf('day').seconds(videoItem.duration).format("H:mm:ss").replace(/^[0:]+(?=\d[\d:]{3})/, '');
                    }
                }

                if (!videoItem.title) {
                    let videoItemInfo = getInfoFromPath(`src/${videoItem.target}`)
                    if (!videoItemInfo.lesson) {
                        continue
                    }

                    if (videoItemInfo.day) {
                        let read = metaMarked(fs.readFileSync(`${SOURCE_DIR}${videoItemInfo.language}/${videoItemInfo.quarterly}/${videoItemInfo.lesson}/${videoItemInfo.day}.md`, "utf-8"))
                        videoItem.title = read.meta.title
                    } else {
                        let lesson = yamljs.load(`${SOURCE_DIR}${videoItemInfo.language}/${videoItemInfo.quarterly}/${videoItemInfo.lesson}/info.yml`)
                        videoItem.title = lesson.title
                    }
                }

                videoInfo.clips.push(videoItem)

                if (mode === "keep" && fs.pathExistsSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}`)) {
                    let stats = fs.statSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}`);
                    if (stats.size > 0) {
                        fs.outputFileSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/.keep`, "");
                    }
                }

                if (mode === "gen" && !fs.pathExistsSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/`)) {
                    curlConfig += `
url = "${clip.src}"
output = "video/video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}"
-C -
--create-dirs
-L
`
                }
            }

            if (videoInfo.clips.length) {

                videoInfo.clips = videoInfo.clips.sort(function(a, b){
                    if (a.targetIndex < b.targetIndex) {
                        return a.targetIndex.length < b.targetIndex.length ? 1 : -1;
                    }

                    if (a.targetIndex > b.targetIndex) {
                        return a.targetIndex.length > b.targetIndex.length ? -1 : 1;
                    }
                    return 0;
                })

                videoAPIJson.push(videoInfo)
            }
        }

        if (mode === "sync") {<commit_msg>Sorting videos desc by default<commit_after>#!/usr/bin/env node

let firebase = require("firebase-admin"),
    glob = require("glob"),
    yamljs = require("yamljs"),
    fs = require("fs-extra"),
    crypto = require('crypto'),
    metaMarked = require("meta-marked"),
    path = require("path"),
    moment = require("moment");

const { getCompilationQuarterValue, getInfoFromPath } = require('./deploy-helper')

let argv = require("optimist").usage("Compile & deploy video - DON'T USE IF YOU DON'T KNOW WHAT IT DOES\n" +
    "Usage: $0 -b [string]")
    .alias({"b": "branch"})
    .describe({
        "b": "branch",
        "l": "language",
        "q": "quarter",
        "m": "mode"
    })
    .demand(["b"])
    .argv;

let branch = argv.b,
    compile_language = argv.l || "*",
    compile_quarter = argv.q || getCompilationQuarterValue(),
    mode = argv.m || "sync";

let API_HOST = "https://sabbath-school.adventech.io/api/",
    MEDIA_HOST = "https://sabbath-school-media.adventech.io/",
    API_VERSION = "v1",
    SOURCE_DIR = "src/",
    SOURCE_VIDEO_FILE = "video.yml",
    SOURCE_COVER_FILE = "cover.png",
    DIST_DIR = "dist/api/" + API_VERSION + "/",
    FIREBASE_DATABASE_VIDEO = "/api/" + API_VERSION + "/video";

let db
if (branch.toLowerCase() === "master") {
    API_HOST = "https://sabbath-school.adventech.io/api/";
    MEDIA_HOST = "https://sabbath-school-media.adventech.io/";
    firebase.initializeApp({
        databaseURL: "https://blistering-inferno-8720.firebaseio.com",
        credential: firebase.credential.cert(require('./deploy-creds.json')),
        databaseAuthVariableOverride: {
            uid: "deploy"
        }
    });
    db = firebase.database();
} else if (branch.toLowerCase() === "stage") {
    API_HOST = "https://sabbath-school-stage.adventech.io/api/";
    MEDIA_HOST = "https://sabbath-school-media-stage.adventech.io/";
    firebase.initializeApp({
        databaseURL: "https://sabbath-school-stage.firebaseio.com",
        credential: firebase.credential.cert(require('./deploy-creds-stage.json')),
        databaseAuthVariableOverride: {
            uid: "deploy"
        }
    });
    db = firebase.database();
} else {
    firebase = {
        app: function () {
            return {
                delete: function () {}
            }
        }
    };
    db = {
        ref: function () {
            return {
                set: function () {},
                child: function () {
                    return {
                        set: function (a) {},
                        once: function () {}
                    }
                }
            }
        },
        goOffline: function () {}
    }
}

let videoAPI = async function (mode) {
    const allowedVideoItemKeys = ['title', 'src', 'thumbnail', 'target', 'duration']
    console.log('Deploying video API');

    let videos = glob.sync(`${SOURCE_DIR}/${compile_language}/${compile_quarter}/${SOURCE_VIDEO_FILE}`);
    let videoAPIJson = []

    let curlConfig = ""

    for (let video of videos) {
        let videoSource = yamljs.load(`${video}`),
            info = getInfoFromPath(video);

        for (let artist of videoSource.video) {
            let videoInfo = {
                artist: artist.artist,
                clips: []
            }

            if (artist.thumbnail) {
                videoInfo.thumbnail = artist.thumbnail
            }

            for (let clip of artist.clips) {
                if (!clip['target'] || !clip['src']) { continue }

                let videoItem = {
                    artist: videoInfo.artist
                }

                videoItem.id = crypto.createHash('sha256').update(artist.artist + clip['target'] + clip['src']).digest('hex');

                for (let k of Object.keys(clip)) {
                    if (allowedVideoItemKeys.indexOf(k) >= 0) {
                        videoItem[k] = clip[k]
                    }
                }

                let extname = path.extname(videoItem.src)

                if (!extname.length || extname.length <= 1) {
                    extname = ".mp4"
                }

                videoItem.src = `${MEDIA_HOST}video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}`

                videoItem.targetIndex = videoItem.target.replace(/\//g, '-')

                if (!videoItem.thumbnail) {
                    if (artist.thumbnail && !/^http/.test(artist.thumbnail.trim())) {
                        artist.thumbnail = `${API_HOST}${API_VERSION}/images/${artist.thumbnail}`
                    }
                    videoItem.thumbnail = artist.thumbnail || `${API_HOST}${API_VERSION}/${info.language}/quarterlies/${info.quarterly}/${SOURCE_COVER_FILE}`
                }

                if (videoItem.duration) {
                    if (typeof videoItem.duration === 'number') {
                        videoItem.duration = moment("2015-01-01").startOf('day').seconds(videoItem.duration).format("H:mm:ss").replace(/^[0:]+(?=\d[\d:]{3})/, '');
                    }
                }

                if (!videoItem.title) {
                    let videoItemInfo = getInfoFromPath(`src/${videoItem.target}`)
                    if (!videoItemInfo.lesson) {
                        continue
                    }

                    if (videoItemInfo.day) {
                        let read = metaMarked(fs.readFileSync(`${SOURCE_DIR}${videoItemInfo.language}/${videoItemInfo.quarterly}/${videoItemInfo.lesson}/${videoItemInfo.day}.md`, "utf-8"))
                        videoItem.title = read.meta.title
                    } else {
                        let lesson = yamljs.load(`${SOURCE_DIR}${videoItemInfo.language}/${videoItemInfo.quarterly}/${videoItemInfo.lesson}/info.yml`)
                        videoItem.title = lesson.title
                    }
                }

                videoInfo.clips.push(videoItem)

                if (mode === "keep" && fs.pathExistsSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}`)) {
                    let stats = fs.statSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}`);
                    if (stats.size > 0) {
                        fs.outputFileSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/.keep`, "");
                    }
                }

                if (mode === "gen" && !fs.pathExistsSync(`video/video/${info.language}/${info.quarterly}/${videoItem.id}/`)) {
                    curlConfig += `
url = "${clip.src}"
output = "video/video/${info.language}/${info.quarterly}/${videoItem.id}/${videoItem.id}${extname}"
-C -
--create-dirs
-L
`
                }
            }

            if (videoInfo.clips.length) {

                videoInfo.clips = videoInfo.clips.sort(function(a, b){
                    if (a.targetIndex < b.targetIndex) {
                        return a.targetIndex.length < b.targetIndex.length ? -1 : 1;
                    }

                    if (a.targetIndex > b.targetIndex) {
                        return a.targetIndex.length > b.targetIndex.length ? 1 : -1;
                    }
                    return 0;
                })

                videoAPIJson.push(videoInfo)
            }
        }

        if (mode === "sync") {
{'fraction_non_alphanumeric': 0.1576610424209304, 'fraction_numerical': 0.003962019263610902, 'mean_word_length': 1.7076012576289994, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 16, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4390583', 'n_tokens_mistral': 4446, 'n_tokens_neox': 4179, 'n_words': 648}