<commit_before>/// <reference path="../typings/node/node.d.ts"/>
/// <reference path="../built/pxtlib.d.ts"/>
/// <reference path="../built/pxtsim.d.ts"/>


(global as any).pxt = pxt;

import * as nodeutil from './nodeutil';
nodeutil.init();

import * as fs from 'fs';
import * as path from 'path';
import * as child_process from 'child_process';

import U = pxt.Util;
import Cloud = pxt.Cloud;

import * as server from './server';
import * as uploader from './uploader';

let forceCloudBuild = process.env["KS_FORCE_CLOUD"] === "yes"

function initTargetCommands() {
    let cmdsjs = nodeutil.targetDir + '/built/cmds.js';
    if (fs.existsSync(cmdsjs)) {
        console.log(`loading cli extensions...`)
        let cli = require(cmdsjs)
        if (cli.deployCoreAsync) {
            pxt.commands.deployCoreAsync = cli.deployCoreAsync
        }
    }
}

let prevExports = (global as any).savedModuleExports
if (prevExports) {
    module.exports = prevExports
}

export interface UserConfig {
    accessToken?: string;
    localToken?: string;
    noAutoBuild?: boolean;
    noAutoStart?: boolean;
    localBuild?: boolean;
}

let reportDiagnostic = reportDiagnosticSimply;

function reportDiagnostics(diagnostics: ts.pxt.KsDiagnostic[]): void {
    for (const diagnostic of diagnostics) {
        reportDiagnostic(diagnostic);
    }
}

function reportDiagnosticSimply(diagnostic: ts.pxt.KsDiagnostic): void {
    let output = "";

    if (diagnostic.fileName) {
        output += `${diagnostic.fileName}(${diagnostic.line + 1},${diagnostic.character + 1}): `;
    }

    const category = ts.DiagnosticCategory[diagnostic.category].toLowerCase();
    output += `${category} TS${diagnostic.code}: ${ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")}`;
    console.log(output);
}

function fatal(msg: string): Promise<any> {
    console.log("Fatal error:", msg)
    throw new Error(msg)
}

let globalConfig: UserConfig = {}

function homePxtDir() {
    return path.join(process.env["HOME"] || process.env["UserProfile"], ".pxt")
}

function cacheDir() {
    return path.join(homePxtDir(), "cache")
}

function configPath() {
    return path.join(homePxtDir(), "config.json")
}

let homeDirsMade = false
function mkHomeDirs() {
    if (homeDirsMade) return
    homeDirsMade = true
    if (!fs.existsSync(homePxtDir()))
        fs.mkdirSync(homePxtDir())
    if (!fs.existsSync(cacheDir()))
        fs.mkdirSync(cacheDir())
}

function saveConfig() {
    mkHomeDirs()
    fs.writeFileSync(configPath(), JSON.stringify(globalConfig, null, 4) + "\n")
}

function initConfig() {
    let atok: string = process.env["CLOUD_ACCESS_TOKEN"]
    if (fs.existsSync(configPath())) {
        let config = <UserConfig>readJson(configPath())
        globalConfig = config
        if (!atok && config.accessToken) {
            atok = config.accessToken
        }
    }

    if (atok) {
        let mm = /^(https?:.*)\?access_token=([\w\.]+)/.exec(atok)
        if (!mm) {
            fatal("Invalid accessToken format, expecting something like 'https://example.com/?access_token=0abcd.XXXX'")
        }
        Cloud.apiRoot = mm[1].replace(/\/$/, "").replace(/\/api$/, "") + "/api/"
        Cloud.accessToken = mm[2]
    }
}

export function loginAsync(access_token: string) {
    if (/^http/.test(access_token)) {
        globalConfig.accessToken = access_token
        saveConfig()
        if (process.env["CLOUD_ACCESS_TOKEN"])
            console.log("You have $CLOUD_ACCESS_TOKEN set; this overrides what you've specified here.")
    } else {
        let root = Cloud.apiRoot.replace(/api\/$/, "")
        console.log("USAGE:")
        console.log(`  pxt login https://example.com/?access_token=...`)
        console.log(`Go to ${root}oauth/gettoken to obtain the token.`)
        return fatal("Bad usage")
    }

    return Promise.resolve()
}

export function apiAsync(path: string, postArguments?: string): Promise<void> {
    if (postArguments == "delete") {
        return Cloud.privateDeleteAsync(path)
            .then(resp => console.log(resp))
    }

    if (postArguments == "-") {
        return nodeutil.readResAsync(process.stdin)
            .then(buf => buf.toString("utf8"))
            .then(str => apiAsync(path, str))
    }

    let dat = postArguments ? JSON.parse(postArguments) : null
    if (dat)
        console.log("POST", "/api/" + path, JSON.stringify(dat, null, 2))

    return Cloud.privateRequestAsync({
        url: path,
        data: dat
    })
        .then(resp => {
            if (resp.json)
                console.log(JSON.stringify(resp.json, null, 2))
            else console.log(resp.text)
        })
}

function uploadFileAsync(path: string) {
    let buf = fs.readFileSync(path)
    let mime = U.getMime(path)
    console.log("Upload", path)
    return Cloud.privatePostAsync("upload/files", {
        filename: path,
        encoding: "base64",
        content: buf.toString("base64"),
        contentType: mime
    })
        .then(resp => {
            console.log(resp)
        })
}

export function ptrAsync(path: string, target?: string) {
    // in MinGW when you say 'pxt ptr /foo/bar' on command line you get C:/MinGW/msys/1.0/foo/bar instead of '/foo/bar'
    let mingwRx = /^[a-z]:\/.*?MinGW.*?1\.0\//i

    path = path.replace(mingwRx, "/")
    path = nodeutil.sanitizePath(path)

    if (!target) {
        return Cloud.privateGetAsync(nodeutil.pathToPtr(path))
            .then(r => {
                console.log(r)
                return r
            })
    }

    if (target == "delete") {
        return Cloud.privateDeleteAsync(nodeutil.pathToPtr(path))
            .then(() => {
                console.log("Pointer " + path + " deleted.")
            })
    }

    if (target == "refresh") {
        return Cloud.privatePostAsync(nodeutil.pathToPtr(path), {})
            .then(r => {
                console.log(r)
                return r
            })
    }

    let ptr = {
        path: path,
        releaseid: "",
        redirect: "",
        scriptid: "",
        artid: "",
        htmlartid: "",
        userplatform: ["pxt-cli"],
    }

    target = target.replace(mingwRx, "/")

    return (/^[a-z]+$/.test(target) ? Cloud.privateGetAsync(target) : Promise.reject(""))
        .then(r => {
            if (r.kind == "script")
                ptr.scriptid = target
            else if (r.kind == "art")
                ptr.artid = target
            else if (r.kind == "release")
                ptr.releaseid = target
            else {
                U.userError("Don't know how to set pointer to this publication type: " + r.kind)
            }
        }, e => {
            if (/^(\/|http)/.test(target)) {
                console.log("Assuming redirect for: " + target)
                ptr.redirect = target
            } else {
                U.userError("Don't know how to set pointer to: " + target)
            }
        })
        .then(() => Cloud.privatePostAsync("pointers", ptr))
        .then(r => {
            console.log(r)
            return r
        })
}

function allFiles(top: string, maxDepth = 4): string[] {
    let res: string[] = []
    for (let p of fs.readdirSync(top)) {
        if (p[0] == ".") continue;
        let inner = top + "/" + p
        let st = fs.statSync(inner)
        if (st.isDirectory()) {
            if (maxDepth > 1)
                U.pushRange(res, allFiles(inner, maxDepth - 1))
        } else {
            res.push(inner)
        }
    }
    return res
}

function onlyExts(files: string[], exts: string[]) {
    return files.filter(f => exts.indexOf(path.extname(f)) >= 0)
}

function pxtFileList(pref: string) {
    return allFiles(pref + "webapp/public")
        .concat(onlyExts(allFiles(pref + "built/web", 1), [".js", ".css"]))
        .concat(allFiles(pref + "built/web/fonts", 1))

}

export function uploadrelAsync(label?: string) {
    return uploadCoreAsync({
        label: label,
        pkgversion: pkgVersion(),
        fileList: pxtFileList("")
    })
}

function semverCmp(a: string, b: string) {
    let parse = (s: string) => {
        let v = s.split(/\./).map(parseInt)
        return v[0] * 100000000 + v[1] * 10000 + v[2]
    }
    return parse(a) - parse(b)
}

let readJson = nodeutil.readJson;

function travisAsync() {
    forceCloudBuild = true

    let rel = process.env.TRAVIS_TAG || ""
    let atok = process.env.NPM_ACCESS_TOKEN

    if (/^v\d/.test(rel) && atok) {
        let npmrc = path.join(process.env.HOME, ".npmrc")
        console.log(`Setting up ${npmrc}`)
        let cfg = "//registry.npmjs.org/:_authToken=" + atok + "\n"
        fs.writeFileSync(npmrc, cfg)
    }

    console.log("TRAVIS_TAG:", rel)

    let branch = process.env.TRAVIS_BRANCH || "local"
    let latest = branch == "master" ? "latest" : "git-" + branch

    let pkg = readJson("package.json")
    if (pkg["name"] == "pxt-core") {
        if (rel)
            return uploadrelAsync("release/" + rel)
                .then(() => runNpmAsync("publish"))
        else
            return uploadrelAsync("release/" + latest)
    } else {
        return buildTargetAsync()
            .then(() => {
                let trg = readLocalPxTarget()
                if (rel)
                    return uploadtrgAsync(trg.id + "/" + rel)
                        .then(() => runNpmAsync("publish"))
                else
                    return uploadtrgAsync(trg.id + "/" + latest, "release/latest")
            })
    }
}

function bumpKsDepAsync() {
    let pkg = readJson("package.json")
    if (pkg["name"] == "pxt-core") return Promise.resolve(pkg)

    let gitPull = Promise.resolve()

    if (fs.existsSync("node_modules/pxt-core/.git")) {
        gitPull = spawnAsync({
            cmd: "git",
            args: ["pull"],
            cwd: "node_modules/pxt-core"
        })
    }

    return gitPull
        .then(() => {
            let kspkg = readJson("node_modules/pxt-core/package.json")
            let currVer = pkg["dependencies"]["pxt-core"]
            let newVer = kspkg["version"]
            if (currVer == newVer) {
                console.log(`Referenced pxt-core dep up to date: ${currVer}`)
                return pkg
            }

            console.log(`Bumping pxt-core dep version: ${currVer} -> ${newVer}`)
            if (currVer != "*" && semverCmp(currVer, newVer) > 0) {
                U.userError("Trying to downgrade pxt-core.")
            }
            pkg["dependencies"]["pxt-core"] = newVer
            fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n")
            return runGitAsync("commit", "-m", `Bump pxt-core to ${newVer}`, "--", "package.json")
                .then(() => pkg)
        })
}

function bumpAsync() {
    return Promise.resolve()
        .then(() => runGitAsync("pull"))
        .then(() => bumpKsDepAsync())
        .then(() => runNpmAsync("version", "patch"))
        .then(() => runGitAsync("push", "--tags"))
        .then(() => runGitAsync("push"))
}

function runGitAsync(...args: string[]) {
    return spawnAsync({
        cmd: "git",
        args: args,
        cwd: "."
    })
}

function runNpmAsync(...args: string[]) {
    return spawnAsync({
        cmd: addCmd("npm"),
        args: args,
        cwd: "."
    })
}

function pkgVersion() {
    let ver = readJson("package.json")["version"]
    let info = travisInfo()
    if (!info.tag)
        ver += "-" + (info.commit ? info.commit.slice(0, 6) : "local")
    return ver
}

function targetFileList() {
    let lst = onlyExts(allFiles("built", 1), [".js", ".css", ".json", ".webmanifest"])
        .concat(allFiles("sim/public"))
    // the cloud only accepts *.json and sim* files in targets
    return lst.filter(fn => /\.json$/.test(fn) || /[\/\\]sim[^\\\/]*$/.test(fn))
}

export function uploadtrgAsync(label?: string, apprel?: string) {
    if (!apprel) {
        let pkg = readJson("node_modules/pxt-core/package.json")
        apprel = "release/v" + pkg.version
    }
    return Promise.resolve()
        .then(() => Cloud.privateGetAsync(apprel))
        .then(r => r.kind == "release" ? r : null, e => null)
        .then(r => r || Cloud.privateGetAsync(nodeutil.pathToPtr(apprel))
            .then(ptr => Cloud.privateGetAsync(ptr.releaseid)))
        .then(r => r.kind == "release" ? r : null)
        .then<pxt.Cloud.JsonRelease>(r => r, e => {
            console.log("Cannot find release: " + apprel)
            process.exit(1)
        })
        .then(r => {
            console.log(`Uploading target against: ${apprel} /${r.id}`);
            return uploadCoreAsync({
                label: label,
                fileList: targetFileList(),
                pkgversion: pkgVersion(),
                baserelease: r.id,
                fileContent: {}
            })
        })
}

interface UploadOptions {
    fileList: string[];
    pkgversion: string;
    baserelease?: string;
    fileContent?: U.Map<string>;
    label?: string
    legacyLabel?: boolean;
    target?: string;
    localDir?: string;
}

function uploadCoreAsync(opts: UploadOptions) {
    let liteId = "<none>"

    let replacements: U.Map<string> = {
        "/sim/simulator.html": "@simUrl@",
        "/sim/sim.webmanifest": "@relprefix@webmanifest",
        "/embed.js": "@relprefix@embed",
        "/cdn/": "@pxtCdnUrl@",
        "/sim/": "@targetCdnUrl@",
        "data-manifest=\"\"": "@manifest@",
        "var pxtConfig = null": "var pxtConfig = @cfg@",
    }


    if (opts.localDir) {
        let cfg: pxt.WebConfig = {
            "relprefix": opts.localDir,
            "workerjs": opts.localDir + "worker.js",
            "tdworkerjs": opts.localDir + "tdworker.js",
            "pxtVersion": opts.pkgversion,
            "pxtRelId": "",
            "pxtCdnUrl": opts.localDir,
            "targetVersion": opts.pkgversion,
            "targetRelId": "",
            "targetCdnUrl": opts.localDir,
            "targetId": opts.target,
            "simUrl": opts.localDir + "simulator.html",
            "runUrl": opts.localDir + "run.html",
            "isStatic": true,
        }
        replacements = {
            "/embed.js": opts.localDir + "embed.js",
            "/cdn/": opts.localDir,
            "/sim/": opts.localDir,
            "@workerjs@": `${opts.localDir}worker.js\n# ver ${new Date().toString()}`,
            //"data-manifest=\"\"": `manifest="${opts.localDir}release.manifest"`,
            "var pxtConfig = null": "var pxtConfig = " + JSON.stringify(cfg, null, 4),
        }
    }

    let replFiles = [
        "index.html",
        "embed.js",
        "run.html",
        "release.manifest",
        "worker.js",
        "tdworker.js",
        "simulator.html",
        "sim.manifest",
        "sim.webmanifest",
    ]

    nodeutil.mkdirP("built/uploadrepl")

    let uploadFileAsync = (p: string) => {
        let rdf: Promise<Buffer> = null
        if (opts.fileContent) {
            let s = U.lookup(opts.fileContent, p)
            if (s != null)
                rdf = Promise.resolve(new Buffer(s, "utf8"))
        }
        if (!rdf) {
            if (!fs.existsSync(p))
                return;
            rdf = readFileAsync(p)
        }

        return rdf
            .then((data: Buffer) => {
                // Strip the leading directory name, unless we are uploading a single file.
                let fileName = p.replace(/^.*(built\/web\/|\w+\/public\/|built\/)/, "")
                let mime = U.getMime(p)
                let isText = /^(text\/.*|application\/.*(javascript|json))$/.test(mime)
                let content = ""

                if (isText) {
                    content = data.toString("utf8")
                    if (replFiles.indexOf(fileName) >= 0) {
                        for (let from of Object.keys(replacements)) {
                            content = U.replaceAll(content, from, replacements[from])
                        }
                        if (opts.localDir) {
                            data = new Buffer(content, "utf8")
                        } else {
                            // save it for developer inspection
                            fs.writeFileSync("built/uploadrepl/" + fileName, content)
                        }
                    } else if (fileName == "target.json" && opts.localDir) {
                        let trg: pxt.TargetBundle = JSON.parse(content)
                        for (let e of trg.appTheme.docMenu)
                            if (e.path[0] == "/") {
                                e.path = opts.localDir + "docs" + e.path + ".html"
                            }
                        trg.appTheme.logoUrl = opts.localDir
                        trg.appTheme.homeUrl = opts.localDir
                        data = new Buffer(JSON.stringify(trg, null, 2), "utf8")
                    }
                } else {
                    content = data.toString("base64")
                }

                if (opts.localDir) {
                    let fn = path.join(builtPackaged + opts.localDir, fileName)
                    nodeutil.mkdirP(path.dirname(fn))
                    return writeFileAsync(fn, data)
                }

                return Cloud.privatePostAsync(liteId + "/files", {
                    encoding: isText ? "utf8" : "base64",
                    filename: fileName,
                    contentType: mime,
                    content,
                })
                    .then(resp => {
                        console.log(fileName, mime)
                    })
            })
    }

    if (opts.localDir)
        return Promise.map(opts.fileList, uploadFileAsync, { concurrency: 15 })
            .then(() => {
                console.log("Release files written to", path.resolve(builtPackaged + opts.localDir))
            })

    let info = travisInfo()
    return Cloud.privatePostAsync("releases", {
        baserelease: opts.baserelease,
        pkgversion: opts.pkgversion,
        commit: info.commitUrl,
        branch: info.tag || info.branch,
        buildnumber: process.env['TRAVIS_BUILD_NUMBER'],
        target: pxt.appTarget ? pxt.appTarget.id : "",
        type: opts.baserelease ? "target" : "base",
    })
        .then(resp => {
            console.log(resp)
            liteId = resp.id
            return Promise.map(opts.fileList, uploadFileAsync, { concurrency: 15 })
        })
        .then(() => {
            if (!opts.label) return Promise.resolve()
            else if (opts.legacyLabel) return Cloud.privatePostAsync(liteId + "/label", { name: opts.label })
            else return Cloud.privatePostAsync("pointers", {
                path: nodeutil.sanitizePath(opts.label),
                releaseid: liteId
            })
                .then(() => {
                    // tag release/v0.1.2 also as release/beta
                    let beta = opts.label.replace(/\/v\d.*/, "/beta")
                    if (beta == opts.label) return Promise.resolve()
                    else {
                        console.log("Alos tagging with " + beta)
                        return Cloud.privatePostAsync("pointers", {
                            path: nodeutil.sanitizePath(beta),
                            releaseid: liteId
                        })
                    }
                })
        })
        .then(() => {
            console.log("All done; tagged with " + opts.label)
        })
}

function readLocalPxTarget() {
    if (!fs.existsSync("pxtarget.json")) {
        console.error("This command requires pxtarget.json in current directory.")
        process.exit(1)
    }
    nodeutil.targetDir = process.cwd()
    let cfg: pxt.TargetBundle = readJson("pxtarget.json")
    return cfg
}

function forEachBundledPkgAsync(f: (pkg: pxt.MainPackage) => Promise<void>) {
    let cfg = readLocalPxTarget()
    let prev = process.cwd()

    return Promise.mapSeries(cfg.bundleddirs, (dirname) => {
        process.chdir(path.join(nodeutil.targetDir, dirname))
        mainPkg = new pxt.MainPackage(new Host())
        return f(mainPkg);
    })
        .finally(() => process.chdir(prev))
        .then(() => { })
}

export function publishTargetAsync() {
    return forEachBundledPkgAsync((pkg) => {
        return pkg.publishAsync()
    })
}

export function spawnAsync(opts: {
    cmd: string,
    args: string[],
    cwd: string,
    shell?: boolean
}) {
    let info = opts.cmd + " " + opts.args.join(" ")
    if (opts.cwd != ".") info = "cd " + opts.cwd + "; " + info
    console.log("[run] " + info)
    return new Promise<void>((resolve, reject) => {
        let ch = child_process.spawn(opts.cmd, opts.args, {
            cwd: opts.cwd,
            env: process.env,
            stdio: "inherit",
            shell: opts.shell || false
        } as any)
        ch.on('close', (code: number) => {
            if (code != 0)
                reject(new Error("Exit code: " + code + " from " + info))
            resolve()
        });
    })
}

function ghpSetupRepoAsync() {
    function getreponame() {
        let cfg = fs.readFileSync("built/gh-pages/.git/config", "utf8")
        let m = /^\s*url\s*=\s*.*github.*\/([^\/\s]+)$/mi.exec(cfg)
        if (!m) U.userError("cannot determine GitHub repo name")
        return m[1].replace(/\.git$/, "")
    }
    if (fs.existsSync("built/gh-pages")) {
        console.log("Skipping init of built/gh-pages; you can delete it first to get full re-init")
        return Promise.resolve(getreponame())
    }

    cpR(".git", "built/gh-pages/.git")
    return ghpGitAsync("checkout", "gh-pages")
        .then(() => getreponame(), (e: any) => {
            U.userError("No gh-pages branch. Try 'pxt ghpinit' first.")
        })
}

function ghpGitAsync(...args: string[]) {
    return spawnAsync({
        cmd: "git",
        cwd: "built/gh-pages",
        args: args
    })
}

export function ghpPushAsync() {
    let repoName = ""
    return ghpSetupRepoAsync()
        .then(name => staticpkgAsync((repoName = name)))
        .then(() => {
            cpR(builtPackaged + "/" + repoName, "built/gh-pages")
        })
        .then(() => ghpGitAsync("add", "."))
        .then(() => ghpGitAsync("commit", "-m", "Auto-push"))
        .then(() => ghpGitAsync("push"))
}

export function ghpInitAsync() {
    if (fs.existsSync("built/gh-pages"))
        U.userError("built/gh-pages already exists")
    cpR(".git", "built/gh-pages/.git")
    return ghpGitAsync("checkout", "gh-pages")
        .then(() => U.userError("gh-pages branch already exists"), (e: any) => { })
        .then(() => ghpGitAsync("checkout", "--orphan", "gh-pages"))
        .then(() => ghpGitAsync("rm", "-rf", "."))
        .then(() => {
            fs.writeFileSync("built/gh-pages/index.html", "Under construction.")
            return ghpGitAsync("add", ".")
        })
        .then(() => ghpGitAsync("commit", "-m", "Initial."))
        .then(() => ghpGitAsync("push", "--set-upstream", "origin", "gh-pages"))
}

function maxMTimeAsync(dirs: string[]) {
    let max = 0
    return Promise.map(dirs, dn => readDirAsync(dn)
        .then(files => Promise.map(files, fn => statAsync(path.join(dn, fn))
            .then(st => {
                max = Math.max(st.mtime.getTime(), max)
            }))))
        .then(() => max)
}

export function buildTargetAsync(): Promise<void> {
    return simshimAsync()
        .then(() => buildFolderAsync('sim'))
        .then(buildTargetCoreAsync)
        .then(() => buildFolderAsync('cmds', true))
        .then(() => buildFolderAsync('server', true))
}

function buildFolderAsync(p: string, optional?: boolean): Promise<void> {
    if (!fs.existsSync(p + "/tsconfig.json")) {
        if (!optional) U.userError(`${p}/tsconfig.json not found`);
        return Promise.resolve()
    }

    console.log(`building ${p}...`)
    dirsToWatch.push(p)
    return spawnAsync({
        cmd: "node",
        args: ["../node_modules/typescript/bin/tsc"],
        cwd: p
    })
}

function addCmd(name: string) {
    return name + (/win/.test(process.platform) ? ".cmd" : "")
}

function buildPxtAsync(): Promise<string[]> {
    let ksd = "node_modules/pxt-core"
    if (!fs.existsSync(ksd + "/pxtlib/main.ts")) return Promise.resolve([]);

    console.log(`building ${ksd}...`);
    return spawnAsync({
        cmd: addCmd("jake"),
        args: [],
        cwd: ksd
    }).then(() => {
        console.log("local pxt-core built.")
        return [ksd]
    }, e => {
        buildFailed("local pxt-core build FAILED")
        return [ksd]
    });
}

let dirsToWatch: string[] = []

function travisInfo() {
    return {
        branch: process.env['TRAVIS_BRANCH'],
        tag: process.env['TRAVIS_TAG'],
        commit: process.env['TRAVIS_COMMIT'],
        commitUrl: !process.env['TRAVIS_COMMIT'] ? undefined :
            "https://github.com/" + process.env['TRAVIS_REPO_SLUG'] + "/commits/" + process.env['TRAVIS_COMMIT'],
    }
}

function buildWebManifest(cfg: pxt.TargetBundle) {
    let webmanifest: any = {
        "lang": "en",
        "dir": "ltr",
        "name": cfg.name,
        "short_name": cfg.name,
        "icons": [
            {
                "src": "\/static\/icons\/android-chrome-36x36.png",
                "sizes": "36x36",
                "type": "image\/png",
                "density": 0.75
            },
            {
                "src": "\/static\/icons\/android-chrome-48x48.png",
                "sizes": "48x48",
                "type": "image\/png",
                "density": 1
            },
            {
                "src": "\/static\/icons\/android-chrome-72x72.png",
                "sizes": "72x72",
                "type": "image\/png",
                "density": 1.5
            },
            {
                "src": "\/static\/icons\/android-chrome-96x96.png",
                "sizes": "96x96",
                "type": "image\/png",
                "density": 2
            },
            {
                "src": "\/static\/icons\/android-chrome-144x144.png",
                "sizes": "144x144",
                "type": "image\/png",
                "density": 3
            },
            {
                "src": "\/static\/icons\/android-chrome-192x192.png",
                "sizes": "192x192",
                "type": "image\/png",
                "density": 4
            }
        ],
        "scope": "/",
        "start_url": "/",
        "display": "standalone",
        "orientation": "landscape"
    }
    let diskManifest: any = {}
    if (fs.existsSync("webmanifest.json"))
        diskManifest = nodeutil.readJson("webmanifest.json")
    U.jsonCopyFrom(webmanifest, diskManifest)
    return webmanifest;
}

function saveThemeJson(cfg: pxt.TargetBundle) {
    cfg.appTheme.id = cfg.id
    cfg.appTheme.title = cfg.title
    cfg.appTheme.name = cfg.name

    // expand logo
    let logos = (cfg.appTheme as any as U.Map<string>);
    Object.keys(logos)
        .filter(k => /logo$/i.test(k) && /^\.\//.test(logos[k]))
        .forEach(k => {
            let fn = path.join('./docs', logos[k]);
            console.log(`importing ${fn}`)
            let b = fs.readFileSync(fn)
            let mimeType = '';
            if (/\.svg$/i.test(fn)) mimeType = "image/svg+xml";
            else if (/\.png$/i.test(fn)) mimeType = "image/png";
            else if (/\.jpe?g$/i.test(fn)) mimeType = "image/jpeg";
            if (mimeType) logos[k] = `data:${mimeType};base64,${b.toString('base64')}`;
            else logos[k] = b.toString('utf8');
        })

    cfg.appTheme.locales = {}

    let lpath = "docs/_locales"
    if (fs.existsSync(lpath)) {
        for (let loc of fs.readdirSync(lpath)) {
            let fn = lpath + "/" + loc + "/_theme.json"
            if (fs.existsSync(fn))
                cfg.appTheme.locales[loc.toLowerCase()] = readJson(fn)
        }
    }

    nodeutil.mkdirP("built");
    fs.writeFileSync("built/theme.json", JSON.stringify(cfg.appTheme, null, 2))
}

function buildTargetCoreAsync() {
    let cfg = readLocalPxTarget()
    cfg.bundledpkgs = {}
    pxt.appTarget = cfg;
    let statFiles: U.Map<number> = {}
    dirsToWatch = cfg.bundleddirs.slice()
    dirsToWatch.push("sim"); // simulator
    dirsToWatch.push("sim/public"); // simulator
    console.log("building target.json...")
    return forEachBundledPkgAsync(pkg =>
        pkg.filesToBePublishedAsync()
            .then(res => {
                cfg.bundledpkgs[pkg.config.name] = res
            })
            .then(testForBuildTargetAsync))
        .then(() => {
            let info = travisInfo()
            cfg.versions = {
                branch: info.branch,
                tag: info.tag,
                commits: info.commitUrl,
                target: readJson("package.json")["version"],
                pxt: readJson("node_modules/pxt-core/package.json")["version"],
            }

            saveThemeJson(cfg)

            let webmanifest = buildWebManifest(cfg)
            fs.writeFileSync("built/target.json", JSON.stringify(cfg, null, 2))
            pxt.appTarget = cfg; // make sure we're using the latest version
            let targetlight = U.flatClone(cfg)
            delete targetlight.bundleddirs
            delete targetlight.bundledpkgs
            delete targetlight.appTheme
            fs.writeFileSync("built/targetlight.json", JSON.stringify(targetlight, null, 2))
            fs.writeFileSync("built/sim.webmanifest", JSON.stringify(webmanifest, null, 2))
        })
        .then(() => {
            console.log("target.json built.")
        })
}

function buildAndWatchAsync(f: () => Promise<string[]>): Promise<void> {
    let currMtime = Date.now()
    return f()
        .then(dirs => {
            if (globalConfig.noAutoBuild) return
            console.log('watching ' + dirs.join(', ') + '...');
            let loop = () => {
                Promise.delay(1000)
                    .then(() => maxMTimeAsync(dirs))
                    .then(num => {
                        if (num > currMtime) {
                            currMtime = num
                            f()
                                .then(d => {
                                    dirs = d
                                    U.nextTick(loop)
                                })
                        } else {
                            U.nextTick(loop)
                        }
                    })
            }
            U.nextTick(loop)
        })

}

function buildFailed(msg: string) {
    console.log("")
    console.log("***")
    console.log("*** Build failed: " + msg)
    console.log("***")
    console.log("")
}

function buildAndWatchTargetAsync() {
    if (!fs.existsSync("sim/tsconfig.json")) {
        console.log("No sim/tsconfig.json; assuming npm installed package")
        return Promise.resolve()
    }

    return buildAndWatchAsync(() => buildPxtAsync()
        .then(() => buildTargetAsync().then(r => { }, e => {
            buildFailed(e.message)
        }))
        .then(() => uploader.checkDocsAsync())
        .then(() => [path.resolve("node_modules/pxt-core")].concat(dirsToWatch)));
}

function cpR(src: string, dst: string, maxDepth = 8) {
    src = path.resolve(src)
    let files = allFiles(src, maxDepth)
    let dirs: U.Map<boolean> = {}
    for (let f of files) {
        let bn = f.slice(src.length)
        let dd = path.join(dst, bn)
        let dir = path.dirname(dd)
        if (!U.lookup(dirs, dir)) {
            nodeutil.mkdirP(dir)
            dirs[dir] = true
        }
        let buf = fs.readFileSync(f)
        fs.writeFileSync(dd, buf)
    }
}

let builtPackaged = "built/packaged"

function renderDocs(localDir: string) {
    let dst = path.resolve(builtPackaged + localDir)

    cpR("node_modules/pxt-core/docfiles", dst + "/docfiles")
    if (fs.existsSync("docfiles"))
        cpR("docfiles", dst + "/docfiles")

    let webpath = localDir
    let docsTemplate = fs.readFileSync(dst + "/docfiles/template.html", "utf8")
    docsTemplate = U.replaceAll(docsTemplate, "/cdn/", webpath)
    docsTemplate = U.replaceAll(docsTemplate, "/docfiles/", webpath + "docfiles/")
    docsTemplate = U.replaceAll(docsTemplate, "/--embed", webpath + "embed.js")

    let dirs: U.Map<boolean> = {}
    for (let f of allFiles("docs", 8)) {
        let dd = path.join(dst, f)
        let dir = path.dirname(dd)
        if (!U.lookup(dirs, dir)) {
            nodeutil.mkdirP(dir)
            dirs[dir] = true
        }
        let buf = fs.readFileSync(f)
        if (/\.md$/.test(f)) {
            let str = buf.toString("utf8")
            let path = f.slice(5).split(/\//)
            let bc = path.map((e, i) => {
                return {
                    href: "/" + path.slice(0, i + 1).join("/"),
                    name: e
                }
            })
            let html = pxt.docs.renderMarkdown(docsTemplate, str, pxt.appTarget.appTheme, null, bc)
            html = html.replace(/(<a[^<>]*)\shref="(\/[^<>"]*)"/g, (f, beg, url) => {
                return beg + ` href="${webpath}docs${url}.html"`
            })
            buf = new Buffer(html, "utf8")
            dd = dd.slice(0, dd.length - 3) + ".html"
        }
        fs.writeFileSync(dd, buf)
    }
    console.log("Docs written.")
}

export function staticpkgAsync(label?: string) {
    let dir = label ? "/" + label + "/" : "/"
    return Promise.resolve()
        .then(() => uploadCoreAsync({
            label: label || "main",
            pkgversion: "0.0.0",
            fileList: pxtFileList("node_modules/pxt-core/").concat(targetFileList()),
            localDir: dir
        }))
        .then(() => renderDocs(dir))
}

export function serveAsync(arg?: string) {
    forceCloudBuild = !globalConfig.localBuild
    let justServe = false
    let packaged = false
    if (arg == "-yt") {
        forceCloudBuild = false
    } else if (arg == "-cloud") {
        forceCloudBuild = true
    } else if (arg == "-just") {
        justServe = true
    } else if (arg == "-pkg") {
        justServe = true
        packaged = true
    }
    if (!globalConfig.localToken) {
        globalConfig.localToken = U.guidGen();
        saveConfig()
    }
    let localToken = globalConfig.localToken;
    if (!fs.existsSync("pxtarget.json")) {
        let upper = path.join(__dirname, "../../..")
        if (fs.existsSync(path.join(upper, "pxtarget.json"))) {
            console.log("going to " + upper)
            process.chdir(upper)
        } else {
            U.userError("Cannot find pxtarget.json to serve.")
        }
    }
    return (justServe ? Promise.resolve() : buildAndWatchTargetAsync())
        .then(() => server.serveAsync({
            localToken: localToken,
            autoStart: !globalConfig.noAutoStart,
            packaged: packaged
        }))
}<commit_msg>detect that typescript is missing<commit_after>/// <reference path="../typings/node/node.d.ts"/>
/// <reference path="../built/pxtlib.d.ts"/>
/// <reference path="../built/pxtsim.d.ts"/>


(global as any).pxt = pxt;

import * as nodeutil from './nodeutil';
nodeutil.init();

import * as fs from 'fs';
import * as path from 'path';
import * as child_process from 'child_process';

import U = pxt.Util;
import Cloud = pxt.Cloud;

import * as server from './server';
import * as uploader from './uploader';

let forceCloudBuild = process.env["KS_FORCE_CLOUD"] === "yes"

function initTargetCommands() {
    let cmdsjs = nodeutil.targetDir + '/built/cmds.js';
    if (fs.existsSync(cmdsjs)) {
        console.log(`loading cli extensions...`)
        let cli = require(cmdsjs)
        if (cli.deployCoreAsync) {
            pxt.commands.deployCoreAsync = cli.deployCoreAsync
        }
    }
}

let prevExports = (global as any).savedModuleExports
if (prevExports) {
    module.exports = prevExports
}

export interface UserConfig {
    accessToken?: string;
    localToken?: string;
    noAutoBuild?: boolean;
    noAutoStart?: boolean;
    localBuild?: boolean;
}

let reportDiagnostic = reportDiagnosticSimply;

function reportDiagnostics(diagnostics: ts.pxt.KsDiagnostic[]): void {
    for (const diagnostic of diagnostics) {
        reportDiagnostic(diagnostic);
    }
}

function reportDiagnosticSimply(diagnostic: ts.pxt.KsDiagnostic): void {
    let output = "";

    if (diagnostic.fileName) {
        output += `${diagnostic.fileName}(${diagnostic.line + 1},${diagnostic.character + 1}): `;
    }

    const category = ts.DiagnosticCategory[diagnostic.category].toLowerCase();
    output += `${category} TS${diagnostic.code}: ${ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")}`;
    console.log(output);
}

function fatal(msg: string): Promise<any> {
    console.log("Fatal error:", msg)
    throw new Error(msg)
}

let globalConfig: UserConfig = {}

function homePxtDir() {
    return path.join(process.env["HOME"] || process.env["UserProfile"], ".pxt")
}

function cacheDir() {
    return path.join(homePxtDir(), "cache")
}

function configPath() {
    return path.join(homePxtDir(), "config.json")
}

let homeDirsMade = false
function mkHomeDirs() {
    if (homeDirsMade) return
    homeDirsMade = true
    if (!fs.existsSync(homePxtDir()))
        fs.mkdirSync(homePxtDir())
    if (!fs.existsSync(cacheDir()))
        fs.mkdirSync(cacheDir())
}

function saveConfig() {
    mkHomeDirs()
    fs.writeFileSync(configPath(), JSON.stringify(globalConfig, null, 4) + "\n")
}

function initConfig() {
    let atok: string = process.env["CLOUD_ACCESS_TOKEN"]
    if (fs.existsSync(configPath())) {
        let config = <UserConfig>readJson(configPath())
        globalConfig = config
        if (!atok && config.accessToken) {
            atok = config.accessToken
        }
    }

    if (atok) {
        let mm = /^(https?:.*)\?access_token=([\w\.]+)/.exec(atok)
        if (!mm) {
            fatal("Invalid accessToken format, expecting something like 'https://example.com/?access_token=<KEY>'")
        }
        Cloud.apiRoot = mm[1].replace(/\/$/, "").replace(/\/api$/, "") + "/api/"
        Cloud.accessToken = mm[2]
    }
}

export function loginAsync(access_token: string) {
    if (/^http/.test(access_token)) {
        globalConfig.accessToken = access_token
        saveConfig()
        if (process.env["CLOUD_ACCESS_TOKEN"])
            console.log("You have $CLOUD_ACCESS_TOKEN set; this overrides what you've specified here.")
    } else {
        let root = Cloud.apiRoot.replace(/api\/$/, "")
        console.log("USAGE:")
        console.log(`  pxt login https://example.com/?access_token=...`)
        console.log(`Go to ${root}oauth/gettoken to obtain the token.`)
        return fatal("Bad usage")
    }

    return Promise.resolve()
}

export function apiAsync(path: string, postArguments?: string): Promise<void> {
    if (postArguments == "delete") {
        return Cloud.privateDeleteAsync(path)
            .then(resp => console.log(resp))
    }

    if (postArguments == "-") {
        return nodeutil.readResAsync(process.stdin)
            .then(buf => buf.toString("utf8"))
            .then(str => apiAsync(path, str))
    }

    let dat = postArguments ? JSON.parse(postArguments) : null
    if (dat)
        console.log("POST", "/api/" + path, JSON.stringify(dat, null, 2))

    return Cloud.privateRequestAsync({
        url: path,
        data: dat
    })
        .then(resp => {
            if (resp.json)
                console.log(JSON.stringify(resp.json, null, 2))
            else console.log(resp.text)
        })
}

function uploadFileAsync(path: string) {
    let buf = fs.readFileSync(path)
    let mime = U.getMime(path)
    console.log("Upload", path)
    return Cloud.privatePostAsync("upload/files", {
        filename: path,
        encoding: "base64",
        content: buf.toString("base64"),
        contentType: mime
    })
        .then(resp => {
            console.log(resp)
        })
}

export function ptrAsync(path: string, target?: string) {
    // in MinGW when you say 'pxt ptr /foo/bar' on command line you get C:/MinGW/msys/1.0/foo/bar instead of '/foo/bar'
    let mingwRx = /^[a-z]:\/.*?MinGW.*?1\.0\//i

    path = path.replace(mingwRx, "/")
    path = nodeutil.sanitizePath(path)

    if (!target) {
        return Cloud.privateGetAsync(nodeutil.pathToPtr(path))
            .then(r => {
                console.log(r)
                return r
            })
    }

    if (target == "delete") {
        return Cloud.privateDeleteAsync(nodeutil.pathToPtr(path))
            .then(() => {
                console.log("Pointer " + path + " deleted.")
            })
    }

    if (target == "refresh") {
        return Cloud.privatePostAsync(nodeutil.pathToPtr(path), {})
            .then(r => {
                console.log(r)
                return r
            })
    }

    let ptr = {
        path: path,
        releaseid: "",
        redirect: "",
        scriptid: "",
        artid: "",
        htmlartid: "",
        userplatform: ["pxt-cli"],
    }

    target = target.replace(mingwRx, "/")

    return (/^[a-z]+$/.test(target) ? Cloud.privateGetAsync(target) : Promise.reject(""))
        .then(r => {
            if (r.kind == "script")
                ptr.scriptid = target
            else if (r.kind == "art")
                ptr.artid = target
            else if (r.kind == "release")
                ptr.releaseid = target
            else {
                U.userError("Don't know how to set pointer to this publication type: " + r.kind)
            }
        }, e => {
            if (/^(\/|http)/.test(target)) {
                console.log("Assuming redirect for: " + target)
                ptr.redirect = target
            } else {
                U.userError("Don't know how to set pointer to: " + target)
            }
        })
        .then(() => Cloud.privatePostAsync("pointers", ptr))
        .then(r => {
            console.log(r)
            return r
        })
}

function allFiles(top: string, maxDepth = 4): string[] {
    let res: string[] = []
    for (let p of fs.readdirSync(top)) {
        if (p[0] == ".") continue;
        let inner = top + "/" + p
        let st = fs.statSync(inner)
        if (st.isDirectory()) {
            if (maxDepth > 1)
                U.pushRange(res, allFiles(inner, maxDepth - 1))
        } else {
            res.push(inner)
        }
    }
    return res
}

function onlyExts(files: string[], exts: string[]) {
    return files.filter(f => exts.indexOf(path.extname(f)) >= 0)
}

function pxtFileList(pref: string) {
    return allFiles(pref + "webapp/public")
        .concat(onlyExts(allFiles(pref + "built/web", 1), [".js", ".css"]))
        .concat(allFiles(pref + "built/web/fonts", 1))

}

export function uploadrelAsync(label?: string) {
    return uploadCoreAsync({
        label: label,
        pkgversion: pkgVersion(),
        fileList: pxtFileList("")
    })
}

function semverCmp(a: string, b: string) {
    let parse = (s: string) => {
        let v = s.split(/\./).map(parseInt)
        return v[0] * 100000000 + v[1] * 10000 + v[2]
    }
    return parse(a) - parse(b)
}

let readJson = nodeutil.readJson;

function travisAsync() {
    forceCloudBuild = true

    let rel = process.env.TRAVIS_TAG || ""
    let atok = process.env.NPM_ACCESS_TOKEN

    if (/^v\d/.test(rel) && atok) {
        let npmrc = path.join(process.env.HOME, ".npmrc")
        console.log(`Setting up ${npmrc}`)
        let cfg = "//registry.npmjs.org/:_authToken=" + atok + "\n"
        fs.writeFileSync(npmrc, cfg)
    }

    console.log("TRAVIS_TAG:", rel)

    let branch = process.env.TRAVIS_BRANCH || "local"
    let latest = branch == "master" ? "latest" : "git-" + branch

    let pkg = readJson("package.json")
    if (pkg["name"] == "pxt-core") {
        if (rel)
            return uploadrelAsync("release/" + rel)
                .then(() => runNpmAsync("publish"))
        else
            return uploadrelAsync("release/" + latest)
    } else {
        return buildTargetAsync()
            .then(() => {
                let trg = readLocalPxTarget()
                if (rel)
                    return uploadtrgAsync(trg.id + "/" + rel)
                        .then(() => runNpmAsync("publish"))
                else
                    return uploadtrgAsync(trg.id + "/" + latest, "release/latest")
            })
    }
}

function bumpKsDepAsync() {
    let pkg = readJson("package.json")
    if (pkg["name"] == "pxt-core") return Promise.resolve(pkg)

    let gitPull = Promise.resolve()

    if (fs.existsSync("node_modules/pxt-core/.git")) {
        gitPull = spawnAsync({
            cmd: "git",
            args: ["pull"],
            cwd: "node_modules/pxt-core"
        })
    }

    return gitPull
        .then(() => {
            let kspkg = readJson("node_modules/pxt-core/package.json")
            let currVer = pkg["dependencies"]["pxt-core"]
            let newVer = kspkg["version"]
            if (currVer == newVer) {
                console.log(`Referenced pxt-core dep up to date: ${currVer}`)
                return pkg
            }

            console.log(`Bumping pxt-core dep version: ${currVer} -> ${newVer}`)
            if (currVer != "*" && semverCmp(currVer, newVer) > 0) {
                U.userError("Trying to downgrade pxt-core.")
            }
            pkg["dependencies"]["pxt-core"] = newVer
            fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n")
            return runGitAsync("commit", "-m", `Bump pxt-core to ${newVer}`, "--", "package.json")
                .then(() => pkg)
        })
}

function bumpAsync() {
    return Promise.resolve()
        .then(() => runGitAsync("pull"))
        .then(() => bumpKsDepAsync())
        .then(() => runNpmAsync("version", "patch"))
        .then(() => runGitAsync("push", "--tags"))
        .then(() => runGitAsync("push"))
}

function runGitAsync(...args: string[]) {
    return spawnAsync({
        cmd: "git",
        args: args,
        cwd: "."
    })
}

function runNpmAsync(...args: string[]) {
    return spawnAsync({
        cmd: addCmd("npm"),
        args: args,
        cwd: "."
    })
}

function pkgVersion() {
    let ver = readJson("package.json")["version"]
    let info = travisInfo()
    if (!info.tag)
        ver += "-" + (info.commit ? info.commit.slice(0, 6) : "local")
    return ver
}

function targetFileList() {
    let lst = onlyExts(allFiles("built", 1), [".js", ".css", ".json", ".webmanifest"])
        .concat(allFiles("sim/public"))
    // the cloud only accepts *.json and sim* files in targets
    return lst.filter(fn => /\.json$/.test(fn) || /[\/\\]sim[^\\\/]*$/.test(fn))
}

export function uploadtrgAsync(label?: string, apprel?: string) {
    if (!apprel) {
        let pkg = readJson("node_modules/pxt-core/package.json")
        apprel = "release/v" + pkg.version
    }
    return Promise.resolve()
        .then(() => Cloud.privateGetAsync(apprel))
        .then(r => r.kind == "release" ? r : null, e => null)
        .then(r => r || Cloud.privateGetAsync(nodeutil.pathToPtr(apprel))
            .then(ptr => Cloud.privateGetAsync(ptr.releaseid)))
        .then(r => r.kind == "release" ? r : null)
        .then<pxt.Cloud.JsonRelease>(r => r, e => {
            console.log("Cannot find release: " + apprel)
            process.exit(1)
        })
        .then(r => {
            console.log(`Uploading target against: ${apprel} /${r.id}`);
            return uploadCoreAsync({
                label: label,
                fileList: targetFileList(),
                pkgversion: pkgVersion(),
                baserelease: r.id,
                fileContent: {}
            })
        })
}

interface UploadOptions {
    fileList: string[];
    pkgversion: string;
    baserelease?: string;
    fileContent?: U.Map<string>;
    label?: string
    legacyLabel?: boolean;
    target?: string;
    localDir?: string;
}

function uploadCoreAsync(opts: UploadOptions) {
    let liteId = "<none>"

    let replacements: U.Map<string> = {
        "/sim/simulator.html": "@simUrl@",
        "/sim/sim.webmanifest": "@relprefix@webmanifest",
        "/embed.js": "@relprefix@embed",
        "/cdn/": "@pxtCdnUrl@",
        "/sim/": "@targetCdnUrl@",
        "data-manifest=\"\"": "@manifest@",
        "var pxtConfig = null": "var pxtConfig = @cfg@",
    }


    if (opts.localDir) {
        let cfg: pxt.WebConfig = {
            "relprefix": opts.localDir,
            "workerjs": opts.localDir + "worker.js",
            "tdworkerjs": opts.localDir + "tdworker.js",
            "pxtVersion": opts.pkgversion,
            "pxtRelId": "",
            "pxtCdnUrl": opts.localDir,
            "targetVersion": opts.pkgversion,
            "targetRelId": "",
            "targetCdnUrl": opts.localDir,
            "targetId": opts.target,
            "simUrl": opts.localDir + "simulator.html",
            "runUrl": opts.localDir + "run.html",
            "isStatic": true,
        }
        replacements = {
            "/embed.js": opts.localDir + "embed.js",
            "/cdn/": opts.localDir,
            "/sim/": opts.localDir,
            "@workerjs@": `${opts.localDir}worker.js\n# ver ${new Date().toString()}`,
            //"data-manifest=\"\"": `manifest="${opts.localDir}release.manifest"`,
            "var pxtConfig = null": "var pxtConfig = " + JSON.stringify(cfg, null, 4),
        }
    }

    let replFiles = [
        "index.html",
        "embed.js",
        "run.html",
        "release.manifest",
        "worker.js",
        "tdworker.js",
        "simulator.html",
        "sim.manifest",
        "sim.webmanifest",
    ]

    nodeutil.mkdirP("built/uploadrepl")

    let uploadFileAsync = (p: string) => {
        let rdf: Promise<Buffer> = null
        if (opts.fileContent) {
            let s = U.lookup(opts.fileContent, p)
            if (s != null)
                rdf = Promise.resolve(new Buffer(s, "utf8"))
        }
        if (!rdf) {
            if (!fs.existsSync(p))
                return;
            rdf = readFileAsync(p)
        }

        return rdf
            .then((data: Buffer) => {
                // Strip the leading directory name, unless we are uploading a single file.
                let fileName = p.replace(/^.*(built\/web\/|\w+\/public\/|built\/)/, "")
                let mime = U.getMime(p)
                let isText = /^(text\/.*|application\/.*(javascript|json))$/.test(mime)
                let content = ""

                if (isText) {
                    content = data.toString("utf8")
                    if (replFiles.indexOf(fileName) >= 0) {
                        for (let from of Object.keys(replacements)) {
                            content = U.replaceAll(content, from, replacements[from])
                        }
                        if (opts.localDir) {
                            data = new Buffer(content, "utf8")
                        } else {
                            // save it for developer inspection
                            fs.writeFileSync("built/uploadrepl/" + fileName, content)
                        }
                    } else if (fileName == "target.json" && opts.localDir) {
                        let trg: pxt.TargetBundle = JSON.parse(content)
                        for (let e of trg.appTheme.docMenu)
                            if (e.path[0] == "/") {
                                e.path = opts.localDir + "docs" + e.path + ".html"
                            }
                        trg.appTheme.logoUrl = opts.localDir
                        trg.appTheme.homeUrl = opts.localDir
                        data = new Buffer(JSON.stringify(trg, null, 2), "utf8")
                    }
                } else {
                    content = data.toString("base64")
                }

                if (opts.localDir) {
                    let fn = path.join(builtPackaged + opts.localDir, fileName)
                    nodeutil.mkdirP(path.dirname(fn))
                    return writeFileAsync(fn, data)
                }

                return Cloud.privatePostAsync(liteId + "/files", {
                    encoding: isText ? "utf8" : "base64",
                    filename: fileName,
                    contentType: mime,
                    content,
                })
                    .then(resp => {
                        console.log(fileName, mime)
                    })
            })
    }

    if (opts.localDir)
        return Promise.map(opts.fileList, uploadFileAsync, { concurrency: 15 })
            .then(() => {
                console.log("Release files written to", path.resolve(builtPackaged + opts.localDir))
            })

    let info = travisInfo()
    return Cloud.privatePostAsync("releases", {
        baserelease: opts.baserelease,
        pkgversion: opts.pkgversion,
        commit: info.commitUrl,
        branch: info.tag || info.branch,
        buildnumber: process.env['TRAVIS_BUILD_NUMBER'],
        target: pxt.appTarget ? pxt.appTarget.id : "",
        type: opts.baserelease ? "target" : "base",
    })
        .then(resp => {
            console.log(resp)
            liteId = resp.id
            return Promise.map(opts.fileList, uploadFileAsync, { concurrency: 15 })
        })
        .then(() => {
            if (!opts.label) return Promise.resolve()
            else if (opts.legacyLabel) return Cloud.privatePostAsync(liteId + "/label", { name: opts.label })
            else return Cloud.privatePostAsync("pointers", {
                path: nodeutil.sanitizePath(opts.label),
                releaseid: liteId
            })
                .then(() => {
                    // tag release/v0.1.2 also as release/beta
                    let beta = opts.label.replace(/\/v\d.*/, "/beta")
                    if (beta == opts.label) return Promise.resolve()
                    else {
                        console.log("Alos tagging with " + beta)
                        return Cloud.privatePostAsync("pointers", {
                            path: nodeutil.sanitizePath(beta),
                            releaseid: liteId
                        })
                    }
                })
        })
        .then(() => {
            console.log("All done; tagged with " + opts.label)
        })
}

function readLocalPxTarget() {
    if (!fs.existsSync("pxtarget.json")) {
        console.error("This command requires pxtarget.json in current directory.")
        process.exit(1)
    }
    nodeutil.targetDir = process.cwd()
    let cfg: pxt.TargetBundle = readJson("pxtarget.json")
    return cfg
}

function forEachBundledPkgAsync(f: (pkg: pxt.MainPackage) => Promise<void>) {
    let cfg = readLocalPxTarget()
    let prev = process.cwd()

    return Promise.mapSeries(cfg.bundleddirs, (dirname) => {
        process.chdir(path.join(nodeutil.targetDir, dirname))
        mainPkg = new pxt.MainPackage(new Host())
        return f(mainPkg);
    })
        .finally(() => process.chdir(prev))
        .then(() => { })
}

export function publishTargetAsync() {
    return forEachBundledPkgAsync((pkg) => {
        return pkg.publishAsync()
    })
}

export function spawnAsync(opts: {
    cmd: string,
    args: string[],
    cwd?: string,
    shell?: boolean
}) {
    let info = opts.cmd + " " + opts.args.join(" ")
    if (opts.cwd && opts.cwd != ".") info = "cd " + opts.cwd + "; " + info
    console.log("[run] " + info)
    return new Promise<void>((resolve, reject) => {
        let ch = child_process.spawn(opts.cmd, opts.args, {
            cwd: opts.cwd,
            env: process.env,
            stdio: "inherit",
            shell: opts.shell || false
        } as any)
        ch.on('close', (code: number) => {
            if (code != 0)
                reject(new Error("Exit code: " + code + " from " + info))
            resolve()
        });
    })
}

function ghpSetupRepoAsync() {
    function getreponame() {
        let cfg = fs.readFileSync("built/gh-pages/.git/config", "utf8")
        let m = /^\s*url\s*=\s*.*github.*\/([^\/\s]+)$/mi.exec(cfg)
        if (!m) U.userError("cannot determine GitHub repo name")
        return m[1].replace(/\.git$/, "")
    }
    if (fs.existsSync("built/gh-pages")) {
        console.log("Skipping init of built/gh-pages; you can delete it first to get full re-init")
        return Promise.resolve(getreponame())
    }

    cpR(".git", "built/gh-pages/.git")
    return ghpGitAsync("checkout", "gh-pages")
        .then(() => getreponame(), (e: any) => {
            U.userError("No gh-pages branch. Try 'pxt ghpinit' first.")
        })
}

function ghpGitAsync(...args: string[]) {
    return spawnAsync({
        cmd: "git",
        cwd: "built/gh-pages",
        args: args
    })
}

export function ghpPushAsync() {
    let repoName = ""
    return ghpSetupRepoAsync()
        .then(name => staticpkgAsync((repoName = name)))
        .then(() => {
            cpR(builtPackaged + "/" + repoName, "built/gh-pages")
        })
        .then(() => ghpGitAsync("add", "."))
        .then(() => ghpGitAsync("commit", "-m", "Auto-push"))
        .then(() => ghpGitAsync("push"))
}

export function ghpInitAsync() {
    if (fs.existsSync("built/gh-pages"))
        U.userError("built/gh-pages already exists")
    cpR(".git", "built/gh-pages/.git")
    return ghpGitAsync("checkout", "gh-pages")
        .then(() => U.userError("gh-pages branch already exists"), (e: any) => { })
        .then(() => ghpGitAsync("checkout", "--orphan", "gh-pages"))
        .then(() => ghpGitAsync("rm", "-rf", "."))
        .then(() => {
            fs.writeFileSync("built/gh-pages/index.html", "Under construction.")
            return ghpGitAsync("add", ".")
        })
        .then(() => ghpGitAsync("commit", "-m", "Initial."))
        .then(() => ghpGitAsync("push", "--set-upstream", "origin", "gh-pages"))
}

function maxMTimeAsync(dirs: string[]) {
    let max = 0
    return Promise.map(dirs, dn => readDirAsync(dn)
        .then(files => Promise.map(files, fn => statAsync(path.join(dn, fn))
            .then(st => {
                max = Math.max(st.mtime.getTime(), max)
            }))))
        .then(() => max)
}

export function buildTargetAsync(): Promise<void> {
    return simshimAsync()
        .then(() => buildFolderAsync('sim'))
        .then(buildTargetCoreAsync)
        .then(() => buildFolderAsync('cmds', true))
        .then(() => buildFolderAsync('server', true))
}

function buildFolderAsync(p: string, optional?: boolean): Promise<void> {
    if (!fs.existsSync(p + "/tsconfig.json")) {
        if (!optional) U.userError(`${p}/tsconfig.json not found`);
        return Promise.resolve()
    }

    console.log(`building ${p}...`)
    dirsToWatch.push(p)
    return spawnAsync({
        cmd: "node",
        args: ["../node_modules/typescript/bin/tsc"],
        cwd: p
    })
}

function addCmd(name: string) {
    return name + (/win/.test(process.platform) ? ".cmd" : "")
}

function buildPxtAsync(): Promise<string[]> {
    let ksd = "node_modules/pxt-core"
    if (!fs.existsSync(ksd + "/pxtlib/main.ts")) return Promise.resolve([]);

    console.log(`building ${ksd}...`);
    return spawnAsync({
        cmd: addCmd("jake"),
        args: [],
        cwd: ksd
    }).then(() => {
        console.log("local pxt-core built.")
        return [ksd]
    }, e => {
        buildFailed("local pxt-core build FAILED")
        return [ksd]
    });
}

let dirsToWatch: string[] = []

function travisInfo() {
    return {
        branch: process.env['TRAVIS_BRANCH'],
        tag: process.env['TRAVIS_TAG'],
        commit: process.env['TRAVIS_COMMIT'],
        commitUrl: !process.env['TRAVIS_COMMIT'] ? undefined :
            "https://github.com/" + process.env['TRAVIS_REPO_SLUG'] + "/commits/" + process.env['TRAVIS_COMMIT'],
    }
}

function buildWebManifest(cfg: pxt.TargetBundle) {
    let webmanifest: any = {
        "lang": "en",
        "dir": "ltr",
        "name": cfg.name,
        "short_name": cfg.name,
        "icons": [
            {
                "src": "\/static\/icons\/android-chrome-36x36.png",
                "sizes": "36x36",
                "type": "image\/png",
                "density": 0.75
            },
            {
                "src": "\/static\/icons\/android-chrome-48x48.png",
                "sizes": "48x48",
                "type": "image\/png",
                "density": 1
            },
            {
                "src": "\/static\/icons\/android-chrome-72x72.png",
                "sizes": "72x72",
                "type": "image\/png",
                "density": 1.5
            },
            {
                "src": "\/static\/icons\/android-chrome-96x96.png",
                "sizes": "96x96",
                "type": "image\/png",
                "density": 2
            },
            {
                "src": "\/static\/icons\/android-chrome-144x144.png",
                "sizes": "144x144",
                "type": "image\/png",
                "density": 3
            },
            {
                "src": "\/static\/icons\/android-chrome-192x192.png",
                "sizes": "192x192",
                "type": "image\/png",
                "density": 4
            }
        ],
        "scope": "/",
        "start_url": "/",
        "display": "standalone",
        "orientation": "landscape"
    }
    let diskManifest: any = {}
    if (fs.existsSync("webmanifest.json"))
        diskManifest = nodeutil.readJson("webmanifest.json")
    U.jsonCopyFrom(webmanifest, diskManifest)
    return webmanifest;
}

function saveThemeJson(cfg: pxt.TargetBundle) {
    cfg.appTheme.id = cfg.id
    cfg.appTheme.title = cfg.title
    cfg.appTheme.name = cfg.name

    // expand logo
    let logos = (cfg.appTheme as any as U.Map<string>);
    Object.keys(logos)
        .filter(k => /logo$/i.test(k) && /^\.\//.test(logos[k]))
        .forEach(k => {
            let fn = path.join('./docs', logos[k]);
            console.log(`importing ${fn}`)
            let b = fs.readFileSync(fn)
            let mimeType = '';
            if (/\.svg$/i.test(fn)) mimeType = "image/svg+xml";
            else if (/\.png$/i.test(fn)) mimeType = "image/png";
            else if (/\.jpe?g$/i.test(fn)) mimeType = "image/jpeg";
            if (mimeType) logos[k] = `data:${mimeType};base64,${b.toString('base64')}`;
            else logos[k] = b.toString('utf8');
        })

    cfg.appTheme.locales = {}

    let lpath = "docs/_locales"
    if (fs.existsSync(lpath)) {
        for (let loc of fs.readdirSync(lpath)) {
            let fn = lpath + "/" + loc + "/_theme.json"
            if (fs.existsSync(fn))
                cfg.appTheme.locales[loc.toLowerCase()] = readJson(fn)
        }
    }

    nodeutil.mkdirP("built");
    fs.writeFileSync("built/theme.json", JSON.stringify(cfg.appTheme, null, 2))
}

function buildTargetCoreAsync() {
    let cfg = readLocalPxTarget()
    cfg.bundledpkgs = {}
    pxt.appTarget = cfg;
    let statFiles: U.Map<number> = {}
    dirsToWatch = cfg.bundleddirs.slice()
    dirsToWatch.push("sim"); // simulator
    dirsToWatch.push("sim/public"); // simulator
    console.log("building target.json...")
    return forEachBundledPkgAsync(pkg =>
        pkg.filesToBePublishedAsync()
            .then(res => {
                cfg.bundledpkgs[pkg.config.name] = res
            })
            .then(testForBuildTargetAsync))
        .then(() => {
            let info = travisInfo()
            cfg.versions = {
                branch: info.branch,
                tag: info.tag,
                commits: info.commitUrl,
                target: readJson("package.json")["version"],
                pxt: readJson("node_modules/pxt-core/package.json")["version"],
            }

            saveThemeJson(cfg)

            let webmanifest = buildWebManifest(cfg)
            fs.writeFileSync("built/target.json", JSON.stringify(cfg, null, 2))
            pxt.appTarget = cfg; // make sure we're using the latest version
            let targetlight = U.flatClone(cfg)
            delete targetlight.bundleddirs
            delete targetlight.bundledpkgs
            delete targetlight.appTheme
            fs.writeFileSync("built/targetlight.json", JSON.stringify(targetlight, null, 2))
            fs.writeFileSync("built/sim.webmanifest", JSON.stringify(webmanifest, null, 2))
        })
        .then(() => {
            console.log("target.json built.")
        })
}

function buildAndWatchAsync(f: () => Promise<string[]>): Promise<void> {
    let currMtime = Date.now()
    return f()
        .then(dirs => {
            if (globalConfig.noAutoBuild) return
            console.log('watching ' + dirs.join(', ') + '...');
            let loop = () => {
                Promise.delay(1000)
                    .then(() => maxMTimeAsync(dirs))
                    .then(num => {
                        if (num > currMtime) {
                            currMtime = num
                            f()
                                .then(d => {
                                    dirs = d
                                    U.nextTick(loop)
                                })
                        } else {
                            U.nextTick(loop)
                        }
                    })
            }
            U.nextTick(loop)
        })

}

function buildFailed(msg: string) {
    console.log("")
    console.log("***")
    console.log("*** Build failed: " + msg)
    console.log("***")
    console.log("")
}

function buildAndWatchTargetAsync() {
    if (!fs.existsSync("sim/tsconfig.json")) {
        console.log("No sim/tsconfig.json; assuming npm installed package")
        return Promise.resolve()
    }

    return buildAndWatchAsync(() => buildPxtAsync()
        .then(() => buildTargetAsync().then(r => { }, e => {
            buildFailed(e.message)
        }))
        .then(() => uploader.checkDocsAsync())
        .then(() => [path.resolve("node_modules/pxt-core")].concat(dirsToWatch)));
}

function cpR(src: string, dst: string, maxDepth = 8) {
    src = path.resolve(src)
    let files = allFiles(src, maxDepth)
    let dirs: U.Map<boolean> = {}
    for (let f of files) {
        let bn = f.slice(src.length)
        let dd = path.join(dst, bn)
        let dir = path.dirname(dd)
        if (!U.lookup(dirs, dir)) {
            nodeutil.mkdirP(dir)
            dirs[dir] = true
        }
        let buf = fs.readFileSync(f)
        fs.writeFileSync(dd, buf)
    }
}

let builtPackaged = "built/packaged"

function renderDocs(localDir: string) {
    let dst = path.resolve(builtPackaged + localDir)

    cpR("node_modules/pxt-core/docfiles", dst + "/docfiles")
    if (fs.existsSync("docfiles"))
        cpR("docfiles", dst + "/docfiles")

    let webpath = localDir
    let docsTemplate = fs.readFileSync(dst + "/docfiles/template.html", "utf8")
    docsTemplate = U.replaceAll(docsTemplate, "/cdn/", webpath)
    docsTemplate = U.replaceAll(docsTemplate, "/docfiles/", webpath + "docfiles/")
    docsTemplate = U.replaceAll(docsTemplate, "/--embed", webpath + "embed.js")

    let dirs: U.Map<boolean> = {}
    for (let f of allFiles("docs", 8)) {
        let dd = path.join(dst, f)
        let dir = path.dirname(dd)
        if (!U.lookup(dirs, dir)) {
            nodeutil.mkdirP(dir)
            dirs[dir] = true
        }
        let buf = fs.readFileSync(f)
        if (/\.md$/.test(f)) {
            let str = buf.toString("utf8")
            let path = f.slice(5).split(/\//)
            let bc = path.map((e, i) => {
                return {
                    href: "/" + path.slice(0, i + 1).join("/"),
                    name: e
                }
            })
            let html = pxt.docs.renderMarkdown(docsTemplate, str, pxt.appTarget.appTheme, null, bc)
            html = html.replace(/(<a[^<>]*)\shref="(\/[^<>"]*)"/g, (f, beg, url) => {
                return beg + ` href="${webpath}docs${url}.html"`
            })
            buf = new Buffer(html, "utf8")
            dd = dd.slice(0, dd.length - 3) + ".html"
        }
        fs.writeFileSync(dd, buf)
    }
    console.log("Docs written.")
}

export function staticpkgAsync(label?: string) {
    let dir = label ? "/" + label + "/" : "/"
    return Promise.resolve()
        .then(() => uploadCoreAsync({
            label: label || "main",
            pkgversion: "0.0.0",
            fileList: pxtFileList("node_modules/pxt-core/").concat(targetFileList()),
            localDir: dir
        }))
        .then(() => renderDocs(dir))
}

export function serveAsync(arg?: string) {
    forceCloudBuild = !globalConfig.localBuild
    let justServe = false
    let packaged = false
    if (arg == "-yt") {
        forceCloudBuild = false
    } else if (arg == "-cloud") {
        forceCloudBuild = true
    } else if (arg == "-just") {
        justServe = true
    } else if (arg == "-pkg") {
        justServe = true
        packaged = true
    }
    if (!globalConfig.localToken) {
        globalConfig.localToken = U.guidGen();
        saveConfig()
    }
    let localToken = globalConfig.localToken;
    if (!fs.existsSync("pxtarget.json")) {
        let upper = path.join(__dirname, "../../..")
        if (fs.existsSync(path.join(upper, "pxtarget.json"))) {
            console.log("going to " + upper)
            process.chdir(upper)
        } else {
            U.userError("Cannot find pxtarget.json to serve.")
        }
    }
    if (!fs.existsSync("node_modules/typescript")) {
        U.userError("Oops, typescript does not seem to be installed, did you run 'npm install'?");
    }
    return (justServe ? Promise.resolve() : buildAndWatchTargetAsync())
        .then(() => server.serveAsync({
            localToken: localToken,
            autoStart: !globalConfig.noAutoStart,
            packaged: packaged
        }))
}
{'fraction_non_alphanumeric': 0.15863880357091598, 'fraction_numerical': 0.004664580761833683, 'mean_word_length': 1.8446903196831748, 'pattern_counts': {'":': 118, '<': 54, '<?xml version=': 0, '>': 256, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6087071', 'n_tokens_mistral': 21901, 'n_tokens_neox': 21099, 'n_words': 4262}