<commit_before>/// <reference path="../../localtypings/projectheader.d.ts"/>
namespace pxt.Cloud {
    import Util = pxtc.Util;

    // hit /api/ to stay on same domain and avoid CORS
    export let apiRoot = (pxt.BrowserUtils.isLocalHost() || Util.isNodeJS) ? "https://www.makecode.com/api/" : "/api/";
    export let accessToken = "";
    export let localToken = "";
    let _isOnline = true;
    export let onOffline = () => { };

    function offlineError(url: string) {
        let e: any = new Error(Util.lf("Cannot access {0} while offline", url));
        e.isOffline = true;
        return Promise.delay(1000).then(() => Promise.reject(e))
    }

    export function hasAccessToken() {
        return !!accessToken
    }

    export function localRequestAsync(path: string, data?: any) {
        return U.requestAsync({
            url: "/api/" + path,
            headers: { "Authorization": Cloud.localToken },
            method: data ? "POST" : "GET",
            data: data || undefined,
            allowHttpErrors: true
        })
    }

    export function useCdnApi() {
        return pxt.webConfig && !pxt.webConfig.isStatic
            && !BrowserUtils.isLocalHost() && !!pxt.webConfig.cdnUrl
    }

    export function cdnApiUrl(url: string) {
        url = url.replace(/^\//, '');
        if (!useCdnApi())
            return apiRoot + url;

        const d = new Date()
        const timestamp = d.getUTCFullYear() + ("0" + (d.getUTCMonth() + 1)).slice(-2) + ("0" + d.getUTCDate()).slice(-2)
        if (url.indexOf("?") < 0)
            url += "?"
        else
            url += "&"
        url += "cdn=" + timestamp
        // url = url.replace("?", "$")
        return pxt.webConfig.cdnUrl + "/api/" + url
    }

    export function apiRequestWithCdnAsync(options: Util.HttpRequestOptions) {
        if (!useCdnApi())
            return privateRequestAsync(options)
        options.url = cdnApiUrl(options.url)
        return Util.requestAsync(options)
            .catch(e => handleNetworkError(options, e))
    }

    function handleNetworkError(options: Util.HttpRequestOptions, e: any) {
        if (e.statusCode == 0) {
            if (_isOnline) {
                _isOnline = false;
                onOffline();
            }
            return offlineError(options.url)
        } else {
            return Promise.reject(e)
        }
    }

    export function privateRequestAsync(options: Util.HttpRequestOptions) {
        options.url = pxt.webConfig?.isStatic && !options.forceLiveEndpoint ? pxt.webConfig.relprefix + options.url : apiRoot + options.url;
        options.allowGzipPost = true
        if (!Cloud.isOnline() && !pxt.BrowserUtils.isPxtElectron()) {
            return offlineError(options.url);
        }
        if (!options.headers) options.headers = {}
        if (pxt.BrowserUtils.isLocalHost()) {
            if (Cloud.localToken)
                options.headers["Authorization"] = Cloud.localToken;
        } else if (accessToken) {
            options.headers["x-td-access-token"] = accessToken
        }
        return Util.requestAsync(options)
            .catch(e => handleNetworkError(options, e))
    }

    export function privateGetTextAsync(path: string, headers?: pxt.Map<string>): Promise<string> {
        return privateRequestAsync({ url: path, headers }).then(resp => resp.text)
    }

    export function privateGetAsync(path: string, forceLiveEndpoint: boolean = false): Promise<any> {
        return privateRequestAsync({ url: path, forceLiveEndpoint }).then(resp => resp.json)
    }

    export function downloadTargetConfigAsync(): Promise<pxt.TargetConfig> {
        if (!Cloud.isOnline()) // offline
            return Promise.resolve(undefined);

        const targetVersion = pxt.appTarget.versions && pxt.appTarget.versions.target;
        const url = pxt.webConfig && pxt.webConfig.isStatic ? `targetconfig.json` : `config/${pxt.appTarget.id}/targetconfig${targetVersion ? `/v${targetVersion}` : ''}`;
        if (pxt.BrowserUtils.isLocalHost())
            return localRequestAsync(url).then(r => r ? r.json : undefined)
        else
            return apiRequestWithCdnAsync({ url }).then(r => r.json)
    }

    export function downloadScriptFilesAsync(id: string) {
        return privateRequestAsync({ url: id + "/text", forceLiveEndpoint: true }).then(resp => {
            return JSON.parse(resp.text)
        })
    }

    // 1h check on markdown content if not on development server
    const MARKDOWN_EXPIRATION = pxt.BrowserUtils.isLocalHostDev() ? 1 : 1 * 60 * 60 * 1000;
    export function markdownAsync(docid: string, locale?: string): Promise<string> {
        locale = locale || pxt.Util.userLanguage();
        const live = pxt.Util.localizeLive;
        const branch = "";
        return pxt.BrowserUtils.translationDbAsync()
            .then(db => db.getAsync(locale, docid, "")
                .then(entry => {
                    if (entry && Date.now() - entry.time > MARKDOWN_EXPIRATION)
                        // background update,
                        downloadMarkdownAsync(docid, locale, live, entry.etag)
                            .then(r => db.setAsync(locale, docid, branch, r.etag, undefined, r.md || entry.md))
                            .catch(() => { }) // swallow errors
                            .done();
                    // return cached entry
                    if (entry && entry.md)
                        return entry.md;
                    // download and cache
                    else return downloadMarkdownAsync(docid, locale, live)
                        .then(r => db.setAsync(locale, docid, branch, r.etag, undefined, r.md)
                            .then(() => r.md))
                        .catch(() => ""); // no translation
                }))
    }

    function downloadMarkdownAsync(docid: string, locale?: string, live?: boolean, etag?: string): Promise<{ md: string; etag?: string; }> {
        const packaged = pxt.webConfig?.isStatic;
        const targetVersion = pxt.appTarget.versions && pxt.appTarget.versions.target || '?';
        let url: string;<commit_msg>if cached markdown > a week old, wait for new markdown (#7430)<commit_after>/// <reference path="../../localtypings/projectheader.d.ts"/>
namespace pxt.Cloud {
    import Util = pxtc.Util;

    // hit /api/ to stay on same domain and avoid CORS
    export let apiRoot = (pxt.BrowserUtils.isLocalHost() || Util.isNodeJS) ? "https://www.makecode.com/api/" : "/api/";
    export let accessToken = "";
    export let localToken = "";
    let _isOnline = true;
    export let onOffline = () => { };

    function offlineError(url: string) {
        let e: any = new Error(Util.lf("Cannot access {0} while offline", url));
        e.isOffline = true;
        return Promise.delay(1000).then(() => Promise.reject(e))
    }

    export function hasAccessToken() {
        return !!accessToken
    }

    export function localRequestAsync(path: string, data?: any) {
        return U.requestAsync({
            url: "/api/" + path,
            headers: { "Authorization": Cloud.localToken },
            method: data ? "POST" : "GET",
            data: data || undefined,
            allowHttpErrors: true
        })
    }

    export function useCdnApi() {
        return pxt.webConfig && !pxt.webConfig.isStatic
            && !BrowserUtils.isLocalHost() && !!pxt.webConfig.cdnUrl
    }

    export function cdnApiUrl(url: string) {
        url = url.replace(/^\//, '');
        if (!useCdnApi())
            return apiRoot + url;

        const d = new Date()
        const timestamp = d.getUTCFullYear() + ("0" + (d.getUTCMonth() + 1)).slice(-2) + ("0" + d.getUTCDate()).slice(-2)
        if (url.indexOf("?") < 0)
            url += "?"
        else
            url += "&"
        url += "cdn=" + timestamp
        // url = url.replace("?", "$")
        return pxt.webConfig.cdnUrl + "/api/" + url
    }

    export function apiRequestWithCdnAsync(options: Util.HttpRequestOptions) {
        if (!useCdnApi())
            return privateRequestAsync(options)
        options.url = cdnApiUrl(options.url)
        return Util.requestAsync(options)
            .catch(e => handleNetworkError(options, e))
    }

    function handleNetworkError(options: Util.HttpRequestOptions, e: any) {
        if (e.statusCode == 0) {
            if (_isOnline) {
                _isOnline = false;
                onOffline();
            }
            return offlineError(options.url)
        } else {
            return Promise.reject(e)
        }
    }

    export function privateRequestAsync(options: Util.HttpRequestOptions) {
        options.url = pxt.webConfig?.isStatic && !options.forceLiveEndpoint ? pxt.webConfig.relprefix + options.url : apiRoot + options.url;
        options.allowGzipPost = true
        if (!Cloud.isOnline() && !pxt.BrowserUtils.isPxtElectron()) {
            return offlineError(options.url);
        }
        if (!options.headers) options.headers = {}
        if (pxt.BrowserUtils.isLocalHost()) {
            if (Cloud.localToken)
                options.headers["Authorization"] = Cloud.localToken;
        } else if (accessToken) {
            options.headers["x-td-access-token"] = accessToken
        }
        return Util.requestAsync(options)
            .catch(e => handleNetworkError(options, e))
    }

    export function privateGetTextAsync(path: string, headers?: pxt.Map<string>): Promise<string> {
        return privateRequestAsync({ url: path, headers }).then(resp => resp.text)
    }

    export function privateGetAsync(path: string, forceLiveEndpoint: boolean = false): Promise<any> {
        return privateRequestAsync({ url: path, forceLiveEndpoint }).then(resp => resp.json)
    }

    export function downloadTargetConfigAsync(): Promise<pxt.TargetConfig> {
        if (!Cloud.isOnline()) // offline
            return Promise.resolve(undefined);

        const targetVersion = pxt.appTarget.versions && pxt.appTarget.versions.target;
        const url = pxt.webConfig && pxt.webConfig.isStatic ? `targetconfig.json` : `config/${pxt.appTarget.id}/targetconfig${targetVersion ? `/v${targetVersion}` : ''}`;
        if (pxt.BrowserUtils.isLocalHost())
            return localRequestAsync(url).then(r => r ? r.json : undefined)
        else
            return apiRequestWithCdnAsync({ url }).then(r => r.json)
    }

    export function downloadScriptFilesAsync(id: string) {
        return privateRequestAsync({ url: id + "/text", forceLiveEndpoint: true }).then(resp => {
            return JSON.parse(resp.text)
        })
    }

    // 1h check on markdown content if not on development server
    const MARKDOWN_EXPIRATION = pxt.BrowserUtils.isLocalHostDev() ? 0 : 1 * 60 * 60 * 1000;
    // 1w check don't use cached version and wait for new content
    const FORCE_MARKDOWN_UPDATE = MARKDOWN_EXPIRATION * 24 * 7;
    export function markdownAsync(docid: string, locale?: string): Promise<string> {
        locale = locale || pxt.Util.userLanguage();
        const live = pxt.Util.localizeLive;
        const branch = "";
        return pxt.BrowserUtils.translationDbAsync()
            .then(db => db.getAsync(locale, docid, "")
                .then(entry => {
                    const timeDiff = Date.now() - (entry?.time || 0);
                    const shouldFetchInBackground = timeDiff > MARKDOWN_EXPIRATION;
                    const shouldWaitForNewContent = timeDiff > FORCE_MARKDOWN_UPDATE;

                    if (!shouldWaitForNewContent && entry) {
                        if (entry && shouldFetchInBackground) {
                            // background update,
                            downloadMarkdownAsync(docid, locale, live, entry.etag)
                                .then(r => db.setAsync(locale, docid, branch, r.etag, undefined, r.md || entry.md))
                                .catch(() => { }) // swallow errors
                                .done();
                        }
                        // return cached entry
                        if (entry.md)
                            return entry.md;
                    }
                    // download and cache
                    return downloadMarkdownAsync(docid, locale, live)
                        .then(r => db.setAsync(locale, docid, branch, r.etag, undefined, r.md)
                            .then(() => r.md))
                        .catch(() => ""); // no translation
                }))
    }

    function downloadMarkdownAsync(docid: string, locale?: string, live?: boolean, etag?: string): Promise<{ md: string; etag?: string; }> {
        const packaged = pxt.webConfig?.isStatic;
        const targetVersion = pxt.appTarget.versions && pxt.appTarget.versions.target || '?';
        let url: string;
{'fraction_non_alphanumeric': 0.12991787250684395, 'fraction_numerical': 0.004301916308173641, 'mean_word_length': 2.055927342256214, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 53, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6402442', 'n_tokens_mistral': 3637, 'n_tokens_neox': 3492, 'n_words': 815}