<commit_before>import path from 'path';
import fs from 'fs';
import Koa from 'koa';
import mount from 'koa-mount';
import helmet from 'koa-helmet';
import koa_logger from 'koa-logger';
import requestTime from './requesttimings';
import hardwareStats from './hardwarestats';
import cluster from 'cluster';
import os from 'os';
import prod_logger from './prod_logger';
import favicon from 'koa-favicon';
import staticCache from 'koa-static-cache';
import useRedirects from './redirects';
import useOauthLogin from './api/oauth';
import useGeneralApi from './api/general';
import useAccountRecoveryApi from './api/account_recovery';
import useNotificationsApi from './api/notifications';
import useEnterAndConfirmEmailPages from './sign_up_pages/enter_confirm_email';
import useEnterAndConfirmMobilePages from './sign_up_pages/enter_confirm_mobile';
import useUserJson from './json/user_json';
import usePostJson from './json/post_json';
import isBot from 'koa-isbot';
import session from '@steem/crypto-session';
import csrf from 'koa-csrf';
import flash from 'koa-flash';
import minimist from 'minimist';
import Grant from 'grant-koa';
import config from 'config';
import { routeRegex } from 'app/ResolveRoute';
import secureRandom from 'secure-random';
import userIllegalContent from 'app/utils/userIllegalContent';
import koaLocale from 'koa-locale';

if(cluster.isMaster)
    console.log('application server starting, please wait.');

const grant = new Grant(config.grant);
// import uploadImage from 'server/upload-image' //medium-editor

const app = new Koa();
app.name = 'Steemit app';
const env = process.env.NODE_ENV || 'development';
// cache of a thousand days
const cacheOpts = { maxAge: 86400000, gzip: true };

// set number of processes equal to number of cores
// (unless passed in as an env var)
const numProcesses = process.env.NUM_PROCESSES || os.cpus().length;

app.use(requestTime(numProcesses));

app.keys = [config.get('session_key')];

const crypto_key = config.get('server_session_secret');
session(app, {
    maxAge: 1000 * 3600 * 24 * 60,
    crypto_key,
    key: config.get('session_cookie_key')
});
csrf(app);

app.use(mount(grant));
app.use(flash({ key: 'flash' }));
koaLocale(app);

function convertEntriesToArrays(obj) {
    return Object.keys(obj).reduce((result, key) => {
            result[key] = obj[key].split(/\s+/);
    return result;
}, {});
}

const service_worker_js_content = fs
    .readFileSync(path.join(__dirname, './service-worker.js'))
    .toString();

// some redirects and health status
app.use(function* (next) {

    if (this.method === 'GET' && this.url === '/.well-known/healthcheck.json') {
        this.status = 200;
        this.body = {status: 'ok'};
        return;
    }

    // redirect to home page/feed if known account
    if (this.method === 'GET' && this.url === '/' && this.session.a) {
        this.status = 302;
        this.redirect(`/@${this.session.a}/feed`);
        return;
    }
    // normalize user name url from cased params
    if (
        this.method === 'GET' &&
        (routeRegex.UserProfile1.test(this.url) ||
        routeRegex.PostNoCategory.test(this.url) || routeRegex.Post.test(this.url)
        )
    ) {
        const p = this.originalUrl.toLowerCase();
        let userCheck = "";
        if (routeRegex.Post.test(this.url)) {
            userCheck = p.split("/")[2].slice(1);
        } else {
            userCheck = p.split("/")[1].slice(1);
        }
        if (userIllegalContent.includes(userCheck)) {
            console.log('Illegal content user found blocked', userCheck);
            this.status = 451;
            return;
        }
        if (p !== this.originalUrl) {
            this.status = 301;
            this.redirect(p);
            return;
        }
    }
    // normalize top category filtering from cased params
    if (this.method === 'GET' && routeRegex.CategoryFilters.test(this.url)) {
        const p = this.originalUrl.toLowerCase();
        if (p !== this.originalUrl) {
            this.status = 301;
            this.redirect(p);
            return;
        }
    }
    // // do not enter unless session uid & verified phone
    // if (this.url === '/create_account' && !this.session.uid) {
    //     this.status = 302;
    //     this.redirect('/enter_email');
    //     return;
    // }
    // remember ch, cn, r url params in the session and remove them from url
    if (this.method === 'GET' && /\?[^\w]*(ch=|cn=|r=)/.test(this.url)) {
        let redir = this.url.replace(/((ch|cn|r)=[^&]+)/gi, r => {
                const p = r.split('=');
        if (p.length === 2) this.session[p[0]] = p[1];
        return '';
    });
        redir = redir.replace(/&&&?/, '');
        redir = redir.replace(/\?&?$/, '');
        console.log(`server redirect ${this.url} -> ${redir}`);
        this.status = 302;
        this.redirect(redir);
    } else {
        yield next;
    }
});

// load production middleware
if (env === 'production') {
    app.use(require('koa-conditional-get')());
    app.use(require('koa-etag')());
    app.use(require('koa-compressor')());
}

// Logging
if (env === 'production') {
    app.use(prod_logger());
} else {
    app.use(koa_logger());
}

app.use(helmet());

app.use(
    mount(
        '/static',
        staticCache(path.join(__dirname, '../app/assets/static'), cacheOpts)
    )
);

app.use(
    mount('/robots.txt', function*() {
        this.set('Cache-Control', 'public, max-age=86400000');
        this.type = 'text/plain';
        this.body = 'User-agent: *\nAllow: /';
    })
);

app.use(
    mount('/service-worker.js', function*() {
        this.set('Cache-Control', 'public, max-age=7200000');
        this.type = 'application/javascript';
        // TODO: use APP_URL from client_config.js
        // actually use a config value for it
        this.body = service_worker_js_content.replace(
            /\{DEFAULT_URL\}/i,
            'https://' + this.request.header.host
        );
    })
);

// set user's uid - used to identify users in logs and some other places<commit_msg>FIx redundant Strict-Transport-Security response header<commit_after>import path from 'path';
import fs from 'fs';
import Koa from 'koa';
import mount from 'koa-mount';
import helmet from 'koa-helmet';
import koa_logger from 'koa-logger';
import requestTime from './requesttimings';
import hardwareStats from './hardwarestats';
import cluster from 'cluster';
import os from 'os';
import prod_logger from './prod_logger';
import favicon from 'koa-favicon';
import staticCache from 'koa-static-cache';
import useRedirects from './redirects';
import useOauthLogin from './api/oauth';
import useGeneralApi from './api/general';
import useAccountRecoveryApi from './api/account_recovery';
import useNotificationsApi from './api/notifications';
import useEnterAndConfirmEmailPages from './sign_up_pages/enter_confirm_email';
import useEnterAndConfirmMobilePages from './sign_up_pages/enter_confirm_mobile';
import useUserJson from './json/user_json';
import usePostJson from './json/post_json';
import isBot from 'koa-isbot';
import session from '@steem/crypto-session';
import csrf from 'koa-csrf';
import flash from 'koa-flash';
import minimist from 'minimist';
import Grant from 'grant-koa';
import config from 'config';
import { routeRegex } from 'app/ResolveRoute';
import secureRandom from 'secure-random';
import userIllegalContent from 'app/utils/userIllegalContent';
import koaLocale from 'koa-locale';

if(cluster.isMaster)
    console.log('application server starting, please wait.');

const grant = new Grant(config.grant);
// import uploadImage from 'server/upload-image' //medium-editor

const app = new Koa();
app.name = 'Steemit app';
const env = process.env.NODE_ENV || 'development';
// cache of a thousand days
const cacheOpts = { maxAge: 86400000, gzip: true };

// set number of processes equal to number of cores
// (unless passed in as an env var)
const numProcesses = process.env.NUM_PROCESSES || os.cpus().length;

app.use(requestTime(numProcesses));

app.keys = [config.get('session_key')];

const crypto_key = config.get('server_session_secret');
session(app, {
    maxAge: 1000 * 3600 * 24 * 60,
    crypto_key,
    key: config.get('session_cookie_key')
});
csrf(app);

app.use(mount(grant));
app.use(flash({ key: 'flash' }));
koaLocale(app);

function convertEntriesToArrays(obj) {
    return Object.keys(obj).reduce((result, key) => {
            result[key] = obj[key].split(/\s+/);
    return result;
}, {});
}

const service_worker_js_content = fs
    .readFileSync(path.join(__dirname, './service-worker.js'))
    .toString();

// some redirects and health status
app.use(function* (next) {

    if (this.method === 'GET' && this.url === '/.well-known/healthcheck.json') {
        this.status = 200;
        this.body = {status: 'ok'};
        return;
    }

    // redirect to home page/feed if known account
    if (this.method === 'GET' && this.url === '/' && this.session.a) {
        this.status = 302;
        this.redirect(`/@${this.session.a}/feed`);
        return;
    }
    // normalize user name url from cased params
    if (
        this.method === 'GET' &&
        (routeRegex.UserProfile1.test(this.url) ||
        routeRegex.PostNoCategory.test(this.url) || routeRegex.Post.test(this.url)
        )
    ) {
        const p = this.originalUrl.toLowerCase();
        let userCheck = "";
        if (routeRegex.Post.test(this.url)) {
            userCheck = p.split("/")[2].slice(1);
        } else {
            userCheck = p.split("/")[1].slice(1);
        }
        if (userIllegalContent.includes(userCheck)) {
            console.log('Illegal content user found blocked', userCheck);
            this.status = 451;
            return;
        }
        if (p !== this.originalUrl) {
            this.status = 301;
            this.redirect(p);
            return;
        }
    }
    // normalize top category filtering from cased params
    if (this.method === 'GET' && routeRegex.CategoryFilters.test(this.url)) {
        const p = this.originalUrl.toLowerCase();
        if (p !== this.originalUrl) {
            this.status = 301;
            this.redirect(p);
            return;
        }
    }
    // // do not enter unless session uid & verified phone
    // if (this.url === '/create_account' && !this.session.uid) {
    //     this.status = 302;
    //     this.redirect('/enter_email');
    //     return;
    // }
    // remember ch, cn, r url params in the session and remove them from url
    if (this.method === 'GET' && /\?[^\w]*(ch=|cn=|r=)/.test(this.url)) {
        let redir = this.url.replace(/((ch|cn|r)=[^&]+)/gi, r => {
                const p = r.split('=');
        if (p.length === 2) this.session[p[0]] = p[1];
        return '';
    });
        redir = redir.replace(/&&&?/, '');
        redir = redir.replace(/\?&?$/, '');
        console.log(`server redirect ${this.url} -> ${redir}`);
        this.status = 302;
        this.redirect(redir);
    } else {
        yield next;
    }
});

// load production middleware
if (env === 'production') {
    app.use(require('koa-conditional-get')());
    app.use(require('koa-etag')());
    app.use(require('koa-compressor')());
}

// Logging
if (env === 'production') {
    app.use(prod_logger());
} else {
    app.use(koa_logger());
}

app.use(helmet({
    hsts: false
}));

app.use(
    mount(
        '/static',
        staticCache(path.join(__dirname, '../app/assets/static'), cacheOpts)
    )
);

app.use(
    mount('/robots.txt', function*() {
        this.set('Cache-Control', 'public, max-age=86400000');
        this.type = 'text/plain';
        this.body = 'User-agent: *\nAllow: /';
    })
);

app.use(
    mount('/service-worker.js', function*() {
        this.set('Cache-Control', 'public, max-age=7200000');
        this.type = 'application/javascript';
        // TODO: use APP_URL from client_config.js
        // actually use a config value for it
        this.body = service_worker_js_content.replace(
            /\{DEFAULT_URL\}/i,
            'https://' + this.request.header.host
        );
    })
);

// set user's uid - used to identify users in logs and some other places
{'fraction_non_alphanumeric': 0.16177799687782435, 'fraction_numerical': 0.010516802234820474, 'mean_word_length': 2.799001248439451, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2445255', 'n_tokens_mistral': 3932, 'n_tokens_neox': 3813, 'n_words': 917}