<commit_before>﻿import * as mongodb from "mongodb";
import * as http from "http";
import {IConfig, IServer, IResponse, IRender, IGetRenders} from "modepress-api";
import * as winston from "winston";
import * as express from "express";
import * as bodyParser from "body-parser";
import {Controller} from "./controller";
import {UsersService} from "../users-service"
import {RendersModel} from "../models/renders-model";
import {ModelInstance, Model} from "../models/model";
import * as net from "net";
import * as url from "url";
import * as jsdom from "jsdom";
import {okJson, errJson} from "../serializers";

/**
* Sets up a prerender server and saves the rendered html requests to mongodb.
* These saved HTML documents can then be sent to web crawlers who cannot interpret javascript.
*/
export default class PageRenderer extends Controller
{
    private renderQueryFlag: string;
    private expiration: number;

    // googlebot, yahoo, and bingbot are not in this list because
    // we support _escaped_fragment_ and want to ensure people aren't
    // penalized for cloaking.
    private static crawlerUserAgents : Array<string> = [
        // 'googlebot',
        // 'yahoo',
        // 'bingbot',
        'baiduspider',
        'facebookexternalhit',
        'twitterbot',
        'rogerbot',
        'linkedinbot',
        'embedly',
        'quora link preview',
        'showyoubot',
        'outbrain',
        'pinterest',
        'developers.google.com/+/web/snippet',
        'slackbot',
        'vkShare',
        'W3C_Validator'
    ];

    private static extensionsToIgnore: Array<string> = [
        '.js',
        '.css',
        '.xml',
        '.less',
        '.png',
        '.jpg',
        '.jpeg',
        '.gif',
        '.pdf',
        '.doc',
        '.txt',
        '.ico',
        '.rss',
        '.zip',
        '.mp3',
        '.rar',
        '.exe',
        '.wmv',
        '.doc',
        '.avi',
        '.ppt',
        '.mpg',
        '.mpeg',
        '.tif',
        '.wav',
        '.mov',
        '.psd',
        '.ai',
        '.xls',
        '.mp4',
        '.m4a',
        '.swf',
        '.dat',
        '.dmg',
        '.iso',
        '.flv',
        '.m4v',
        '.torrent'
    ];

    /**
	* Creates a new instance of the email controller
	* @param {IServer} server The server configuration options
    * @param {IConfig} config The configuration options
    * @param {express.Express} e The express instance of this server
	*/
    constructor(server: IServer, config: IConfig, e: express.Express)
    {
        super([ Model.registerModel( RendersModel) ]);

        if (!config.enableAjaxRendering)
            return;

        this.renderQueryFlag = "__render__request";
        e.use(this.processBotRequest.bind(this));
        this.expiration = config.ajaxRenderExpiration * 1000;

        var router = express.Router();
        router.use(bodyParser.urlencoded({ 'extended': true }));
        router.use(bodyParser.json());
        router.use(bodyParser.json({ type: 'application/vnd.api+json' }));

        router.get("/", <any>[this.authenticateAdmin.bind(this), this.getRenders.bind(this)]);
        router.get("/preview/:id", <any>[this.previewRender.bind(this)]);
        router.delete("/:id", <any>[this.authenticateAdmin.bind(this), this.removeRender.bind(this)]);
        router.delete("/clear", <any>[this.authenticateAdmin.bind(this), this.clearRenders.bind(this)]);

        // Register the path
        e.use("/api/renders", router);
    }

    /**
    * Strips the html page of any script tags
    * @param {string} html
    */<commit_msg>Bug fix: renders Clear function given highrt priority<commit_after>﻿import * as mongodb from "mongodb";
import * as http from "http";
import {IConfig, IServer, IResponse, IRender, IGetRenders} from "modepress-api";
import * as winston from "winston";
import * as express from "express";
import * as bodyParser from "body-parser";
import {Controller} from "./controller";
import {UsersService} from "../users-service"
import {RendersModel} from "../models/renders-model";
import {ModelInstance, Model} from "../models/model";
import * as net from "net";
import * as url from "url";
import * as jsdom from "jsdom";
import {okJson, errJson} from "../serializers";

/**
* Sets up a prerender server and saves the rendered html requests to mongodb.
* These saved HTML documents can then be sent to web crawlers who cannot interpret javascript.
*/
export default class PageRenderer extends Controller
{
    private renderQueryFlag: string;
    private expiration: number;

    // googlebot, yahoo, and bingbot are not in this list because
    // we support _escaped_fragment_ and want to ensure people aren't
    // penalized for cloaking.
    private static crawlerUserAgents : Array<string> = [
        // 'googlebot',
        // 'yahoo',
        // 'bingbot',
        'baiduspider',
        'facebookexternalhit',
        'twitterbot',
        'rogerbot',
        'linkedinbot',
        'embedly',
        'quora link preview',
        'showyoubot',
        'outbrain',
        'pinterest',
        'developers.google.com/+/web/snippet',
        'slackbot',
        'vkShare',
        'W3C_Validator'
    ];

    private static extensionsToIgnore: Array<string> = [
        '.js',
        '.css',
        '.xml',
        '.less',
        '.png',
        '.jpg',
        '.jpeg',
        '.gif',
        '.pdf',
        '.doc',
        '.txt',
        '.ico',
        '.rss',
        '.zip',
        '.mp3',
        '.rar',
        '.exe',
        '.wmv',
        '.doc',
        '.avi',
        '.ppt',
        '.mpg',
        '.mpeg',
        '.tif',
        '.wav',
        '.mov',
        '.psd',
        '.ai',
        '.xls',
        '.mp4',
        '.m4a',
        '.swf',
        '.dat',
        '.dmg',
        '.iso',
        '.flv',
        '.m4v',
        '.torrent'
    ];

    /**
	* Creates a new instance of the email controller
	* @param {IServer} server The server configuration options
    * @param {IConfig} config The configuration options
    * @param {express.Express} e The express instance of this server
	*/
    constructor(server: IServer, config: IConfig, e: express.Express)
    {
        super([ Model.registerModel( RendersModel) ]);

        if (!config.enableAjaxRendering)
            return;

        this.renderQueryFlag = "__render__request";
        e.use(this.processBotRequest.bind(this));
        this.expiration = config.ajaxRenderExpiration * 1000;

        var router = express.Router();
        router.use(bodyParser.urlencoded({ 'extended': true }));
        router.use(bodyParser.json());
        router.use(bodyParser.json({ type: 'application/vnd.api+json' }));

        router.get("/", <any>[this.authenticateAdmin.bind(this), this.getRenders.bind(this)]);
        router.get("/preview/:id", <any>[this.previewRender.bind(this)]);
        router.delete("/clear", <any>[this.authenticateAdmin.bind(this), this.clearRenders.bind(this)]);
        router.delete("/:id", <any>[this.authenticateAdmin.bind(this), this.removeRender.bind(this)]);

        // Register the path
        e.use("/api/renders", router);
    }

    /**
    * Strips the html page of any script tags
    * @param {string} html
    */
{'fraction_non_alphanumeric': 0.1500826901874311, 'fraction_numerical': 0.0024807056229327455, 'mean_word_length': 2.2310774710596615, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6866159', 'n_tokens_mistral': 2202, 'n_tokens_neox': 2131, 'n_words': 573}