<commit_before>var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var request = require('superagent');
var session = require('express-session');
var index = require('./routes/index');
var weibo = require('./routes/weibo');
var v1 = require('./routes/v1');
// 设置与安全相关的HTTP头的中间件
var helmet = require('helmet');
// express的消息提示中间件
var flash = require('express-flash');

// 定时器
var schedule = require('node-schedule');

// 各种工具类
var dbUtils = require('./utils/dbUtils');
var bingUtils = require('./utils/bingUtils');
var mailUtils = require('./utils/mailUtils');
var qiniuUtils = require('./utils/qiniuUtils');
var weiboUtils = require('./utils/weiboUtils');

var app = express();
app.use('/static', express.static(path.join(__dirname, 'static')));
app.use(favicon(__dirname + '/static/images/bing.ico'));
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'pug');
app.enable('trust proxy');
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
    extended: true
}));
app.use(cookieParser('bing.ioliu.cn'));
app.use(session({
    secret: 'bing app', //secret的值建议使用随机字符串
    cookie: {
        secure: true,
        maxAge: 60 * 30 * 1000 // 过期时间（毫秒）
    },
    resave: false
}));
// 设置日志
app.use(logger('combined', {
    skip: function(req, res) { return res.statusCode < 400 }
}));
// 启用 helmet 
app.use(helmet());
app.use(flash());

// 每天 00:00,00:10,00:20 检测bing数据
schedule.scheduleJob('0 0,10,20 0 * * *', function() {
    var date = new Date();
    var year = date.getFullYear();
    var month = date.getMonth() + 1;
    var day = date.getDate();
    var now = year + '' + (month < 10 ? '0' + month : month) + '' + (day < 10 ? '0' + day : day);
    // 查询是否已经抓取并插入数据库，如果已插入就不重复抓取
    dbUtils.get('bing', {
        body: {
            enddate: now
        }
    }, function(rows) {
        if (rows.length === 0) {
            bingUtils.fetchPicture({}, function(data) {
                dbUtils.set('bing', data, function(rows) {
                    data.id = rows.insertId || 0;
                    mailUtils.send({
                        message: '从Bing抓取成功',
                        title: '从Bing抓取成功',
                        stack: JSON.stringify(data, '', 4)
                    });
                })
            });
        }
    });
});<commit_msg>fix response header<commit_after>var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');
var request = require('superagent');
var session = require('express-session');
var index = require('./routes/index');
var weibo = require('./routes/weibo');
var v1 = require('./routes/v1');
// express的消息提示中间件
var flash = require('express-flash');

// 定时器
var schedule = require('node-schedule');

// 各种工具类
var dbUtils = require('./utils/dbUtils');
var bingUtils = require('./utils/bingUtils');
var mailUtils = require('./utils/mailUtils');
var qiniuUtils = require('./utils/qiniuUtils');
var weiboUtils = require('./utils/weiboUtils');

var app = express();
app.use('/static', express.static(path.join(__dirname, 'static')));
app.use(favicon(__dirname + '/static/images/bing.ico'));
// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'pug');
app.enable('trust proxy');
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({
    extended: true
}));
app.use(cookieParser('bing.ioliu.cn'));
app.use(session({
    secret: 'bing app', //secret的值建议使用随机字符串
    cookie: {
        secure: true,
        maxAge: 60 * 30 * 1000 // 过期时间（毫秒）
    },
    resave: false
}));
// 设置日志
app.use(logger('combined', {
    skip: function(req, res) { return res.statusCode < 400 }
}));
app.use(flash());

// 每天 00:00,00:10,00:20 检测bing数据
schedule.scheduleJob('0 0,10,20 0 * * *', function() {
    var date = new Date();
    var year = date.getFullYear();
    var month = date.getMonth() + 1;
    var day = date.getDate();
    var now = year + '' + (month < 10 ? '0' + month : month) + '' + (day < 10 ? '0' + day : day);
    // 查询是否已经抓取并插入数据库，如果已插入就不重复抓取
    dbUtils.get('bing', {
        body: {
            enddate: now
        }
    }, function(rows) {
        if (rows.length === 0) {
            bingUtils.fetchPicture({}, function(data) {
                dbUtils.set('bing', data, function(rows) {
                    data.id = rows.insertId || 0;
                    mailUtils.send({
                        message: '从Bing抓取成功',
                        title: '从Bing抓取成功',
                        stack: JSON.stringify(data, '', 4)
                    });
                })
            });
        }
    });
});
{'fraction_non_alphanumeric': 0.184394250513347, 'fraction_numerical': 0.01724845995893224, 'mean_word_length': 2.749807544264819, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '426583', 'n_tokens_mistral': 1724, 'n_tokens_neox': 1671, 'n_words': 337}