<commit_before>
const path = require("path");
const fs=require('fs');
const mv = require('mv');

const async = require('async');
const settings = require('./settings.js');
const uri_pics =settings.uri_pics;
const port = 4200;
const root_len= settings.pic_root.length;

const gm = require('gm');
const NodeCache = require( "node-cache" );
//缓存失效时间设置为1天, 每小时检查一次,不使用clone
const fsCache = new NodeCache({ stdTTL: 24*3600, checkperiod: 3600, useClones:false });
const path_tbn = settings.thumbnails_uri + settings.thumbnails_size+path.sep;
const prefixWH = 'PWH-';

function archiveTask(){
    var schedule = require('node-schedule');
    var rule = new schedule.RecurrenceRule();  
    rule.minute = 42;
    var j = schedule.scheduleJob(settings.cron_archive, function(){  
        console.log('----archive check---');
        archive(function(){
            console.log('----archive check end---');           
        })
    });
}

//检查是否存在昨天目录,如果没有，生成昨天的目录，并将upload/p /n 移动到该目录下
function archive(func){
    let dt = new Date(); // Today!
    dt.setDate(dt.getDate() - 1); // Yesterday!
    let path_yesterday = path.join(settings.pic_root,
        settings.pic_archive,
        dt.getFullYear().toString(),
        (dt.getMonth()+1).toString(),
        dt.getDate().toString());
    //console.log(path_yesterday);
    fs.stat(path_yesterday, function(err, stat) {
        //目录已经存在
        if(stat&&stat.isDirectory()){
            console.log('archive folder:'+ path_yesterday+' exists.');
            return func();
        } 
        //移动upload/p upload/n
        let path_src = path.join(
            settings.pic_root,
            settings.pic_upload);
        mv(path.join(path_src,'p'),
            path.join(path_yesterday,'p'), {mkdirp: true}, function(err) {
            if(err){ 
                //无须告警
                //console.log(err);
            }
            func();
            //未通过的不用归档,避免share到公网
        });            
    });
}

//cache原始图或抽点图,tbn_len=0:原始图, tbn_len>0 抽点图 
function cacheFile(fpath,func) {
  //非抽点图
  if(fs.existsSync(fpath)) {
    let data = fs.readFileSync(fpath);
    if(settings.fs_cache){
      fsCache.set(fpath,data);
    }
    if(func){
      func(data);
    }<commit_msg>简化archive的path<commit_after>
const path = require("path");
const fs=require('fs');
const mv = require('mv');

const async = require('async');
const settings = require('./settings.js');
const uri_pics =settings.uri_pics;
const port = 4200;
const root_len= settings.pic_root.length;

const gm = require('gm');
const NodeCache = require( "node-cache" );
//缓存失效时间设置为1天, 每小时检查一次,不使用clone
const fsCache = new NodeCache({ stdTTL: 24*3600, checkperiod: 3600, useClones:false });
const path_tbn = settings.thumbnails_uri + settings.thumbnails_size+path.sep;
const prefixWH = 'PWH-';

function archiveTask(){
    var schedule = require('node-schedule');
    var rule = new schedule.RecurrenceRule();  
    rule.minute = 42;
    var j = schedule.scheduleJob(settings.cron_archive, function(){  
        console.log('----archive check---');
        archive(function(){
            console.log('----archive check end---');           
        })
    });
}

//检查是否存在昨天目录,如果没有，生成昨天的目录，并将upload/p /n 移动到该目录下
function archive(func){
    let dt = new Date(); // Today!
    dt.setDate(dt.getDate() - 1); // Yesterday!
    let path_yesterday = path.join(settings.pic_root,
        settings.pic_archive,
        dt.getFullYear().toString(),
        (dt.getMonth()+1).toString(),
        dt.getDate().toString());
    //console.log(path_yesterday);
    fs.stat(path_yesterday, function(err, stat) {
        //目录已经存在
        if(stat&&stat.isDirectory()){
            console.log('archive folder:'+ path_yesterday+' exists.');
            return func();
        } 
        //移动upload/p upload/n
        let path_src = path.join(
            settings.pic_root,
            settings.pic_upload);
        mv(path.join(path_src,'p'),path_yesterday, {mkdirp: true}, function(err) {
            if(!err){ 
                //无须告警
                console.log('move pics from:'+path_src+' to:'+path_yesterday);
            }
            func();
            //未通过的不用归档,避免share到公网
        });            
    });
}

//cache原始图或抽点图,tbn_len=0:原始图, tbn_len>0 抽点图 
function cacheFile(fpath,func) {
  //非抽点图
  if(fs.existsSync(fpath)) {
    let data = fs.readFileSync(fpath);
    if(settings.fs_cache){
      fsCache.set(fpath,data);
    }
    if(func){
      func(data);
    }
{'fraction_non_alphanumeric': 0.15622895622895622, 'fraction_numerical': 0.009427609427609427, 'mean_word_length': 2.5365079365079364, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '540997', 'n_tokens_mistral': 1599, 'n_tokens_neox': 1606, 'n_words': 268}