<commit_before>const schedule = require('node-schedule');
const fs = require('fs');
const ROOT_DIR = `./photosTemp`;
const FINISH_DIR = `./photosFinish`;
const XLSX = require('xlsx');
const Jimp = require('jimp');
const axios = require('axios');


const jobFn = async function(){
  console.log('ocr job running!');
  const ary = fs.readdirSync(ROOT_DIR);
  const dir = ary[ary.length - 1];
  if(!dir){
    console.log('job complete!(no folder)');
    return;
  }
  const filePath = `${FINISH_DIR}/${dir}.xlsx`;
  const isFileExists = fs.existsSync(filePath);
  if(isFileExists){
    const wb = await XLSX.readFile(filePath);
    let ws = wb.Sheets.order;
    let j =0;
    let target;
    const keys = [];
    do {
      j++;
      target = ws['H'+j];
      if(target){
        const l = target.v.split('/');
        keys.push(l[l.length-1]);
      }
    } while(target);
    const { files, base64String } = await quickstart(`${ROOT_DIR}/${dir}`, FINISH_DIR, (file)=>!keys.includes(file));
    if(files.length > 0){
      console.log('update', files);
      const result = await visonAPI(base64String);
      const json =  XLSX.utils.sheet_to_json(ws);
      const json_tmp = regexpSplit(result, files.map(file=>file.replace(ROOT_DIR, `http://172.16.17.32:8080/${ROOT_DIR.replace('./','')}`)));
      json.push(...json_tmp);
      ws = await XLSX.utils.json_to_sheet(json);
      j =0;
      do {
        j++;
        target = ws['H'+j];
        if(target){
          target.l = { Target:target.v, Tooltip:"link to image" };
          target.v = '原圖連結';
        }
      } while(target);
      await XLSX.writeFile({
        SheetNames: ['order'],
        Sheets: {
          'order':ws
        }
      }, filePath);
    }
  } else {

    const { files, base64String } = await quickstart(`${ROOT_DIR}/${dir}`, FINISH_DIR);
    console.log('insert', files);
    const result = await visonAPI(base64String);
    const json =  regexpSplit(result, files.map(file=>file.replace(ROOT_DIR, `http://3172.16.31.10:8080/${ROOT_DIR.replace('./','')}`)));
    const ws = XLSX.utils.json_to_sheet(json);
    let target;
    let j =0;
    do {
      j++;
      target = ws['H'+j];
      if(target){
        target.l = { Target:target.v, Tooltip:"link to image" };
        target.v = '原圖連結';
      }
    } while(target);
    await XLSX.writeFile({
      SheetNames: ['order'],
      Sheets: {
        'order':ws
      }
    }, filePath);
  }
  console.log('job complete!');
};
schedule.scheduleJob('* * * * *', jobFn);


async function quickstart(sourcePath, destPath, filterFn = (file, idx) => {
  return true;
}) {
  let files = fs.readdirSync(sourcePath);

  async function cutX(path) {
    const jimp = await Jimp.read(`${sourcePath}/${path}`);

    let cutL = 0;
    for (let i = 5; i < 100; i++) {
      const v = await jimp.getPixelColor(i, jimp.getHeight() / 1.5);
      const { r, g, b } = Jimp.intToRGBA(v);
      console.log(path, i, Jimp.intToRGBA(v));
      if (r <= 115 && g <= 126 && b <= 123) {
        // console.log(path, i, Jimp.intToRGBA(v));
        cutL = i;
        break;
      }
    }

    let cutR = 0;
    for (let j = jimp.getWidth() - 1; j > 0; j--) {
      // console.log(j);
      const v = await jimp.getPixelColor(j, jimp.getHeight() / 1.5);
      const { r, g, b } = Jimp.intToRGBA(v);
      // console.log(path, j, Jimp.intToRGBA(v));
      if (r < 95 && g < 95 && b < 95) {
        // console.log(path, j, Jimp.intToRGBA(v));
        cutR = j;
        break;
      }
    }

    return { jimp, cutL, cutR };
  }

  const fileFilter = files.filter((file)=>file.endsWith('.jpg')).filter(filterFn).filter((file, idx)=>idx<21);
  const images = await Promise.all(fileFilter.map(async (path) => {

    //去邊距<commit_msg>fix schedule<commit_after>const schedule = require('node-schedule');
const fs = require('fs');
const ROOT_DIR = `./photosTemp`;
const FINISH_DIR = `./photosFinish`;
const XLSX = require('xlsx');
const Jimp = require('jimp');
const axios = require('axios');


const jobFn = async function(){
  console.log('ocr job running!');
  const ary = fs.readdirSync(ROOT_DIR);
  const dir = ary[ary.length - 1];
  if(!dir){
    console.log('job complete!(no folder)');
    return;
  }
  const filePath = `${FINISH_DIR}/${dir}.xlsx`;
  const isFileExists = fs.existsSync(filePath);
  if(isFileExists){
    const wb = await XLSX.readFile(filePath);
    let ws = wb.Sheets.order;
    let j =0;
    let target;
    const keys = [];
    do {
      j++;
      target = ws['H'+j];
      if(target){
        const l = target.v.split('/');
        keys.push(l[l.length-1]);
      }
    } while(target);
    const { files, base64String } = await quickstart(`${ROOT_DIR}/${dir}`, FINISH_DIR, (file)=>!keys.includes(file));
    if(files.length > 0){
      console.log('update', files);
      const result = await visonAPI(base64String);
      const json =  XLSX.utils.sheet_to_json(ws);
      const json_tmp = regexpSplit(result, files.map(file=>file.replace(ROOT_DIR, `http://35.206.254.19:8080/${ROOT_DIR.replace('./','')}`)));
      json.push(...json_tmp);
      ws = await XLSX.utils.json_to_sheet(json);
      j =0;
      do {
        j++;
        target = ws['H'+j];
        if(target){
          target.l = { Target:target.v, Tooltip:"link to image" };
          target.v = '原圖連結';
        }
      } while(target);
      await XLSX.writeFile({
        SheetNames: ['order'],
        Sheets: {
          'order':ws
        }
      }, filePath);
    }
  } else {

    const { files, base64String } = await quickstart(`${ROOT_DIR}/${dir}`, FINISH_DIR);
    console.log('insert', files);
    const result = await visonAPI(base64String);
    const json =  regexpSplit(result, files.map(file=>file.replace(ROOT_DIR, `http://35.206.254.19:8080/${ROOT_DIR.replace('./','')}`)));
    const ws = XLSX.utils.json_to_sheet(json);
    let target;
    let j =0;
    do {
      j++;
      target = ws['H'+j];
      if(target){
        target.l = { Target:target.v, Tooltip:"link to image" };
        target.v = '原圖連結';
      }
    } while(target);
    await XLSX.writeFile({
      SheetNames: ['order'],
      Sheets: {
        'order':ws
      }
    }, filePath);
  }
  console.log('job complete!');
};
schedule.scheduleJob('* * * * *', jobFn);


async function quickstart(sourcePath, destPath, filterFn = (file, idx) => {
  return true;
}) {
  let files = fs.readdirSync(sourcePath);

  async function cutX(path) {
    const jimp = await Jimp.read(`${sourcePath}/${path}`);

    let cutL = 0;
    for (let i = 5; i < 100; i++) {
      const v = await jimp.getPixelColor(i, jimp.getHeight() / 1.5);
      const { r, g, b } = Jimp.intToRGBA(v);
      // console.log(path, i, Jimp.intToRGBA(v));
      if (r <= 115 && g <= 126 && b <= 123) {
        // console.log(path, i, Jimp.intToRGBA(v));
        cutL = i;
        break;
      }
    }

    let cutR = 0;
    for (let j = jimp.getWidth() - 1; j > 0; j--) {
      // console.log(j);
      const v = await jimp.getPixelColor(j, jimp.getHeight() / 1.5);
      const { r, g, b } = Jimp.intToRGBA(v);
      // console.log(path, j, Jimp.intToRGBA(v));
      if (r < 95 && g < 95 && b < 95) {
        // console.log(path, j, Jimp.intToRGBA(v));
        cutR = j;
        break;
      }
    }

    return { jimp, cutL, cutR };
  }

  const fileFilter = files.filter((file)=>file.endsWith('.jpg')).filter(filterFn).filter((file, idx)=>idx<21);
  const images = await Promise.all(fileFilter.map(async (path) => {

    //去邊距
{'fraction_non_alphanumeric': 0.18503146338197884, 'fraction_numerical': 0.01887802918730754, 'mean_word_length': 2.508689525598873, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3780303', 'n_tokens_mistral': 2824, 'n_tokens_neox': 2711, 'n_words': 558}