<commit_before>/*
smf - read Simple Machine Forum RSS feeds & mail the articles
      for SMF 2.0.x

Copyright 2011-2018 <NAME>
@license MIT
*/

// for each feed in database
//   fetch URL, category, lasttime
//   for each message in feed
//     if new
//       fetch message
//       mail message
//       record new last for feed

let sqlite3 = require("sqlite3").verbose();
let http = require("http");
let process = require("process");
let url = require("url");
let parser = require("xml2json");
let cheerio = require("cheerio");
let nodemailer = require("nodemailer");
let fs = require("fs");
let ini = require("ini");
let Log = require("log");
let log = new Log(Log.DEBUG, fs.createWriteStream("smf.log", { flags: "a" }));

let config = ini.parse(fs.readFileSync("./smf.rc", "utf-8"));
exports.config = config;

let { cookie } = config.smf;

let db = new sqlite3.Database(config.database.database);
let smtpConfig = {
  host: config.email.host,
  port: config.email.port,
  ignoreTLS: true
};
if (config.email.user) {
  smtpConfig.auth = {
    user: config.email.user,
    pass: config.email.pass
  };
}
exports.mailer = nodemailer.createTransport(smtpConfig);

let feeds = [];
let items = [];
let lastdate = new Date("1970-1-1");

let decodeEntity = (m, p1) => String.fromCharCode(parseInt(p1, 10));

let unHTMLEntities = function(a) {
  a = unescape(a);
  a = a.replace(/&amp;&#35;/g, "&#");
  a = a.replace(/&quot;/g, '"');
  a = a.replace(/&apos;/g, "'");
  a = a.replace(/&lt;/g, "<");
  a = a.replace(/&gt;/g, ">");
  a = a.replace(/&amp;/g, "&");
  a = a.replace(/&#(\d+);/g, decodeEntity);
  return a;
};

let isoDateString = function(d) {
  let pad = function(n) {
    return n >= 10 ? n : `0${n}`;
  };

  return (
    d.getUTCFullYear() +
    "-" +
    pad(d.getUTCMonth() + 1) +
    "-" +
    pad(d.getUTCDate()) +
    "T" +
    pad(d.getUTCHours()) +
    ":" +
    pad(d.getUTCMinutes()) +
    ":" +
    pad(d.getUTCSeconds()) +
    "Z"
  );
};

var processItems = function() {
  let processPage = function(page) {
    let $ = cheerio.load(page);
    // look through all the div.post_wrapper for one that contains
    // a subject for the desired message number
    let $h5 = $(`#subject_${u.hash.replace("#msg", "")}`);
    let $el = $h5.closest(".post_wrapper");
    let from = $el
      .find('div.poster a[title^="View the profile of"]')
      .eq(0)
      .text();
    let $post = $el.find("div.post").eq(0);
    $("div.quote", $post).attr(
      "style",
      "color: #000; background-color: #d7daec; margin: 1px; padding: 6px; font-size: 1em; line-height: 1.5em; font-style: italic; font-family: Georgia, Times, serif;"
    );
    $("div.quoteheader,div.codeheader", $post).attr(
      "style",
      "color: #000; text-decoration: none; font-style: normal; font-weight: bold; font-size: 1em; line-height: 1.2em; padding-bottom: 4px;"
    );
    $(".meaction", $post).attr("style", "color: red;");
    $("embed", $post).each(function() {
      let src = decodeURIComponent($(this).attr("src"));
      log.debug(`    embed: ${src}`);
      return $(this).replaceWith(`<p><a href="${src}">${src}</a></p>`);
    });
    let $attachments = $el.find("div.attachments");
    if ($attachments) {
      $attachments.attr("style", "font-size: 0.8em;");
      $("a", $attachments)
        .prop("onclick", null);
      $post.append($attachments);
    }
    let post = $post.html();
    let isodate = isoDateString(d);
    log.debug(`From: ${from}`);
    log.debug(`Subject: [${item.category}] ${unHTMLEntities(item.title)}`);
    log.debug(`Date: ${isodate} Lastdate: ${isoDateString(lastdate)}`);
    return exports.mailer.sendMail(
      {
        from: `"${from}" ${exports.config.email.sender}`,
        to: exports.config.email.to,
        subject: `[${item.category}] ${unHTMLEntities(item.title.trim())}`,
        html: `<html><head></head><body><div><p><b>From:</b> ${from}<br /><b>Date:</b> ${
          item.pubDate
        }</p><div>${post}</div><p><a href="${
          item.link.replace('http:', 'https:')
        }">Original message</a></p></div></body></html>`
      },
      function(error) {
        if (error) {
          log.debug(`>>failed ${isodate}`);
          log.debug(error);
          return process.exit(1);
        } else {
          log.debug(`>>sent ${isodate} for ${item.category}`);
          let st = db.prepare("UPDATE feeds SET last=(?) WHERE category=(?)");
          st.run(isodate, item.category);
          return st.finalize(function() {
            log.debug(`db ${item.category} <- ${isodate}`);
            process.nextTick(processItems);
          });
        }
      }
    );
  };<commit_msg>Limit character width<commit_after>/*
smf - read Simple Machine Forum RSS feeds & mail the articles
      for SMF 2.0.x

Copyright 2011-2018 <NAME>
@license MIT
*/

// for each feed in database
//   fetch URL, category, lasttime
//   for each message in feed
//     if new
//       fetch message
//       mail message
//       record new last for feed

let sqlite3 = require("sqlite3").verbose();
let http = require("http");
let process = require("process");
let url = require("url");
let parser = require("xml2json");
let cheerio = require("cheerio");
let nodemailer = require("nodemailer");
let fs = require("fs");
let ini = require("ini");
let Log = require("log");
let log = new Log(Log.DEBUG, fs.createWriteStream("smf.log", { flags: "a" }));

let config = ini.parse(fs.readFileSync("./smf.rc", "utf-8"));
exports.config = config;

let { cookie } = config.smf;

let db = new sqlite3.Database(config.database.database);
let smtpConfig = {
  host: config.email.host,
  port: config.email.port,
  ignoreTLS: true
};
if (config.email.user) {
  smtpConfig.auth = {
    user: config.email.user,
    pass: config.email.pass
  };
}
exports.mailer = nodemailer.createTransport(smtpConfig);

let feeds = [];
let items = [];
let lastdate = new Date("1970-1-1");

let decodeEntity = (m, p1) => String.fromCharCode(parseInt(p1, 10));

let unHTMLEntities = function(a) {
  a = unescape(a);
  a = a.replace(/&amp;&#35;/g, "&#");
  a = a.replace(/&quot;/g, '"');
  a = a.replace(/&apos;/g, "'");
  a = a.replace(/&lt;/g, "<");
  a = a.replace(/&gt;/g, ">");
  a = a.replace(/&amp;/g, "&");
  a = a.replace(/&#(\d+);/g, decodeEntity);
  return a;
};

let isoDateString = function(d) {
  let pad = function(n) {
    return n >= 10 ? n : `0${n}`;
  };

  return (
    d.getUTCFullYear() +
    "-" +
    pad(d.getUTCMonth() + 1) +
    "-" +
    pad(d.getUTCDate()) +
    "T" +
    pad(d.getUTCHours()) +
    ":" +
    pad(d.getUTCMinutes()) +
    ":" +
    pad(d.getUTCSeconds()) +
    "Z"
  );
};

var processItems = function() {
  let processPage = function(page) {
    let $ = cheerio.load(page);
    // look through all the div.post_wrapper for one that contains
    // a subject for the desired message number
    let $h5 = $(`#subject_${u.hash.replace("#msg", "")}`);
    let $el = $h5.closest(".post_wrapper");
    let from = $el
      .find('div.poster a[title^="View the profile of"]')
      .eq(0)
      .text();
    let $post = $el.find("div.post").eq(0);
    $("div.quote", $post).attr(
      "style",
      "color: #000; background-color: #d7daec; margin: 1px; padding: 6px; font-size: 1em; line-height: 1.5em; font-style: italic; font-family: Georgia, Times, serif;"
    );
    $("div.quoteheader,div.codeheader", $post).attr(
      "style",
      "color: #000; text-decoration: none; font-style: normal; font-weight: bold; font-size: 1em; line-height: 1.2em; padding-bottom: 4px;"
    );
    $(".meaction", $post).attr("style", "color: red;");
    $("embed", $post).each(function() {
      let src = decodeURIComponent($(this).attr("src"));
      log.debug(`    embed: ${src}`);
      return $(this).replaceWith(`<p><a href="${src}">${src}</a></p>`);
    });
    let $attachments = $el.find("div.attachments");
    if ($attachments) {
      $attachments.attr("style", "font-size: 0.8em;");
      $("a", $attachments)
        .prop("onclick", null);
      $post.append($attachments);
    }
    let post = $post.html();
    let isodate = isoDateString(d);
    log.debug(`From: ${from}`);
    log.debug(`Subject: [${item.category}] ${unHTMLEntities(item.title)}`);
    log.debug(`Date: ${isodate} Lastdate: ${isoDateString(lastdate)}`);
    return exports.mailer.sendMail(
      {
        from: `"${from}" ${exports.config.email.sender}`,
        to: exports.config.email.to,
        subject: `[${item.category}] ${unHTMLEntities(item.title.trim())}`,
        html: `<html><head></head><body><div><p><b>From:</b> ${from}<br /><b>Date:</b> ${
          item.pubDate
        }</p><div style="max-width:72ch;">${post}</div>
        <p><a href="${item.link.replace('http:', 'https:')
        }">Original message</a></p></div></body></html>`
      },
      function(error) {
        if (error) {
          log.debug(`>>failed ${isodate}`);
          log.debug(error);
          return process.exit(1);
        } else {
          log.debug(`>>sent ${isodate} for ${item.category}`);
          let st = db.prepare("UPDATE feeds SET last=(?) WHERE category=(?)");
          st.run(isodate, item.category);
          return st.finalize(function() {
            log.debug(`db ${item.category} <- ${isodate}`);
            process.nextTick(processItems);
          });
        }
      }
    );
  };
{'fraction_non_alphanumeric': 0.21096594559862925, 'fraction_numerical': 0.011779824373527523, 'mean_word_length': 2.912442396313364, 'pattern_counts': {'":': 4, '<': 59, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6279207', 'n_tokens_mistral': 3352, 'n_tokens_neox': 3289, 'n_words': 687}