<commit_before>#!/bin/env node

var TelegramBot = require('node-telegram-bot-api');
var request     = require('request');
var cheerio     = require('cheerio');
var CronJob     = require('cron').CronJob;

var token = process.env.TGPACKTPUBTOKEN || '';
var bot   = new TelegramBot(token, {polling: true});

function scrapPacktpub(msg) {
  var sendTo = (msg == null ? process.env.TGIDDEVABS : msg.chat.id);   //Dev@AbS
  var url    = 'https://www.packtpub.com/packt/offers/free-learning';

  request(url, function(error, response, html) {
    if(!error) {
      var $ = cheerio.load(html);

      var cover, title, link, details;
      cover   = $("div[class='dotd-main-book-image float-left'] a img").attr("src");
      title   = $("div[class='dotd-title'] h2").html();
      link    = $("div[class='float-left free-ebook'] a").attr("href");
      details = $("div[class='dotd-main-book-image float-left'] a").attr("href");

      (function(url, sendTo, title, cover, link) {
        request(url, function(error, response, html2) {
          if(!error) {
            var $ = cheerio.load(html2);

            var dt, pages;
            dt    = $("time[itemprop='datePublished']").html();
            pages = $("span[itemprop='numberOfPages']").html();

            var det = "";
            var lines = [];
            if (dt) {
              lines.push(dt);
            }
            if (pages) {
              lines.push("(" + pages + " pages)");
            }
            if (lines.length > 0) {
              det = "\n● " + lines.join(" ");
            }

            var reqcover = request("http:"+cover);

            var options = {
              caption: "● " + title.trim() + det,
              parse_mode: 'HTML',
              reply_markup: JSON.stringify({
                inline_keyboard: [[{ text: "Claim Your Free eBook", url: "https://www.packtpub.com"+link}]]
              })
            };
            bot.sendPhoto(sendTo, reqcover, options).then(function (sended) {
              delete options.chat_id;
              delete options.photo;
              bot.sendPhoto(process.env.TGIDGRPA, sended.photo[0].file_id, options);  //MORE GRP
            });

          }
        });
      })("https://www.packtpub.com" + details, sendTo, title, cover, link);
    }
  });
}


new CronJob('30 8 * * *', function() {
  scrapPacktpub(null);
}, null, true, 'Asia/Kuala_Lumpur');

bot.onText(/\/packtpub$/i, function (msg, m) {
  scrapPacktpub(msg);
});
<commit_msg>append packtpub free learning url<commit_after>#!/bin/env node

var TelegramBot = require('node-telegram-bot-api');
var request     = require('request');
var cheerio     = require('cheerio');
var CronJob     = require('cron').CronJob;

var token = process.env.TGPACKTPUBTOKEN || '';
var bot   = new TelegramBot(token, {polling: true});

function scrapPacktpub(msg) {
  var sendTo = (msg == null ? process.env.TGIDDEVABS : msg.chat.id);   //Dev@AbS
  var url    = 'https://www.packtpub.com/packt/offers/free-learning';

  request(url, function(error, response, html) {
    if(!error) {
      var $ = cheerio.load(html);

      var cover, title, link, details;
      cover   = $("div[class='dotd-main-book-image float-left'] a img").attr("src");
      title   = $("div[class='dotd-title'] h2").html();
      link    = $("div[class='float-left free-ebook'] a").attr("href");
      details = $("div[class='dotd-main-book-image float-left'] a").attr("href");

      (function(url, sendTo, title, cover, link) {
        request(url, function(error, response, html2) {
          if(!error) {
            var $ = cheerio.load(html2);

            var dt, pages;
            dt    = $("time[itemprop='datePublished']").html();
            pages = $("span[itemprop='numberOfPages']").html();

            var det = "";
            var lines = [];
            if (dt) {
              lines.push(dt);
            }
            if (pages) {
              lines.push("(" + pages + " pages)");
            }
            if (lines.length > 0) {
              det = "\n● " + lines.join(" ");
            }

            var reqcover = request("http:"+cover);

            var options = {
              caption: "● " + title.trim() + det + "\nhttps://www.packtpub.com/packt/offers/free-learning",
              parse_mode: 'HTML',
              reply_markup: JSON.stringify({
                inline_keyboard: [[{ text: "Claim Your Free eBook", url: "https://www.packtpub.com"+link}]]
              })
            };
            bot.sendPhoto(sendTo, reqcover, options).then(function (sended) {
              delete options.chat_id;
              delete options.photo;
              bot.sendPhoto(process.env.TGIDGRPA, sended.photo[0].file_id, options);  //MORE GRP
            });

          }
        });
      })("https://www.packtpub.com" + details, sendTo, title, cover, link);
    }
  });
}


new CronJob('30 8 * * *', function() {
  scrapPacktpub(null);
}, null, true, 'Asia/Kuala_Lumpur');

bot.onText(/\/packtpub$/i, function (msg, m) {
  scrapPacktpub(msg);
});

{'fraction_non_alphanumeric': 0.17098145285935085, 'fraction_numerical': 0.0030911901081916537, 'mean_word_length': 1.9149774774774775, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 7, 'lorem ipsum': 0, 'www.': 7, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2173128', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1621, 'n_words': 311}