<commit_before>// Fetch images from remote servers and store their size to post.image_info
//
'use strict';


const _        = require('lodash');
const Promise  = require('bluebird');
const co       = require('bluebird-co').co;
const get_size = require('probe-image-size');

// a number of times the task can be re-created if image fetch errors out
const MAX_RETRIES = 2;


module.exports = function (N) {

  N.wire.on('init:jobs', function register_forum_post_images_fetch() {
    N.queue.registerWorker({
      name: 'forum_post_images_fetch',

      // 5 minute delay by default
      postponeDelay: 5 * 60 * 1000,

      timeout: 120000,

      taskID(taskData) {
        return taskData.post_id;
      },

      * process() {
        let update        = {};
        let needs_rebuild = false;
        let needs_restart = false;
        let post_id       = this.data.post_id;
        let retry_count   = this.data.retry || 0;
        let flush_promise;
        let interval;

        // Put pending data from "update" object into a database.
        //
        function flush_data() {
          if (_.isEmpty(update)) return Promise.resolve();

          return N.models.forum.Post.findById(post_id).then(post => {
            if (!post) return;

            let updateData = { $set: {} };

            Object.keys(update).forEach(key => {
              if (_.isObject(post.image_info) && post.image_info[key] === null) {
                updateData.$set['image_info.' + key] = update[key];
              }
            });

            update = {};

            return N.models.forum.Post.update({ _id: post._id }, updateData);
          });
        }

        // write image info into database once every 10 sec
        // (in addition to writing after all images are retrieved)
        interval = setInterval(function () {
          flush_promise = flush_data();
        }, 10000);

        let post = yield N.models.forum.Post.findById(post_id);

        if (!post || !_.isObject(post.image_info)) return;

        const extendDeadline = _.throttle(() => {
          this.setDeadline();
        }, 10000);

        yield Promise.map(Object.keys(post.image_info), co.wrap(function* (key) {
          extendDeadline();

          // if it's not an external image (e.g. attachment), skip
          if (!key.match(/^url:/)) return;

          // if it's already loaded, skip
          if (post.image_info[key]) return;

          // key is "prefix"+"url with replaced dots", example:
          // url:http://example．com/foo．jpg
          let url = key.slice(4).replace(/．/g, '.');

          let result;

          try {
            result = yield Promise.fromCallback(cb => get_size(url, cb));
          } catch (err) {
            // if we can't parse file or status code is 4xx, this request is final
            let url_failed = (err.code === 'ECONTENT') ||
                             (err.status && err.status >= 400 && err.status < 500);
<commit_msg>Removed promise wrapper - not needed anymore<commit_after>// Fetch images from remote servers and store their size to post.image_info
//
'use strict';


const _        = require('lodash');
const Promise  = require('bluebird');
const co       = require('bluebird-co').co;
const get_size = require('probe-image-size');

// a number of times the task can be re-created if image fetch errors out
const MAX_RETRIES = 2;


module.exports = function (N) {

  N.wire.on('init:jobs', function register_forum_post_images_fetch() {
    N.queue.registerWorker({
      name: 'forum_post_images_fetch',

      // 5 minute delay by default
      postponeDelay: 5 * 60 * 1000,

      timeout: 120000,

      taskID(taskData) {
        return taskData.post_id;
      },

      * process() {
        let update        = {};
        let needs_rebuild = false;
        let needs_restart = false;
        let post_id       = this.data.post_id;
        let retry_count   = this.data.retry || 0;
        let flush_promise;
        let interval;

        // Put pending data from "update" object into a database.
        //
        function flush_data() {
          if (_.isEmpty(update)) return Promise.resolve();

          return N.models.forum.Post.findById(post_id).then(post => {
            if (!post) return;

            let updateData = { $set: {} };

            Object.keys(update).forEach(key => {
              if (_.isObject(post.image_info) && post.image_info[key] === null) {
                updateData.$set['image_info.' + key] = update[key];
              }
            });

            update = {};

            return N.models.forum.Post.update({ _id: post._id }, updateData);
          });
        }

        // write image info into database once every 10 sec
        // (in addition to writing after all images are retrieved)
        interval = setInterval(function () {
          flush_promise = flush_data();
        }, 10000);

        let post = yield N.models.forum.Post.findById(post_id);

        if (!post || !_.isObject(post.image_info)) return;

        const extendDeadline = _.throttle(() => {
          this.setDeadline();
        }, 10000);

        yield Promise.map(Object.keys(post.image_info), co.wrap(function* (key) {
          extendDeadline();

          // if it's not an external image (e.g. attachment), skip
          if (!key.match(/^url:/)) return;

          // if it's already loaded, skip
          if (post.image_info[key]) return;

          // key is "prefix"+"url with replaced dots", example:
          // url:http://example．com/foo．jpg
          let url = key.slice(4).replace(/．/g, '.');

          let result;

          try {
            result = yield get_size(url);
          } catch (err) {
            // if we can't parse file or status code is 4xx, this request is final
            let url_failed = (err.code === 'ECONTENT') ||
                             (err.status && err.status >= 400 && err.status < 500);

{'fraction_non_alphanumeric': 0.1285690150363237, 'fraction_numerical': 0.012164216928535226, 'mean_word_length': 1.916256157635468, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6192155', 'n_tokens_mistral': 1801, 'n_tokens_neox': 1666, 'n_words': 457}