<commit_before>import ProcessorBase from '../base/processor.js';

import Logger from '../util/logger.js';

class ImageProcessor extends ProcessorBase {

  constructor() {
    super();

    this.processingDownloads = 0;
    this.concurrencyLimit = 10;
    this.images = [];
    this.strategy = null;
  }

  init(strategy) {
    Logger.instance.info('[ImageProcessor][%s] Init ...', process.pid);

    this.strategy = strategy;
  }

  async start(taskUrl) {
    Logger.instance.info('[ImageProcessor][%s] Start to process task: %s', process.pid, taskUrl);

    try {
      let albums = await this.strategy.parseAlbums(taskUrl);
      Logger.instance.info('[ImageProcessor][%s] Albums to be downloaded:', process.pid, albums);

      for (let albumUrl of albums) {
        let albumImages = await this.strategy.parseImages(albumUrl);
        this.images = [...this.images, ...albumImages];
      }
      Logger.instance.info('[ImageProcessor][%s] Images to be downloaded:', process.pid, this.images);

      if (this.images.length > 0) {
        await this.strategy.ensureOutputDir();
      }
    } catch (err) {
      return new Promise((resolve, reject) => {
        reject(err);<commit_msg>Reduce log, make easy to see.<commit_after>import ProcessorBase from '../base/processor.js';

import Logger from '../util/logger.js';

class ImageProcessor extends ProcessorBase {

  constructor() {
    super();

    this.processingDownloads = 0;
    this.concurrencyLimit = 10;
    this.images = [];
    this.strategy = null;
  }

  init(strategy) {
    Logger.instance.info('[ImageProcessor][%s] Init ...', process.pid);

    this.strategy = strategy;
  }

  async start(taskUrl) {
    Logger.instance.info('[ImageProcessor][%s] Start to process task: %s', process.pid, taskUrl);

    try {
      let albums = await this.strategy.parseAlbums(taskUrl);
      Logger.instance.info('[ImageProcessor][%s] Albums to be downloaded: %d', process.pid, albums.length);

      for (let albumUrl of albums) {
        let albumImages = await this.strategy.parseImages(albumUrl);
        this.images = [...this.images, ...albumImages];
      }
      Logger.instance.info('[ImageProcessor][%s] Images to be downloaded: %d', process.pid, this.images.length);

      if (this.images.length > 0) {
        await this.strategy.ensureOutputDir();
      }
    } catch (err) {
      return new Promise((resolve, reject) => {
        reject(err);
{'fraction_non_alphanumeric': 0.14652317880794702, 'fraction_numerical': 0.0033112582781456954, 'mean_word_length': 3.0621848739495796, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5958858', 'n_tokens_mistral': 746, 'n_tokens_neox': 721, 'n_words': 166}