<commit_before>import Promise from 'bluebird';
import Danbooru from 'danbooru';
import path from 'path';
import fs from 'fs';
import got from 'got';

const req = async (url, opt) => {
  opt.headers = {                         // eslint-disable-line no-param-reassign
    'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko',
  };
  const res = await got(url, opt);
  return res.body;
};


const getImages = (tags, unsafe) => {
  const danbooru = unsafe ? new Danbooru() : new Danbooru.Safebooru();

  return new Promise(resolve =>
    danbooru.posts(tags)
      .then(posts =>
        resolve(
          posts.map(post => `http://danbooru.donmai.us${post.raw.file_url}`),
        ),
      ),
  );
};<commit_msg>[module][fix] danbooru - get ALL images<commit_after>import Promise from 'bluebird';
import Danbooru from 'danbooru';
import path from 'path';
import fs from 'fs';
import got from 'got';
import _ from 'lodash';

const req = async (url, opt) => {
  opt.headers = {                         // eslint-disable-line no-param-reassign
    'user-agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; Touch; rv:11.0) like Gecko',
  };
  const res = await got(url, opt);
  return res.body;
};

const getImages = async (tags, unsafe) => {
  const danbooru = unsafe ? new Danbooru() : new Danbooru.Safebooru();
  const count = await danbooru.requestJson('GET counts/posts.json', { tags });
  let results = [];

  if (!count || !count.counts || !count.counts.posts) {
    return [];
  }
  if (count.counts.posts < 100) results = await danbooru.posts({ limit: 100, page: 1, tags });
  else {
    const queries = [];
    for (let i = 1, len = Math.ceil(count.counts.posts / 100); i < len + 1; i += 1) {
      queries.push(danbooru.posts({ limit: 100, page: i, tags }));
    }
    results = await Promise.all(queries);
  }

  return _.flattenDeep(results).map(post => `http://danbooru.donmai.us${post.raw.file_url}`);
};
{'fraction_non_alphanumeric': 0.16436957631444615, 'fraction_numerical': 0.019397651863195507, 'mean_word_length': 3.0329218106995883, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7356334', 'n_tokens_mistral': 731, 'n_tokens_neox': 698, 'n_words': 181}