<commit_before>const fs = require('fs');
const path = require('path');
const getBishengConfig = require('../utils/get-bisheng-config');
const getThemeConfig = require('../utils/get-theme-config');
const sourceData = require('../utils/source-data');
const resolvePlugins = require('../utils/resolve-plugins');
const context = require('../context');
const boss = require('./common/boss');

module.exports = function bishengDataLoader(/* content */) {
  if (this.cacheable) {
    this.cacheable();
  }

  const bishengConfig = getBishengConfig(context.config);
  const themeConfig = getThemeConfig(bishengConfig.theme);

  const markdown = sourceData.generate(bishengConfig.source, bishengConfig.transformers);
  const browserPlugins = resolvePlugins(themeConfig.plugins, 'browser');
  const pluginsString = browserPlugins
          .map(plugin => `[require('${plugin[0]}'), ${JSON.stringify(plugin[1])}]`)
          .join(',\n');

  const callback = this.async();

  const picked = {};
  const pickedPromises = []; // Flag to remind loader that job is done.
  if (themeConfig.pick) {
    const nodePlugins = resolvePlugins(themeConfig.plugins, 'node');
    sourceData.traverse(markdown, (filename) => {
      const fileContent = fs.readFileSync(path.join(process.cwd(), filename)).toString();
      pickedPromises.push(new Promise((resolve) => {
        boss.queue({
          filename,
          content: fileContent,
          plugins: nodePlugins,
          transformers: bishengConfig.transformers,
          isBuild: context.isBuild,
          callback(err, result) {
            const parsedMarkdown = JSON.parse(result);

            Object.keys(themeConfig.pick).forEach((key) => {
              if (!picked[key]) {
                picked[key] = [];
              }

              const picker = themeConfig.pick[key];<commit_msg>fix: should support function and so on<commit_after>const fs = require('fs');
const path = require('path');
const getBishengConfig = require('../utils/get-bisheng-config');
const getThemeConfig = require('../utils/get-theme-config');
const sourceData = require('../utils/source-data');
const resolvePlugins = require('../utils/resolve-plugins');
const context = require('../context');
const boss = require('./common/boss');

module.exports = function bishengDataLoader(/* content */) {
  if (this.cacheable) {
    this.cacheable();
  }

  const bishengConfig = getBishengConfig(context.config);
  const themeConfig = getThemeConfig(bishengConfig.theme);

  const markdown = sourceData.generate(bishengConfig.source, bishengConfig.transformers);
  const browserPlugins = resolvePlugins(themeConfig.plugins, 'browser');
  const pluginsString = browserPlugins
          .map(plugin => `[require('${plugin[0]}'), ${JSON.stringify(plugin[1])}]`)
          .join(',\n');

  const callback = this.async();

  const picked = {};
  const pickedPromises = []; // Flag to remind loader that job is done.
  if (themeConfig.pick) {
    const nodePlugins = resolvePlugins(themeConfig.plugins, 'node');
    sourceData.traverse(markdown, (filename) => {
      const fileContent = fs.readFileSync(path.join(process.cwd(), filename)).toString();
      pickedPromises.push(new Promise((resolve) => {
        boss.queue({
          filename,
          content: fileContent,
          plugins: nodePlugins,
          transformers: bishengConfig.transformers,
          isBuild: context.isBuild,
          callback(err, result) {
            const parsedMarkdown = eval(`(${result})`);

            Object.keys(themeConfig.pick).forEach((key) => {
              if (!picked[key]) {
                picked[key] = [];
              }

              const picker = themeConfig.pick[key];
{'fraction_non_alphanumeric': 0.14791779340183883, 'fraction_numerical': 0.001081665765278529, 'mean_word_length': 3.28125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6608466', 'n_tokens_mistral': 1035, 'n_tokens_neox': 1025, 'n_words': 211}