<commit_before>import path from 'path';
import { writeFile } from 'fs-extra';
import puppeteerCore from 'puppeteer-core';
import express from 'express';
import getPort from 'get-port';
import { logger } from '@storybook/node-logger';

const read = async url => {
  const puppeteer = await usePuppeteer();
  const browser = await puppeteer.launch();
  const page = await browser.newPage();

  await page.goto(url);

  await page.waitForFunction(
    'window.__STORYBOOK_STORY_STORE__ && window.__STORYBOOK_STORY_STORE__.extract && window.__STORYBOOK_STORY_STORE__.extract()'
  );
  const data = JSON.parse(
    await page.evaluate(async () => {
      // eslint-disable-next-line no-undef
      const d = window.__STORYBOOK_STORY_STORE__.extract();

      const result = Object.entries(d).reduce(
        (acc, [k, v]) => ({
          ...acc,
          [k]: {
            ...v,
            parameters: {
              globalArgs: v.parameters.globalArgs,
              globalArgTypes: v.parameters.globalArgTypes,
              options: v.parameters.options,
              args: v.parameters.args,
              argTypes: v.parameters.argTypes,
              framework: v.parameters.framework,
              fileName: v.parameters.fileName,
              docsOnly: v.parameters.docsOnly,
            },
          },
        }),
        {}
      );
      return JSON.stringify(result, null, 2);
    })
  );

  setImmediate(() => {
    browser.close();
  });
  return data;
};

const useLocation = async path => {
  if (path.match(/^http/)) {
    return [path, async () => {}];
  }

  const app = express();

  app.use(express.static(path));

  const port = await getPort();

  return new Promise((resolve, reject) => {
    const server = app.listen(port, e => {
      if (e) {
        reject(e);
      }

      const result = `http://localhost:${port}/iframe.html`;

      logger.info(`connecting to: ${result}`);

      resolve([result, server.close.bind(server)]);
    });
  });
};

const usePuppeteer = async () => {
  try {
    // eslint-disable-next-line import/no-extraneous-dependencies, global-require
    require('puppeteer');

    return puppeteerCore;
  } catch (e) {
    // it's not installed
    logger.info('installing puppeteer...');
    // eslint-disable-next-line global-require, import/no-dynamic-require
    return require(path.join(require.resolve('puppeteer-core'), 'install.js'));
  }
};

export async function extract(input, targetPath) {
  if (input && targetPath) {
    const [location, exit] = await useLocation(input);

    const stories = await read(location);
<commit_msg>FIX wait for script to finish<commit_after>import path from 'path';
import { writeFile } from 'fs-extra';
import puppeteerCore from 'puppeteer-core';
import express from 'express';
import getPort from 'get-port';
import { logger } from '@storybook/node-logger';

const read = async url => {
  const puppeteer = await usePuppeteer();
  const browser = await puppeteer.launch();
  const page = await browser.newPage();

  await page.goto(url);

  await page.waitForFunction(
    'window.__STORYBOOK_STORY_STORE__ && window.__STORYBOOK_STORY_STORE__.extract && window.__STORYBOOK_STORY_STORE__.extract()'
  );
  const data = JSON.parse(
    await page.evaluate(async () => {
      // eslint-disable-next-line no-undef
      const d = window.__STORYBOOK_STORY_STORE__.extract();

      const result = Object.entries(d).reduce(
        (acc, [k, v]) => ({
          ...acc,
          [k]: {
            ...v,
            parameters: {
              globalArgs: v.parameters.globalArgs,
              globalArgTypes: v.parameters.globalArgTypes,
              options: v.parameters.options,
              args: v.parameters.args,
              argTypes: v.parameters.argTypes,
              framework: v.parameters.framework,
              fileName: v.parameters.fileName,
              docsOnly: v.parameters.docsOnly,
            },
          },
        }),
        {}
      );
      return JSON.stringify(result, null, 2);
    })
  );

  setImmediate(() => {
    browser.close();
  });
  return data;
};

const useLocation = async path => {
  if (path.match(/^http/)) {
    return [path, async () => {}];
  }

  const app = express();

  app.use(express.static(path));

  const port = await getPort();

  return new Promise((resolve, reject) => {
    const server = app.listen(port, e => {
      if (e) {
        reject(e);
      }

      const result = `http://localhost:${port}/iframe.html`;

      logger.info(`connecting to: ${result}`);

      resolve([result, server.close.bind(server)]);
    });
  });
};

const usePuppeteer = async () => {
  try {
    // eslint-disable-next-line import/no-extraneous-dependencies, global-require
    require('puppeteer');

    return puppeteerCore;
  } catch (e) {
    // it's not installed
    logger.info('installing puppeteer...');
    return new Promise((resolve, reject) => {
      // eslint-disable-next-line global-require
      require('child_process').exec(
        `node ${path.join(require.resolve('puppeteer-core'), 'install.js')}`,
        error => (error ? reject(error) : resolve(puppeteerCore))
      );
    });
  }
};

export async function extract(input, targetPath) {
  if (input && targetPath) {
    const [location, exit] = await useLocation(input);

    const stories = await read(location);

{'fraction_non_alphanumeric': 0.1470753130256027, 'fraction_numerical': 0.00037376191366099795, 'mean_word_length': 2.5561461794019933, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '883476', 'n_tokens_mistral': 1654, 'n_tokens_neox': 1627, 'n_words': 347}