<commit_before>import WebMiddle, { PropTypes } from 'webmiddle';
import phantom from 'phantom';

// promise-based implementation of http://stackoverflow.com/a/19070446
function waitForFn(config) {
  config._start = config._start || new Date();

  if (config.timeout && new Date - config._start > config.timeout) {
    const errorMessage = 'waitForFn timedout ' + (new Date - config._start) + 'ms';
    if (config.debug) console.log(errorMessage);
    return Promise.reject(errorMessage);
  }

  return config.check().then(result => {
    if (result) {
      if (config.debug) console.log('waitForFn success ' + (new Date - config._start) + 'ms');
      return result;
    }

    return new Promise((resolve, reject) => {
      setTimeout(() => {
        waitForFn(config).then(resolve, reject);
      }, config.interval || 0);
    });
  });
}

const Browser =
({ name, contentType, url, method = 'GET', body = {}, httpHeaders = {}, cookies = {}, waitFor }) => {
  // TODO: cookies
  return new Promise((resolve, reject) => {
    let sitepage = null;
    let phInstance = null;
    const pageResponses = {};
    let finalUrl = url;

    if (typeof body === 'object' && body !== null) {
      // body as string
      if (httpHeaders && httpHeaders['Content-Type'] === 'application/json') {
        body = JSON.stringify(body);
      } else {
        // default: convert to form data
        body = Object.keys(body).reduce((list, prop) => {
          const value = body[prop];
          list.push(`${encodeURIComponent(prop)}=${encodeURIComponent(value)}`);
          return list;
        }, []).join('&');
      }
    }

    phantom.create()
    .then(instance => {
      phInstance = instance;
      return instance.createPage();
    })
    .then(page => {
      sitepage = page;

      //page.customHeaders = httpHeaders; // TODO: doesn't seem to work

      // track final response (after redirects)
      // https://github.com/ariya/phantomjs/issues/10185#issuecomment-38856203
      page.on('onResourceReceived', (response) => {
        pageResponses[response.url] = response;
      });
      page.on('onUrlChanged', (targetUrl) => {
        finalUrl = targetUrl;
      });

      // TODO: are httpHeaders still sent in case of redirect?
      const settings = {
        operation: method,
        headers: httpHeaders,
        data: body,
      };

      return page.open(url, settings);
    })
    .then(status => {
      const pageResponse = pageResponses[finalUrl];

      if (status !== 'success' || pageResponse.status !== 200) {
        throw pageResponse;
      }

      if (waitFor) {
        return waitForFn({
          debug: true,  // optional
          interval: 0,  // optional
          timeout: 10000,  // optional
          check: () => sitepage.evaluate((waitFor) =>
            document.querySelector(waitFor) !== null
          , waitFor),
        });
      }
    })
    .then(() => sitepage.property('content'))
    .then(content => {
      sitepage.close();
      phInstance.exit();

      resolve({ name, contentType, content });
    })
    .catch(error => {
      phInstance.exit();
      reject(error);
    });
  });<commit_msg>Keep json & virtual resource content as javascript object<commit_after>import WebMiddle, { PropTypes } from 'webmiddle';
import phantom from 'phantom';

// promise-based implementation of http://stackoverflow.com/a/19070446
function waitForFn(config) {
  config._start = config._start || new Date();

  if (config.timeout && new Date - config._start > config.timeout) {
    const errorMessage = 'waitForFn timedout ' + (new Date - config._start) + 'ms';
    if (config.debug) console.log(errorMessage);
    return Promise.reject(errorMessage);
  }

  return config.check().then(result => {
    if (result) {
      if (config.debug) console.log('waitForFn success ' + (new Date - config._start) + 'ms');
      return result;
    }

    return new Promise((resolve, reject) => {
      setTimeout(() => {
        waitForFn(config).then(resolve, reject);
      }, config.interval || 0);
    });
  });
}

const Browser =
({ name, contentType, url, method = 'GET', body = {}, httpHeaders = {}, cookies = {}, waitFor }) => {
  // TODO: cookies
  return new Promise((resolve, reject) => {
    let sitepage = null;
    let phInstance = null;
    const pageResponses = {};
    let finalUrl = url;

    if (typeof body === 'object' && body !== null) {
      // body as string
      if (httpHeaders && httpHeaders['Content-Type'] === 'application/json') {
        body = JSON.stringify(body);
      } else {
        // default: convert to form data
        body = Object.keys(body).reduce((list, prop) => {
          const value = body[prop];
          list.push(`${encodeURIComponent(prop)}=${encodeURIComponent(value)}`);
          return list;
        }, []).join('&');
      }
    }

    phantom.create()
    .then(instance => {
      phInstance = instance;
      return instance.createPage();
    })
    .then(page => {
      sitepage = page;

      //page.customHeaders = httpHeaders; // TODO: doesn't seem to work

      // track final response (after redirects)
      // https://github.com/ariya/phantomjs/issues/10185#issuecomment-38856203
      page.on('onResourceReceived', (response) => {
        pageResponses[response.url] = response;
      });
      page.on('onUrlChanged', (targetUrl) => {
        finalUrl = targetUrl;
      });

      // TODO: are httpHeaders still sent in case of redirect?
      const settings = {
        operation: method,
        headers: httpHeaders,
        data: body,
      };

      return page.open(url, settings);
    })
    .then(status => {
      const pageResponse = pageResponses[finalUrl];

      if (status !== 'success' || pageResponse.status !== 200) {
        throw pageResponse;
      }

      if (waitFor) {
        return waitForFn({
          debug: true,  // optional
          interval: 0,  // optional
          timeout: 10000,  // optional
          check: () => sitepage.evaluate((waitFor) =>
            document.querySelector(waitFor) !== null
          , waitFor),
        });
      }
    })
    .then(() => sitepage.property('content'))
    .then(content => {
      sitepage.close();
      phInstance.exit();

      resolve({
        name,
        contentType,
        content: (contentType === 'application/json') ? JSON.parse(content) : content,
      });
    })
    .catch(error => {
      phInstance.exit();
      reject(error);
    });
  });
{'fraction_non_alphanumeric': 0.1535366042408296, 'fraction_numerical': 0.00959603776505185, 'mean_word_length': 2.3516597510373445, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 37, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1363893', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1856, 'n_words': 433}