<commit_before>const { Scheduler: PownScheduler } = require('@pown/request/lib/scheduler')

class Scheduler extends PownScheduler {
  /**
   * The Pown Scheduler does not throw in normal circumstances no matter the
   * error. This method changes this behaviour slightly. Error is thrown when
   * the response contains an error or even when the response code itself is
   * an error, i.e. 500. This is done in order to minimise further error
   * checking when programming a transform. If the request contains and error
   * the transform will simply bail out by the means of the exception.
   *
   * It is tempting to overload the request method but we do not do that here
   * because the scheduler could be used by other libraries and as a result this
   * specific behaviour might be unexpected side-effect.
   */
  async tryRequest(request) {
    if (request.headers && !request.headers['user-agent']) {
      request = {
        ...request,

        headers: { ...request.headers, 'user-agent': 'pown' },
      }
    }

    const response = await this.request(request)

    if (response.info.error) {
      // NOTE: in the context of Recon the scheduler also prints error messages for better monitoring of complex tasks

      console.error(
        `${response.method || 'GET'}`,
        response.uri,
        '->',
        response.info.error
      )

      // NOTE: as per the above, we throw errors to bail out as soon as we can

      throw response.info.error
    } else {
      console.debug(
        `${response.method || 'GET'}`,
        response.uri,
        '->',
        response.responseCode,
        Buffer.from(response.responseBody).slice(0, 512).toString('base64')
      )
    }

    if (response.responseCode >= 500) {
      // NOTE: as per the above, we throw errors to bail out as soon as we can

      throw new Error(
        `Cannot request ${response.method} ${response.uri} -> ${response.responseCode}`
      )
    }

    if (request.toJson || request.toJSON) {
      return JSON.parse(response.responseBody)
    } else {
      return response
    }
  }
}

module.exports = { Scheduler }<commit_msg>Better request handling.<commit_after>const { Scheduler: PownScheduler } = require('@pown/request/lib/scheduler')

class Scheduler extends PownScheduler {
  /**
   * The Pown Scheduler does not throw in normal circumstances no matter the
   * error. This method changes this behaviour slightly. Error is thrown when
   * the response contains an error or even when the response code itself is
   * an error, i.e. 500. This is done in order to minimise further error
   * checking when programming a transform. If the request contains and error
   * the transform will simply bail out by the means of the exception.
   *
   * It is tempting to overload the request method but we do not do that here
   * because the scheduler could be used by other libraries and as a result this
   * specific behaviour might be unexpected side-effect.
   */
  async tryRequest(request) {
    if (request.headers && !request.headers['user-agent']) {
      request = {
        ...request,

        headers: { ...request.headers, 'user-agent': 'pown' },
      }
    }

    const response = await this.request(request)

    if (response.info.error) {
      // NOTE: in the context of Recon the scheduler also prints error messages for better monitoring of complex tasks

      console.error(
        `${response.method || 'GET'}`,
        response.uri,
        '->',
        response.info.error
      )

      // NOTE: as per the above, we throw errors to bail out as soon as we can

      throw response.info.error
    } else {
      console.debug(
        `${response.method || 'GET'}`,
        response.uri,
        '->',
        response.responseCode,
        Buffer.from(response.responseBody).slice(0, 512).toString('base64')
      )
    }

    if (response.responseCode < 200 || response.responseCode > 299) {
      // NOTE: as per the above, we throw errors to bail out as soon as we can

      throw new Error(
        `Cannot request ${response.method} ${response.uri} -> ${response.responseCode}`
      )
    }

    if (request.toJson || request.toJSON) {
      return JSON.parse(response.responseBody)
    } else {
      return response
    }
  }
}

module.exports = { Scheduler }
{'fraction_non_alphanumeric': 0.09807111317685335, 'fraction_numerical': 0.006274692075296305, 'mean_word_length': 2.524979524979525, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1761090', 'n_tokens_mistral': 1192, 'n_tokens_neox': 1151, 'n_words': 442}