<commit_before>const Config = require('../config.json');
const fetch = require('node-fetch');

module.exports = class Fetcher {
  get(url) {
    const options = { method: 'GET' };
    return this.makeRequest(url, options);
  }

  post(url, params) {
    const body = [];
    for (const key in params) {
      body.push(key + '=' + encodeURIComponent(params[key]));
    }
    const options = { method: 'POST', body: body.join('&') };
    return this.makeRequest(url, options);
  }

  makeRequest(url, options) {
    if (typeof options.headers === 'undefined') {
      options.headers = {};
    }
    options.headers['User-Agent'] = Config.user_agent;
    return new Promise((resolve, reject) => {
      fetch(url, options).then((response) => {
        if (response.headers.get('Content-Type') === 'text/html') {
          this.handleHtmlResponse(response, resolve, reject);
        } else {
          this.handleJsonResponse(response, resolve, reject);
        }
      });
    });
  }

  handleHtmlResponse(response, resolve, reject) {
    response.text().then((body) => {
      if (response.ok) {
        resolve({ body: body });
      } else {
        const status = this.getStatus(response);
        reject({ url: url, status: status, body: body });
      }
    }).catch((error) => {
      console.error('failed to parse html response', error);
      reject({ error: error });
    });
  }

  handleJsonResponse(response, resolve, reject) {
    response.json().then((json) => {
      if (response.ok) {
        resolve(json);
      } else {
        json.url = url;
        json.status = this.getStatus(response);
        reject(json);
      }
    }).catch((error) => {
      console.error('failed to parse json response', error);
      reject({ error: error });
    });
  }

  getStatus(response) {
    return response.status + ' ' + response.statusText;
  }
}<commit_msg>Fix missing url variable in fetcher<commit_after>const Config = require('../config.json');
const fetch = require('node-fetch');

module.exports = class Fetcher {
  get(url) {
    const options = { method: 'GET' };
    return this.makeRequest(url, options);
  }

  post(url, params) {
    const body = [];
    for (const key in params) {
      body.push(key + '=' + encodeURIComponent(params[key]));
    }
    const options = { method: 'POST', body: body.join('&') };
    return this.makeRequest(url, options);
  }

  makeRequest(url, options) {
    if (typeof options.headers === 'undefined') {
      options.headers = {};
    }
    options.headers['User-Agent'] = Config.user_agent;
    return new Promise((resolve, reject) => {
      fetch(url, options).then((response) => {
        if (response.headers.get('Content-Type') === 'text/html') {
          this.handleHtmlResponse(response, url, resolve, reject);
        } else {
          this.handleJsonResponse(response, url, resolve, reject);
        }
      });
    });
  }

  handleHtmlResponse(response, url, resolve, reject) {
    response.text().then((body) => {
      if (response.ok) {
        resolve({ body: body });
      } else {
        const status = this.getStatus(response);
        reject({ url: url, status: status, body: body });
      }
    }).catch((error) => {
      console.error('failed to parse html response', error);
      reject({ error: error });
    });
  }

  handleJsonResponse(response, url, resolve, reject) {
    response.json().then((json) => {
      if (response.ok) {
        resolve(json);
      } else {
        json.url = url;
        json.status = this.getStatus(response);
        reject(json);
      }
    }).catch((error) => {
      console.error('failed to parse json response', error);
      reject({ error: error });
    });
  }

  getStatus(response) {
    return response.status + ' ' + response.statusText;
  }
}
{'fraction_non_alphanumeric': 0.16578669482576558, 'fraction_numerical': 0.0, 'mean_word_length': 2.594876660341556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6227938', 'n_tokens_mistral': 1129, 'n_tokens_neox': 1121, 'n_words': 247}