<commit_before>import request from 'request'
import cheerio from 'cheerio'

export default class GoogleScraper {
  constructor(options) {
    this.options = options;
  }

  get getGoogleLinks() {
    return new Promise((resolve, reject) => {
      this.getHtml().then((body) => {
        return resolve(this.extractLink(body))
      }).catch((err) => {
        return reject(err);
      })
    })
  }

  getHtml() {
    return new Promise((resolve, reject) => {
      request(`http://www.google.${this.options.tld}/search?hl=${this.options.language}&num=${this.options.results}&q=${this.options.keyword}`, (err, res, body) => {
        if (err) {
          return reject(err)
        } else if (res.statusCode !== 200) {
          const error = new Error(`Unexpected status code: ${res.statusCode}`)
          error.res = res
          return reject(error)
        }
        return resolve(body)
      })
    })
  }

  extractLink(html) { // eslint-disable-line class-methods-use-this
    const arrayLinks = [];
    const $ = cheerio.load(html);
    $('h3.r a').each((i, link) => {
      const linkClean = $(link).attr('href').match('(?=http|https).*(?=&sa)')
      if (linkClean !== null) {
        arrayLinks.push(linkClean[0]);
      }
    })
    return arrayLinks;
  }

}<commit_msg>use config file<commit_after>import request from 'request'
import cheerio from 'cheerio'
import * as config from '../config'

export default class GoogleScraper {

  constructor(options) {
    this.options = options;
  }

  get getGoogleLinks() {
    return new Promise((resolve, reject) => {
      this.getHtml().then((body) => {
        return resolve(this.extractLink(body))
      }).catch((err) => {
        return reject(err);
      })
    })
  }

  getHtml() {
    const option = {
      url: config.urlSearch(this.options.tld, this.options.language, this.options.results, this.options.keyword),
    }
    return new Promise((resolve, reject) => {
      request(option, (err, res, body) => {
        if (err) {
          return reject(err)
        } else if (res.statusCode !== 200) {
          const error = new Error(`Unexpected status code: ${res.statusCode}`)
          error.res = res
          return reject(error)
        }
        return resolve(body)
      })
    })
  }

  extractLink(html) { // eslint-disable-line class-methods-use-this
    const arrayLinks = [];
    const $ = cheerio.load(html);
    $('h3.r a').each((i, link) => {
      const linkClean = $(link).attr('href').match('(?=http|https).*(?=&sa)')
      if (linkClean) {
        arrayLinks.push(linkClean[0]);
      }
    })
    return arrayLinks;
  }

}
{'fraction_non_alphanumeric': 0.16545939763629433, 'fraction_numerical': 0.003812428516965307, 'mean_word_length': 2.5033377837116153, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5953024', 'n_tokens_mistral': 831, 'n_tokens_neox': 812, 'n_words': 165}