<commit_before>import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import * as request from 'request';
import * as xml2js from 'xml2js';

import {Config} from '../config';
import {HttpUtils} from '../http_utils';

export abstract class ConfigSource {
  ostype = Config.osType();
  osarch = Config.osArch();
  out_dir: string = Config.getSeleniumDir();
  opt_ignoreSSL: boolean;
  opt_proxy: string;

  abstract getUrl(version: string, opt_proxy?: string, opt_ignoreSSL?: boolean):
      Promise<{url: string, version: string}>;
  abstract getVersionList(): Promise<string[]>;
}

export abstract class XmlConfigSource extends ConfigSource {
  constructor(public name: string, public xmlUrl: string) {
    super();
  }

  protected getFileName(): string {
    try {
      fs.statSync(this.out_dir);
    } catch (e) {
      fs.mkdirSync(this.out_dir);
    }
    return path.resolve(this.out_dir, this.name + '-response.xml');
  }

  protected getXml(): Promise<any> {
    let fileName = this.getFileName();
    let content = this.readResponse();
    if (content != null) {
      return Promise.resolve(content);
    }
    return this.requestXml().then(text => {
      let xml = this.convertXml2js(text);
      fs.writeFileSync(fileName, text);
      return xml;
    });
  }

  private readResponse(): any {
    let fileName = this.getFileName();
    try {
      let contents = fs.readFileSync(fileName).toString();
      let timestamp = new Date(fs.statSync(fileName).mtime).getTime();

      let now = Date.now();
      if (now - 36000000 < timestamp) {
        return this.convertXml2js(contents);
      }
    } catch (err) {
      return null;
    }
  }

  private requestXml(): Promise<string> {
    return new Promise<string>((resolve, reject) => {
      let options = HttpUtils.initOptions(this.xmlUrl);
      options = HttpUtils.optionsSSL(options, this.opt_ignoreSSL);
      options = HttpUtils.optionsProxy(options, this.xmlUrl, this.opt_proxy);

      let req = request(options);
      req.on('response', response => {
        if (response.statusCode === 200) {
          // logger.info('curl -v ' + options.url);
          let output = '';
          response.on('data', (data) => {
            output += data;
          });
          response.on('end', () => {
            resolve(output);
          });

        } else {
          reject(new Error('response status code is not 200'));
        }
      });
    });
  }

  private convertXml2js(xml: string): any {
    let retResult: any = null;
    xml2js.parseString(xml, (err, result) => {
      retResult = result;
    });
    return retResult;
  }
}

export abstract class JsonConfigSource extends ConfigSource {
  constructor(public name: string, public jsonUrl: string) {
    super();
  }

  protected getFileName(): string {
    try {
      fs.statSync(this.out_dir);
    } catch (e) {
      fs.mkdirSync(this.out_dir);
    }
    return path.resolve(this.out_dir, this.name + '-response.json');
  }

  protected abstract getJson(): Promise<string>;
}

export abstract class GithubApiConfigSource extends JsonConfigSource {
  constructor(name: string, url: string) {
    super(name, url);
  }

  /**
   * This is an unauthenticated request and since Github limits the rate, we will cache this
   * to a file. { timestamp: number, response: response }. We will check the timestamp and renew
   * this request if the file is older than an hour.
   */
  getJson(): Promise<any> {
    let fileName = this.getFileName();
    let content = this.readResponse();
    if (content != null) {
      return Promise.resolve(JSON.parse(content));
    } else {
      return this.requestJson().then(body => {
        let json = JSON.parse(body);
        fs.writeFileSync(fileName, JSON.stringify(json, null, '  '));
        return json;
      });
    }
  }

  private requestJson(): Promise<any> {
    return new Promise<any>((resolve, reject) => {
      let options = HttpUtils.initOptions(this.jsonUrl);
      options = HttpUtils.optionsSSL(options, this.opt_ignoreSSL);
      options = HttpUtils.optionsProxy(options, this.jsonUrl, this.opt_proxy);
      options = HttpUtils.optionsHeader(options, 'Host', 'api.github.com');
      options = HttpUtils.optionsHeader(options, 'User-Agent', 'request');

      let req = request(options);
      req.on('response', response => {
        if (response.statusCode === 200) {
          // logger.info('curl -v ' + options.url);
          let output = '';
          response.on('data', (data) => {
            output += data;
          });
          response.on('end', () => {
            resolve(output);
          });

        } else {
          reject(new Error('response status code is not 200'));
        }
      })
    });
  }

  private readResponse(): any {
    let fileName = this.getFileName();
    try {<commit_msg>feat(gecko): Improve error message when Github api limit reached. (#217)<commit_after>import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import * as request from 'request';
import * as xml2js from 'xml2js';

import {Config} from '../config';
import {HttpUtils} from '../http_utils';

export abstract class ConfigSource {
  ostype = Config.osType();
  osarch = Config.osArch();
  out_dir: string = Config.getSeleniumDir();
  opt_ignoreSSL: boolean;
  opt_proxy: string;

  abstract getUrl(version: string, opt_proxy?: string, opt_ignoreSSL?: boolean):
      Promise<{url: string, version: string}>;
  abstract getVersionList(): Promise<string[]>;
}

export abstract class XmlConfigSource extends ConfigSource {
  constructor(public name: string, public xmlUrl: string) {
    super();
  }

  protected getFileName(): string {
    try {
      fs.statSync(this.out_dir);
    } catch (e) {
      fs.mkdirSync(this.out_dir);
    }
    return path.resolve(this.out_dir, this.name + '-response.xml');
  }

  protected getXml(): Promise<any> {
    let fileName = this.getFileName();
    let content = this.readResponse();
    if (content != null) {
      return Promise.resolve(content);
    }
    return this.requestXml().then(text => {
      let xml = this.convertXml2js(text);
      fs.writeFileSync(fileName, text);
      return xml;
    });
  }

  private readResponse(): any {
    let fileName = this.getFileName();
    try {
      let contents = fs.readFileSync(fileName).toString();
      let timestamp = new Date(fs.statSync(fileName).mtime).getTime();

      let now = Date.now();
      if (now - 36000000 < timestamp) {
        return this.convertXml2js(contents);
      }
    } catch (err) {
      return null;
    }
  }

  private requestXml(): Promise<string> {
    return new Promise<string>((resolve, reject) => {
      let options = HttpUtils.initOptions(this.xmlUrl);
      options = HttpUtils.optionsSSL(options, this.opt_ignoreSSL);
      options = HttpUtils.optionsProxy(options, this.xmlUrl, this.opt_proxy);

      let req = request(options);
      req.on('response', response => {
        if (response.statusCode === 200) {
          // logger.info('curl -v ' + options.url);
          let output = '';
          response.on('data', (data) => {
            output += data;
          });
          response.on('end', () => {
            resolve(output);
          });

        } else {
          reject(new Error('response status code is not 200'));
        }
      });
    });
  }

  private convertXml2js(xml: string): any {
    let retResult: any = null;
    xml2js.parseString(xml, (err, result) => {
      retResult = result;
    });
    return retResult;
  }
}

export abstract class JsonConfigSource extends ConfigSource {
  constructor(public name: string, public jsonUrl: string) {
    super();
  }

  protected getFileName(): string {
    try {
      fs.statSync(this.out_dir);
    } catch (e) {
      fs.mkdirSync(this.out_dir);
    }
    return path.resolve(this.out_dir, this.name + '-response.json');
  }

  protected abstract getJson(): Promise<string>;
}

export abstract class GithubApiConfigSource extends JsonConfigSource {
  constructor(name: string, url: string) {
    super(name, url);
  }

  /**
   * This is an unauthenticated request and since Github limits the rate, we will cache this
   * to a file. { timestamp: number, response: response }. We will check the timestamp and renew
   * this request if the file is older than an hour.
   */
  getJson(): Promise<any> {
    let fileName = this.getFileName();
    let content = this.readResponse();
    if (content != null) {
      return Promise.resolve(JSON.parse(content));
    } else {
      return this.requestJson().then(body => {
        let json = JSON.parse(body);
        fs.writeFileSync(fileName, JSON.stringify(json, null, '  '));
        return json;
      });
    }
  }

  private requestJson(): Promise<any> {
    return new Promise<any>((resolve, reject) => {
      let options = HttpUtils.initOptions(this.jsonUrl);
      options = HttpUtils.optionsSSL(options, this.opt_ignoreSSL);
      options = HttpUtils.optionsProxy(options, this.jsonUrl, this.opt_proxy);
      options = HttpUtils.optionsHeader(options, 'Host', 'api.github.com');
      options = HttpUtils.optionsHeader(options, 'User-Agent', 'request');

      let req = request(options);
      req.on('response', response => {
        if (response.statusCode === 200) {
          // logger.info('curl -v ' + options.url);
          let output = '';
          response.on('data', (data) => {
            output += data;
          });
          response.on('end', () => {
            resolve(output);
          });

        } else if (response.statusCode == 403 && response.headers['x-ratelimit-remaining'] == 0) {
          reject(new Error('Failed to make Github request, rate limit reached.'));
        } else {
          reject(new Error('response status code is not 200.  It was ' + response.statusCode));
        }
      })
    });
  }

  private readResponse(): any {
    let fileName = this.getFileName();
    try {
{'fraction_non_alphanumeric': 0.1383843452082288, 'fraction_numerical': 0.005920722528850978, 'mean_word_length': 2.6993318485523385, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 22}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5499165', 'n_tokens_mistral': 2982, 'n_tokens_neox': 2903, 'n_words': 745}