<commit_before>var _ = require('underscore');
var cheerio = require('cheerio');
var request = require('request');
var util = require('util');

var h = require('../helper');
var log = require('../log');
var Plugin = require('../plugin');
var Queue = require('../queue');
var session = require('../session');

// Still working in progress!
//
// TODO: star/submissions/submission
// FIXME: why [ERROR] Error: read ECONNRESET [0]??
//
// [Usage]
//
// https://github.com/skygragon/leetcode-cli-plugins/blob/master/docs/lintcode.md
//
const plugin = new Plugin(15, 'lintcode', '2018.05.29',
    'Plugin to talk with lintcode APIs.');

const config = {
  URL_PROBLEMS:       'https://www.lintcode.com/api/problems/?page=$page',
  URL_PROBLEM:        'https://www.lintcode.com/problem/$slug/description',
  URL_PROBLEM_DETAIL: 'https://www.lintcode.com/api/problems/detail/?unique_name_or_alias=$slug&_format=detail',
  URL_PROBLEM_CODE:   'https://www.lintcode.com/api/problems/$id/reset/?language=$lang',
  URL_TEST:           'https://www.lintcode.com/api/submissions/',
  URL_TEST_VERIFY:    'https://www.lintcode.com/api/submissions/refresh/?id=$id&is_test_submission=true',
  URL_SUBMIT_VERIFY:  'https://www.lintcode.com/api/submissions/refresh/?id=$id',
  URL_LOGIN:          'https://www.lintcode.com/api/accounts/signin/?next=%2F'
};

// FIXME: add more langs
const LANGS = [
  {value: 'cpp', text: 'C++'},
  {value: 'java', text: 'Java'},
  {value: 'python', text: 'Python'}
];

var spin;

function signOpts(opts, user) {
  opts.headers.Cookie = 'sessionid=' + user.sessionId +
                        ';csrftoken=' + user.sessionCSRF + ';';
  opts.headers['x-csrftoken'] = user.sessionCSRF;
}

function makeOpts(url) {
  const opts = {
    url:     url,
    headers: {}
  };
  if (session.isLogin())
    signOpts(opts, session.getUser());
  return opts;
}

function checkError(e, resp, expectedStatus) {
  if (!e && resp && resp.statusCode !== expectedStatus) {
    const code = resp.statusCode;
    log.debug('http error: ' + code);

    if (code === 403 || code === 401) {
      e = session.errors.EXPIRED;
    } else {
      e = {msg: 'http error', statusCode: code};
    }
  }
  return e;
}

function _split(s, delim) {
  return (s || '').split(delim).map(function(x) {
    return x.trim();
  }).filter(function(x) {
    return x.length > 0;
  });
}

function _strip(s) {
  s = s.replace(/^<pre><code>/, '').replace(/<\/code><\/pre>$/, '');
  return util.inspect(s.trim());
}

plugin.getProblems = function(cb) {
  log.debug('running lintcode.getProblems');

  var problems = [];
  const getPage = function(page, queue, cb) {
    plugin.getPageProblems(page, function(e, _problems, ctx) {
      if (!e) {
        problems = problems.concat(_problems);
        queue.tasks = _.reject(queue.tasks, x => ctx.pages > 0 && x > ctx.pages);
      }
      return cb(e);
    });
  };

  const pages = _.range(1, 100);
  const q = new Queue(pages, {}, getPage);
  spin = h.spin('Downloading problems');
  q.run(null, function(e, ctx) {
    spin.stop();
    problems = _.sortBy(problems, x => -x.id);
    return cb(e, problems);
  });
};

plugin.getPageProblems = function(page, cb) {
  log.debug('running lintcode.getPageProblems: ' + page);
  const opts = makeOpts(config.URL_PROBLEMS.replace('$page', page));

  spin.text = 'Downloading page ' + page;
  request(opts, function(e, resp, body) {
    e = checkError(e, resp, 200);
    if (e) return cb(e);

    const ctx = {};
    const json = JSON.parse(body);
    const problems = json.problems.map(function(p, a) {
      const problem = {
        id:        p.id, 
        fid:       p.id,
        name:      p.title,
        slug:      p.unique_name,
        category:  'lintcode',
        level:     h.levelToName(p.level),
        locked:    false,
        percent:   p.accepted_rate,
        starred:   p.is_favorited,
        companies: p.company_tags,
        tags:      []
      };
      problem.link = config.URL_PROBLEM.replace('$slug', problem.slug);
      switch (p.user_status) {<commit_msg>[lintcode] fixes missing level names<commit_after>var _ = require('underscore');
var cheerio = require('cheerio');
var request = require('request');
var util = require('util');

var h = require('../helper');
var log = require('../log');
var Plugin = require('../plugin');
var Queue = require('../queue');
var session = require('../session');

// Still working in progress!
//
// TODO: star/submissions/submission
// FIXME: why [ERROR] Error: read ECONNRESET [0]??
//
// [Usage]
//
// https://github.com/skygragon/leetcode-cli-plugins/blob/master/docs/lintcode.md
//
const plugin = new Plugin(15, 'lintcode', '2018.05.29',
    'Plugin to talk with lintcode APIs.');

const config = {
  URL_PROBLEMS:       'https://www.lintcode.com/api/problems/?page=$page',
  URL_PROBLEM:        'https://www.lintcode.com/problem/$slug/description',
  URL_PROBLEM_DETAIL: 'https://www.lintcode.com/api/problems/detail/?unique_name_or_alias=$slug&_format=detail',
  URL_PROBLEM_CODE:   'https://www.lintcode.com/api/problems/$id/reset/?language=$lang',
  URL_TEST:           'https://www.lintcode.com/api/submissions/',
  URL_TEST_VERIFY:    'https://www.lintcode.com/api/submissions/refresh/?id=$id&is_test_submission=true',
  URL_SUBMIT_VERIFY:  'https://www.lintcode.com/api/submissions/refresh/?id=$id',
  URL_LOGIN:          'https://www.lintcode.com/api/accounts/signin/?next=%2F'
};

// FIXME: add more langs
const LANGS = [
  {value: 'cpp', text: 'C++'},
  {value: 'java', text: 'Java'},
  {value: 'python', text: 'Python'}
];

const LEVELS = {
  0: 'Naive',
  1: 'Easy',
  2: 'Medium',
  3: 'Hard',
  4: 'Super'
};

var spin;

function signOpts(opts, user) {
  opts.headers.Cookie = 'sessionid=' + user.sessionId +
                        ';csrftoken=' + user.sessionCSRF + ';';
  opts.headers['x-csrftoken'] = user.sessionCSRF;
}

function makeOpts(url) {
  const opts = {
    url:     url,
    headers: {}
  };
  if (session.isLogin())
    signOpts(opts, session.getUser());
  return opts;
}

function checkError(e, resp, expectedStatus) {
  if (!e && resp && resp.statusCode !== expectedStatus) {
    const code = resp.statusCode;
    log.debug('http error: ' + code);

    if (code === 403 || code === 401) {
      e = session.errors.EXPIRED;
    } else {
      e = {msg: 'http error', statusCode: code};
    }
  }
  return e;
}

function _split(s, delim) {
  return (s || '').split(delim).map(function(x) {
    return x.trim();
  }).filter(function(x) {
    return x.length > 0;
  });
}

function _strip(s) {
  s = s.replace(/^<pre><code>/, '').replace(/<\/code><\/pre>$/, '');
  return util.inspect(s.trim());
}

plugin.getProblems = function(cb) {
  log.debug('running lintcode.getProblems');

  var problems = [];
  const getPage = function(page, queue, cb) {
    plugin.getPageProblems(page, function(e, _problems, ctx) {
      if (!e) {
        problems = problems.concat(_problems);
        queue.tasks = _.reject(queue.tasks, x => ctx.pages > 0 && x > ctx.pages);
      }
      return cb(e);
    });
  };

  const pages = _.range(1, 100);
  const q = new Queue(pages, {}, getPage);
  spin = h.spin('Downloading problems');
  q.run(null, function(e, ctx) {
    spin.stop();
    problems = _.sortBy(problems, x => -x.id);
    return cb(e, problems);
  });
};

plugin.getPageProblems = function(page, cb) {
  log.debug('running lintcode.getPageProblems: ' + page);
  const opts = makeOpts(config.URL_PROBLEMS.replace('$page', page));

  spin.text = 'Downloading page ' + page;
  request(opts, function(e, resp, body) {
    e = checkError(e, resp, 200);
    if (e) return cb(e);

    const ctx = {};
    const json = JSON.parse(body);
    const problems = json.problems.map(function(p, a) {
      const problem = {
        id:        p.id, 
        fid:       p.id,
        name:      p.title,
        slug:      p.unique_name,
        category:  'lintcode',
        level:     LEVELS[p.level],
        locked:    false,
        percent:   p.accepted_rate,
        starred:   p.is_favorited,
        companies: p.company_tags,
        tags:      []
      };
      problem.link = config.URL_PROBLEM.replace('$slug', problem.slug);
      switch (p.user_status) {
{'fraction_non_alphanumeric': 0.18234936092513696, 'fraction_numerical': 0.007181984175289105, 'mean_word_length': 2.9576107899807322, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 18, 'lorem ipsum': 0, 'www.': 16, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1414735', 'n_tokens_mistral': 3011, 'n_tokens_neox': 2912, 'n_words': 572}