<commit_before>var _ = require('underscore');
var cheerio = require('cheerio');
var log = require('loglevel');
var request = require('request');

var config = require('./config');
var h = require('./helper');

// update options with user credentials
function signOpts(opts, user) {
  opts.headers.Cookie = 'PHPSESSID=' + user.sessionId +
                        ';csrftoken=' + user.sessionCSRF + ';';
  opts.headers['X-CSRFToken'] = user.sessionCSRF;
}

function makeOpts(url) {
  var opts = {url: url, headers: {}, _expectedStatus: 200};
  var core = require('./core');
  if (core.isLogin()) signOpts(opts, core.getUser());
  return opts;
}

function checkError(e, resp, expectedStatus, msg) {
  if (e) return e;

  if (resp && resp.statusCode !== expectedStatus) {
    if (resp.statusCode === 403) {
      msg = msg || 'session expired, please login again';

      var core = require('./core');
      core.logout();
    }

    return {
      msg:        msg || 'http error',
      statusCode: resp.statusCode
    };
  }
}

// leetcode.com is limiting one session alive in the same time,
// which means once you login on web, your cli session will get
// expired immediately. In that case we will try to re-login in
// the backend to give a seamless user experience.
function requestWithReLogin(opts, cb) {
  if (!config.AUTO_LOGIN)
    return request(opts, cb);

  var core = require('./core');
  var user = core.getUser();

  request(opts, function(e, resp, body) {
    e = checkError(e, resp, opts._expectedStatus);

    // not 403: transparently pass down
    if (!e || e.statusCode !== 403)
      return cb(e, resp, body);

    // if 403: try re-login
    log.debug('session expired, auto re-login...');

    core.login(user, function(e2, user) {
      if (e2) return cb(e, resp, body);

      log.debug('login successfully, cont\'d...');
      signOpts(opts, user);

      request(opts, cb);
    });
  });
}

var leetcodeClient = {};

leetcodeClient.getProblems = function(cb) {
  var opts = makeOpts(config.URL_PROBLEMS);

  requestWithReLogin(opts, function(e, resp, body) {
    e = checkError(e, resp, 200);
    if (e) return cb(e);

    var json = JSON.parse(body);

    // leetcode permits anonymous access to the problem list
    // while we require login first to make a better experience.
    if (json.user_name.length === 0)
      return cb('session expired, please login again');

    var problems = json.stat_status_pairs
        .filter(function(p) {
          return !p.stat.question__hide;
        })
        .map(function(p) {
          return {
            state:   p.status || 'None',
            id:      p.stat.question_id,
            name:    p.stat.question__title,
            key:     p.stat.question__title_slug,
            link:    config.URL_PROBLEM.replace('$id', p.stat.question__title_slug),
            locked:  p.paid_only,
            percent: p.stat.total_acs * 100 / p.stat.total_submitted,
            level:   h.levelToName(p.difficulty.level),
            starred: p.is_favor
          };
        });

    return cb(null, problems);
  });
};

// hacking ;P
var aceCtrl = {
  init: function() {
    return Array.prototype.slice.call(arguments);
  }
};

leetcodeClient.getProblem = function(problem, cb) {
  request(problem.link, function(e, resp, body) {
    e = checkError(e, resp, 200);
    if (e) return cb(e);

    var $ = cheerio.load(body);
    var info = $('div[class="question-info text-info"] ul li strong');

    problem.totalAC = $(info[0]).text();
    problem.totalSubmit = $(info[1]).text();
    problem.desc = $('meta[property="og:description"]').attr('content');

    var raw = $('div[ng-controller="AceCtrl as aceCtrl"]').attr('ng-init');
    if (!raw)
      return cb('failed to load' + (problem.locked ? ' locked ' : ' ') +
                'problem!');
<commit_msg>allow paid user to show locked problems<commit_after>var _ = require('underscore');
var cheerio = require('cheerio');
var log = require('loglevel');
var request = require('request');

var config = require('./config');
var h = require('./helper');

// update options with user credentials
function signOpts(opts, user) {
  opts.headers.Cookie = 'PHPSESSID=' + user.sessionId +
                        ';csrftoken=' + user.sessionCSRF + ';';
  opts.headers['X-CSRFToken'] = user.sessionCSRF;
}

function makeOpts(url) {
  var opts = {url: url, headers: {}, _expectedStatus: 200};
  var core = require('./core');
  if (core.isLogin()) signOpts(opts, core.getUser());
  return opts;
}

function checkError(e, resp, expectedStatus, msg) {
  if (e) return e;

  if (resp && resp.statusCode !== expectedStatus) {
    if (resp.statusCode === 403) {
      msg = msg || 'session expired, please login again';

      var core = require('./core');
      core.logout();
    }

    return {
      msg:        msg || 'http error',
      statusCode: resp.statusCode
    };
  }
}

// leetcode.com is limiting one session alive in the same time,
// which means once you login on web, your cli session will get
// expired immediately. In that case we will try to re-login in
// the backend to give a seamless user experience.
function requestWithReLogin(opts, cb) {
  if (!config.AUTO_LOGIN)
    return request(opts, cb);

  var core = require('./core');
  var user = core.getUser();

  request(opts, function(e, resp, body) {
    e = checkError(e, resp, opts._expectedStatus);

    // not 403: transparently pass down
    if (!e || e.statusCode !== 403)
      return cb(e, resp, body);

    // if 403: try re-login
    log.debug('session expired, auto re-login...');

    core.login(user, function(e2, user) {
      if (e2) return cb(e, resp, body);

      log.debug('login successfully, cont\'d...');
      signOpts(opts, user);

      request(opts, cb);
    });
  });
}

var leetcodeClient = {};

leetcodeClient.getProblems = function(cb) {
  var opts = makeOpts(config.URL_PROBLEMS);

  requestWithReLogin(opts, function(e, resp, body) {
    e = checkError(e, resp, 200);
    if (e) return cb(e);

    var json = JSON.parse(body);

    // leetcode permits anonymous access to the problem list
    // while we require login first to make a better experience.
    if (json.user_name.length === 0)
      return cb('session expired, please login again');

    var problems = json.stat_status_pairs
        .filter(function(p) {
          return !p.stat.question__hide;
        })
        .map(function(p) {
          return {
            state:   p.status || 'None',
            id:      p.stat.question_id,
            name:    p.stat.question__title,
            key:     p.stat.question__title_slug,
            link:    config.URL_PROBLEM.replace('$id', p.stat.question__title_slug),
            locked:  p.paid_only,
            percent: p.stat.total_acs * 100 / p.stat.total_submitted,
            level:   h.levelToName(p.difficulty.level),
            starred: p.is_favor
          };
        });

    return cb(null, problems);
  });
};

// hacking ;P
var aceCtrl = {
  init: function() {
    return Array.prototype.slice.call(arguments);
  }
};

leetcodeClient.getProblem = function(problem, cb) {
  var opts = makeOpts();
  opts.url = problem.link;
  request(opts, function(e, resp, body) {
    e = checkError(e, resp, 200);
    // FIXME: if session expired, this will still return 200
    if (e) return cb(e);

    var $ = cheerio.load(body);
    var info = $('div[class="question-info text-info"] ul li strong');

    problem.totalAC = $(info[0]).text();
    problem.totalSubmit = $(info[1]).text();
    problem.desc = $('meta[property="og:description"]').attr('content');

    var raw = $('div[ng-controller="AceCtrl as aceCtrl"]').attr('ng-init');
    if (!raw)
      return cb('failed to load' + (problem.locked ? ' locked ' : ' ') +
                'problem!');

{'fraction_non_alphanumeric': 0.15111054050584158, 'fraction_numerical': 0.00783155732443189, 'mean_word_length': 2.721930243669374, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1320900', 'n_tokens_mistral': 2554, 'n_tokens_neox': 2442, 'n_words': 656}