<commit_before>#!/usr/bin/env node
'use strict';

const fs        = require('fs');
const path      = require('path');
const fetch     = require('node-fetch');
const cheerio   = require('cheerio');
const coroutine = require('co');
const Mocha     = require('mocha');

const HOST = 'https://leetcode.com';
const OUTPUT = path.resolve(__dirname, '../README.md');

const Categories = ['Algorithms']; // Database, Shell
const Difficulties = ['Easy', 'Medium', 'Hard'];

function mochaTest (file) {
  return new Promise((resolve, reject) => {
    const mocha = new Mocha();
    mocha.addFile(file);
    mocha.run(result => {
      resolve(result === 0);
    });
  });
}

coroutine(function * () {

  // 1. http request
  const response = [];
  for (let i = 0; i < Categories.length; i++) {
    const category = Categories[i];
    const url = HOST + '/problemset/' + category.toLowerCase();
    const data = yield fetch(url, { timeout: 10 * 1000 }).then(res => {
      if (res.status !== 200) {
        throw new Error(`request to ${url} failed, status code = ${res.status} (${res.statusText})`);
      }
      return res.text().then(html => new Object({ category, html }));
    });
    response.push(data);
  }

  // 2. response => problem list
  const problemList = response.reduce((list, data) => {
    const $ = cheerio.load(data.html);
    $('#problemList tbody tr').each((index, tr) => {
      const td = $(tr).find('td');
      const no         = Number.parseInt(td.eq(1).text().trim(), 10);
      const title      = td.eq(2).find('a').text().trim();
      const uri        = td.eq(2).find('a').attr('href').trim();
      const lock       = td.eq(2).find('i').attr('class') ? true : false;
      const acceptance = td.eq(3).text().trim();
      const difficulty = td.eq(6).text().trim();
      list[no] = { no, category: data.category, title, uri, lock, acceptance, difficulty };
    });
    return list;
  }, [])
  .filter(problem => problem !== null); // filter the empty item from list


  // 3. generate statistics
  let statistics = {};
  for (let i = 0; i < problemList.length; i++) {
    const name       = problemList[i].uri.split('/')[2];
    const category   = problemList[i].category;
    const difficulty = problemList[i].difficulty;

    // state
    let state = false;
    try {
      state = yield mochaTest(path.resolve(__dirname, '../src', name + '.js'));
    } catch (e) {
      if (e.code !== 'MODULE_NOT_FOUND') {
        throw e;
      }
    }

    // add name and state
    Object.assign(problemList[i], {
      name, state
    });

    // Difficulty Statistics
    if (!statistics.hasOwnProperty(difficulty)) {
      statistics[difficulty] = { total: 0, solve: 0 };
    }
    statistics[difficulty].total += 1;
    statistics[difficulty].solve += state ? 1 : 0;

    // Category Statistics
    if (!statistics.hasOwnProperty(category)) {
      statistics[category] = { total: 0, solve: 0 };
    }
    statistics[category].total += 1;
    statistics[category].solve += state ? 1 : 0;
  }

  // 4. generate markdown
  const difficultyStatistics = Difficulties.map(difficulty => {
    const total = statistics[difficulty].total;
    const solve = statistics[difficulty].solve;

    return `* ${difficulty} (${solve}/${total})`;
  }).join('\n');

  const leetcodeTable = problemList.map(p => {
    return [
      '',
      p.category,
      p.no,
      `[${p.title}](${HOST + p.uri})${p.lock ? ' :blue_book:' : ''}`,
      p.difficulty,
      p.state ? `[✓](src/${p.name}.js)` : '✘',
      ''
    ].join(' | ').trim();
  }).join('\n');

  const markdown = [
    '#LeetCode',
    '',
    '[![][dependency-img]][dependency-url]',
    '[![][travis-img]][travis-url]',
    '',
    difficultyStatistics,
    '',
    '| Category | #     | Title | Difficulty | State |',
    '| :---     | :---: | :---  | :---       | :---: |',
    leetcodeTable,
    '',
    '[dependency-url]: https://david-dm.org/zlargon/leetcode',
    '[dependency-img]: https://img.shields.io/david/zlargon/leetcode.svg',
    '',
    '[travis-url]: https://travis-ci.org/zlargon/leetcode',
    '[travis-img]: https://img.shields.io/travis/zlargon/leetcode/master.svg',<commit_msg>update script "update_readme.js" for new website<commit_after>#!/usr/bin/env node
'use strict';

const fs = require('fs');
const path = require('path');
const co = require('co');
const fetch = require('node-fetch');
const Mocha = require('mocha');

const HOST = 'https://leetcode.com';
const OUTPUT = path.resolve(__dirname, '../README.md');

const Categories = ['Algorithms']; // Database, Shell
const Difficulties = ['Easy', 'Medium', 'Hard'];

function mochaTest (file) {
  return new Promise((resolve, reject) => {
    const mocha = new Mocha();
    mocha.addFile(file);
    mocha.run(result => {
      resolve(result === 0);
    });
  });
}

co(function * () {

  // 1. http request
  const problemList = [];
  for (let i = 0; i < Categories.length; i++) {
    const category = Categories[i];
    const url = HOST + '/api/problems/' + category.toLowerCase();
    const data = yield fetch(url, { timeout: 10 * 1000 }).then(res => {
      if (res.status !== 200) {
        throw new Error(`request to ${url} failed, status code = ${res.status} (${res.statusText})`);
      }
      return res.json().then(json => {
        json.stat_status_pairs.forEach(x => {
          problemList.unshift({
            id: x.stat.question_id,
            category,
            title: x.stat.question__title,
            name: x.stat.question__title_slug,
            url: HOST + '/problems/' + x.stat.question__title_slug + '/',
            lock: x.paid_only,
            difficulty: x.difficulty.level - 1
          });
        });
      });
    });
  }

  // 3. generate statistics
  const diff_total = [0, 0, 0];
  const diff_solve = [0, 0, 0];

  for (let i = 0; i < problemList.length; i++) {
    const p = problemList[i];
    p.state = false;

    try {
      p.state = yield mochaTest(path.resolve(__dirname, '../src', p.name + '.js'));
    } catch (e) {
      if (e.code !== 'MODULE_NOT_FOUND') {
        throw e;
      }
    }

    // statistics
    diff_total[p.difficulty] += 1;
    if (p.state) {
      diff_solve[p.difficulty] += 1;
    }
  }

  // 4. generate markdown
  const statistics = Difficulties.map((diff, i) => {
    return `* ${diff} (${diff_solve[i]}/${diff_total[i]})`;
  }).join('\n');

  const leetcodeTable = problemList.map(p => {
    return [
      '',
      p.category,
      p.id,
      `[${p.title}](${p.url})${p.lock ? ' :blue_book:' : ''}`,
      Difficulties[p.difficulty],
      p.state ? `[✓](src/${p.name}.js)` : '✘',
      ''
    ].join(' | ').trim();
  }).join('\n');

  const markdown = [
    '# LeetCode',
    '',
    '[![][dependency-img]][dependency-url]',
    '[![][travis-img]][travis-url]',
    '',
    statistics,
    '',
    '| Category | #     | Title | Difficulty | State |',
    '| :---     | :---: | :---  | :---       | :---: |',
    leetcodeTable,
    '',
    '[dependency-url]: https://david-dm.org/zlargon/leetcode',
    '[dependency-img]: https://img.shields.io/david/zlargon/leetcode.svg',
    '',
    '[travis-url]: https://travis-ci.org/zlargon/leetcode',
    '[travis-img]: https://img.shields.io/travis/zlargon/leetcode/master.svg',
{'fraction_non_alphanumeric': 0.20253863134657837, 'fraction_numerical': 0.008140176600441501, 'mean_word_length': 2.5447432762836186, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 20, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6505850', 'n_tokens_mistral': 2546, 'n_tokens_neox': 2498, 'n_words': 496}