<commit_before>var request = require('request');
var parse = require('./parseWikiText');
var XRegExp = require('xregexp').XRegExp;

var page = "Warsaw";
var apiURL = "http://en.wikipedia.org/w/api.php?format=json&action=query&prop=revisions&rvprop=content&titles=" + page;

request.get(apiURL, function(error, data, body){
  if (error) {
    console.log('ERROR: ', error);
    return;
  }

  var content = JSON.parse(body);
  content = content.query.pages;

  var page = Object.keys(content);
  content = content[page].revisions[0]['*'];

  var startingPointRegex = /\{\{\s*[Ii]nfobox/;

  var macz = content.match(startingPointRegex).index;

  var end = parse(content.substr(macz, content.length));

  content = content.substr(macz, end);

  var regex = XRegExp("'^\*+\s*(\[\['
                '((?P<wiki_link>[^]]+?)(\|))?'
                 '(?P<lang_name>.+?)'
                 '\]\])'", "s");

  var result = XRegExp.exec(content, regex);


  console.log(result);
  //console.log(content.substr(macz, content.length);

});<commit_msg>[WIKI-INFOBOX] - :christmas_tree: parse links in infobox<commit_after>var request = require('request');
var parse = require('./parseWikiText');
var XRegExp = require('xregexp').XRegExp;

var page = "Warsaw";
var apiURL = "http://en.wikipedia.org/w/api.php?format=json&action=query&prop=revisions&rvprop=content&titles=" + page;

request.get(apiURL, function(error, data, body){
  if (error) {
    console.log('ERROR: ', error);
    return;
  }

  var content = JSON.parse(body);
  content = content.query.pages;

  var page = Object.keys(content);
  content = content[page].revisions[0]['*'];

  var startingPointRegex = /\{\{\s*[Ii]nfobox/;

  var macz = content.match(startingPointRegex).index;

  var end = parse(content.substr(macz, content.length));

  content = content.substr(macz, end);

  var regex = XRegExp("/\[\[(.+?)\]\]/", 'sig');


  var result = content.match(/\[\[(.+?)\]\]/ig);


  console.log(result);
  //console.log(content.substr(macz, content.length);

});
{'fraction_non_alphanumeric': 0.20376797223599405, 'fraction_numerical': 0.0009915716410510659, 'mean_word_length': 3.9704433497536944, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '992704', 'n_tokens_mistral': 732, 'n_tokens_neox': 695, 'n_words': 124}