<commit_before>var fs = require('fs')
var chalk = require('chalk')
var got = require('got')
var mkdirp = require('mkdirp')
var _ = require('lodash')
var ProgressBar = require('progress')
var urlDl = require('./download.js')
var requestretry = require('requestretry')
var glob = require('matched')
var vc = require('version_compare')
var config = require('./config.js')
var log = require('winston')

var parseString = require('xml2js').parseString

var EuPMCVersion = '5.2.2'

var EuPmc = function (opts) {
  var eupmc = this
  this.baseurl = 'https://www.ebi.ac.uk/' +
                 'europepmc/webservices/rest/search/'
  this.opts = opts || {}
  eupmc.first = true
  eupmc.hitlimit = eupmc.opts.hitlimit ? eupmc.opts.hitlimit : 0
  eupmc.hitcount = 0
  eupmc.residualhits = 0
  eupmc.allresults = []
  eupmc.nextCursorMark = '*' // we always get back the first page
  eupmc.pagesize = '1000'
  eupmc.unfillledPage = false
}

EuPmc.prototype.search = function (query) {
  var eupmc = this

  if (!eupmc.opts.all) {
    query += ' OPEN_ACCESS:y'
  }
  var options = { resulttype: 'core', pageSize: eupmc.pagesize }
  eupmc.queryurl = eupmc.buildQuery(query, options)

  if (eupmc.opts.restart) {
    fs.readFile('eupmc_results.json', (err, data) => {
      if ((err) && (err.code === 'ENOENT')) {<commit_msg>Update EUPMC API to 5.3.2 (January 2018). Update query URL syntax.<commit_after>var fs = require('fs')
var chalk = require('chalk')
var got = require('got')
var mkdirp = require('mkdirp')
var _ = require('lodash')
var ProgressBar = require('progress')
var urlDl = require('./download.js')
var requestretry = require('requestretry')
var glob = require('matched')
var vc = require('version_compare')
var config = require('./config.js')
var log = require('winston')

var parseString = require('xml2js').parseString

var EuPMCVersion = '5.3.2'

var EuPmc = function (opts) {
  var eupmc = this
  this.baseurl = 'https://www.ebi.ac.uk/' +
                 'europepmc/webservices/rest/search?'
  this.opts = opts || {}
  eupmc.first = true
  eupmc.hitlimit = eupmc.opts.hitlimit ? eupmc.opts.hitlimit : 0
  eupmc.hitcount = 0
  eupmc.residualhits = 0
  eupmc.allresults = []
  eupmc.nextCursorMark = '*' // we always get back the first page
  eupmc.pagesize = '1000'
  eupmc.unfillledPage = false
}

EuPmc.prototype.search = function (query) {
  var eupmc = this

  if (!eupmc.opts.all) {
    query += ' OPEN_ACCESS:y'
  }
  var options = { resulttype: 'core', pageSize: eupmc.pagesize }
  eupmc.queryurl = eupmc.buildQuery(query, options)

  if (eupmc.opts.restart) {
    fs.readFile('eupmc_results.json', (err, data) => {
      if ((err) && (err.code === 'ENOENT')) {
{'fraction_non_alphanumeric': 0.15712682379349047, 'fraction_numerical': 0.010849233071455293, 'mean_word_length': 3.7411347517730498, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2699640', 'n_tokens_mistral': 1005, 'n_tokens_neox': 958, 'n_words': 218}