<commit_before>var request = require('request')
  , fs = require('fs')
  , url = require('url')
  , async = require('async')
  , path = require('path')
  , temp = require('temp')
  , tar = require('tar')
  , once = require('once')
  , pubmed = require('./pubmed').pubmed
  , Client = require('ftp')
  , xml2js = require('xml2js')
  , DecompressZip = require('decompress-zip')
  , zlib = require('zlib')
  , traverse = require('traverse')
  , recursiveReaddir = require('recursive-readdir')
  , DOMParser = require('xmldom').DOMParser
  , tools = require('./lib/tools');

process.maxTickDepth = 10000;
// to avoid warnings when using nextTick
// https://groups.google.com/forum/#!topic/nodejs/9_uM04IDNWg

temp.track();

module.exports = oapmc;

/**
 * 'this' is an Ldpm instance
 */
function oapmc(uri, opts, callback){

  callback = once(callback);

  if(arguments.length === 2){
    callback = opts;
    opts = {};
  }

  var that = this;
  var pmcid = tools.extractBetween(uri, 'PMC');

  // check url
  if (uri.slice(0, 53) === 'http://www.pubmedcentral.nih.gov/utils/oa/oa.fcgi?id=' ){

    // 0. Preliminary fetches
    that.logHttp('GET', uri);
    // Fetch the url of the tar.gz of the article
    request(uri, function(error, response, oaContentBody){
      if(error) return callback(error);

      that.logHttp(response.statusCode, uri);

      if(response.statusCode >= 400){
        var err = new Error(oaContentBody);
        err.code = response.statusCode;
        return callback(err);
      }

      // First way to get the name of the main article: from the pdf name in oaContentBody
      // that contains pdf and tar.gz
      mainArticleName = extractPdfName(oaContentBody).replace(/\./g, '-');

      var conversionUrl = 'http://www.pubmedcentral.nih.gov/utils/idconv/v1.0/?ids=' + 'PMC' + pmcid + '&format=json';
      that.logHttp('GET', conversionUrl);
      // For PMC article, the idconv api returns {pmid,pmcid,doi} when given any of the three.
      request(conversionUrl, function(error, response, idConversionBody) {
        if(error) return callback(error);

        that.logHttp(response.statusCode,conversionUrl);

        if(response.statusCode >= 400){
          var err = new Error(idConversionBody);
          err.code = response.statusCode;
          return callback(err);
        }

        var res = JSON.parse(idConversionBody);
        var doi = res['records'][0]['doi'];
        var pmid = res['records'][0]['pmid'];
        if(pmid==undefined){
          // OAPMC entries do not all have a PMID (eg PMC3875093)
          opts.noPubmed = true;
        }

        // 1. Fetch : resources, xml, and pubmed metadata
        // a. resources
        fetchTar(tools.extractBetween(oaContentBody, 'href="', '"').slice(27), that, function(err, files){
          if(err) return callback(err);

          // Second way to get the name of the main article: from the name of the nxml file
          // in the tar.gz. OAPMC entries always have at least a pdf or an nxml.
          if(mainArticleName===undefined){
            mainArticleName = extractNXMLName(files).replace(/\./g, '-');
          }

          // b. xml
          fetchXml('http://www.pubmedcentral.nih.gov/oai/oai.cgi?verb=GetRecord&identifier=oai:pubmedcentral.nih.gov:' + pmcid + '&metadataPrefix=pmc', that, function(err, xml){
            if(err) return callback(err);

            // c. pubmed metadata
            fetchPubmedMetadata('http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id='+pmid+'&rettype=abstract&retmode=xml', that, opts, function(err,pubmedPkg){
              if(err) return callback(err);

              var pkg = { version: '0.0.0' };

              // 2. Parse and complete pkg
              // a. resources: identify different encodings, substitute plos urls to contentPaths
              parseResources(pkg, files, doi, that, function(err,pkg){
                if(err) return callback(err);
                // b. xml: get captions, citations, authors, publishers etc from the xml
                parseXml(xml,pkg,pmcid,mainArticleName,that,opts,function(err,pkg){
                  if(err) return callback(err);
                  var artInd = tools.getArtInd(pkg); // index of the main article in pkg.article

                  // 3. Convert xml + pkg to html
                  // a. two steps conversion of the xml articleBody: xml -> json -> html
                  var jsonBody = xml2json(xml);
                  tools.json2html(that, jsonBody, pkg, function(err, htmlBody){
                    if(err) return callback(err);

                    // b. if formulas have been inlined as base 64 in the text,
                    // they're removed from the pkg resources
                    removeInlineFormulas(pkg, that, function(err,pkg){
                      if(err) return callback(err);

                      // c. integrate the html article as a resource of the pkg
                      fs.writeFile(path.join(that.root, pkg.article[artInd].name + '.html'), htmlBody, function(err){
                        if(err) return callback(err);

                        that.paths2resources([path.join(that.root,pkg.article[artInd].name + '.html')], function(err,resources){
                          if(err) return callback(err);
                          pkg.article[artInd].encoding.push(resources.article[0].encoding[0]);

                          // d. extract pubmed annotations, adapt the target, and add to the pkg
                          tools.addPubmedAnnotations(pkg, pubmedPkg, that, function(err,pkg){
                            if(err) return callback(err);
                            callback(null, pkg);
                          });

                        });
                      });
                    });
                  });
                });
              });

            });
          });
        });


      });
    });
  } else {
    callback(new Error('unrecognized uri'));
  }

};

function extractPdfName(body){
  if(body.indexOf('format="pdf"')>-1){
    var tmp = tools.extractBetween(body, 'format="pdf"');
    var href = tools.extractBetween(tmp, 'href="','"');
    var tmp = path.basename(href.slice(6),path.extname(href.slice(6)));
    return tmp.slice(0,tmp.lastIndexOf('.'));
  } else {
    return null;
  }
};

function extractNXMLName(files){
  var name;
  files.forEach(function(f){
    if(path.extname(path.basename(f))==='.nxml'){
      name = path.basename(path.basename(f),path.extname(path.basename(f)));
    }
  })
  return name;
}

function fetchTar(uri, ldpm, callback){
  // return the list of files contained in the tar.gz of the article,
  // and move them to the current directory

  var root = ldpm.root;
  var c = new Client();

  ldpm.logHttp('GET', uri);
  c.on('ready', function() {
    temp.mkdir('__ldpmTmp', function(err, dirPath) {
      c.get(uri, function(err, stream) {
        if (err) return callback(err);<commit_msg>dealing with cases where there's no pdf<commit_after>var request = require('request')
  , fs = require('fs')
  , url = require('url')
  , async = require('async')
  , path = require('path')
  , temp = require('temp')
  , tar = require('tar')
  , once = require('once')
  , pubmed = require('./pubmed').pubmed
  , Client = require('ftp')
  , xml2js = require('xml2js')
  , DecompressZip = require('decompress-zip')
  , zlib = require('zlib')
  , traverse = require('traverse')
  , recursiveReaddir = require('recursive-readdir')
  , DOMParser = require('xmldom').DOMParser
  , tools = require('./lib/tools');

process.maxTickDepth = 10000;
// to avoid warnings when using nextTick
// https://groups.google.com/forum/#!topic/nodejs/9_uM04IDNWg

temp.track();

module.exports = oapmc;

/**
 * 'this' is an Ldpm instance
 */
function oapmc(uri, opts, callback){

  callback = once(callback);

  if(arguments.length === 2){
    callback = opts;
    opts = {};
  }

  var that = this;
  var pmcid = tools.extractBetween(uri, 'PMC');

  // check url
  if (uri.slice(0, 53) === 'http://www.pubmedcentral.nih.gov/utils/oa/oa.fcgi?id=' ){

    // 0. Preliminary fetches
    that.logHttp('GET', uri);
    // Fetch the url of the tar.gz of the article
    request(uri, function(error, response, oaContentBody){
      if(error) return callback(error);

      that.logHttp(response.statusCode, uri);

      if(response.statusCode >= 400){
        var err = new Error(oaContentBody);
        err.code = response.statusCode;
        return callback(err);
      }

      // First way to get the name of the main article: from the pdf name in oaContentBody
      // that contains pdf and tar.gz
      mainArticleName = extractPdfName(oaContentBody);

      var conversionUrl = 'http://www.pubmedcentral.nih.gov/utils/idconv/v1.0/?ids=' + 'PMC' + pmcid + '&format=json';
      that.logHttp('GET', conversionUrl);
      // For PMC article, the idconv api returns {pmid,pmcid,doi} when given any of the three.
      request(conversionUrl, function(error, response, idConversionBody) {
        if(error) return callback(error);

        that.logHttp(response.statusCode,conversionUrl);

        if(response.statusCode >= 400){
          var err = new Error(idConversionBody);
          err.code = response.statusCode;
          return callback(err);
        }

        var res = JSON.parse(idConversionBody);
        var doi = res['records'][0]['doi'];
        var pmid = res['records'][0]['pmid'];
        if(pmid==undefined){
          // OAPMC entries do not all have a PMID (eg PMC3875093)
          opts.noPubmed = true;
        }

        // 1. Fetch : resources, xml, and pubmed metadata
        // a. resources
        fetchTar(tools.extractBetween(oaContentBody, 'href="', '"').slice(27), that, function(err, files){
          if(err) return callback(err);

          // Second way to get the name of the main article: from the name of the nxml file
          // in the tar.gz. OAPMC entries always have at least a pdf or an nxml.
          if(mainArticleName===undefined){
            mainArticleName = extractNXMLName(files);
          }

          // b. xml
          fetchXml('http://www.pubmedcentral.nih.gov/oai/oai.cgi?verb=GetRecord&identifier=oai:pubmedcentral.nih.gov:' + pmcid + '&metadataPrefix=pmc', that, function(err, xml){
            if(err) return callback(err);

            // c. pubmed metadata
            fetchPubmedMetadata('http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id='+pmid+'&rettype=abstract&retmode=xml', that, opts, function(err,pubmedPkg){
              if(err) return callback(err);

              var pkg = { version: '0.0.0' };

              // 2. Parse and complete pkg
              // a. resources: identify different encodings, substitute plos urls to contentPaths
              parseResources(pkg, files, doi, that, function(err,pkg){
                if(err) return callback(err);
                // b. xml: get captions, citations, authors, publishers etc from the xml
                parseXml(xml,pkg,pmcid,mainArticleName,that,opts,function(err,pkg){
                  if(err) return callback(err);
                  var artInd = tools.getArtInd(pkg); // index of the main article in pkg.article

                  // 3. Convert xml + pkg to html
                  // a. two steps conversion of the xml articleBody: xml -> json -> html
                  var jsonBody = xml2json(xml);
                  tools.json2html(that, jsonBody, pkg, function(err, htmlBody){
                    if(err) return callback(err);

                    // b. if formulas have been inlined as base 64 in the text,
                    // they're removed from the pkg resources
                    removeInlineFormulas(pkg, that, function(err,pkg){
                      if(err) return callback(err);

                      // c. integrate the html article as a resource of the pkg
                      fs.writeFile(path.join(that.root, pkg.article[artInd].name + '.html'), htmlBody, function(err){
                        if(err) return callback(err);

                        that.paths2resources([path.join(that.root,pkg.article[artInd].name + '.html')], function(err,resources){
                          if(err) return callback(err);
                          if(pkg.article[artInd].encoding==undefined){
                            pkg.article[artInd].encoding = [];
                          }
                          pkg.article[artInd].encoding.push(resources.article[0].encoding[0]);

                          // d. extract pubmed annotations, adapt the target, and add to the pkg
                          tools.addPubmedAnnotations(pkg, pubmedPkg, that, function(err,pkg){
                            if(err) return callback(err);
                            callback(null, pkg);
                          });

                        });
                      });
                    });
                  });
                });
              });

            });
          });
        });


      });
    });
  } else {
    callback(new Error('unrecognized uri'));
  }

};

function extractPdfName(body){
  if(body.indexOf('format="pdf"')>-1){
    var tmp = tools.extractBetween(body, 'format="pdf"');
    var href = tools.extractBetween(tmp, 'href="','"');
    var tmp = path.basename(href.slice(6),path.extname(href.slice(6)));
    return tmp.slice(0,tmp.lastIndexOf('.')).replace(/\./g, '-');
  } else {
    return null;
  }
};

function extractNXMLName(files){
  var name;
  files.forEach(function(f){
    if(path.extname(path.basename(f))==='.nxml'){
      name = path.basename(path.basename(f),path.extname(path.basename(f))).replace(/\./g, '-');
    }
  })
  return name;
}

function fetchTar(uri, ldpm, callback){
  // return the list of files contained in the tar.gz of the article,
  // and move them to the current directory

  var root = ldpm.root;
  var c = new Client();

  ldpm.logHttp('GET', uri);
  c.on('ready', function() {
    temp.mkdir('__ldpmTmp', function(err, dirPath) {
      c.get(uri, function(err, stream) {
        if (err) return callback(err);
{'fraction_non_alphanumeric': 0.13839920386693205, 'fraction_numerical': 0.0072504975831674725, 'mean_word_length': 2.184472612041648, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 36}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5062885', 'n_tokens_mistral': 4340, 'n_tokens_neox': 4083, 'n_words': 997}