<commit_before>'use strict';

//
// This is a wrapper for the Stanford Parser
// http://nlp.stanford.edu/software/lex-parser.shtml
//
// It parses 'wordsAndTags' and 'penn' formats.
//
// To parse stuff the following command can be used from the
// ./external/stanford-parser directory:
//
//     $ java -mx200m -cp "stanford-parser.jar" \
//     edu.stanford.nlp.parser.lexparser.LexicalizedParser \
//     -outputFormat "wordsAndTags,penn" \
//     models/englishPCFG.caseless.ser.gz <INPUT-FILE>
//
// Online demo can be accessed at:
// http://nlp.stanford.edu:8080/parser/index.jsp
//

var os = require('os');
var path = require('path');
var fs = require('fs');
var execSync = require('child_process').execSync;

var javaOpts = '-mx200m';
var outputFormat = 'wordsAndTags,penn';
var parser = 'models/englishPCFG.caseless.ser.gz';

function StanfordParser() {

}

Object.defineProperties(StanfordParser.prototype, {
    wordsAndTags: {
        set: function(raw) {
            this._wordsAndTags = raw;
        }
    },
    penn: {
        set: function(raw) {
            this._penn = parsePenn(raw);
        }
    }
});

// Returns true if at least one verb exists in the sentence
StanfordParser.prototype.hasVerb = function() {
    return /\S\/VB/.test(this._wordsAndTags);
}

StanfordParser.parseSentencesSync = function(sentences) {
    var instances = [];
    var tmpfile = path.resolve(os.tmpdir(), 'nlp-parser-sentences.txt');
    var parserWd = path.resolve(__dirname, '../external/stanford-parser');

    // Create a temp file containing the given sentences
    var fd = fs.openSync(tmpfile, 'w');
    fs.writeSync(fd, sentences.join('\n'));
    fs.closeSync(fd);

    var cmd = 'java ' + javaOpts +
        ' -cp "stanford-parser.jar" ' +
        'edu.stanford.nlp.parser.lexparser.LexicalizedParser ' +
        '-outputFormat "' + outputFormat + '" ' +
        parser + ' ' + tmpfile;

    try {
        var output = execSync(cmd, {
            cwd: parserWd,
            encoding: 'utf8',
            stdio: [null]<commit_msg>Update from external to vendor in code<commit_after>'use strict';

//
// This is a wrapper for the Stanford Parser
// http://nlp.stanford.edu/software/lex-parser.shtml
//
// It parses 'wordsAndTags' and 'penn' formats.
//
// To parse stuff the following command can be used from the
// ./vendor/stanford-parser directory:
//
//     $ java -mx200m -cp "stanford-parser.jar" \
//     edu.stanford.nlp.parser.lexparser.LexicalizedParser \
//     -outputFormat "wordsAndTags,penn" \
//     models/englishPCFG.caseless.ser.gz <INPUT-FILE>
//
// Online demo can be accessed at:
// http://nlp.stanford.edu:8080/parser/index.jsp
//

var os = require('os');
var path = require('path');
var fs = require('fs');
var execSync = require('child_process').execSync;

var javaOpts = '-mx200m';
var outputFormat = 'wordsAndTags,penn';
var parser = 'models/englishPCFG.caseless.ser.gz';

function StanfordParser() {

}

Object.defineProperties(StanfordParser.prototype, {
    wordsAndTags: {
        set: function(raw) {
            this._wordsAndTags = raw;
        }
    },
    penn: {
        set: function(raw) {
            this._penn = parsePenn(raw);
        }
    }
});

// Returns true if at least one verb exists in the sentence
StanfordParser.prototype.hasVerb = function() {
    return /\S\/VB/.test(this._wordsAndTags);
}

StanfordParser.parseSentencesSync = function(sentences) {
    var instances = [];
    var tmpfile = path.resolve(os.tmpdir(), 'nlp-parser-sentences.txt');
    var parserWd = path.resolve(__dirname, '../vendor/stanford-parser');

    // Create a temp file containing the given sentences
    var fd = fs.openSync(tmpfile, 'w');
    fs.writeSync(fd, sentences.join('\n'));
    fs.closeSync(fd);

    var cmd = 'java ' + javaOpts +
        ' -cp "stanford-parser.jar" ' +
        'edu.stanford.nlp.parser.lexparser.LexicalizedParser ' +
        '-outputFormat "' + outputFormat + '" ' +
        parser + ' ' + tmpfile;

    try {
        var output = execSync(cmd, {
            cwd: parserWd,
            encoding: 'utf8',
            stdio: [null]
{'fraction_non_alphanumeric': 0.14859926918392205, 'fraction_numerical': 0.00535931790499391, 'mean_word_length': 2.682511210762332, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7440794', 'n_tokens_mistral': 1354, 'n_tokens_neox': 1317, 'n_words': 291}