<commit_before>var levelgraph, levelgraphN3, restify, uuid, http, url, Q, db, server, doProxy, proxyHelper, config, tr, serviceConfig, localSuggestHelper, IDBASE;

levelup = require ('level');
levelgraph = require('levelgraph');
levelgraphN3 = require('levelgraph-n3');
N3 = require('n3');
restify = require('restify');
uuid = require('uuid');
url = require('url');
Q = require('q');
http = require('q-io/http');
tr = require('./translate');

IDBASE = 'http://example.org/id';

server = restify.createServer();
server.use(restify.bodyParser({ mapParams: false }));

/**
 * To configure using a separate file, create config.js in this
 * directory.  See below for what attributes module.exports can take.
 */
try {
    config = require('./config');
} catch (ex) {
    config = {
        'store': {
            'path': './bfstore'
        },
        'listen': 8888,
        'staticDir': '.'
    }
}

serviceConfig = {
    'agrovoc': {
        'config': {
            'host': 'foris.fao.org',
            'path': '/agrovoc/term/find',
            'queryArgs': '&hits=8&match=freeText&suggestions=20&language=EN&relationshipType[]=alternative&relationshipType[]=broader&callback=',
            'arg': 'q'
        }
    },
    'fast': {
        'config': {
            'host': 'fast.oclc.org',
            'path': '/searchfast/fastsuggest',
            'queryArgs': '&queryIndex=suggestall&queryReturn=suggestall%2Cidroot%2Cauth%2Ctag%2Ctype%2Craw%2Cbreaker%2Cindicator&rows=10',
            'arg': 'query'
        }
    },
    'viaf': {
        'config': {
            'host': 'viaf.org',
            'path': '/viaf/AutoSuggest',
            'queryArgs': '',
            'arg': 'query'
        }
    },
    'lc': {
        'config': {
            'host': 'id.loc.gov',
            'path': '/suggest/',
            'queryArgs': '',
            'arg': 'q'
        },
        'subjects': {
            'branch': '/authorities/subjects'
        },
        'geo': {
            'branch': '/vocabulary/geographicAreas'
        },
        'names': {
            'branch': '/authorities/names'
        },
        'lang': {
            'branch': '/vocabulary/iso639-2'
        },
        'works': {
            'branch': '/authorities/names'
        },
        'mediaTypes': {
            'branch': '/vocabulary/mediaTypes'
        },
        'contentTypes': {
            'branch': '/vocabulary/contentTypes'
        },
        'carrier': {
            'branch': '/vocabulary/carriers'
        }
    },
    'local': {
        'works': {
            'types': [
                'http://bibframe.org/vocab/Work',
                'http://bibframe.org/vocab/Book',
                'http://bibframe.org/vocab/proposed/EBook',
                'http://bibframe.org/vocab/proposed/PhysicalBook',
                'http://bibframe.org/vocab/Article',
                'http://bibframe.org/vocab/proposed/EArticle',
                'http://bibframe.org/vocab/proposed/PhysicalArticle',
                'http://bibframe.org/vocab/Painting'
            ]
        },
        'agents': {
            'types': [
                'http://bibframe.org/vocab/Family',
                'http://bibframe.org/vocab/Jurisdiction',
                'http://bibframe.org/vocab/Meeting',
                'http://bibframe.org/vocab/Organization',
                'http://bibframe.org/vocab/Person',
                'http://bibframe.org/vocab/rda/Agent'
            ]
        },
        'lang': {
            'types': [
                'http://bibframe.org/vocab/LanguageEntity'
            ]
        },
        'providers': {
            'types': [
                'http://bibframe.org/vocab/PublisherEvent',
                'http://bibframe.org/vocab/ManufactureEvent',
                'http://bibframe.org/vocab/ProducerEvent',
                'http://bibframe.org/vocab/DistributeEvent'
            ]
        },
        'geo': {
            'types': [
                'http://bibframe.org/vocab/rda/Place',
                'http://bibframe.org/vocab/Place'
            ]
        },
        'subjects': {
            'types': [
                'http://bibframe.org/vocab/Topic',
                'http://bibframe.org/vocab/TemporalConcept',
                'http://bibframe.org/vocab/ClassificationEntity',
                'http://bibframe.org/vocab/rda/Authority',
                'http://bibframe.org/vocab/rda/Period',
                'http://bibframe.org/vocab/rda/Technique',
                'http://bibframe.org/vocab/rda/WorkType'
            ]
        }
    }
};

db = levelgraphN3(levelgraph(
    levelup(config.store.path, {
        createIfMissing: true,
        errorIfExists: false,
        compression: true
    }),
    { joinAlgorithm: "basic" }
));

// Store the n3 propety of an incoming JSON object, because just
// getting the raw body is apparently more difficult; may come in
// handy, possibly also pass a list of IDs to save, and delete
// existing nodes with those IDs before saving.
server.put('/resource/new', function newResource(req, res, next) {
    db.n3.put(req.body.n3, function putResult(err) {
        var success = (typeof err === "undefined" || err === null) ? true : false;
        res.send(201, {'success': success});
    });
    return next();
});

localSuggestHelper = function(qin, types) {
    var i, query, deferred, answer;
    answer = [];
    deferred = Q.defer();

    query = [{
        subject: db.v("s"),
        predicate: db.v("p"),
        object: db.v("o"),
        filter: function predFilter(triple) {<commit_msg>make PUT respond with 500 if error<commit_after>var levelgraph, levelgraphN3, restify, uuid, http, url, Q, db, server, doProxy, proxyHelper, config, tr, serviceConfig, localSuggestHelper, IDBASE;

levelup = require ('level');
levelgraph = require('levelgraph');
levelgraphN3 = require('levelgraph-n3');
N3 = require('n3');
restify = require('restify');
uuid = require('uuid');
url = require('url');
Q = require('q');
http = require('q-io/http');
tr = require('./translate');
// sort of silence Node warnings, appears to be a levelgraph usage that
// triggers too many listeners warnings.
require('events').EventEmitter.prototype._maxListeners = 20;

IDBASE = 'http://example.org/id';

server = restify.createServer();
server.use(restify.bodyParser({ mapParams: false }));

/**
 * To configure using a separate file, create config.js in this
 * directory.  See below for what attributes module.exports can take.
 */
try {
    config = require('./config');
} catch (ex) {
    config = {
        'store': {
            'path': './bfstore'
        },
        'listen': 8888,
        'staticDir': '.'
    }
}

serviceConfig = {
    'agrovoc': {
        'config': {
            'host': 'foris.fao.org',
            'path': '/agrovoc/term/find',
            'queryArgs': '&hits=8&match=freeText&suggestions=20&language=EN&relationshipType[]=alternative&relationshipType[]=broader&callback=',
            'arg': 'q'
        }
    },
    'fast': {
        'config': {
            'host': 'fast.oclc.org',
            'path': '/searchfast/fastsuggest',
            'queryArgs': '&queryIndex=suggestall&queryReturn=suggestall%2Cidroot%2Cauth%2Ctag%2Ctype%2Craw%2Cbreaker%2Cindicator&rows=10',
            'arg': 'query'
        }
    },
    'viaf': {
        'config': {
            'host': 'viaf.org',
            'path': '/viaf/AutoSuggest',
            'queryArgs': '',
            'arg': 'query'
        }
    },
    'lc': {
        'config': {
            'host': 'id.loc.gov',
            'path': '/suggest/',
            'queryArgs': '',
            'arg': 'q'
        },
        'subjects': {
            'branch': '/authorities/subjects'
        },
        'geo': {
            'branch': '/vocabulary/geographicAreas'
        },
        'names': {
            'branch': '/authorities/names'
        },
        'lang': {
            'branch': '/vocabulary/iso639-2'
        },
        'works': {
            'branch': '/authorities/names'
        },
        'mediaTypes': {
            'branch': '/vocabulary/mediaTypes'
        },
        'contentTypes': {
            'branch': '/vocabulary/contentTypes'
        },
        'carrier': {
            'branch': '/vocabulary/carriers'
        }
    },
    'local': {
        'works': {
            'types': [
                'http://bibframe.org/vocab/Work',
                'http://bibframe.org/vocab/Book',
                'http://bibframe.org/vocab/proposed/EBook',
                'http://bibframe.org/vocab/proposed/PhysicalBook',
                'http://bibframe.org/vocab/Article',
                'http://bibframe.org/vocab/proposed/EArticle',
                'http://bibframe.org/vocab/proposed/PhysicalArticle',
                'http://bibframe.org/vocab/Painting'
            ]
        },
        'agents': {
            'types': [
                'http://bibframe.org/vocab/Family',
                'http://bibframe.org/vocab/Jurisdiction',
                'http://bibframe.org/vocab/Meeting',
                'http://bibframe.org/vocab/Organization',
                'http://bibframe.org/vocab/Person',
                'http://bibframe.org/vocab/rda/Agent'
            ]
        },
        'lang': {
            'types': [
                'http://bibframe.org/vocab/LanguageEntity'
            ]
        },
        'providers': {
            'types': [
                'http://bibframe.org/vocab/PublisherEvent',
                'http://bibframe.org/vocab/ManufactureEvent',
                'http://bibframe.org/vocab/ProducerEvent',
                'http://bibframe.org/vocab/DistributeEvent'
            ]
        },
        'geo': {
            'types': [
                'http://bibframe.org/vocab/rda/Place',
                'http://bibframe.org/vocab/Place'
            ]
        },
        'subjects': {
            'types': [
                'http://bibframe.org/vocab/Topic',
                'http://bibframe.org/vocab/TemporalConcept',
                'http://bibframe.org/vocab/ClassificationEntity',
                'http://bibframe.org/vocab/rda/Authority',
                'http://bibframe.org/vocab/rda/Period',
                'http://bibframe.org/vocab/rda/Technique',
                'http://bibframe.org/vocab/rda/WorkType'
            ]
        }
    }
};

db = levelgraphN3(levelgraph(
    levelup(config.store.path, {
        createIfMissing: true,
        errorIfExists: false,
        compression: true
    }),
    { joinAlgorithm: "basic" }
));

// Store the n3 propety of an incoming JSON object, because just
// getting the raw body is apparently more difficult; may come in
// handy, possibly also pass a list of IDs to save, and delete
// existing nodes with those IDs before saving.
server.put('/resource/new', function newResource(req, res, next) {
    db.n3.put(req.body.n3, function putResult(err) {
        if (typeof err === "undefined" || err === null) {
            res.send(201, {'success': true});
        } else {
            console.log(err);
            res.send(500, {'success': false});
        }
    });
    // This would allow things to proceed as it appears the put() call
    // doesn't ever call its callback.
    // res.send(201, {'success': true});
    return next();
});

localSuggestHelper = function(qin, types) {
    var i, query, deferred, answer;
    answer = [];
    deferred = Q.defer();

    query = [{
        subject: db.v("s"),
        predicate: db.v("p"),
        object: db.v("o"),
        filter: function predFilter(triple) {
{'fraction_non_alphanumeric': 0.1640529709008538, 'fraction_numerical': 0.006534239414532148, 'mean_word_length': 1.7653577451216573, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4171445', 'n_tokens_mistral': 3701, 'n_tokens_neox': 3527, 'n_words': 597}