<commit_before>var greyhoundIngestService = module.exports = {};

var q = require('q');
var Xray = require('x-ray');
var xray = Xray();

greyhoundIngestService.grvUrl = "https://fasttrack.grv.org.au";

greyhoundIngestService.getExternalGreyhoundInfo = function(greyhound){
    if (greyhound && greyhound.name){
        var url = greyhoundIngestService.buildGRVGreyhoundUrl(greyhound.name);
        var selector = {
            id: "a[rel='dog-details']@href",
            sireName: "a[rel='View-Sire']",
            damName: "a[rel='View-Dam']"
        };
        return greyhoundIngestService.downloadInfo(url, selector).then(greyhoundIngestService.grvPostProcess);
    } else {
        q.reject("cannot retieve info on invalid greyhound or greyhound name")
    }
};

greyhoundIngestService.buildGRVGreyhoundUrl = function(name){
    return greyhoundIngestService.grvUrl +"/Dog/Search?Dog=" + encodeURI(name) + "&ExactMatch=true&Search=Search";
};

greyhoundIngestService.grvPostProcess = function(result){
    result.source = "fasttrack";
    if (result.id){
        result.url = result.id;
        result.id = result.id.replace('https://fasttrack.grv.org.au/Dog/Details/-','');
    }

    return q(result);
};

greyhoundIngestService.downloadInfo = function(url, selector){
    var deferred = q.defer();
    xray(url, selector)(function(err, result) {
        if(err){
            deferred.reject(err);
        } else {
            deferred.resolve(result);
        }
    });
    return deferred.promise;
};<commit_msg>Added ref to the external info lookup<commit_after>var greyhoundIngestService = module.exports = {};

var q = require('q');
var Xray = require('x-ray');
var xray = Xray();
var logger = require('winston');

greyhoundIngestService.grvUrl = "https://fasttrack.grv.org.au";

greyhoundIngestService.getExternalGreyhoundInfo = function(greyhound){
    if (greyhound && greyhound.name){
        var url = greyhoundIngestService.buildGRVGreyhoundUrl(greyhound.name);
        var selector = {
            url: "a[rel='dog-details']@href",
            sireName: "a[rel='View-Sire']",
            damName: "a[rel='View-Dam']"
        };
        return greyhoundIngestService.downloadInfo(url, selector).then(greyhoundIngestService.addRef);
    } else {
        q.reject("cannot retrieve info on invalid greyhound or greyhound name")
    }
};

greyhoundIngestService.buildGRVGreyhoundUrl = function(name){
    return greyhoundIngestService.grvUrl +"/Dog/Search?Dog=" + encodeURI(name) + "&ExactMatch=true&Search=Search";
};

greyhoundIngestService.addRef = function(result){
    var ref = {};
    ref.source = "fasttrack";
    if (result.url){
        ref.url = result.url;
        ref.id = result.id.replace('https://fasttrack.grv.org.au/Dog/Details/','');
    }
    result.ref = ref;
    return q(result);
};

greyhoundIngestService.downloadInfo = function(url, selector){
    var deferred = q.defer();
    logger.info("API request [" + url + "]");
    xray(url, selector)(function(err, result) {
        if(err){
            deferred.reject(err);
        } else {
            deferred.resolve(result);
        }
    });
    return deferred.promise;
};
{'fraction_non_alphanumeric': 0.14601769911504425, 'fraction_numerical': 0.0, 'mean_word_length': 3.2828146143437076, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1674703', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1035, 'n_words': 170}