<commit_before>var config;
var creds = require(process.env.HOME + '/.googlePlayCreds.json');
var PlayMusic = require('playmusic');
var mkdirp = require('mkdirp');
var https = require('https');
var send = require('send');
var url = require('url');
var fs = require('fs');

var gmusicBackend = {};

var gmusicDownload = function(startUrl, songID, callback, errCallback) {
    var doDownload = function(streamUrl) {
        console.log('downloading song ' + songID);
        var filePath = config.songCachePath + '/gmusic/' + songID + '.mp3';
        var songFd = fs.openSync(filePath, 'w');

        var req = https.request(streamUrl, function(res) {

            res.on('data', function(chunk) {
                fs.writeSync(songFd, chunk, 0, chunk.length, null);
            });
            res.on('end', function() {
                if(res.statusCode === 302) { // redirect
                    console.log('redirected. retrying with new URL');
                    fs.closeSync(songFd);
                    fs.unlinkSync(config.songCachePath + '/gmusic/' + songID + '.mp3');
                    gmusicDownload(res.headers.location, songID, callback, errCallback);
                } else if(res.statusCode === 200) {
                    console.log('download finished ' + songID);
                    fs.closeSync(songFd);
                    if(callback)
                        callback();
                } else {
                    console.log('ERROR: unknown status code ' + res.statusCode);
                    fs.closeSync(songFd);
                    fs.unlinkSync(config.songCachePath + '/gmusic/' + songID + '.mp3');
                    if(errCallback)
                        errCallback('error while pre-caching ' + songID);
                }
            });
        });
        req.on('error', function(e) {
            console.log('error ' + e + ' while fetching! reconnecting in 5s...');
            setTimeout(function() {
                gmusicBackend.init(function() {
                    console.log('error while fetching! now reconnected to gmusic');
                    gmusicBackend.pm.getStreamUrl(songID, function(streamUrl) {
                        gmusicDownload(streamUrl, songID, callback, errCallback);
                    });
                });
            }, 5000);
        });
        req.end();
    };

    if(startUrl) {
        doDownload(startUrl);
    } else {
        gmusicBackend.pm.getStreamUrl(songID, function(streamUrl) {
            doDownload(streamUrl);
        });
    }
};

// cache songID to disk.
// on success: callback must be called
// on failure: errCallback must be called with error message
gmusicBackend.prepareSong = function(songID, callback, errCallback) {
    var filePath = config.songCachePath + '/gmusic/' + songID + '.mp3';

    if(fs.existsSync(filePath)) {
        // song was found from cache
        if(callback)
            callback();
        return;
    } else {
        // song had to be downloaded
        gmusicDownload(null, songID, callback, errCallback);
    }
};
<commit_msg>gmusic: remove race condition where song could possibly be downloaded twice<commit_after>var config;
var creds = require(process.env.HOME + '/.googlePlayCreds.json');
var PlayMusic = require('playmusic');
var mkdirp = require('mkdirp');
var https = require('https');
var send = require('send');
var url = require('url');
var fs = require('fs');

var gmusicBackend = {};

var gmusicDownload = function(startUrl, songID, callback, errCallback) {
    var doDownload = function(streamUrl) {
        console.log('downloading song ' + songID);
        var filePath = config.songCachePath + '/gmusic/' + songID + '.mp3';
        var songFd = fs.openSync(filePath, 'w');

        var req = https.request(streamUrl, function(res) {

            res.on('data', function(chunk) {
                fs.writeSync(songFd, chunk, 0, chunk.length, null);
            });
            res.on('end', function() {
                if(res.statusCode === 302) { // redirect
                    console.log('redirected. retrying with new URL');
                    fs.closeSync(songFd);
                    fs.unlinkSync(config.songCachePath + '/gmusic/' + songID + '.mp3');
                    gmusicDownload(res.headers.location, songID, callback, errCallback);
                } else if(res.statusCode === 200) {
                    console.log('download finished ' + songID);
                    fs.closeSync(songFd);
                    if(callback)
                        callback();
                } else {
                    console.log('ERROR: unknown status code ' + res.statusCode);
                    fs.closeSync(songFd);
                    fs.unlinkSync(config.songCachePath + '/gmusic/' + songID + '.mp3');
                    if(errCallback)
                        errCallback('error while pre-caching ' + songID);
                }
            });
        });
        req.on('error', function(e) {
            console.log('error ' + e + ' while fetching! reconnecting in 5s...');
            setTimeout(function() {
                gmusicBackend.init(function() {
                    console.log('error while fetching! now reconnected to gmusic');
                    gmusicBackend.pm.getStreamUrl(songID, function(streamUrl) {
                        gmusicDownload(streamUrl, songID, callback, errCallback);
                    });
                });
            }, 5000);
        });
        req.end();
    };

    if(startUrl) {
        doDownload(startUrl);
    } else {
        gmusicBackend.pm.getStreamUrl(songID, function(streamUrl) {
            doDownload(streamUrl);
        });
    }
};

var pendingSongs = {};
// cache songID to disk.
// on success: callback must be called
// on failure: errCallback must be called with error message
gmusicBackend.prepareSong = function(songID, callback, errCallback) {
    var filePath = config.songCachePath + '/gmusic/' + songID + '.mp3';

    // song is already downloading
    if(pendingSongs[songID]) {
        return;
    }

    if(fs.existsSync(filePath)) {
        // song was found from cache
        if(callback)
            callback();
        return;
    } else {
        // song had to be downloaded
        pendingSongs[songID] = true;
        gmusicDownload(null, songID, function() {
            delete(pendingSongs[songID]);
            callback();
        }, function() {
            delete(pendingSongs[songID]);
            errCallback();
        });
    }
};

{'fraction_non_alphanumeric': 0.12767925983037778, 'fraction_numerical': 0.004934464148033924, 'mean_word_length': 1.8298429319371727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1310101', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1722, 'n_words': 390}