<commit_before>'use strict';

var util = require('util');
var path = require('path');
var fs = require('fs');
var _ = require('lodash');
var Q = require('q');

var logger = require('./logger.js');
var db = require('./data.js');
var spotify = require('./spotify.js');

var processingQueue = {
    queue: []
};
var running = false;

var cachePath = path.join(path.dirname(require.main.filename), 'data', 'queue.txt');

var eventLoop = function () {
    if (!running) {
        return;
    }

    if (processingQueue.queue.length === 0) {
        console.log('empty queue');

        setTimeout(eventLoop, 1000);
        return;
    }

    // Pop an item off of the queue and process it
    processPlaylist(processingQueue.queue.pop())
    .then(function () {
        setTimeout(eventLoop, 1000);
    });
};

/**
 * Process the passed in playlist
 *
 * @param {Object} playlist The playlist to process
 * @return {Promise}
 */
function processPlaylist(spotifyPlaylist) {
    logger.info('processing ' + spotifyPlaylist.name);
    return db.findByUri(spotifyPlaylist.uri)
    .then(function (playlist) {
        if (!playlist) {
           return importPlaylist(spotifyPlaylist);
        } else {
            return importPlaylist(spotifyPlaylist);
        }
    })
    .fail(function (err) {
        logger.error('processItem error');
        console.log(err);
        logger.error(err);
        logger.error(err.stack);
    });
}

function importPlaylist(playlist) {
    return db.create({
        name: playlist.name,
        uri: playlist.uri
    })
    .then(function (node) {
        return db.addLabel(node.id, 'Playlist')
        .then(function () {
            return spotify.getTrackData(playlist.tracks);
        })
        .then(function processPlaylistItems (tracks) {
            if (tracks.length === 0) {
                return;
            }

            var processItem = function (track) {
                return importPlaylistTrack(track)
                .then(function (trackNode) {
                    return db.createRelationship(node, trackNode, 'MEMBER', {})
                    .then(function () {
                        if (tracks.length === 0) {
                            return;
                        } else {
                            console.log('keep going');
                            return processItem(tracks.pop().track);
                        }
                    });
                })
            }

            return processItem(tracks.pop().track);
        });
    });
}

function importPlaylistTrack(track) {
    var artists = track.artists;
    var album = track.album;

    return importArtists(artists)
    .then(function (artistNodes) {
        return importAlbum(album, artistNodes)
        .then(function (albumNode) {
            return importTrack(track, artistNodes, albumNode);
        });
    });
}

function importArtists(artists) {
    var nodes = [];

    var importArtist = function (artist) {
        return db.importObject({
            name: artist.name,
            uri: artist.href
        }, 'Artist')
        .then(function (artistNode) {
            nodes.push(artistNode);
            if (artists.length === 0) {
                return nodes;
            } else {
                return importArtist(artists.pop());
            }
        });
    };

    return importArtist(artists.pop());
}

function importAlbum(album, artistNodes) {
    return db.importObject({
        name: album.name,
        released: album.released,
        uri: album.href
    }, 'Album')
    .then(function (albumNode) {
        var promises = [];

        _.each(artistNodes, function (artistNode) {
            promises.push(db.createRelationship(artistNode, albumNode, 'RELEASED'));
        });

        return Q.allSettled(promises)
        .then(function (results) {<commit_msg>Set position of items in playlists. Use shift() instead of pop()<commit_after>'use strict';

var util = require('util');
var path = require('path');
var fs = require('fs');
var _ = require('lodash');
var Q = require('q');

var logger = require('./logger.js');
var db = require('./data.js');
var spotify = require('./spotify.js');

var processingQueue = {
    queue: []
};
var running = false;

var cachePath = path.join(path.dirname(require.main.filename), 'data', 'queue.txt');

var eventLoop = function () {
    if (!running) {
        return;
    }

    if (processingQueue.queue.length === 0) {
        console.log('empty queue');

        setTimeout(eventLoop, 1000);
        return;
    }

    // Pop an item off of the queue and process it
    processPlaylist(processingQueue.queue.shift())
    .then(function () {
        setTimeout(eventLoop, 1000);
    });
};

/**
 * Process the passed in playlist
 *
 * @param {Object} playlist The playlist to process
 * @return {Promise}
 */
function processPlaylist(spotifyPlaylist) {
    logger.info('processing ' + spotifyPlaylist.name);
    return db.findByUri(spotifyPlaylist.uri)
    .then(function (playlist) {
        if (!playlist) {
           return importPlaylist(spotifyPlaylist);
        } else {
            return importPlaylist(spotifyPlaylist);
        }
    })
    .fail(function (err) {
        logger.error('processItem error');
        console.log(err);
        logger.error(err);
        logger.error(err.stack);
    });
}

function importPlaylist(playlist) {
    return db.create({
        name: playlist.name,
        uri: playlist.uri
    })
    .then(function (node) {
        return db.addLabel(node.id, 'Playlist')
        .then(function () {
            return spotify.getTrackData(playlist.tracks);
        })
        .then(function processPlaylistItems (tracks) {
            var counter = 1;
            if (tracks.length === 0) {
                return;
            }

            var processItem = function (track) {
                return importPlaylistTrack(track)
                .then(function (trackNode) {
                    return db.createRelationship(node, trackNode, 'MEMBER', { position: counter})
                    .then(function () {
                        if (tracks.length === 0) {
                            return;
                        } else {
                            counter++;
                            return processItem(tracks.shift().track);
                        }
                    });
                })
            }

            return processItem(tracks.shift().track);
        });
    });
}

function importPlaylistTrack(track) {
    var artists = track.artists;
    var album = track.album;

    return importArtists(artists)
    .then(function (artistNodes) {
        return importAlbum(album, artistNodes)
        .then(function (albumNode) {
            return importTrack(track, artistNodes, albumNode);
        });
    });
}

function importArtists(artists) {
    var nodes = [];

    var importArtist = function (artist) {
        return db.importObject({
            name: artist.name,
            uri: artist.href
        }, 'Artist')
        .then(function (artistNode) {
            nodes.push(artistNode);
            if (artists.length === 0) {
                return nodes;
            } else {
                return importArtist(artists.shift());
            }
        });
    };

    return importArtist(artists.shift());
}

function importAlbum(album, artistNodes) {
    return db.importObject({
        name: album.name,
        released: album.released,
        uri: album.href
    }, 'Album')
    .then(function (albumNode) {
        var promises = [];

        _.each(artistNodes, function (artistNode) {
            promises.push(db.createRelationship(artistNode, albumNode, 'RELEASED'));
        });

        return Q.allSettled(promises)
        .then(function (results) {
{'fraction_non_alphanumeric': 0.12818861118268488, 'fraction_numerical': 0.003220819376449369, 'mean_word_length': 1.8966417910447761, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7095793', 'n_tokens_mistral': 2218, 'n_tokens_neox': 2135, 'n_words': 458}