<commit_before>var path        = require("path");
var querystring = require("querystring");
var crypto      = require("crypto");
var underscore  = require("underscore");
var app         = require(path.join(__dirname, "..", "..", "..", "config", "app"));
var settings    = require(path.join(app.root, "config", "settings"));
var redis       = require(path.join(app.root, "config", "redis"));
var RateLimiter = require(path.join(app.root, "app", "lib", "rate_limiter"));
var HttpJson    = require(path.join(app.root, "app", "lib", "http_json"));
var Spotbox     = require(path.join(app.root, "app", "lib", "spotbox"));

var rateLimiter = new RateLimiter(1000.0 / 3); // 3 requests per second

var apiSettings = {
  host: "api.acoustid.org",
  version: "v2"
};

function request(method, data, hollaback) {
  var options = {
    host: apiSettings.host,
    path: "/" + apiSettings.version + "/lookup"
  };

  var query = querystring.stringify(data) + "&meta=recordings+releasegroups+releases";
  rateLimiter.queue(function() {
    if (method === "POST") {
      options.headers = {"Content-Type": "application/x-www-form-urlencoded"};
      HttpJson.post(options, query, hollaback);
    } else {
      options.path += "?" + query;
      HttpJson.get(options, hollaback);
    }
  });
};

function cachedRequest(method, params, cacheKey, hollaback) {
  redis.get(cacheKey, function(error, resultString) {
    if (error) {
      hollaback(error);
    } else if (resultString) {
      hollaback(null, JSON.parse(resultString));
    } else {
      request(method, params, function(error, json) {
        if (error) {
          hollaback(error);
        } else {
          hollaback(null, json);
          redis.set(cacheKey, JSON.stringify(json));
        }
      });
    }
  });
};

function buildRecordingTracks(recording) {
  var artists = recording.artists;
  return underscore.map(recording.releasegroups, function(group) {
    return {
      provider: "acoustid",
      ids: {
        music_brainz: recording.id
      },
      name: recording.title,
      track_number: group.releases[0].track_count,
      length: recording.duration,
      artists: artists,
      album: {
        name: group.title,
        id: group.id,
        released: group.releases[0].date.year
      }
    };
  });
};

function bestResult(data) {
  var result = underscore.sortBy(data.results, function(result) {
    return -result.score;
  })[0];

  return result;
};

var Acoustid = function() {};

Acoustid.fingerprintLookup = function(data, hollaback) {
  var sha = crypto.createHash("sha1");
  sha.update(data.fingerprint);
  sha.update(data.duration);

  var key = Spotbox.namespace("acoustid:fingerprint" + sha.digest("hex"));

  var params = {
    client: settings.acoustid.api_key,
    meta: "recordings+releasegroups+releases",
    duration: data.duration,
    format: "json",
    fingerprint: data.fingerprint
  };

  cachedRequest("POST", params, key, function(error, data) {
    if (error) {
      hollback(error);
    } else {
      if (data.results.length === 0) {
        hollaback();
      } else {
        var result = bestResult(data);
        var acoustidData = {
          acoustid: result.id,
          trackId: result.recordings[0].id,
          albumId: result.recordings[0].releasegroups[0].id
        };

        hollaback(null, acoustidData);
      }
    }
  });
};

Acoustid.groupLookup = function(acoustid, hollaback) {
  var params = {
    client: settings.acoustid.api_key,
    meta: "recordings+releasegroups+releases",
    format: "json",
    trackid: acoustid
  };

  var key = Spotbox.namespace("acoustid:id:" + acoustid);

  cachedRequest("GET", params, key, function(error, data) {
    if (error) {
      hollaback(error);
    } else {
      var result = bestResult(data);
      var recordingTracks = underscore.chain(result.recordings).map(result.recordings, function(recording) {
        return buildRecordingTracks(recording);
      }).flatten().value();

      hollaback(null, recordingTracks);
    }
  });
};

Acoustid.lookup = function(acoustid, trackId, albumId, hollaback) {
  var key = Spotbox.namespace("acoustid:track:" + acoustid + trackId + albumId);

  redis.get(key, function(error, cachedMetadata) {
    if (error) {
      hollaback(error);
    } else if (cachedMetadata) {
      hollback(null, JSON.parse(cachedMetadata));
    } else {
      Acoustid.groupLookup(acoustid, function(error, tracks) {
        if (error) {
          hollaback(error);
        } else {
          var track = underscore.find(tracks, function(track) {
            return track.ids.music_brainz === trackId && track.album.id === albumId;
          });
          redis.set(key, JSON.stringify(track), function(error) {
            hollaback(error, track);
          });
        }
      });
    }
  });
};

module.exports = Acoustid;<commit_msg>Add smart matching to acoustid<commit_after>var path        = require("path");
var querystring = require("querystring");
var crypto      = require("crypto");
var underscore  = require("underscore");
var app         = require(path.join(__dirname, "..", "..", "..", "config", "app"));
var settings    = require(path.join(app.root, "config", "settings"));
var redis       = require(path.join(app.root, "config", "redis"));
var RateLimiter = require(path.join(app.root, "app", "lib", "rate_limiter"));
var HttpJson    = require(path.join(app.root, "app", "lib", "http_json"));
var Spotbox     = require(path.join(app.root, "app", "lib", "spotbox"));

var rateLimiter = new RateLimiter(1000.0 / 3); // 3 requests per second

var apiSettings = {
  host: "api.acoustid.org",
  version: "v2"
};

function request(method, data, hollaback) {
  var options = {
    host: apiSettings.host,
    path: "/" + apiSettings.version + "/lookup"
  };

  var query = querystring.stringify(data) + "&meta=recordings+releasegroups+releases";
  rateLimiter.queue(function() {
    if (method === "POST") {
      options.headers = {"Content-Type": "application/x-www-form-urlencoded"};
      HttpJson.post(options, query, hollaback);
    } else {
      options.path += "?" + query;
      HttpJson.get(options, hollaback);
    }
  });
};

function cachedRequest(method, params, cacheKey, hollaback) {
  redis.get(cacheKey, function(error, resultString) {
    if (error) {
      hollaback(error);
    } else if (resultString) {
      hollaback(null, JSON.parse(resultString));
    } else {
      request(method, params, function(error, json) {
        if (error) {
          hollaback(error);
        } else {
          hollaback(null, json);
          redis.set(cacheKey, JSON.stringify(json));
        }
      });
    }
  });
};

function buildRecordingTracks(recording) {
  var artists = recording.artists;
  return underscore.map(recording.releasegroups, function(group) {
    return {
      provider: "acoustid",
      ids: {
        music_brainz: recording.id
      },
      name: recording.title,
      track_number: group.releases[0].track_count,
      length: recording.duration,
      artists: artists,
      album: {
        name: group.title,
        id: group.id,
        released: group.releases[0].date.year
      }
    };
  });
};

function findBestResult(results) {
  return underscore.sortBy(results, function(recording) {
    return -recording.score;
  })[0];
};


var Acoustid = function() {};

Acoustid.fingerprintLookup = function(data, hollaback) {
  var sha = crypto.createHash("sha1");
  sha.update(data.fingerprint);
  sha.update(data.duration);

  var key = Spotbox.namespace("acoustid:fingerprint:" + sha.digest("hex"));

  var params = {
    client: settings.acoustid.api_key,
    duration: data.duration,
    format: "json",
    fingerprint: data.fingerprint
  };

  cachedRequest("POST", params, key, function(error, data) {
    if (error) {
      hollaback(error);
    } else {
      var result = findBestResult(data.results);
      if (result) {
        hollaback(null, result.id);
      } else {
        hollaback();
      }
    }
  });
};

Acoustid.groupLookup = function(acoustid, hollaback) {
  var params = {
    client: settings.acoustid.api_key,
    format: "json",
    trackid: acoustid
  };

  var key = Spotbox.namespace("acoustid:id:" + acoustid);

  cachedRequest("GET", params, key, function(error, data) {
    if (error) {
      hollaback(error);
    } else {
      var result = findBestResult(data.results);

      var recordingTracks = underscore.reduce(result.recordings, function(memo, recording) {
        try {
          memo = memo.concat(buildRecordingTracks(recording));
        } catch(e) {
          console.log("error building recording track: ", e);
        }
        return memo;
      }, []);

      hollaback(null, recordingTracks);
    }
  });
};

Acoustid.bestMatchLookup = function(acoustidId, sourceTrack, hollaback) {
  Acoustid.groupLookup(acoustidId, function(error, tracks) {
    var filteredTracks = underscore.filter(tracks, function(track) {
      return track.artists && track.artists.length && track.length && track.name;
    });
    if (sourceTrack.name) {
      var orderedTracks = underscore.sortBy(filteredTracks, function(track) {
        var parts = sourceTrack.name.split(/\s/);
        return -underscore.reduce(parts, function(memo, part) {
          if (part.length >= 3) {
            if (track.name.match(part)) {
              memo += part.length;
            };
          }
          return memo;
        }, 0);
      });
      hollaback(null, orderedTracks[0]);
    } else {
      hollaback(null, filteredTracks[0]);
    }
  });
};

Acoustid.lookup = function(acoustidId, trackId, albumId, hollaback) {
  var key = Spotbox.namespace("acoustid:track:" + acoustidId + trackId + albumId);

  redis.get(key, function(error, cachedMetadata) {
    if (error) {
      hollaback(error);
    } else if (cachedMetadata) {
      hollaback(null, JSON.parse(cachedMetadata));
    } else {
      Acoustid.groupLookup(acoustidId, function(error, tracks) {
        if (error) {
          hollaback(error);
        } else {
          var track = underscore.find(tracks, function(track) {
            return track.ids.music_brainz === trackId && track.album.id === albumId;
          });
          redis.set(key, JSON.stringify(track), function(error) {
            hollaback(error, track);
          });
        }
      });
    }
  });
};

module.exports = Acoustid;
{'fraction_non_alphanumeric': 0.1500434488751569, 'fraction_numerical': 0.00308969778893502, 'mean_word_length': 2.8735976065818996, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1852618', 'n_tokens_mistral': 3413, 'n_tokens_neox': 3329, 'n_words': 634}