<commit_before>import alexa from './alexa.js';
import redis from '../db/backend-redis.js';

function setCurrentCrawl(client, crawlKey) {
  return redis.set(client, 'currentCrawl', crawlKey);
}

// fetches all required resources from url, builds an Object and returns
// a promise
function fetchSite(url) {
  // XXX write this part
  throw new Error('not implemented ' + url);
}

// returns key to the saved crawl
function saveCrawl(client, topSites, key) {
  if (!key) {
    key = new Date().toISOString();
  }
  return topSites.forEach((url) => fetchSite(url)
    // XXX change to the current schema
    .then((siteData) => redis.hmset(client, key, url, siteData))
  )
  .then(() => key);
}

// fetches top sites and saves in database
// checks if a crawl is needed
function fetchAlexaTopSites() {
  let client;
  return redis.getClient()
  .then((redisClient) => {
    client = redisClient;
  })
  .then(() => alexa.fetchTopSites())
  .then((topSites) => saveCrawl(client, topSites))
  .then((crawlKey) => setCurrentCrawl(client, crawlKey))
  .then(() => redis.quit(client));
}

export default {
  fetchAlexaTopSites,
};

const test = {
  saveCrawl,
  setCurrentCrawl,
};

export { test };<commit_msg>Keep only urlProcessor.js bits that process urls<commit_after>// fetches all required resources from url, builds an Object and returns
// a promise
function fetchSite(url) {
  // XXX write this part
  throw new Error('not implemented ' + url);
}

export default {
  fetchSite,
};

const test = {
};

export { test };
{'fraction_non_alphanumeric': 0.13759052007899933, 'fraction_numerical': 0.0, 'mean_word_length': 3.840764331210191, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4526823', 'n_tokens_mistral': 510, 'n_tokens_neox': 495, 'n_words': 149}