<commit_before>import cheerio from 'cheerio';

export default function getCorrectUrls(word) {
  return fetch(`http://www.ldoceonline.com/search/?q=${word}`)
    .then(resp => {
      return resp.text()
    })
    .then(body => {
      let $ = cheerio.load(body);
      const urlsNumber =
        $('span.headword')
          .map((i, item) => $(item).text())
          .get()
          .filter(currentWord => currentWord === word)
          .length;

      return urlsNumber;
    })
    .then(urlsNumber => {
      if(urlsNumber < 2) {
        return [`http://www.ldoceonline.com/dictionary/${word}`]
      } else {
        correctUrls = [];
        for(let i = 1; i <= urlsNumber; i++) {
          correctUrls.push(`http://www.ldoceonline.com/dictionary/${word}_${i}`)
        }
        return correctUrls;
      }
    })
    .catch(err => console.log(err));
}<commit_msg>feature: fix urls in getCorrectUrls<commit_after>import cheerio from 'cheerio';

export default function getCorrectUrls(word) {
  return fetch(`https://query.yahooapis.com/v1/public/yql?q=SELECT%20*%20FROM%20html%20WHERE%20url%3D%22http%3A%2F%2Fwww.ldoceonline.com%2Fsearch%2F%3Fq%3D${word}%22&diagnostics=true`)
    .then(resp => {
      return resp.text()
    })
    .then(body => {
      let $ = cheerio.load(body);
      const urlsNumber =
        $('span.headword')
          .map((i, item) => $(item).text())
          .get()
          .filter(currentWord => currentWord === word)
          .length;

      return urlsNumber;
    })
    .then(urlsNumber => {
      if(urlsNumber < 2) {
        return [`https://query.yahooapis.com/v1/public/yql?q=SELECT%20*%20FROM%20html%20WHERE%20url%3D%22http%3A%2F%2Fwww.ldoceonline.com%2Fdictionary%2F${word}%22&diagnostics=true`]
      } else {
        let correctUrls = [];
        for(let i = 1; i <= urlsNumber; i++) {
          correctUrls.push(`https://query.yahooapis.com/v1/public/yql?q=SELECT%20*%20FROM%20html%20WHERE%20url%3D%22http%3A%2F%2Fwww.ldoceonline.com%2Fdictionary%2F${word}_${i}%22&diagnostics=true`)
        }
        return correctUrls;
      }
    })
    .catch(err => console.log(err));
}
{'fraction_non_alphanumeric': 0.1755868544600939, 'fraction_numerical': 0.0323943661971831, 'mean_word_length': 2.719022687609075, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 3, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7432282', 'n_tokens_mistral': 802, 'n_tokens_neox': 763, 'n_words': 99}