<commit_before>import fetch   from 'node-fetch';
import cheerio from 'cheerio';

function fetchRegionShip(lang) {
  return new Promise((resolve, reject) => {
    if (lang !== 'jp' && lang !== 'tw') {
      return reject(new Error('Invalid parameter: lang = ' + lang));
    }

    const host = lang === 'tw' ? 'line-optc.com/tw' : 'onepiece-treasurecruise.com';

    fetch(`http://${host}/ship/`, {
      method: 'GET',
      timeout: 30 * 1000
    })
    .then(res => res.text())
    .then(html => {
      const $ = cheerio.load(html);
      const entry = $('#entry');
      let Ships = [];

      // get ship name
      entry.find('h2.ship_name').each((shipIndex, element) => {
        const name = $(element).text().trim();
        Ships[shipIndex] = { name };
      });

      // get ship img url
      entry.find('img.aligncenter').each((shipIndex, element) => {
        const imgUrl = $(element).attr('src');
        Ships[shipIndex].imgUrl = imgUrl;
      });

      // get ship level text
      entry.find('table.ship_lv').each((shipIndex, shipElement) => {
        let level = [];
        $(shipElement).find('tr td:last-child').each((index, element) => {
          const text = $(element).text().trim();
          level.push({ text });
        });
        Ships[shipIndex].level = level;
      });

      resolve(Ships);
    })
    .catch(e => {
      reject(e);
    });
  });
}

/*
 * fetchShip
 *
 * @return Promise
 */
export default function fetchShip() {
  return Promise.all([
    fetchRegionShip('jp'),
    fetchRegionShip('tw')
  ])
  .then(function ([ ships, tw ]) {
    tw.forEach((ship, index) => {
      const imgUrl = ships[index].imgUrl;
      ships[index] = {
        ...ship,
        imgUrl    // Keep imgUrl from Japan
      };
    });
    return ships;
  });
}
<commit_msg>[lib] rewrite ShipFetch by async/await<commit_after>import fetch   from 'node-fetch';
import cheerio from 'cheerio';

async function fetchRegionShip(lang) {
  if (lang !== 'jp' && lang !== 'tw') {
    throw new Error('Invalid parameter: lang = ' + lang);
  }

  const host = lang === 'tw' ? 'line-optc.com/tw' : 'onepiece-treasurecruise.com';

  const res = await fetch(`http://${host}/ship/`, {
    method: 'GET',
    timeout: 30 * 1000
  });

  if (res.status !== 200) {
    throw new Error(`request to ${HOST} failed: ${res.statusText} (${res.status})`);
  }

  const $ = cheerio.load(await res.text());
  const entry = $('#entry');
  let Ships = [];

  // get ship name
  entry.find('h2.ship_name').each((shipIndex, element) => {
    const name = $(element).text().trim();
    Ships[shipIndex] = { name };
  });

  // get ship img url
  entry.find('img.aligncenter').each((shipIndex, element) => {
    const imgUrl = $(element).attr('src');
    Ships[shipIndex].imgUrl = imgUrl;
  });

  // get ship level text
  entry.find('table.ship_lv').each((shipIndex, shipElement) => {
    let level = [];
    $(shipElement).find('tr td:last-child').each((index, element) => {
      const text = $(element).text().trim();
      level.push({ text });
    });
    Ships[shipIndex].level = level;
  });

  return Ships;
}

/*
 * fetchShip
 *
 * @return Promise
 */
export default async function fetchShip() {
  const ships = await fetchRegionShip('jp');

  (await fetchRegionShip('tw')).forEach((ship, index) => {
    const imgUrl = ships[index].imgUrl;
    ships[index] = {
      ...ship,
      imgUrl    // Keep imgUrl from Japan
    };
  });

  return ships;
}

{'fraction_non_alphanumeric': 0.19044862518089725, 'fraction_numerical': 0.004920405209840811, 'mean_word_length': 2.6112852664576804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4365902', 'n_tokens_mistral': 1143, 'n_tokens_neox': 1117, 'n_words': 245}