<commit_before>const ENTITIES = require('html-entities').AllHtmlEntities;
const PATH = require('path');
const URL = require('url');
const HTTPS = require('https');
const FS = require('fs');
const QUERYSTRING = require('querystring');
const BASE_URL = 'https://www.youtube.com/results?';

// Builds the search query url
exports.buildLink = query => BASE_URL + QUERYSTRING
  .encode({
    search_query: query,
    spf: 'navigate',
    gl: 'US',
    hl: 'en',
  });

exports.buildFromNextpage = nextpageRef => {
  let parsed = URL.parse(nextpageRef, true);
  let overwrites = QUERYSTRING.decode(parsed.search.substr(1));
  return BASE_URL + QUERYSTRING.encode(Object.assign({}, overwrites, {
    spf: 'navigate',
    gl: 'US',
    hl: 'en',
  }));
};

// Start of parsing an item
exports.parseItem = (string, respString, searchString) => {
  const titles = exports.between(string, '<div class="', '"');
  const type = exports.between(titles, 'yt-lockup yt-lockup-tile yt-lockup-', ' ');
  if (type === 'playlist') {
    if (string.includes('yt-pl-icon-mix')) return exports.parseMix(string);
    return exports.parsePlaylist(string);
  } else if (type === 'channel') {
    return exports.parseChannel(string);
  } else if (type === 'video') {
    return exports.parseVideo(string);
  } else if (type === 'movie-vertical-poster') {
    return exports.parseMovie(string);
  } else if (titles === 'search-refinements') {
    return exports.parseRelatedSearches(string);
  } else if (titles.includes('shelf') && string.includes('<div class="compact-shelf')) {
    return exports.parseShelfCompact(string);
  } else if (titles.includes('shelf') && string.includes('<div class="vertical-shelf">')) {
    return exports.parseShelfVertical(string);
  } else if (string.includes('<div class="display-message">No more results</div>')) {
    return null;
  } else if (titles.includes('yt-lockup-clarification')) {
    return null;
  } else {
    const dir = PATH.resolve(__dirname, '../dumps/');
    const file = PATH.resolve(dir, `${Math.random().toString(36).substr(3)}-${Date.now()}.dumb`);
    const cfg = PATH.resolve(__dirname, '../package.json');
    const bugsRef = require(cfg).bugs.url;
    if (!FS.existsSync(dir)) FS.mkdirSync(dir);
    FS.writeFileSync(file, JSON.stringify({ type, searchString, itemString: string, htmlBody: respString }));
    /* eslint-disable no-console */
    console.error(`\n/${'*'.repeat(200)}`);
    console.error(`found an unknwon type |${type}|${titles}|`);
    console.error(`pls post the the files in ${dir} to ${bugsRef}`);
    console.error(`${'*'.repeat(200)}\\`);
    /* eslint-enable no-console */
    return null;
  }
};

exports.parseMix = string => {
  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
  const plistID = exports.removeHtml(exports.between(string, 'data-list-id="', '"'));
  const videoID = exports.removeHtml(exports.between(string, 'data-video-ids="', '"'));
  return {
    type: 'mix',
    title: exports.removeHtml(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
    firstItem: `https://www.youtube.com/watch?v=${videoID}&list=${plistID}`,
    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),
    length: exports.removeHtml(exports.between(string, '<span class="formatted-video-count-label">', '</span>')),
  };
};

// Parse an item of type playlist
exports.parsePlaylist = string => {
  const ownerBox = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
  const cleanID = exports.removeHtml(exports.between(string, 'href="/playlist?list=', '"'));
  return {
    type: 'playlist',
    title: exports.removeHtml(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
    link: `https://www.youtube.com/playlist?list=${cleanID}`,
    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),

    author: {
      name: exports.removeHtml(exports.between(ownerBox, '>', '</a>')),
      ref: URL.resolve(BASE_URL, exports.removeHtml(exports.between(ownerBox, '<a href="', '"'))),
      verified: string.includes('title="Verified"'),
    },

    length: exports.removeHtml(exports.between(string, '<span class="formatted-video-count-label">', '</span>')),
  };
};

// Parse an item of type channel
exports.parseChannel = string => {
  const avatarRaw = exports.between(string, 'data-thumb="', '"');
  const avatar = avatarRaw ? avatarRaw : exports.between(string, 'src="', '"');
  const rawDesc = exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>');
  const rawFollows = exports.between(exports.between(string, 'yt-subscriber-count"', '</span>'), '>');
  return {
    type: 'channel',
    name: exports.removeHtml(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
    channel_id: exports.between(string, 'data-channel-external-id="', '"'),
    link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'href="', '"'))),
    avatar: URL.resolve(BASE_URL, exports.removeHtml(avatar)),
    verified: string.includes('title="Verified"') || string.includes('yt-channel-title-autogenerated'),

    followers: Number(rawFollows.replace(/\.|,/g, '')),
    description_short: exports.removeHtml(rawDesc) || null,
    videos: Number(exports.between(string, '<ul class="yt-lockup-meta-info"><li>', '</li>')
      .split(' ')
      .splice(0, 1)[0]
      .replace(/\.|,/g, ''),
    ),
  };
};

// Parse an item of type video
exports.parseVideo = string => {
  const ownerBox = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
  const metaInfo = exports.between(string, '<div class="yt-lockup-meta ">', '</ul>')
    .replace(/<\/li>/g, '')
    .split('<li>')
    .splice(1);
  const live = string.includes('yt-badge-live" >Live now</span>');
  const live_views = metaInfo[0] ? parseInt(metaInfo[0].replace(/\.|,/g, '')) : 0;
  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
  const rawDesc = exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>');
  return {
    type: 'video',
    live: live,
    title: exports.removeHtml(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
    link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'href="', '"'))),
    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),

    author: {<commit_msg>Parse videos provided by 3rd party correctly<commit_after>const ENTITIES = require('html-entities').AllHtmlEntities;
const PATH = require('path');
const URL = require('url');
const HTTPS = require('https');
const FS = require('fs');
const QUERYSTRING = require('querystring');
const BASE_URL = 'https://www.youtube.com/results?';

// Builds the search query url
exports.buildLink = query => BASE_URL + QUERYSTRING
  .encode({
    search_query: query,
    spf: 'navigate',
    gl: 'US',
    hl: 'en',
  });

exports.buildFromNextpage = nextpageRef => {
  let parsed = URL.parse(nextpageRef, true);
  let overwrites = QUERYSTRING.decode(parsed.search.substr(1));
  return BASE_URL + QUERYSTRING.encode(Object.assign({}, overwrites, {
    spf: 'navigate',
    gl: 'US',
    hl: 'en',
  }));
};

// Start of parsing an item
exports.parseItem = (string, respString, searchString) => {
  const titles = exports.between(string, '<div class="', '"');
  const type = exports.between(titles, 'yt-lockup yt-lockup-tile yt-lockup-', ' ');
  if (type === 'playlist') {
    if (string.includes('yt-pl-icon-mix')) return exports.parseMix(string);
    return exports.parsePlaylist(string);
  } else if (type === 'channel') {
    return exports.parseChannel(string);
  } else if (type === 'video') {
    return exports.parseVideo(string);
  } else if (type === 'movie-vertical-poster') {
    return exports.parseMovie(string);
  } else if (titles === 'search-refinements') {
    return exports.parseRelatedSearches(string);
  } else if (titles.includes('shelf') && string.includes('<div class="compact-shelf')) {
    return exports.parseShelfCompact(string);
  } else if (titles.includes('shelf') && string.includes('<div class="vertical-shelf">')) {
    return exports.parseShelfVertical(string);
  } else if (string.includes('<div class="display-message">No more results</div>')) {
    return null;
  } else if (titles.includes('yt-lockup-clarification')) {
    return null;
  } else {
    const dir = PATH.resolve(__dirname, '../dumps/');
    const file = PATH.resolve(dir, `${Math.random().toString(36).substr(3)}-${Date.now()}.dumb`);
    const cfg = PATH.resolve(__dirname, '../package.json');
    const bugsRef = require(cfg).bugs.url;
    if (!FS.existsSync(dir)) FS.mkdirSync(dir);
    FS.writeFileSync(file, JSON.stringify({ type, searchString, itemString: string, htmlBody: respString }));
    /* eslint-disable no-console */
    console.error(`\n/${'*'.repeat(200)}`);
    console.error(`found an unknwon type |${type}|${titles}|`);
    console.error(`pls post the the files in ${dir} to ${bugsRef}`);
    console.error(`${'*'.repeat(200)}\\`);
    /* eslint-enable no-console */
    return null;
  }
};

exports.parseMix = string => {
  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
  const plistID = exports.removeHtml(exports.between(string, 'data-list-id="', '"'));
  const videoID = exports.removeHtml(exports.between(string, 'data-video-ids="', '"'));
  return {
    type: 'mix',
    title: exports.removeHtml(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
    firstItem: `https://www.youtube.com/watch?v=${videoID}&list=${plistID}`,
    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),
    length: exports.removeHtml(exports.between(string, '<span class="formatted-video-count-label">', '</span>')),
  };
};

// Parse an item of type playlist
exports.parsePlaylist = string => {
  const ownerBox = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
  const cleanID = exports.removeHtml(exports.between(string, 'href="/playlist?list=', '"'));
  return {
    type: 'playlist',
    title: exports.removeHtml(exports.between(exports.between(string, '<h3 class="yt-lockup-title ">', '</a>'), '>')),
    link: `https://www.youtube.com/playlist?list=${cleanID}`,
    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),

    author: {
      name: exports.removeHtml(exports.between(ownerBox, '>', '</a>')),
      ref: URL.resolve(BASE_URL, exports.removeHtml(exports.between(ownerBox, '<a href="', '"'))),
      verified: string.includes('title="Verified"'),
    },

    length: exports.removeHtml(exports.between(string, '<span class="formatted-video-count-label">', '</span>')),
  };
};

// Parse an item of type channel
exports.parseChannel = string => {
  const avatarRaw = exports.between(string, 'data-thumb="', '"');
  const avatar = avatarRaw ? avatarRaw : exports.between(string, 'src="', '"');
  const rawDesc = exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>');
  const rawFollows = exports.between(exports.between(string, 'yt-subscriber-count"', '</span>'), '>');
  return {
    type: 'channel',
    name: exports.removeHtml(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
    channel_id: exports.between(string, 'data-channel-external-id="', '"'),
    link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'href="', '"'))),
    avatar: URL.resolve(BASE_URL, exports.removeHtml(avatar)),
    verified: string.includes('title="Verified"') || string.includes('yt-channel-title-autogenerated'),

    followers: Number(rawFollows.replace(/\.|,/g, '')),
    description_short: exports.removeHtml(rawDesc) || null,
    videos: Number(exports.between(string, '<ul class="yt-lockup-meta-info"><li>', '</li>')
      .split(' ')
      .splice(0, 1)[0]
      .replace(/\.|,/g, ''),
    ),
  };
};

// Parse an item of type video
exports.parseVideo = string => {
  const ownerBox = exports.between(string, '<div class="yt-lockup-byline ">', '</div>');
  const metaInfo = exports.between(string, '<div class="yt-lockup-meta ">', '</ul>')
    .replace(/<\/li>/g, '')
    .split('<li>')
    .splice(1);
  if (metaInfo.length == 1) metaInfo.unshift(null);
  const live = string.includes('yt-badge-live" >Live now</span>');
  const live_views = metaInfo[0] ? parseInt(metaInfo[0].replace(/\.|,/g, '')) : 0;
  const thumbnailRaw = exports.between(string, 'data-thumb="', '"');
  const thumbnail = thumbnailRaw ? thumbnailRaw : exports.between(string, 'src="', '"');
  const rawDesc = exports.between(exports.between(string, '<div class="yt-lockup-description', '</div>'), '>');
  return {
    type: 'video',
    live: live,
    title: exports.removeHtml(exports.between(exports.between(string, '<a href="', '</a>'), '>')),
    link: URL.resolve(BASE_URL, exports.removeHtml(exports.between(string, 'href="', '"'))),
    thumbnail: URL.resolve(BASE_URL, exports.removeHtml(thumbnail)),

    author: {
{'fraction_non_alphanumeric': 0.20362591200530622, 'fraction_numerical': 0.002947895939273344, 'mean_word_length': 4.547833197056419, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 93, 'https://': 6, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5467997', 'n_tokens_mistral': 4457, 'n_tokens_neox': 4400, 'n_words': 836}