<commit_before>const LyricBase = require('../include/lyric-base');
const BlockedError = require('../include/blocked-error');

const keyword = 'genius';

class Lyric extends LyricBase {
  get_by_div_lyrics(html) {
    const prefix = '<div class="lyrics">';
    const suffix = '</div>';

    return this.find_string_by_prefix_suffix(html, prefix, suffix);
  }

  get_by_lyrics_root(html) {
    const prefix = ' Lyrics__Root';
    const suffix = '<div class="SectionLeaderboard';

    let body = this.find_string_by_prefix_suffix(html, prefix, suffix);
    if (body) {
      // remove prefix and suffix
      body = body.replace(new RegExp(`${prefix}.*?>`, 'g'), '');
      body = body.replace(new RegExp(suffix, 'g'), '');

      // add newline for ad block
      body = body.replace(
        /<div class="SidebarAd__Container/g,
        '<br/><div class="'
      );
    }
    return body;
  }

  find_lyric(url, html) {
    let body = this.get_by_lyrics_root(html);
    if (!body) {
      console.log('Failed to get content from .lyrics, try class="lyrics"');
      body = this.get_by_div_lyrics(html);
    }

    let lyric = body;
    lyric = lyric.replace(
      /<br\/><div data-primis-player/g,
      '<br/><br/><div data-primis-player'
    );
    lyric = lyric.replace(/<br\/>/g, '\n');
    lyric = lyric.replace(/<button.*?<\/button>/g, '');
    lyric = lyric.replace(/<label.*?<\/label>/g, '');
    lyric = lyric.replace(/<div class="EmbedForm__Copy.*?<\/div>/g, '');
    lyric = lyric.replace(/<div class="ShareButtons.*?<\/div>/g, '');
    lyric = lyric.replace(/<div class="LyricsEditExplainer__.*?<\/div>/g, '');
    lyric = lyric.replace(/<h2.*?<\/h2>/g, '');
    lyric = this.sanitize_html(lyric);

    this.lyric = lyric;

    return true;
  }

  find_info(url, html) {
    const patterns = {
      title: '<meta content=".*? – (.*?)" property="twitter:title" />',
      artist: '<meta content="(.*?) – .*?" property="twitter:title" />',
    };

    this.fill_song_info(html, patterns);
    return true;
  }

  async parse_page() {
    const { url } = this;

    let html;
    try {
      html = await this.get_html(url);
    } catch (error) {
      if (error.status === 403) {
        throw new BlockedError('genius is blocked');
      }
    }

    this.find_lyric(url, html);
    this.find_info(url, html);

    return true;
  }
}

exports.keyword = keyword;
exports.Lyric = Lyric;

if (require.main === module) {
  (async () => {
    const url = 'https://genius.com/Hollow-coves-coastline-lyrics';
    const object = new Lyric(url);
    const lyric = await object.get();
    console.log(lyric);
  })();
}
<commit_msg>fix genius whitespace<commit_after>const LyricBase = require('../include/lyric-base');
const BlockedError = require('../include/blocked-error');

const keyword = 'genius';

class Lyric extends LyricBase {
  get_by_div_lyrics(html) {
    const prefix = '<div class="lyrics">';
    const suffix = '</div>';

    return this.find_string_by_prefix_suffix(html, prefix, suffix);
  }

  get_by_lyrics_root(html) {
    const prefix = ' Lyrics__Root';
    const suffix = '<div class="SectionLeaderboard';

    let body = this.find_string_by_prefix_suffix(html, prefix, suffix);
    if (body) {
      // remove prefix and suffix
      body = body.replace(new RegExp(`${prefix}.*?>`, 'g'), '');
      body = body.replace(new RegExp(suffix, 'g'), '');

      // add newline for ad block
      body = body.replace(
        /<div class="SidebarAd__Container/g,
        '<br/><div class="'
      );
    }
    return body;
  }

  find_lyric(url, html) {
    let body = this.get_by_lyrics_root(html);
    if (!body) {
      console.log('Failed to get content from .lyrics, try class="lyrics"');
      body = this.get_by_div_lyrics(html);
    }

    let lyric = body;
    lyric = lyric.replace(
      /<div class="PrimisPlayer/,
      '<br/><div class="PrimisPlayer'
    );
    lyric = lyric.replace(/<br\/>/g, '\n');
    lyric = lyric.replace(/<button.*?<\/button>/g, '');
    lyric = lyric.replace(/<label.*?<\/label>/g, '');
    lyric = lyric.replace(/<div class="EmbedForm__Copy.*?<\/div>/g, '');
    lyric = lyric.replace(/<div class="ShareButtons.*?<\/div>/g, '');
    lyric = lyric.replace(/<div class="LyricsEditExplainer__.*?<\/div>/g, '');
    lyric = lyric.replace(/<h2.*?<\/h2>/g, '');
    lyric = this.sanitize_html(lyric);

    this.lyric = lyric;

    return true;
  }

  find_info(url, html) {
    const patterns = {
      title: '<meta content=".*? – (.*?)" property="twitter:title" />',
      artist: '<meta content="(.*?) – .*?" property="twitter:title" />',
    };

    this.fill_song_info(html, patterns);
    return true;
  }

  async parse_page() {
    const { url } = this;

    let html;
    try {
      html = await this.get_html(url);
    } catch (error) {
      if (error.status === 403) {
        throw new BlockedError('genius is blocked');
      }
    }

    this.find_lyric(url, html);
    this.find_info(url, html);

    return true;
  }
}

exports.keyword = keyword;
exports.Lyric = Lyric;

if (require.main === module) {
  (async () => {
    const url = 'https://genius.com/Hollow-coves-coastline-lyrics';
    const object = new Lyric(url);
    const lyric = await object.get();
    console.log(lyric);
  })();
}

{'fraction_non_alphanumeric': 0.18636018237082067, 'fraction_numerical': 0.001899696048632219, 'mean_word_length': 2.831877729257642, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4894574', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1859, 'n_words': 362}