<commit_before>const rp = require('request-promise');

const LyricBase = require('../include/lyric_base');

const keyword = 'lyrical-nonsense';

class Lyric extends LyricBase {
  get_json_lds(html) {
    const prefix = '<script type="application/ld+json">';
    const suffix = '</script>';
    const json_lds = [];
    const first_json_ld = this.find_string_by_prefix_suffix(html, prefix, suffix, false);
    json_lds.push(first_json_ld);

    const pos = html.indexOf(first_json_ld);
    const after_first = html.substring(pos + first_json_ld.length);

    json_lds.push(this.find_string_by_prefix_suffix(after_first, prefix, suffix, false));

    return json_lds.map(JSON.parse);
  }

  get_hash(url) {
    const my_url = new URL(url);
    if (my_url.hash[0] === '#') {
      return my_url.hash.substring(1);
    }
    return '';
  }

  get_lyric_content_block(url, html) {
    const hash = this.get_hash(url) || 'Lyrics';

    const prefix = `<div class="contents" id="${hash}">`;
    const suffix = '</p>\n</div>';

    return this.find_string_by_prefix_suffix(html, prefix, suffix);
  }

  find_lyric(url, html) {
    const block = this.get_lyric_content_block(url, html);
    const prefix = '<div class="olyrictext">';
    const suffix = '</div>';

    let lyric = this.find_string_by_prefix_suffix(block, prefix, suffix);
    if (!lyric) {
      return false;
    }

    lyric = lyric.replace(/<\/p>/g, '\n');
    lyric = this.sanitize_html(lyric);

    this.lyric = lyric;

    return true;
  }

  find_info(url, html) {
    const patterns = {
      title: '"name" : "(.*?) 歌詞"',
      artist: '<th>歌手:</th><td>(.*?)</td>',
      lyricist: '<th>作詞:</th><td>(.*?)</td>',
      composer: '<th>作曲:</th><td>(.*?)</td>',
    };

    this.fill_song_info(html, patterns);

    // some page does not have 曲名、歌手
    if (!this.title) {
      const pattern = { title: '<h1>(.*?)</h1>' };
      this.fill_song_info(html, pattern);
    }<commit_msg>fix lyrical nonsense title<commit_after>const rp = require('request-promise');

const LyricBase = require('../include/lyric_base');

const keyword = 'lyrical-nonsense';

class Lyric extends LyricBase {
  get_json_lds(html) {
    const prefix = '<script type="application/ld+json">';
    const suffix = '</script>';
    const json_lds = [];
    const first_json_ld = this.find_string_by_prefix_suffix(html, prefix, suffix, false);
    json_lds.push(first_json_ld);

    const pos = html.indexOf(first_json_ld);
    const after_first = html.substring(pos + first_json_ld.length);

    json_lds.push(this.find_string_by_prefix_suffix(after_first, prefix, suffix, false));

    return json_lds.map(JSON.parse);
  }

  get_hash(url) {
    const my_url = new URL(url);
    if (my_url.hash[0] === '#') {
      return my_url.hash.substring(1);
    }
    return '';
  }

  get_lyric_content_block(url, html) {
    const hash = this.get_hash(url) || 'Lyrics';

    const prefix = `<div class="contents" id="${hash}">`;
    const suffix = '</p>\n</div>';

    return this.find_string_by_prefix_suffix(html, prefix, suffix);
  }

  find_lyric(url, html) {
    const block = this.get_lyric_content_block(url, html);
    const prefix = '<div class="olyrictext">';
    const suffix = '</div>';

    let lyric = this.find_string_by_prefix_suffix(block, prefix, suffix);
    if (!lyric) {
      return false;
    }

    lyric = lyric.replace(/<\/p>/g, '\n');
    lyric = this.sanitize_html(lyric);

    this.lyric = lyric;

    return true;
  }

  find_info(url, html) {
    const prefix = '"@type": "MusicComposition",';
    const suffix = '"Lyrics" : {';

    const block = this.find_string_by_prefix_suffix(html, prefix, suffix, false);
    if (!block) {
      return false;
    }

    const patterns = {
      title: '"name" : "(.+?)",',
      artist: '"byArtist".*?"name" : "(.+?)",',
      lyricist: '"lyricist".*?"name" : "(.+?)"',
      composer: '"composer".*?"name" : "(.+?)"',
    }
    /*
    const patterns = {
      title: '"name" : "(.*?) 歌詞"',
      artist: '<th>歌手:</th><td>(.*?)</td>',
      lyricist: '<th>作詞:</th><td>(.*?)</td>',
      composer: '<th>作曲:</th><td>(.*?)</td>',
    };
    */

    const line = block.replace(/[\r\n]/g, '');
    this.fill_song_info(line, patterns);

    // some page does not have 曲名、歌手
    if (!this.title) {
      const pattern = { title: '<h1>(.*?)</h1>' };
      this.fill_song_info(html, pattern);
    }
{'fraction_non_alphanumeric': 0.18768529076396806, 'fraction_numerical': 0.0018244013683010262, 'mean_word_length': 3.205177372962608, 'pattern_counts': {'":': 1, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '242035', 'n_tokens_mistral': 1624, 'n_tokens_neox': 1590, 'n_words': 295}