Spaces:

yuekai
/

valle_wenetspeech4tts_demo

Running

File size: 18,775 Bytes

<!DOCTYPE html>
<!-- saved from url=(0033)https://QicongXie.github.io/end2endvc/ -->
<html lang="en-US">

<head>
  <meta http-equiv="Content-Type" content="text/html; charset=UTF-8">


  <!-- Begin Jekyll SEO tag v2.7.1 -->
  <title>WenetSpeech4TTS</title>
  <meta name="generator" content="Jekyll v3.9.0">
  <meta property="og:title" content="title">
  <meta property="og:locale" content="en_US">
  <meta name="twitter:card" content="summary">
  <!-- End Jekyll SEO tag -->

  <meta name="viewport" content="width=device-width, initial-scale=1">
  <meta name="theme-color" content="#157878">
  <link rel="stylesheet" href="style.css">
  <style>
    .method {
      display: inline-block;
      /*             width: 120px; /* Adjust the width as needed */
      font-weight: bold;
    }

    .explanation {
      display: inline-block;
      /*             margin-left: 20px; /* Adjust the margin as needed */
      
    }
      .centered-table {
      width: 50%;
      /* Adjust this value to make the table smaller or larger */
      border-collapse: collapse;
      box-shadow: 0 4px 8px 0 rgba(0, 0, 0, 0.2);
      /* Adds a shadow around the table */
      margin-top: 20px;
      margin-bottom: 20px;
      background: white;
      /* Background color for the table */
    margin-left: auto;
    margin-right: auto;
  }

  .centered-table th,
  .centered-table td {
      border: 1px solid #ddd;
      /* Lighter border color for a softer look */
      text-align: center;
      padding: 10px;
      font-size: 0.85em;
      /* Smaller font size */
  }

  .centered-table th {
      background-color: #f2f2f2;
      /* Light grey background for the header */
      color: #333;
      /* Darker text color for contrast */
  }

  .centered-table caption {
      caption-side: top;
      font-weight: bold;
      margin-bottom: 10px;
      text-align: center;
      font-size: 1.1em;
      /* Slightly larger font size for caption */
  }

  /* Add a color gradient to row hover */
  .centered-table tr:hover {
      background-color: #f5f5f5;
  }
  .model-list {
      list-style-type: none;
      margin: 0;
      padding: 0;
  }

  .model-item {
      display: flex;
      align-items: center;
      margin-bottom: 10px;
  }

  .model-name {
      font-weight: bold;
      margin-right: 5px;
      flex: 0 0 auto;
      /* Prevents flex items from growing or shrinking */
  }

  .model-description {
      flex-grow: 1;
      /* Allows the description to fill the remaining space */
  }

  .basic {
      color: #FF0000;
  }

  .standard {
      color: #FFD700;
  }

  .premium {
      color: #00FF7F;
  }
  .reset {
      color: #cfcecb;
  }
  </style>
</head>

<body data-new-gr-c-s-check-loaded="14.1001.0" data-gr-ext-installed="">
  <section class="page-header">
    <!-- <h1 class="project-name">Demo PAGE</h1> -->
    <!-- <h2 class="project-tagline"></h2> -->


  </section>

  <section class="main-content">
    <h1 id="">
      <center>WenetSpeech4TTS: A 12,800-hour Mandarin TTS Corpus for Large Speech Generation Model Benchmark</center>
    </h1>

    <h3 id="">
       <center>Linhan Ma<sup>1,†</sup>, Dake Guo<sup>1,†</sup>, Kun Song<sup>1</sup>, Yuepeng Jiang<sup>1</sup>, Shuai Wang<sup>2,3</sup>, Liumeng Xue<sup>3</sup>, Weiming Xu<sup>1</sup>,Huan Zhao<sup>1</sup>, Binbin Zhang<sup>4</sup>, Lei Xie<sup>1</sup></center>
      
      <center><sup>1</sup>Audio, Speech and Language Processing Group (ASLP@NPU), School of Computer Science,Northwestern Polytechnical University, Xi’an, China</center>
      <center><sup>2</sup>Shenzhen Research Institute of Big Data,  <sup>3</sup>School of Data Science, The Chinese University of Hong Kong,Shenzhen (CUHK-Shenzhen), China</center>
      <center><sup>4</sup>WeNet Open Source Community, China</center>
    </h3>



    <br>
    <h2 id="abstract"><a name="abstract"></a></h2>
    <p>With the development of large text-to-speech (TTS) models and scale-up of the training data, state-of-the-art TTS
    systems have achieved impressive performance. In this paper, we present <b>WenetSpeech4TTS</b>, a multi-domain
    Mandarin corpus derived from the open-sourced WenetSpeech dataset. Tailored for the text-to-speech tasks, we refined
    WenetSpeech by adjusting segment boundaries, enhancing the audio quality, and eliminating speaker mixing within each
    segment. Following a more accurate transcription process and quality-based data filtering process, the obtained
    WenetSpeech4TTS corpus contains 12,800 hours of paired audio-text data. Furthermore, we have created subsets of
    varying sizes, categorized by segment quality scores to allow for TTS model training and fine-tuning. VALL-E and
    NaturalSpeech 2 systems are trained and fine-tuned on these subsets, establishing benchmarks for the usability of
    WenetSpeech4TTS and the fair comparison of TTS systems. The corpus and corresponding benchmarks will be made publicly
    available to advance research in this field.</p>
    
<center><a href="https://huggingface.co/datasets/Wenetspeech4TTS/WenetSpeech4TTS">Download WenetSpeech4TTS</a></center>
<table class="centered-table">
  <caption>Table 1: WenetSpeech4TTS subsets.</caption>
  <thead>
    <tr>
      <th>Training Subsets</th>
      <th>DNSMOS Threshold</th>
      <th>Hours</th>
      <th>Average Segment Duration (s)</th>
    </tr>
  </thead>
  <tbody>
    <tr>
      <td><span class="premium">Premium</span></td>
      <td>4.0</td>
      <td>945</td>
      <td>8.3</td>
    </tr>
    <tr>
      <td><span class="standard">Standard</span></td>
      <td>3.8</td>
      <td>4,056</td>
      <td>7.5</td>
    </tr>
    <tr>
      <td><span class="basic">Basic</span></td>
      <td>3.6</td>
      <td>7,226</td>
      <td>6.6</td>
    </tr>
    <tr>
      <td><span class="rest">Rest</span></td>
      <td>
        < 3.6</td>
      <td>5,574</td>
      <td>-</td>
    </tr>
    <tr>
      <td>WenetSpeech<sup>1</sup> (orig)</td>
      <td>-</td>
      <td>12,483</td>
      <td>-</td>
    </tr>
  </tbody>
</table>

    <h2>Zero-Shot TTS Samples <a name="Comparison"></a></h2>

    <!-- <p>Models</p>
    <ul>
      <li><b>VALL-E   : </b>VALL-E<sup>1</sup> trained with the WenetSpeech4TTS <i><b>Basic</b></i> subset</li> 
      <li><b>VALL-E S : </b>VALL-E fine-tuning with the WenetSpeech4TTS <i><b>Standard</b></i> subset</li>
      <li><b>VALL-E P : </b>VALL-E fine-tuning with the WenetSpeech4TTS <i><b>Premium</b></i> subset</li>
      <li><b>NS2   : </b> NaturalSpeech 2<sup>2</sup> trained with the WenetSpeech4TTS <i><b>Basic</b></i> subset</li>        
      <li><b>NS2 S : </b> NaturalSpeech 2 fine-tuning with the WenetSpeech4TTS <i><b>Standard</b></i> subset</li>
      <li><b>NS2 P : </b> NaturalSpeech 2 fine-tuning with the WenetSpeech4TTS <i><b>Premium</b></i> subset</li>

    </ul> -->


    <p>Models</p>
    <ul class="model-list">
      <li class="model-item"><span class="model-name">VALL-E:</span> <span class="model-description">VALL-E<sup>2</sup>
          trained with the WenetSpeech4TTS <span class="basic">Basic</span> subset</span></li>
      <li class="model-item"><span class="model-name">VALL-E S:</span> <span class="model-description">VALL-E fine-tuning
          with the WenetSpeech4TTS <span class="standard">Standard</span> subset</span></li>
      <li class="model-item"><span class="model-name">VALL-E P:</span> <span class="model-description">VALL-E fine-tuning
          with the WenetSpeech4TTS <span class="premium">Premium</span> subset</span></li>
      <li class="model-item"><span class="model-name">NS2:</span> <span class="model-description">NaturalSpeech
          2<sup>3</sup> trained with the WenetSpeech4TTS <span class="basic">Basic</span> subset</span></li>
      <li class="model-item"><span class="model-name">NS2 S:</span> <span class="model-description">NaturalSpeech 2
          fine-tuning with the WenetSpeech4TTS <span class="standard">Standard</span> subset</span></li>
      <li class="model-item"><span class="model-name">NS2 P:</span> <span class="model-description">NaturalSpeech 2
          fine-tuning with the WenetSpeech4TTS <span class="premium">Premium</span> subset</span></li>
    </ul>
    <h3>Seen Speakers </h3>
    <table style="max-width:100%;table-layout: fixed;" >
      <tbody id="tbody_seen">
      </tbody>
    </table>
    <h3>Unseen Speakers</h3>
    <table style="max-width:100%;table-layout: fixed;">
      <tbody id="tbody_unseen">
      </tbody>
    </table>
    <!-- <h3>2.2 Long-form Speech</h3>

    <table>
      <tbody id="tbody_long">
      </tbody>
    </table> -->

    <h3>References:</h3>
    <div>
      <div><cite><a href="https://arxiv.org/pdf/2110.03370.pdf">
        [1] B. Zhang, H. Lv, P. Guo, Q. Shao, C. Yang, L. Xie, X. Xu, H. Bu,
        X. Chen, C. Zeng et al., “Wenetspeech: A 10000+ hours multi-
        domain mandarin corpus for speech recognition,” in ICASSP
        2022-2022 IEEE International Conference on Acoustics, Speech
        and Signal Processing (ICASSP). IEEE, 2022, pp. 6182–6186.
      </a></cite></div>
      <div><cite><a href="https://arxiv.org/pdf/2301.02111.pdf">
          [2] C. Wang, S. Chen, Y. Wu, Z. Zhang, L. Zhou, S. Liu, Z. Chen,
          Y. Liu, H. Wang, J. Li, L. He, S. Zhao, and F. Wei, “Neural codec
          language models are zero-shot text to speech synthesizers,” CoRR,
          vol. abs/2301.02111, 2023.</a></cite></div>
      <div><cite><a href="https://arxiv.org/pdf/2304.09116.pdf">
        [3] K. Shen, Z. Ju, X. Tan, E. Liu, Y. Leng, L. He, T. Qin, sheng zhao,
        and J. Bian, “Naturalspeech 2: Latent diffusion models are natural
        and zero-shot speech and singing synthesizers,” in The Twelfth
        International Conference on Learning Representations, 2024.</a></cite></div>
   
    </div>


</html>

<script type="" text/javascript>
function seen_spk() {
  let scenes = [
    ["text18","seen1","应该给千千万万还在路上的创业者致意。(We should pay tribute to the millions of entrepreneurs who are still on their journey.)"],
    ["text19","seen1","准确点说，小森林是一部美食类电影食物佳肴，贯穿了柿子的寒暑交替四十三餐。(To be precise, 'Little Forest' is a gourmet movie that features forty-three meals, showcasing the seasonal changes of persimmons.)"],
    ["text20","seen2","要大力开展全省网吧专项整治工作，加强网络文化内容的整治，深入开展低俗音像制品清查行动。(It is necessary to vigorously carry out special rectification work on internet cafes across the province, strengthen the regulation of online cultural content, and thoroughly conduct a cleanup operation of vulgar audio-visual products.)"],
    ["text22","seen2","因为这是我们法律存在的前提。(Because this is the premise for the existence of our law.)"],
    ["text21","seen3","书记旗县市长乡镇苏木长企事业负责人以及国营农牧场的老兵团们，像赶庙会似的你挤我扛，口里说着，手里记着。(Secretaries, county and town mayors, township and sumu leaders, heads of enterprises and institutions, as well as veteran groups from state-owned farms and pastures, jostled like they were attending a temple fair, talking and taking notes.)"],
    ["text9","seen3","哎，不是有一句话这样说嘛，你永远不知道明天和意外哪个先来。(Alas, isn't there a saying that you never know whether tomorrow or an accident will come first?)"],
    ["text23","seen4","湿热天最麻烦的还属家中的煤气，早上起来，连拌果酱的木铲也发霉了。(On humid and hot days, the biggest nuisance is the gas at home; even the wooden spatula for stirring jam gets moldy.)"],
    ["text5","seen4","这时，朱警官等人才发现小男孩腿脚也异常，根本走不了路。(At this moment, Officer Zhu and others discovered that the little boy's legs were abnormal, and he couldn't walk at all.)"]

  ];
  let models = ["VALL-E", "VALL-E S", "VALL-E P", "NS2", "NS2 S", "NS2 P"];
  let models_path = ["Valle", "Valle_S", "Valle_P", "NS2", "NS2_S", "NS2_P"];
  let data = `
      <colgroup>
        <col style="width: 10%;"> <!-- Reference speaker column -->
        <col style="width: 15%;"> <!-- Text column -->
  `;
  
  // Dynamically adding cols for each model
  models.forEach(function() {
    data += '<col style="width: 12.5%;">'; // Adjust the width as necessary
  });
  
  data += `</colgroup>
      <tr>
        <th style=""><strong>Reference Speaker</strong></th>
        <th style=""><strong>Text</strong></th>
  `;
  
  models.forEach(function(model) {
    data += '<th style=""><strong>' + model + '</strong></th>';
  });
  
  data += '</tr>';
  
  scenes.forEach(function(scene) {
    let file = scene[0];
    let spk = scene[1];
    let text = scene[2];
    let scene_data = "";
  
    scene_data += '<tr>';
    scene_data += '<td style=""><audio controls="" style="width: 100%;" controls src="' + './raw/demos/Ref/' + spk + '_spk.wav' + '"></audio></td>';
    scene_data += '<td style=" font-size: 14px">' + text + '</td>';
    models_path.forEach(function(model) {
      scene_data += '<td style="text-align: center;"><audio style="width: 100%;" controls src="' + './raw/demos/' + model + '/' + spk+'_'+file + '.wav' + '"></audio></td>';
    });
    scene_data += '</tr>';
    data += scene_data;
  });
  
  
  return data;
}

function unseen_spk() {
  let scenes = [
["text5","unseen3","这时，朱警官等人才发现小男孩腿脚也异常，根本走不了路。(At this moment, Officer Zhu and others discovered that the little boy's legs were abnormal, and he couldn't walk at all.)"],
    ["text6","unseen3","哎，我现在兜里只剩八十块钱了，咱们俩就随便看一下，标准件就完了。(Alas, I only have eighty yuan left in my pocket now. Let's just have a casual look, and then we're done with the standard items.)"],
    ["text7","unseen4","按照科学的知识来说这是因为蓝色更加容易散射哦，宝宝以后就会学到的哦。(According to scientific knowledge, this is because blue scatters more easily, and you will learn about this later, baby.)"],
    ["text8","unseen4","他一度梦想参军，但是因没身份而作罢，整日在村内瞎晃。(He once dreamed of joining the army, but gave up due to lack of identification, and wandered aimlessly in the village all day.)"],
    ["text9","unseen5","哎，不是有一句话这样说嘛，你永远不知道明天和意外哪个先来。(Alas, isn't there a saying that you never know whether tomorrow or an accident will come first?)"],
    ["text10","unseen5","当时有点感动还是什么？(Was it a bit touching or something at that time?)"],
    ["text11","unseen6","这些飞机仍然由位于华盛顿州埃弗里特的工厂生产。(These planes are still produced by the factory located in Everett, Washington State.)"],
    ["text12","unseen6","派大军前去争抢，赫赫查拉二话不说，喊出了巴顿的名字，结果手套开始传递给斯科特半路遭到灭霸拦截，真正的杀神从天而降。(A large army was sent to compete, Hehe Chala without hesitation called out Barton's name, resulting in the gauntlet being passed to Scott halfway, only to be intercepted by Thanos, as the true god of death descended from the heavens.)"],
    ["text13","unseen7","以后我们大概不会再见面了。(We probably won't see each other again in the future.)"],
    ["text14","unseen7","而中国却只是将其作为红旗反导系统下的一个补充而已。(And China merely considers it a supplement under the HQ anti-missile system.)"],
    ["text15","unseen8","这一段戏同样也表达了亚瑟说的，我原本以为我的人生是一出悲剧，但其实它是一出戏剧。(This part of the play also expressed what Arthur said, 'I used to think my life was a tragedy, but actually, it's a comedy.')"],
    ["text16","unseen8","常常暴跳如雷，红着眼睛用力攥着洛洛的手，把洛洛的手攥得生疼也不撒手。洛洛吓坏了，她很害怕，那个平日里对她那么好的男友为什么会变得这么可怕？(Often flying into a rage, his eyes red, he would grip Lolo's hand tightly, causing her pain without letting go. Lolo was terrified. She wondered why her boyfriend, who was so good to her on normal days, could become so frightening.)"],
    ["text17","unseen9","或者看一下他的状态。(Or take a look at his condition.)"],
    ["text13","unseen9","以后我们大概不会再见面了。(We probably won't see each other again in the future.)"],
    ["text1","unseen1","目前，我还没有给会所取好名字。(Currently, I haven't come up with a good name for the club yet.)"],
    ["text2","unseen1","史堡村的一名癌症患者。(A cancer patient from Shibu Village.)"],
    ["text3","unseen2","一句话，压缩他们的成熟期。(In a word, it shortens their maturation period.)"],
    ["text4","unseen2","妮可拿着胡萝卜亲自给长颈鹿喂食，并鼓励女儿也跟着做。(Nicole personally feeds the giraffe with carrots and encourages her daughter to do the same.)"]
  ];
  let models = ["VALL-E", "VALL-E S", "VALL-E P", "NS2", "NS2 S", "NS2 P"];
  let models_path = ["Valle", "Valle_S", "Valle_P", "NS2", "NS2_S", "NS2_P"];
  let data = `
      <colgroup>
        <col style="width: 10%;"> <!-- Reference speaker column -->
        <col style="width: 20%;"> <!-- Text column -->
  `;
  
  // Dynamically adding cols for each model
  models.forEach(function() {
    data += '<col style="width: 10%;">'; // Adjust the width as necessary
  });
  
  data += `</colgroup>
      <tr>
        <th style=""><strong>Reference Speaker</strong></th>
        <th style=""><strong>Text</strong></th>
  `;
  
  models.forEach(function(model) {
    data += '<th style=""><strong>' + model + '</strong></th>';
  });
  
  data += '</tr>';
  
  scenes.forEach(function(scene) {
    let file = scene[0];
    let spk = scene[1];
    let text = scene[2];
    let scene_data = "";
  
    scene_data += '<tr>';
    scene_data += '<td style=""><audio controls="" style="width: 100%;" controls src="' + './raw/demos/Ref/' + spk + '_spk.wav' + '"></audio></td>';
    scene_data += '<td style=" font-size: 14px">' + text + '</td>';
    models_path.forEach(function(model) {
      scene_data += '<td style="text-align: center;"><audio style="width: 100%;" controls src="' + './raw/demos/' + model + '/' + spk+'_'+file + '.wav' + '"></audio></td>';
    });
    scene_data += '</tr>';
    data += scene_data;
  });
  
  return data;
}


window.onload = function() {
  document.getElementById('tbody_seen').innerHTML = seen_spk();
  document.getElementById('tbody_unseen').innerHTML = unseen_spk()
}

 
</script>