<commit_before>/*
 Copyright 2016 Nervana Systems Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

#include "gtest/gtest.h"

#include <typeinfo>

#include "provider_factory.hpp"
#include "etl_audio.hpp"
#include "etl_char_map.hpp"
#include "wav_data.hpp"
#include "json.hpp"
#include "util.hpp"
#include "buffer_in.hpp"

using namespace std;
using namespace nervana;

TEST(provider,audio_transcript) {
    nlohmann::json js = {{"media","audio_transcript"},
                         {"data_config",{
                            {"type", "audio"},
                            {"config", {
                              {"max_duration","2000 milliseconds"},
                              {"frame_length","1024 samples"},
                              {"frame_stride","256 samples"},
                              {"sample_freq_hz",44100},
                              {"feature_type","specgram"}
                            }}}},
                         {"target_config",{
                            {"type", "transcript"},
                            {"config", {
                              {"alphabet","ABCDEFGHIJKLMNOPQRSTUVWXYZ .,()"},
                              {"max_length",50}
                            }}}}};

    // Create the config
    auto media = dynamic_pointer_cast<transcribed_audio>(nervana::train_provider_factory::create(js));

    // Get the character map
    auto cmap = media->get_cmap();

    // Get buffer output shapes for data and target. Here there are only 1 input and 1 output.
    const vector<nervana::shape_type>& oshapes = media->get_oshapes();
    size_t dsize = oshapes[0].get_byte_size();
    size_t tsize = oshapes[1].get_byte_size();
    // cout << "dsize is " << dsize << " and tsize is " << tsize << endl;

    size_t batch_size = 128;

    // Generate a simple sine wav
    float sine_freq = 400;
    int16_t sine_ampl = 500;
    sinewave_generator sg{sine_freq, sine_ampl};
    int wav_len_sec = 4, sample_freq = 44100;
    bool stereo = false;

    wav_data wav(sg, wav_len_sec, sample_freq, stereo);
    uint32_t wavbufsize = wav_data::HEADER_SIZE + wav.nbytes();

    vector<char> buf(wavbufsize);
    wav.write_to_buffer(&buf[0], wavbufsize);

    // Generate a fake transcript
    string t1 = "The quick brown fox jumped over the lazy dog";
    vector<char> t1_char(t1.begin(), t1.end());
    // cout << "t1 is: " << t1 << endl;
    // for (int i=0; i<t1.length(); i++) {
    //   cout << (int) cmap[std::toupper(t1[i])] << " ";
    // }
    // cout << endl;

    string t2 = "A much more interesting sentence.";
    vector<char> t2_char(t2.begin(), t2.end());
    // cout << "t2 is: " << t2 << endl;
    // for (int i=0; i<t2.length(); i++) {
    //   cout << (int) cmap[std::toupper(t2[i])] << " ";
    // }
    // cout << endl;

    // Create the input buffer
    buffer_in_array bp({0,0});
    buffer_in& data_p = *bp[0];
    buffer_in& target_p = *bp[1];

    // Fill the input buffer
    for (int i=0; i<batch_size-1; i++) {
        data_p.addItem(buf);
        target_p.addItem(t1_char);
    }
    data_p.addItem(buf);
    target_p.addItem(t2_char);
    EXPECT_EQ(data_p.getItemCount(), batch_size);
    EXPECT_EQ(target_p.getItemCount(), batch_size);

    // Generate an output buffer
    buffer_out_array outBuf({dsize, tsize, 4}, batch_size);

    // Call the provider
    for (int i=0; i<batch_size; i++ ) {
       media->provide(i, bp, outBuf);
    }

    // Check the output data
    // char* data_out = outBuf[1]->getItem(0);


    // Check first target starts with "t"(19) and ends with "g"(6)
    char* target_out = outBuf[1]->getItem(0);
    // cout << "First output is: " << endl;
    for (int i=0; i<t1.length(); i++) {
        auto v = unpack_le<uint8_t>(target_out, i);
        // cout << (int) v << " ";
        ASSERT_EQ(v, cmap[std::toupper(t1[i])]);
    }
    // cout << endl;

    // Check last target starts with "t"(19) and ends with "g"(6)
    target_out = outBuf[1]->getItem(batch_size - 1);
    // cout << "Last output is: " << endl;
    for (int i=0; i<t2.length(); i++) {
        auto v = unpack_le<uint8_t>(target_out, i);
        // cout << (int) v << " ";
        ASSERT_EQ(v, cmap[std::toupper(t2[i])]);
    }
    // cout << endl;

    // Check the first transcript length (should be 44)
    char* len_out = outBuf[2]->getItem(0);
    ASSERT_EQ(unpack_le<uint32_t>(len_out, 0), t1.length());

    // Check the last transcript length is also 44
    len_out = outBuf[2]->getItem(batch_size-1);
    ASSERT_EQ(unpack_le<uint32_t>(len_out, 0), t2.length());
}<commit_msg>iterate through more provided records in test<commit_after>/*
 Copyright 2016 Nervana Systems Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

#include "gtest/gtest.h"

#include <typeinfo>

#include "provider_factory.hpp"
#include "etl_audio.hpp"
#include "etl_char_map.hpp"
#include "wav_data.hpp"
#include "json.hpp"
#include "util.hpp"
#include "buffer_in.hpp"

using namespace std;
using namespace nervana;

TEST(provider,audio_transcript) {
    nlohmann::json js = {{"media","audio_transcript"},
                         {"data_config",{
                            {"type", "audio"},
                            {"config", {
                              {"max_duration","2000 milliseconds"},
                              {"frame_length","1024 samples"},
                              {"frame_stride","256 samples"},
                              {"sample_freq_hz",44100},
                              {"feature_type","specgram"}
                            }}}},
                         {"target_config",{
                            {"type", "transcript"},
                            {"config", {
                              {"alphabet","ABCDEFGHIJKLMNOPQRSTUVWXYZ .,()"},
                              {"max_length",50}
                            }}}}};

    // Create the config
    auto media = dynamic_pointer_cast<transcribed_audio>(nervana::train_provider_factory::create(js));

    // Get the character map
    auto cmap = media->get_cmap();

    size_t batch_size = 128;

    // Generate a simple sine wav
    float sine_freq = 400;
    int16_t sine_ampl = 500;
    sinewave_generator sg{sine_freq, sine_ampl};
    int wav_len_sec = 4, sample_freq = 44100;
    bool stereo = false;

    wav_data wav(sg, wav_len_sec, sample_freq, stereo);
    vector<char> buf(wav_data::HEADER_SIZE + wav.nbytes());
    wav.write_to_buffer(&buf[0], buf.size());

    // Generate alternating fake transcripts
    vector<string> tr {"The quick brown fox jumped over the lazy dog",
                       "A much more interesting sentence."};
    vector<char> tr0_char(tr[0].begin(), tr[0].end());
    vector<char> tr1_char(tr[1].begin(), tr[1].end());

    // Create the input buffer
    buffer_in_array bp({0,0});
    buffer_in& data_p = *bp[0];
    buffer_in& target_p = *bp[1];

    // Fill the input buffer
    for (int i=0; i<batch_size; i++) {
        data_p.addItem(buf);
        target_p.addItem( ( (i % 2) == 0 ? tr0_char : tr1_char) );
    }

    EXPECT_EQ(data_p.getItemCount(), batch_size);
    EXPECT_EQ(target_p.getItemCount(), batch_size);

    // Generate output buffers using shapes from the provider
    buffer_out_array outBuf({media->get_oshapes()[0].get_byte_size(),
                             media->get_oshapes()[1].get_byte_size(),
                             media->get_oshapes()[2].get_byte_size()},
                            batch_size);

    // Call the provider
    for (int i=0; i<batch_size; i++)
    {
        media->provide(i, bp, outBuf);
    }

    // Check target sequences against their source string
    for (int i=0; i<batch_size; i++)
    {
        char* target_out = outBuf[1]->getItem(i);
        auto orig_string = tr[i % 2];
        for (auto c : orig_string)
        {
            ASSERT_EQ(unpack_le<uint8_t>(target_out++),
                      cmap[std::toupper(c)]);
        }
    }

    // Check the transcript lengths match source string length
    for (int i=0; i<batch_size; i++)
    {
        ASSERT_EQ(unpack_le<uint32_t>(outBuf[2]->getItem(i)), tr[i % 2].length());
    }

}
{'fraction_non_alphanumeric': 0.13735417125247634, 'fraction_numerical': 0.020471054369359455, 'mean_word_length': 2.27577505407354, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5882008', 'n_tokens_mistral': 2994, 'n_tokens_neox': 2821, 'n_words': 733}