<commit_before>/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
#include <assert.h>

#include <algorithm>
#include <map>
#include <sstream>
#include <string>

#include "testing/gtest/include/gtest/gtest.h"

#include "webrtc/call.h"
#include "webrtc/frame_callback.h"
#include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
#include "webrtc/modules/video_coding/main/interface/video_coding_defines.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/event_wrapper.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
#include "webrtc/system_wrappers/interface/sleep.h"
#include "webrtc/test/call_test.h"
#include "webrtc/test/direct_transport.h"
#include "webrtc/test/encoder_settings.h"
#include "webrtc/test/fake_audio_device.h"
#include "webrtc/test/fake_decoder.h"
#include "webrtc/test/fake_encoder.h"
#include "webrtc/test/frame_generator.h"
#include "webrtc/test/frame_generator_capturer.h"
#include "webrtc/test/null_transport.h"
#include "webrtc/test/rtcp_packet_parser.h"
#include "webrtc/test/rtp_rtcp_observer.h"
#include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/test/testsupport/gtest_disable.h"
#include "webrtc/test/testsupport/perf_test.h"
#include "webrtc/video/transport_adapter.h"
#include "webrtc/video_encoder.h"

namespace webrtc {

static const unsigned long kSilenceTimeoutMs = 2000;

class EndToEndTest : public test::CallTest {
 public:
  EndToEndTest() {}

  virtual ~EndToEndTest() {
    EXPECT_EQ(NULL, send_stream_);
    EXPECT_TRUE(receive_streams_.empty());
  }

 protected:
  class UnusedTransport : public newapi::Transport {
   private:
    virtual bool SendRtp(const uint8_t* packet, size_t length) override {
      ADD_FAILURE() << "Unexpected RTP sent.";
      return false;
    }

    virtual bool SendRtcp(const uint8_t* packet, size_t length) override {
      ADD_FAILURE() << "Unexpected RTCP sent.";
      return false;
    }
  };

  void DecodesRetransmittedFrame(bool retransmit_over_rtx);
  void ReceivesPliAndRecovers(int rtp_history_ms);
  void RespectsRtcpMode(newapi::RtcpMode rtcp_mode);
  void TestXrReceiverReferenceTimeReport(bool enable_rrtr);
  void TestSendsSetSsrcs(size_t num_ssrcs, bool send_single_ssrc_first);
  void TestRtpStatePreservation(bool use_rtx);
  void TestReceivedFecPacketsNotNacked(const FakeNetworkPipe::Config& config);
};

TEST_F(EndToEndTest, ReceiverCanBeStartedTwice) {
  test::NullTransport transport;
  CreateCalls(Call::Config(&transport), Call::Config(&transport));

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();

  CreateStreams();

  receive_streams_[0]->Start();
  receive_streams_[0]->Start();

  DestroyStreams();
}

TEST_F(EndToEndTest, ReceiverCanBeStoppedTwice) {
  test::NullTransport transport;
  CreateCalls(Call::Config(&transport), Call::Config(&transport));

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();

  CreateStreams();

  receive_streams_[0]->Stop();
  receive_streams_[0]->Stop();

  DestroyStreams();
}

TEST_F(EndToEndTest, RendersSingleDelayedFrame) {
  static const int kWidth = 320;
  static const int kHeight = 240;
  // This constant is chosen to be higher than the timeout in the video_render
  // module. This makes sure that frames aren't dropped if there are no other
  // frames in the queue.
  static const int kDelayRenderCallbackMs = 1000;

  class Renderer : public VideoRenderer {
   public:
    Renderer() : event_(EventWrapper::Create()) {}

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int /*time_to_render_ms*/) override {
      event_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

    EventTypeWrapper Wait() { return event_->Wait(kDefaultTimeoutMs); }

    scoped_ptr<EventWrapper> event_;
  } renderer;

  class TestFrameCallback : public I420FrameCallback {
   public:
    TestFrameCallback() : event_(EventWrapper::Create()) {}

    EventTypeWrapper Wait() { return event_->Wait(kDefaultTimeoutMs); }

   private:
    virtual void FrameCallback(I420VideoFrame* frame) override {
      SleepMs(kDelayRenderCallbackMs);
      event_->Set();
    }

    scoped_ptr<EventWrapper> event_;
  };

  test::DirectTransport sender_transport, receiver_transport;

  CreateCalls(Call::Config(&sender_transport),
              Call::Config(&receiver_transport));

  sender_transport.SetReceiver(receiver_call_->Receiver());
  receiver_transport.SetReceiver(sender_call_->Receiver());

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();

  TestFrameCallback pre_render_callback;
  receive_configs_[0].pre_render_callback = &pre_render_callback;
  receive_configs_[0].renderer = &renderer;

  CreateStreams();
  Start();

  // Create frames that are smaller than the send width/height, this is done to
  // check that the callbacks are done after processing video.
  scoped_ptr<test::FrameGenerator> frame_generator(
      test::FrameGenerator::CreateChromaGenerator(kWidth, kHeight));
  send_stream_->Input()->SwapFrame(frame_generator->NextFrame());
  EXPECT_EQ(kEventSignaled, pre_render_callback.Wait())
      << "Timed out while waiting for pre-render callback.";
  EXPECT_EQ(kEventSignaled, renderer.Wait())
      << "Timed out while waiting for the frame to render.";

  Stop();

  sender_transport.StopSending();
  receiver_transport.StopSending();

  DestroyStreams();
}

TEST_F(EndToEndTest, TransmitsFirstFrame) {
  class Renderer : public VideoRenderer {
   public:
    Renderer() : event_(EventWrapper::Create()) {}

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int /*time_to_render_ms*/) override {
      event_->Set();
    }
    virtual bool IsTextureSupported() const override { return false; }

    EventTypeWrapper Wait() { return event_->Wait(kDefaultTimeoutMs); }

    scoped_ptr<EventWrapper> event_;
  } renderer;

  test::DirectTransport sender_transport, receiver_transport;

  CreateCalls(Call::Config(&sender_transport),
              Call::Config(&receiver_transport));

  sender_transport.SetReceiver(receiver_call_->Receiver());
  receiver_transport.SetReceiver(sender_call_->Receiver());

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();
  receive_configs_[0].renderer = &renderer;

  CreateStreams();
  Start();

  scoped_ptr<test::FrameGenerator> frame_generator(
      test::FrameGenerator::CreateChromaGenerator(
          encoder_config_.streams[0].width, encoder_config_.streams[0].height));
  send_stream_->Input()->SwapFrame(frame_generator->NextFrame());

  EXPECT_EQ(kEventSignaled, renderer.Wait())
      << "Timed out while waiting for the frame to render.";

  Stop();

  sender_transport.StopSending();
  receiver_transport.StopSending();

  DestroyStreams();
}

TEST_F(EndToEndTest, SendsAndReceivesVP9) {
  class VP9Observer : public test::EndToEndTest, public VideoRenderer {
   public:
    VP9Observer()
        : EndToEndTest(2 * kDefaultTimeoutMs),
          encoder_(VideoEncoder::Create(VideoEncoder::kVp9)),
          decoder_(VP9Decoder::Create()),
          frame_counter_(0) {}

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for enough frames to be decoded.";
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      send_config->encoder_settings.encoder = encoder_.get();
      send_config->encoder_settings.payload_name = "VP9";
      send_config->encoder_settings.payload_type = VCM_VP9_PAYLOAD_TYPE;
      encoder_config->streams[0].min_bitrate_bps = 50000;
      encoder_config->streams[0].target_bitrate_bps =
          encoder_config->streams[0].max_bitrate_bps = 2000000;

      (*receive_configs)[0].renderer = this;
      (*receive_configs)[0].decoders.resize(1);
      (*receive_configs)[0].decoders[0].payload_type =
          send_config->encoder_settings.payload_type;
      (*receive_configs)[0].decoders[0].payload_name =
          send_config->encoder_settings.payload_name;
      (*receive_configs)[0].decoders[0].decoder = decoder_.get();
    }

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int time_to_render_ms) override {
      const int kRequiredFrames = 500;
      if (++frame_counter_ == kRequiredFrames)
        observation_complete_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

   private:
    scoped_ptr<webrtc::VideoEncoder> encoder_;
    scoped_ptr<webrtc::VideoDecoder> decoder_;
    int frame_counter_;
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, SendsAndReceivesH264) {
  class H264Observer : public test::EndToEndTest, public VideoRenderer {
   public:
    H264Observer()
        : EndToEndTest(2 * kDefaultTimeoutMs),
          fake_encoder_(Clock::GetRealTimeClock()),
          frame_counter_(0) {}

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for enough frames to be decoded.";
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      send_config->rtp.nack.rtp_history_ms =
          (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      send_config->encoder_settings.encoder = &fake_encoder_;
      send_config->encoder_settings.payload_name = "H264";
      send_config->encoder_settings.payload_type = kFakeSendPayloadType;
      encoder_config->streams[0].min_bitrate_bps = 50000;
      encoder_config->streams[0].target_bitrate_bps =
          encoder_config->streams[0].max_bitrate_bps = 2000000;

      (*receive_configs)[0].renderer = this;
      (*receive_configs)[0].decoders.resize(1);
      (*receive_configs)[0].decoders[0].payload_type =
          send_config->encoder_settings.payload_type;
      (*receive_configs)[0].decoders[0].payload_name =
          send_config->encoder_settings.payload_name;
      (*receive_configs)[0].decoders[0].decoder = &fake_decoder_;
    }

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int time_to_render_ms) override {
      const int kRequiredFrames = 500;
      if (++frame_counter_ == kRequiredFrames)
        observation_complete_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

   private:
    test::FakeH264Decoder fake_decoder_;
    test::FakeH264Encoder fake_encoder_;
    int frame_counter_;
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, ReceiverUsesLocalSsrc) {
  class SyncRtcpObserver : public test::EndToEndTest {
   public:
    SyncRtcpObserver() : EndToEndTest(kDefaultTimeoutMs) {}

    virtual Action OnReceiveRtcp(const uint8_t* packet,
                                 size_t length) override {
      RTCPUtility::RTCPParserV2 parser(packet, length, true);
      EXPECT_TRUE(parser.IsValid());
      uint32_t ssrc = 0;
      ssrc |= static_cast<uint32_t>(packet[4]) << 24;
      ssrc |= static_cast<uint32_t>(packet[5]) << 16;
      ssrc |= static_cast<uint32_t>(packet[6]) << 8;
      ssrc |= static_cast<uint32_t>(packet[7]) << 0;
      EXPECT_EQ(kReceiverLocalSsrc, ssrc);
      observation_complete_->Set();

      return SEND_PACKET;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for a receiver RTCP packet to be sent.";
    }
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, ReceivesAndRetransmitsNack) {
  static const int kNumberOfNacksToObserve = 2;
  static const int kLossBurstSize = 2;
  static const int kPacketsBetweenLossBursts = 9;
  class NackObserver : public test::EndToEndTest {
   public:
    NackObserver()
        : EndToEndTest(kLongTimeoutMs),
          rtp_parser_(RtpHeaderParser::Create()),
          sent_rtp_packets_(0),
          packets_left_to_drop_(0),
          nacks_left_(kNumberOfNacksToObserve) {}

   private:
    virtual Action OnSendRtp(const uint8_t* packet, size_t length) override {
      RTPHeader header;
      EXPECT_TRUE(rtp_parser_->Parse(packet, length, &header));

      // Never drop retransmitted packets.
      if (dropped_packets_.find(header.sequenceNumber) !=
          dropped_packets_.end()) {
        retransmitted_packets_.insert(header.sequenceNumber);
        if (nacks_left_ == 0 &&
            retransmitted_packets_.size() == dropped_packets_.size()) {
          observation_complete_->Set();
        }
        return SEND_PACKET;
      }

      ++sent_rtp_packets_;

      // Enough NACKs received, stop dropping packets.
      if (nacks_left_ == 0)
        return SEND_PACKET;

      // Check if it's time for a new loss burst.
      if (sent_rtp_packets_ % kPacketsBetweenLossBursts == 0)
        packets_left_to_drop_ = kLossBurstSize;

      if (packets_left_to_drop_ > 0) {
        --packets_left_to_drop_;
        dropped_packets_.insert(header.sequenceNumber);
        return DROP_PACKET;
      }

      return SEND_PACKET;
    }

    virtual Action OnReceiveRtcp(const uint8_t* packet,
                                 size_t length) override {
      RTCPUtility::RTCPParserV2 parser(packet, length, true);
      EXPECT_TRUE(parser.IsValid());

      RTCPUtility::RTCPPacketTypes packet_type = parser.Begin();
      while (packet_type != RTCPUtility::kRtcpNotValidCode) {
        if (packet_type == RTCPUtility::kRtcpRtpfbNackCode) {
          --nacks_left_;
          break;
        }
        packet_type = parser.Iterate();
      }
      return SEND_PACKET;
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out waiting for packets to be NACKed, retransmitted and "
             "rendered.";
    }

    scoped_ptr<RtpHeaderParser> rtp_parser_;
    std::set<uint16_t> dropped_packets_;
    std::set<uint16_t> retransmitted_packets_;
    uint64_t sent_rtp_packets_;
    int packets_left_to_drop_;
    int nacks_left_;
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, CanReceiveFec) {
  class FecRenderObserver : public test::EndToEndTest, public VideoRenderer {
   public:
    FecRenderObserver()
        : EndToEndTest(kDefaultTimeoutMs), state_(kFirstPacket) {}

   private:
    virtual Action OnSendRtp(const uint8_t* packet, size_t length) override
        EXCLUSIVE_LOCKS_REQUIRED(crit_) {
      RTPHeader header;
      EXPECT_TRUE(parser_->Parse(packet, length, &header));

      EXPECT_EQ(kRedPayloadType, header.payloadType);
      int encapsulated_payload_type =
          static_cast<int>(packet[header.headerLength]);
      if (encapsulated_payload_type != kFakeSendPayloadType)
        EXPECT_EQ(kUlpfecPayloadType, encapsulated_payload_type);

      if (protected_sequence_numbers_.count(header.sequenceNumber) != 0) {
        // Retransmitted packet, should not count.
        protected_sequence_numbers_.erase(header.sequenceNumber);
        EXPECT_GT(protected_timestamps_.count(header.timestamp), 0u);
        protected_timestamps_.erase(header.timestamp);
        return SEND_PACKET;
      }

      switch (state_) {
        case kFirstPacket:
          state_ = kDropEveryOtherPacketUntilFec;
          break;
        case kDropEveryOtherPacketUntilFec:
          if (encapsulated_payload_type == kUlpfecPayloadType) {
            state_ = kDropNextMediaPacket;
            return SEND_PACKET;
          }
          if (header.sequenceNumber % 2 == 0)
            return DROP_PACKET;
          break;
        case kDropNextMediaPacket:
          if (encapsulated_payload_type == kFakeSendPayloadType) {
            protected_sequence_numbers_.insert(header.sequenceNumber);
            protected_timestamps_.insert(header.timestamp);
            state_ = kDropEveryOtherPacketUntilFec;
            return DROP_PACKET;
          }
          break;
      }

      return SEND_PACKET;
    }

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int time_to_render_ms) override {
      CriticalSectionScoped lock(crit_.get());
      // Rendering frame with timestamp of packet that was dropped -> FEC
      // protection worked.
      if (protected_timestamps_.count(video_frame.timestamp()) != 0)
        observation_complete_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

    enum {
      kFirstPacket,
      kDropEveryOtherPacketUntilFec,
      kDropNextMediaPacket,
    } state_;

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      // TODO(pbos): Run this test with combined NACK/FEC enabled as well.
      // int rtp_history_ms = 1000;
      // (*receive_configs)[0].rtp.nack.rtp_history_ms = rtp_history_ms;
      // send_config->rtp.nack.rtp_history_ms = rtp_history_ms;
      send_config->rtp.fec.red_payload_type = kRedPayloadType;
      send_config->rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;

      (*receive_configs)[0].rtp.fec.red_payload_type = kRedPayloadType;
      (*receive_configs)[0].rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
      (*receive_configs)[0].renderer = this;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out waiting for dropped frames frames to be rendered.";
    }

    std::set<uint32_t> protected_sequence_numbers_ GUARDED_BY(crit_);
    std::set<uint32_t> protected_timestamps_ GUARDED_BY(crit_);
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, ReceivedFecPacketsNotNacked) {
  // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
  // Configure some network delay.
  const int kNetworkDelayMs = 50;
  FakeNetworkPipe::Config config;
  config.queue_delay_ms = kNetworkDelayMs;
  TestReceivedFecPacketsNotNacked(config);
}

void EndToEndTest::TestReceivedFecPacketsNotNacked(
    const FakeNetworkPipe::Config& config) {
  class FecNackObserver : public test::EndToEndTest {
   public:
    explicit FecNackObserver(const FakeNetworkPipe::Config& config)
        : EndToEndTest(kDefaultTimeoutMs, config),
          state_(kDropEveryOtherPacketUntilFec),
          fec_sequence_number_(0),
          has_last_sequence_number_(false),
          last_sequence_number_(0) {}

   private:
    virtual Action OnSendRtp(const uint8_t* packet, size_t length) override {
      RTPHeader header;
      EXPECT_TRUE(parser_->Parse(packet, length, &header));
      EXPECT_EQ(kRedPayloadType, header.payloadType);

      int encapsulated_payload_type =
          static_cast<int>(packet[header.headerLength]);
      if (encapsulated_payload_type != kFakeSendPayloadType)
        EXPECT_EQ(kUlpfecPayloadType, encapsulated_payload_type);

      if (has_last_sequence_number_ &&
          !IsNewerSequenceNumber(header.sequenceNumber,
                                 last_sequence_number_)) {
        // Drop retransmitted packets.
        return DROP_PACKET;
      }
      last_sequence_number_ = header.sequenceNumber;
      has_last_sequence_number_ = true;

      bool fec_packet = encapsulated_payload_type == kUlpfecPayloadType;
      switch (state_) {
        case kDropEveryOtherPacketUntilFec:
          if (fec_packet) {
            state_ = kDropAllMediaPacketsUntilFec;
          } else if (header.sequenceNumber % 2 == 0) {
            return DROP_PACKET;
          }
          break;
        case kDropAllMediaPacketsUntilFec:
          if (!fec_packet)
            return DROP_PACKET;
          fec_sequence_number_ = header.sequenceNumber;
          state_ = kVerifyFecPacketNotInNackList;
          break;
        case kVerifyFecPacketNotInNackList:
          // Continue to drop packets. Make sure no frame can be decoded.
          if (fec_packet || header.sequenceNumber % 2 == 0)
            return DROP_PACKET;
          break;
      }
      return SEND_PACKET;
    }

    virtual Action OnReceiveRtcp(const uint8_t* packet,
                                 size_t length) override {
      if (state_ == kVerifyFecPacketNotInNackList) {
        test::RtcpPacketParser rtcp_parser;
        rtcp_parser.Parse(packet, length);
        std::vector<uint16_t> nacks = rtcp_parser.nack_item()->last_nack_list();
        if (!nacks.empty() &&
            IsNewerSequenceNumber(nacks.back(), fec_sequence_number_)) {
          EXPECT_TRUE(std::find(
              nacks.begin(), nacks.end(), fec_sequence_number_) == nacks.end());
          observation_complete_->Set();
        }
      }
      return SEND_PACKET;
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      // Configure hybrid NACK/FEC.
      send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      send_config->rtp.fec.red_payload_type = kRedPayloadType;
      send_config->rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
      (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      (*receive_configs)[0].rtp.fec.red_payload_type = kRedPayloadType;
      (*receive_configs)[0].rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for FEC packets to be received.";
    }

    enum {
      kDropEveryOtherPacketUntilFec,
      kDropAllMediaPacketsUntilFec,
      kVerifyFecPacketNotInNackList,
    } state_;

    uint16_t fec_sequence_number_;
    bool has_last_sequence_number_;
    uint16_t last_sequence_number_;
  } test(config);

  RunBaseTest(&test);<commit_msg>Let first packet through to avoid getting key frame requests (and no nacks) for EndToEndTest.ReceivedFecPacketsNotNacked.<commit_after>/*
 *  Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */
#include <assert.h>

#include <algorithm>
#include <map>
#include <sstream>
#include <string>

#include "testing/gtest/include/gtest/gtest.h"

#include "webrtc/call.h"
#include "webrtc/frame_callback.h"
#include "webrtc/modules/rtp_rtcp/source/rtcp_utility.h"
#include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h"
#include "webrtc/modules/video_coding/codecs/vp9/include/vp9.h"
#include "webrtc/modules/video_coding/main/interface/video_coding_defines.h"
#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
#include "webrtc/system_wrappers/interface/event_wrapper.h"
#include "webrtc/system_wrappers/interface/scoped_ptr.h"
#include "webrtc/system_wrappers/interface/sleep.h"
#include "webrtc/test/call_test.h"
#include "webrtc/test/direct_transport.h"
#include "webrtc/test/encoder_settings.h"
#include "webrtc/test/fake_audio_device.h"
#include "webrtc/test/fake_decoder.h"
#include "webrtc/test/fake_encoder.h"
#include "webrtc/test/frame_generator.h"
#include "webrtc/test/frame_generator_capturer.h"
#include "webrtc/test/null_transport.h"
#include "webrtc/test/rtcp_packet_parser.h"
#include "webrtc/test/rtp_rtcp_observer.h"
#include "webrtc/test/testsupport/fileutils.h"
#include "webrtc/test/testsupport/gtest_disable.h"
#include "webrtc/test/testsupport/perf_test.h"
#include "webrtc/video/transport_adapter.h"
#include "webrtc/video_encoder.h"

namespace webrtc {

static const unsigned long kSilenceTimeoutMs = 2000;

class EndToEndTest : public test::CallTest {
 public:
  EndToEndTest() {}

  virtual ~EndToEndTest() {
    EXPECT_EQ(NULL, send_stream_);
    EXPECT_TRUE(receive_streams_.empty());
  }

 protected:
  class UnusedTransport : public newapi::Transport {
   private:
    virtual bool SendRtp(const uint8_t* packet, size_t length) override {
      ADD_FAILURE() << "Unexpected RTP sent.";
      return false;
    }

    virtual bool SendRtcp(const uint8_t* packet, size_t length) override {
      ADD_FAILURE() << "Unexpected RTCP sent.";
      return false;
    }
  };

  void DecodesRetransmittedFrame(bool retransmit_over_rtx);
  void ReceivesPliAndRecovers(int rtp_history_ms);
  void RespectsRtcpMode(newapi::RtcpMode rtcp_mode);
  void TestXrReceiverReferenceTimeReport(bool enable_rrtr);
  void TestSendsSetSsrcs(size_t num_ssrcs, bool send_single_ssrc_first);
  void TestRtpStatePreservation(bool use_rtx);
  void TestReceivedFecPacketsNotNacked(const FakeNetworkPipe::Config& config);
};

TEST_F(EndToEndTest, ReceiverCanBeStartedTwice) {
  test::NullTransport transport;
  CreateCalls(Call::Config(&transport), Call::Config(&transport));

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();

  CreateStreams();

  receive_streams_[0]->Start();
  receive_streams_[0]->Start();

  DestroyStreams();
}

TEST_F(EndToEndTest, ReceiverCanBeStoppedTwice) {
  test::NullTransport transport;
  CreateCalls(Call::Config(&transport), Call::Config(&transport));

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();

  CreateStreams();

  receive_streams_[0]->Stop();
  receive_streams_[0]->Stop();

  DestroyStreams();
}

TEST_F(EndToEndTest, RendersSingleDelayedFrame) {
  static const int kWidth = 320;
  static const int kHeight = 240;
  // This constant is chosen to be higher than the timeout in the video_render
  // module. This makes sure that frames aren't dropped if there are no other
  // frames in the queue.
  static const int kDelayRenderCallbackMs = 1000;

  class Renderer : public VideoRenderer {
   public:
    Renderer() : event_(EventWrapper::Create()) {}

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int /*time_to_render_ms*/) override {
      event_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

    EventTypeWrapper Wait() { return event_->Wait(kDefaultTimeoutMs); }

    scoped_ptr<EventWrapper> event_;
  } renderer;

  class TestFrameCallback : public I420FrameCallback {
   public:
    TestFrameCallback() : event_(EventWrapper::Create()) {}

    EventTypeWrapper Wait() { return event_->Wait(kDefaultTimeoutMs); }

   private:
    virtual void FrameCallback(I420VideoFrame* frame) override {
      SleepMs(kDelayRenderCallbackMs);
      event_->Set();
    }

    scoped_ptr<EventWrapper> event_;
  };

  test::DirectTransport sender_transport, receiver_transport;

  CreateCalls(Call::Config(&sender_transport),
              Call::Config(&receiver_transport));

  sender_transport.SetReceiver(receiver_call_->Receiver());
  receiver_transport.SetReceiver(sender_call_->Receiver());

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();

  TestFrameCallback pre_render_callback;
  receive_configs_[0].pre_render_callback = &pre_render_callback;
  receive_configs_[0].renderer = &renderer;

  CreateStreams();
  Start();

  // Create frames that are smaller than the send width/height, this is done to
  // check that the callbacks are done after processing video.
  scoped_ptr<test::FrameGenerator> frame_generator(
      test::FrameGenerator::CreateChromaGenerator(kWidth, kHeight));
  send_stream_->Input()->SwapFrame(frame_generator->NextFrame());
  EXPECT_EQ(kEventSignaled, pre_render_callback.Wait())
      << "Timed out while waiting for pre-render callback.";
  EXPECT_EQ(kEventSignaled, renderer.Wait())
      << "Timed out while waiting for the frame to render.";

  Stop();

  sender_transport.StopSending();
  receiver_transport.StopSending();

  DestroyStreams();
}

TEST_F(EndToEndTest, TransmitsFirstFrame) {
  class Renderer : public VideoRenderer {
   public:
    Renderer() : event_(EventWrapper::Create()) {}

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int /*time_to_render_ms*/) override {
      event_->Set();
    }
    virtual bool IsTextureSupported() const override { return false; }

    EventTypeWrapper Wait() { return event_->Wait(kDefaultTimeoutMs); }

    scoped_ptr<EventWrapper> event_;
  } renderer;

  test::DirectTransport sender_transport, receiver_transport;

  CreateCalls(Call::Config(&sender_transport),
              Call::Config(&receiver_transport));

  sender_transport.SetReceiver(receiver_call_->Receiver());
  receiver_transport.SetReceiver(sender_call_->Receiver());

  CreateSendConfig(1);
  CreateMatchingReceiveConfigs();
  receive_configs_[0].renderer = &renderer;

  CreateStreams();
  Start();

  scoped_ptr<test::FrameGenerator> frame_generator(
      test::FrameGenerator::CreateChromaGenerator(
          encoder_config_.streams[0].width, encoder_config_.streams[0].height));
  send_stream_->Input()->SwapFrame(frame_generator->NextFrame());

  EXPECT_EQ(kEventSignaled, renderer.Wait())
      << "Timed out while waiting for the frame to render.";

  Stop();

  sender_transport.StopSending();
  receiver_transport.StopSending();

  DestroyStreams();
}

TEST_F(EndToEndTest, SendsAndReceivesVP9) {
  class VP9Observer : public test::EndToEndTest, public VideoRenderer {
   public:
    VP9Observer()
        : EndToEndTest(2 * kDefaultTimeoutMs),
          encoder_(VideoEncoder::Create(VideoEncoder::kVp9)),
          decoder_(VP9Decoder::Create()),
          frame_counter_(0) {}

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for enough frames to be decoded.";
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      send_config->encoder_settings.encoder = encoder_.get();
      send_config->encoder_settings.payload_name = "VP9";
      send_config->encoder_settings.payload_type = VCM_VP9_PAYLOAD_TYPE;
      encoder_config->streams[0].min_bitrate_bps = 50000;
      encoder_config->streams[0].target_bitrate_bps =
          encoder_config->streams[0].max_bitrate_bps = 2000000;

      (*receive_configs)[0].renderer = this;
      (*receive_configs)[0].decoders.resize(1);
      (*receive_configs)[0].decoders[0].payload_type =
          send_config->encoder_settings.payload_type;
      (*receive_configs)[0].decoders[0].payload_name =
          send_config->encoder_settings.payload_name;
      (*receive_configs)[0].decoders[0].decoder = decoder_.get();
    }

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int time_to_render_ms) override {
      const int kRequiredFrames = 500;
      if (++frame_counter_ == kRequiredFrames)
        observation_complete_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

   private:
    scoped_ptr<webrtc::VideoEncoder> encoder_;
    scoped_ptr<webrtc::VideoDecoder> decoder_;
    int frame_counter_;
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, SendsAndReceivesH264) {
  class H264Observer : public test::EndToEndTest, public VideoRenderer {
   public:
    H264Observer()
        : EndToEndTest(2 * kDefaultTimeoutMs),
          fake_encoder_(Clock::GetRealTimeClock()),
          frame_counter_(0) {}

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for enough frames to be decoded.";
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      send_config->rtp.nack.rtp_history_ms =
          (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      send_config->encoder_settings.encoder = &fake_encoder_;
      send_config->encoder_settings.payload_name = "H264";
      send_config->encoder_settings.payload_type = kFakeSendPayloadType;
      encoder_config->streams[0].min_bitrate_bps = 50000;
      encoder_config->streams[0].target_bitrate_bps =
          encoder_config->streams[0].max_bitrate_bps = 2000000;

      (*receive_configs)[0].renderer = this;
      (*receive_configs)[0].decoders.resize(1);
      (*receive_configs)[0].decoders[0].payload_type =
          send_config->encoder_settings.payload_type;
      (*receive_configs)[0].decoders[0].payload_name =
          send_config->encoder_settings.payload_name;
      (*receive_configs)[0].decoders[0].decoder = &fake_decoder_;
    }

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int time_to_render_ms) override {
      const int kRequiredFrames = 500;
      if (++frame_counter_ == kRequiredFrames)
        observation_complete_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

   private:
    test::FakeH264Decoder fake_decoder_;
    test::FakeH264Encoder fake_encoder_;
    int frame_counter_;
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, ReceiverUsesLocalSsrc) {
  class SyncRtcpObserver : public test::EndToEndTest {
   public:
    SyncRtcpObserver() : EndToEndTest(kDefaultTimeoutMs) {}

    virtual Action OnReceiveRtcp(const uint8_t* packet,
                                 size_t length) override {
      RTCPUtility::RTCPParserV2 parser(packet, length, true);
      EXPECT_TRUE(parser.IsValid());
      uint32_t ssrc = 0;
      ssrc |= static_cast<uint32_t>(packet[4]) << 24;
      ssrc |= static_cast<uint32_t>(packet[5]) << 16;
      ssrc |= static_cast<uint32_t>(packet[6]) << 8;
      ssrc |= static_cast<uint32_t>(packet[7]) << 0;
      EXPECT_EQ(kReceiverLocalSsrc, ssrc);
      observation_complete_->Set();

      return SEND_PACKET;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for a receiver RTCP packet to be sent.";
    }
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, ReceivesAndRetransmitsNack) {
  static const int kNumberOfNacksToObserve = 2;
  static const int kLossBurstSize = 2;
  static const int kPacketsBetweenLossBursts = 9;
  class NackObserver : public test::EndToEndTest {
   public:
    NackObserver()
        : EndToEndTest(kLongTimeoutMs),
          rtp_parser_(RtpHeaderParser::Create()),
          sent_rtp_packets_(0),
          packets_left_to_drop_(0),
          nacks_left_(kNumberOfNacksToObserve) {}

   private:
    virtual Action OnSendRtp(const uint8_t* packet, size_t length) override {
      RTPHeader header;
      EXPECT_TRUE(rtp_parser_->Parse(packet, length, &header));

      // Never drop retransmitted packets.
      if (dropped_packets_.find(header.sequenceNumber) !=
          dropped_packets_.end()) {
        retransmitted_packets_.insert(header.sequenceNumber);
        if (nacks_left_ == 0 &&
            retransmitted_packets_.size() == dropped_packets_.size()) {
          observation_complete_->Set();
        }
        return SEND_PACKET;
      }

      ++sent_rtp_packets_;

      // Enough NACKs received, stop dropping packets.
      if (nacks_left_ == 0)
        return SEND_PACKET;

      // Check if it's time for a new loss burst.
      if (sent_rtp_packets_ % kPacketsBetweenLossBursts == 0)
        packets_left_to_drop_ = kLossBurstSize;

      if (packets_left_to_drop_ > 0) {
        --packets_left_to_drop_;
        dropped_packets_.insert(header.sequenceNumber);
        return DROP_PACKET;
      }

      return SEND_PACKET;
    }

    virtual Action OnReceiveRtcp(const uint8_t* packet,
                                 size_t length) override {
      RTCPUtility::RTCPParserV2 parser(packet, length, true);
      EXPECT_TRUE(parser.IsValid());

      RTCPUtility::RTCPPacketTypes packet_type = parser.Begin();
      while (packet_type != RTCPUtility::kRtcpNotValidCode) {
        if (packet_type == RTCPUtility::kRtcpRtpfbNackCode) {
          --nacks_left_;
          break;
        }
        packet_type = parser.Iterate();
      }
      return SEND_PACKET;
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out waiting for packets to be NACKed, retransmitted and "
             "rendered.";
    }

    scoped_ptr<RtpHeaderParser> rtp_parser_;
    std::set<uint16_t> dropped_packets_;
    std::set<uint16_t> retransmitted_packets_;
    uint64_t sent_rtp_packets_;
    int packets_left_to_drop_;
    int nacks_left_;
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, CanReceiveFec) {
  class FecRenderObserver : public test::EndToEndTest, public VideoRenderer {
   public:
    FecRenderObserver()
        : EndToEndTest(kDefaultTimeoutMs), state_(kFirstPacket) {}

   private:
    virtual Action OnSendRtp(const uint8_t* packet, size_t length) override
        EXCLUSIVE_LOCKS_REQUIRED(crit_) {
      RTPHeader header;
      EXPECT_TRUE(parser_->Parse(packet, length, &header));

      EXPECT_EQ(kRedPayloadType, header.payloadType);
      int encapsulated_payload_type =
          static_cast<int>(packet[header.headerLength]);
      if (encapsulated_payload_type != kFakeSendPayloadType)
        EXPECT_EQ(kUlpfecPayloadType, encapsulated_payload_type);

      if (protected_sequence_numbers_.count(header.sequenceNumber) != 0) {
        // Retransmitted packet, should not count.
        protected_sequence_numbers_.erase(header.sequenceNumber);
        EXPECT_GT(protected_timestamps_.count(header.timestamp), 0u);
        protected_timestamps_.erase(header.timestamp);
        return SEND_PACKET;
      }

      switch (state_) {
        case kFirstPacket:
          state_ = kDropEveryOtherPacketUntilFec;
          break;
        case kDropEveryOtherPacketUntilFec:
          if (encapsulated_payload_type == kUlpfecPayloadType) {
            state_ = kDropNextMediaPacket;
            return SEND_PACKET;
          }
          if (header.sequenceNumber % 2 == 0)
            return DROP_PACKET;
          break;
        case kDropNextMediaPacket:
          if (encapsulated_payload_type == kFakeSendPayloadType) {
            protected_sequence_numbers_.insert(header.sequenceNumber);
            protected_timestamps_.insert(header.timestamp);
            state_ = kDropEveryOtherPacketUntilFec;
            return DROP_PACKET;
          }
          break;
      }

      return SEND_PACKET;
    }

    virtual void RenderFrame(const I420VideoFrame& video_frame,
                             int time_to_render_ms) override {
      CriticalSectionScoped lock(crit_.get());
      // Rendering frame with timestamp of packet that was dropped -> FEC
      // protection worked.
      if (protected_timestamps_.count(video_frame.timestamp()) != 0)
        observation_complete_->Set();
    }

    virtual bool IsTextureSupported() const override { return false; }

    enum {
      kFirstPacket,
      kDropEveryOtherPacketUntilFec,
      kDropNextMediaPacket,
    } state_;

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      // TODO(pbos): Run this test with combined NACK/FEC enabled as well.
      // int rtp_history_ms = 1000;
      // (*receive_configs)[0].rtp.nack.rtp_history_ms = rtp_history_ms;
      // send_config->rtp.nack.rtp_history_ms = rtp_history_ms;
      send_config->rtp.fec.red_payload_type = kRedPayloadType;
      send_config->rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;

      (*receive_configs)[0].rtp.fec.red_payload_type = kRedPayloadType;
      (*receive_configs)[0].rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
      (*receive_configs)[0].renderer = this;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out waiting for dropped frames frames to be rendered.";
    }

    std::set<uint32_t> protected_sequence_numbers_ GUARDED_BY(crit_);
    std::set<uint32_t> protected_timestamps_ GUARDED_BY(crit_);
  } test;

  RunBaseTest(&test);
}

TEST_F(EndToEndTest, ReceivedFecPacketsNotNacked) {
  // At low RTT (< kLowRttNackMs) -> NACK only, no FEC.
  // Configure some network delay.
  const int kNetworkDelayMs = 50;
  FakeNetworkPipe::Config config;
  config.queue_delay_ms = kNetworkDelayMs;
  TestReceivedFecPacketsNotNacked(config);
}

void EndToEndTest::TestReceivedFecPacketsNotNacked(
    const FakeNetworkPipe::Config& config) {
  class FecNackObserver : public test::EndToEndTest {
   public:
    explicit FecNackObserver(const FakeNetworkPipe::Config& config)
        : EndToEndTest(kDefaultTimeoutMs, config),
          state_(kFirstPacket),
          fec_sequence_number_(0),
          has_last_sequence_number_(false),
          last_sequence_number_(0) {}

   private:
    virtual Action OnSendRtp(const uint8_t* packet, size_t length) override {
      RTPHeader header;
      EXPECT_TRUE(parser_->Parse(packet, length, &header));
      EXPECT_EQ(kRedPayloadType, header.payloadType);

      int encapsulated_payload_type =
          static_cast<int>(packet[header.headerLength]);
      if (encapsulated_payload_type != kFakeSendPayloadType)
        EXPECT_EQ(kUlpfecPayloadType, encapsulated_payload_type);

      if (has_last_sequence_number_ &&
          !IsNewerSequenceNumber(header.sequenceNumber,
                                 last_sequence_number_)) {
        // Drop retransmitted packets.
        return DROP_PACKET;
      }
      last_sequence_number_ = header.sequenceNumber;
      has_last_sequence_number_ = true;

      bool fec_packet = encapsulated_payload_type == kUlpfecPayloadType;
      switch (state_) {
        case kFirstPacket:
          state_ = kDropEveryOtherPacketUntilFec;
          break;
        case kDropEveryOtherPacketUntilFec:
          if (fec_packet) {
            state_ = kDropAllMediaPacketsUntilFec;
          } else if (header.sequenceNumber % 2 == 0) {
            return DROP_PACKET;
          }
          break;
        case kDropAllMediaPacketsUntilFec:
          if (!fec_packet)
            return DROP_PACKET;
          fec_sequence_number_ = header.sequenceNumber;
          state_ = kVerifyFecPacketNotInNackList;
          break;
        case kVerifyFecPacketNotInNackList:
          // Continue to drop packets. Make sure no frame can be decoded.
          if (fec_packet || header.sequenceNumber % 2 == 0)
            return DROP_PACKET;
          break;
      }
      return SEND_PACKET;
    }

    virtual Action OnReceiveRtcp(const uint8_t* packet,
                                 size_t length) override {
      if (state_ == kVerifyFecPacketNotInNackList) {
        test::RtcpPacketParser rtcp_parser;
        rtcp_parser.Parse(packet, length);
        std::vector<uint16_t> nacks = rtcp_parser.nack_item()->last_nack_list();
        if (!nacks.empty() &&
            IsNewerSequenceNumber(nacks.back(), fec_sequence_number_)) {
          EXPECT_TRUE(std::find(
              nacks.begin(), nacks.end(), fec_sequence_number_) == nacks.end());
          observation_complete_->Set();
        }
      }
      return SEND_PACKET;
    }

    virtual void ModifyConfigs(
        VideoSendStream::Config* send_config,
        std::vector<VideoReceiveStream::Config>* receive_configs,
        VideoEncoderConfig* encoder_config) override {
      // Configure hybrid NACK/FEC.
      send_config->rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      send_config->rtp.fec.red_payload_type = kRedPayloadType;
      send_config->rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
      (*receive_configs)[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
      (*receive_configs)[0].rtp.fec.red_payload_type = kRedPayloadType;
      (*receive_configs)[0].rtp.fec.ulpfec_payload_type = kUlpfecPayloadType;
    }

    virtual void PerformTest() override {
      EXPECT_EQ(kEventSignaled, Wait())
          << "Timed out while waiting for FEC packets to be received.";
    }

    enum {
      kFirstPacket,
      kDropEveryOtherPacketUntilFec,
      kDropAllMediaPacketsUntilFec,
      kVerifyFecPacketNotInNackList,
    } state_;

    uint16_t fec_sequence_number_;
    bool has_last_sequence_number_;
    uint16_t last_sequence_number_;
  } test(config);

  RunBaseTest(&test);
{'fraction_non_alphanumeric': 0.10016888557453996, 'fraction_numerical': 0.009694470642423179, 'mean_word_length': 3.515151515151515, 'pattern_counts': {'":': 0, '<': 123, '<?xml version=': 0, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6624753', 'n_tokens_mistral': 14648, 'n_tokens_neox': 14093, 'n_words': 2593}