<commit_before>#include "output_ebml.h"
#include <mist/ebml_socketglue.h>
#include <mist/riff.h>
#include <mist/opus.h>

namespace Mist{
  OutEBML::OutEBML(Socket::Connection &conn) : HTTPOutput(conn){
    currentClusterTime = 0;
    newClusterTime = 0;
    segmentSize = 0xFFFFFFFFFFFFFFFFull;
    tracksSize = 0;
    infoSize = 0;
    cuesSize = 0;
    seekheadSize = 0;
    doctype = "matroska";
    if (config->getString("target").size()){
      if (config->getString("target").find(".webm") != std::string::npos){doctype = "webm";}
      initialize();
      if (myMeta.vod){
        calcVodSizes();
      }
      if (!streamName.size()){
        WARN_MSG("Recording unconnected EBML output to file! Cancelled.");
        conn.close();
        return;
      }
      if (config->getString("target") == "-"){
        parseData = true;
        wantRequest = false;
        INFO_MSG("Outputting %s to stdout in EBML format", streamName.c_str());
        return;
      }
      if (!myMeta.tracks.size()){
        INFO_MSG("Stream not available - aborting");
        conn.close();
        return;
      }
      if (connectToFile(config->getString("target"))){
        parseData = true;
        wantRequest = false;
        INFO_MSG("Recording %s to %s in EBML format", streamName.c_str(),
                 config->getString("target").c_str());
        return;
      }
      conn.close();
    }
  }

  void OutEBML::init(Util::Config *cfg){
    HTTPOutput::init(cfg);
    capa["name"] = "EBML";
    capa["friendly"] = "WebM/MKV over HTTP";
    capa["desc"] = "Pseudostreaming in MKV and WebM (EBML) formats over HTTP";
    capa["url_rel"] = "/$.webm";
    capa["url_match"].append("/$.mkv");
    capa["url_match"].append("/$.webm");
    capa["codecs"][0u][0u].append("H264");
    capa["codecs"][0u][0u].append("HEVC");
    capa["codecs"][0u][0u].append("VP8");
    capa["codecs"][0u][0u].append("VP9");
    capa["codecs"][0u][0u].append("theora");
    capa["codecs"][0u][0u].append("MPEG2");
    capa["codecs"][0u][0u].append("AV1");
    capa["codecs"][0u][1u].append("AAC");
    capa["codecs"][0u][1u].append("vorbis");
    capa["codecs"][0u][1u].append("opus");
    capa["codecs"][0u][1u].append("PCM");
    capa["codecs"][0u][1u].append("ALAW");
    capa["codecs"][0u][1u].append("ULAW");
    capa["codecs"][0u][1u].append("MP2");
    capa["codecs"][0u][1u].append("MP3");
    capa["codecs"][0u][1u].append("FLOAT");
    capa["codecs"][0u][1u].append("AC3");
    capa["codecs"][0u][1u].append("DTS");
    capa["codecs"][0u][2u].append("+JSON");
    capa["methods"][0u]["handler"] = "http";
    capa["methods"][0u]["type"] = "html5/video/webm";
    capa["methods"][0u]["priority"] = 9;
    //EBML will only work with VP8/VP9/Opus except on Chrome
    JSON::Value blacklistNonChrome = JSON::fromString("[[\"blacklist\"], [\"whitelist\",[\"Chrome\",\"Chromium\"]], [\"blacklist\",[\"Edge\",\"OPR/\"]], [\"blacklist\",[\"Android\"]]]");
    capa["exceptions"]["codec:H264"] = blacklistNonChrome;
    capa["exceptions"]["codec:HEVC"] = blacklistNonChrome;
    capa["exceptions"]["codec:theora"] = blacklistNonChrome;
    capa["exceptions"]["codec:MPEG2"] = blacklistNonChrome;
    capa["exceptions"]["codec:AAC"] = blacklistNonChrome;
    capa["exceptions"]["codec:vorbis"] = blacklistNonChrome;
    capa["exceptions"]["codec:PCM"] = blacklistNonChrome;
    capa["exceptions"]["codec:ALAW"] = blacklistNonChrome;
    capa["exceptions"]["codec:ULAW"] = blacklistNonChrome;
    capa["exceptions"]["codec:MP2"] = blacklistNonChrome;
    capa["exceptions"]["codec:MP3"] = blacklistNonChrome;
    capa["exceptions"]["codec:FLOAT"] = blacklistNonChrome;
    capa["exceptions"]["codec:AC3"] = blacklistNonChrome;
    capa["exceptions"]["codec:DTS"] = blacklistNonChrome;
    capa["push_urls"].append("/*.mkv");
    capa["push_urls"].append("/*.webm");

    JSON::Value opt;
    opt["arg"] = "string";
    opt["default"] = "";
    opt["arg_num"] = 1;
    opt["help"] = "Target filename to store EBML file as, or - for stdout.";
    cfg->addOption("target", opt);
  }

  bool OutEBML::isRecording(){return config->getString("target").size();}

  /// Calculates the size of a Cluster (contents only) and returns it.
  /// Bases the calculation on the currently selected tracks and the given start/end time for the cluster.
  uint32_t OutEBML::clusterSize(uint64_t start, uint64_t end){
    if (start == end){++end;}
    uint32_t sendLen = EBML::sizeElemUInt(EBML::EID_TIMECODE, start);
    for (std::set<long unsigned int>::iterator it = selectedTracks.begin();
         it != selectedTracks.end(); it++){
      DTSC::Track &thisTrack = myMeta.tracks[*it];
      uint32_t firstPart = 0;
      unsigned long long int prevParts = 0;
      uint64_t curMS = 0;
      for (std::deque<DTSC::Key>::iterator it2 = thisTrack.keys.begin();
           it2 != thisTrack.keys.end(); it2++){
        if (it2->getTime() > start && it2 != thisTrack.keys.begin()){break;}
        firstPart += prevParts;
        prevParts = it2->getParts();
        curMS = it2->getTime();
      }
      size_t maxParts = thisTrack.parts.size();
      for (size_t i = firstPart; i < maxParts; i++){
        if (curMS >= end){break;}
        if (curMS >= start){
          uint32_t blkLen = EBML::sizeSimpleBlock(thisTrack.trackID, thisTrack.parts[i].getSize());
          sendLen += blkLen;
        }
        curMS += thisTrack.parts[i].getDuration();
      }
    }
    return sendLen;
  }

  void OutEBML::sendNext(){
    if (thisPacket.getTime() >= newClusterTime){
      currentClusterTime = thisPacket.getTime();
      if (myMeta.vod){
        //In case of VoD, clusters are aligned with the main track fragments
        //EXCEPT when they are more than 30 seconds long, because clusters are limited to -32 to 32 seconds.
        DTSC::Track &Trk = myMeta.tracks[getMainSelectedTrack()];<commit_msg>Added liveSeek to EBML output to stay near live point<commit_after>#include "output_ebml.h"
#include <mist/ebml_socketglue.h>
#include <mist/riff.h>
#include <mist/opus.h>

namespace Mist{
  OutEBML::OutEBML(Socket::Connection &conn) : HTTPOutput(conn){
    currentClusterTime = 0;
    newClusterTime = 0;
    segmentSize = 0xFFFFFFFFFFFFFFFFull;
    tracksSize = 0;
    infoSize = 0;
    cuesSize = 0;
    seekheadSize = 0;
    doctype = "matroska";
    if (config->getString("target").size()){
      if (config->getString("target").find(".webm") != std::string::npos){doctype = "webm";}
      initialize();
      if (myMeta.vod){
        calcVodSizes();
      }
      if (!streamName.size()){
        WARN_MSG("Recording unconnected EBML output to file! Cancelled.");
        conn.close();
        return;
      }
      if (config->getString("target") == "-"){
        parseData = true;
        wantRequest = false;
        INFO_MSG("Outputting %s to stdout in EBML format", streamName.c_str());
        return;
      }
      if (!myMeta.tracks.size()){
        INFO_MSG("Stream not available - aborting");
        conn.close();
        return;
      }
      if (connectToFile(config->getString("target"))){
        parseData = true;
        wantRequest = false;
        INFO_MSG("Recording %s to %s in EBML format", streamName.c_str(),
                 config->getString("target").c_str());
        return;
      }
      conn.close();
    }
  }

  void OutEBML::init(Util::Config *cfg){
    HTTPOutput::init(cfg);
    capa["name"] = "EBML";
    capa["friendly"] = "WebM/MKV over HTTP";
    capa["desc"] = "Pseudostreaming in MKV and WebM (EBML) formats over HTTP";
    capa["url_rel"] = "/$.webm";
    capa["url_match"].append("/$.mkv");
    capa["url_match"].append("/$.webm");
    capa["codecs"][0u][0u].append("H264");
    capa["codecs"][0u][0u].append("HEVC");
    capa["codecs"][0u][0u].append("VP8");
    capa["codecs"][0u][0u].append("VP9");
    capa["codecs"][0u][0u].append("theora");
    capa["codecs"][0u][0u].append("MPEG2");
    capa["codecs"][0u][0u].append("AV1");
    capa["codecs"][0u][1u].append("AAC");
    capa["codecs"][0u][1u].append("vorbis");
    capa["codecs"][0u][1u].append("opus");
    capa["codecs"][0u][1u].append("PCM");
    capa["codecs"][0u][1u].append("ALAW");
    capa["codecs"][0u][1u].append("ULAW");
    capa["codecs"][0u][1u].append("MP2");
    capa["codecs"][0u][1u].append("MP3");
    capa["codecs"][0u][1u].append("FLOAT");
    capa["codecs"][0u][1u].append("AC3");
    capa["codecs"][0u][1u].append("DTS");
    capa["codecs"][0u][2u].append("+JSON");
    capa["methods"][0u]["handler"] = "http";
    capa["methods"][0u]["type"] = "html5/video/webm";
    capa["methods"][0u]["priority"] = 9;
    //EBML will only work with VP8/VP9/Opus except on Chrome
    JSON::Value blacklistNonChrome = JSON::fromString("[[\"blacklist\"], [\"whitelist\",[\"Chrome\",\"Chromium\"]], [\"blacklist\",[\"Edge\",\"OPR/\"]], [\"blacklist\",[\"Android\"]]]");
    capa["exceptions"]["codec:H264"] = blacklistNonChrome;
    capa["exceptions"]["codec:HEVC"] = blacklistNonChrome;
    capa["exceptions"]["codec:theora"] = blacklistNonChrome;
    capa["exceptions"]["codec:MPEG2"] = blacklistNonChrome;
    capa["exceptions"]["codec:AAC"] = blacklistNonChrome;
    capa["exceptions"]["codec:vorbis"] = blacklistNonChrome;
    capa["exceptions"]["codec:PCM"] = blacklistNonChrome;
    capa["exceptions"]["codec:ALAW"] = blacklistNonChrome;
    capa["exceptions"]["codec:ULAW"] = blacklistNonChrome;
    capa["exceptions"]["codec:MP2"] = blacklistNonChrome;
    capa["exceptions"]["codec:MP3"] = blacklistNonChrome;
    capa["exceptions"]["codec:FLOAT"] = blacklistNonChrome;
    capa["exceptions"]["codec:AC3"] = blacklistNonChrome;
    capa["exceptions"]["codec:DTS"] = blacklistNonChrome;
    capa["push_urls"].append("/*.mkv");
    capa["push_urls"].append("/*.webm");

    JSON::Value opt;
    opt["arg"] = "string";
    opt["default"] = "";
    opt["arg_num"] = 1;
    opt["help"] = "Target filename to store EBML file as, or - for stdout.";
    cfg->addOption("target", opt);
  }

  bool OutEBML::isRecording(){return config->getString("target").size();}

  /// Calculates the size of a Cluster (contents only) and returns it.
  /// Bases the calculation on the currently selected tracks and the given start/end time for the cluster.
  uint32_t OutEBML::clusterSize(uint64_t start, uint64_t end){
    if (start == end){++end;}
    uint32_t sendLen = EBML::sizeElemUInt(EBML::EID_TIMECODE, start);
    for (std::set<long unsigned int>::iterator it = selectedTracks.begin();
         it != selectedTracks.end(); it++){
      DTSC::Track &thisTrack = myMeta.tracks[*it];
      uint32_t firstPart = 0;
      unsigned long long int prevParts = 0;
      uint64_t curMS = 0;
      for (std::deque<DTSC::Key>::iterator it2 = thisTrack.keys.begin();
           it2 != thisTrack.keys.end(); it2++){
        if (it2->getTime() > start && it2 != thisTrack.keys.begin()){break;}
        firstPart += prevParts;
        prevParts = it2->getParts();
        curMS = it2->getTime();
      }
      size_t maxParts = thisTrack.parts.size();
      for (size_t i = firstPart; i < maxParts; i++){
        if (curMS >= end){break;}
        if (curMS >= start){
          uint32_t blkLen = EBML::sizeSimpleBlock(thisTrack.trackID, thisTrack.parts[i].getSize());
          sendLen += blkLen;
        }
        curMS += thisTrack.parts[i].getDuration();
      }
    }
    return sendLen;
  }

  void OutEBML::sendNext(){
    if (thisPacket.getTime() >= newClusterTime){
      if (liveSeek()){return;}
      currentClusterTime = thisPacket.getTime();
      if (myMeta.vod){
        //In case of VoD, clusters are aligned with the main track fragments
        //EXCEPT when they are more than 30 seconds long, because clusters are limited to -32 to 32 seconds.
        DTSC::Track &Trk = myMeta.tracks[getMainSelectedTrack()];
{'fraction_non_alphanumeric': 0.19534765260208847, 'fraction_numerical': 0.016979370065370574, 'mean_word_length': 3.5552977571539057, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2330666', 'n_tokens_mistral': 4179, 'n_tokens_neox': 4133, 'n_words': 722}