<commit_before>/// \file dtscfix.cpp
/// Contains the code that will attempt to fix the metadata contained in an DTSC file.

#include <string>
#include <mist/dtsc.h>
#include <mist/json.h>
#include <mist/config.h>

///\brief Holds everything unique to converters.
namespace Converters {
  class HeaderEntryDTSC {
    public:
      HeaderEntryDTSC() : totalSize(0), lastKeyTime(-5001), trackID(0), firstms(0x7FFFFFFF), lastms(0), keynum(0) {}
      long long int totalSize;
      std::vector<long long int> parts;
      long long int lastKeyTime;
      long long int trackID;
      long long int firstms;
      long long int lastms;
      long long int keynum;
      std::string type;
  };

  ///\brief Reads a DTSC file and attempts to fix the metadata in it.
  ///\param conf The current configuration of the program.
  ///\return The return code for the fixed program.
  int DTSCFix(Util::Config & conf){
    DTSC::File F(conf.getString("filename"));
    JSON::Value oriheader = F.getFirstMeta();
    JSON::Value meta = F.getMeta();
    JSON::Value pack;

    if ( !oriheader.isMember("moreheader")){
      std::cerr << "This file is too old to fix - please reconvert." << std::endl;
      return 1;
    }
    if (oriheader["moreheader"].asInt() > 0 && !conf.getBool("force")){
      if (meta.isMember("tracks") && meta["tracks"].size() > 0){
        bool isFixed = true;
        for ( JSON::ObjIter trackIt = meta["tracks"].ObjBegin(); trackIt != meta["tracks"].ObjEnd(); trackIt ++) {
          if ( !trackIt->second.isMember("frags") || !trackIt->second.isMember("keynum")){
            isFixed = false;
          }
        }
        if (isFixed){
          std::cerr << "This file was already fixed or doesn't need fixing - cancelling." << std::endl;
          return 0;
        }
      }
    }
    meta.removeMember("isFixed");
    meta.removeMember("keytime");
    meta.removeMember("keybpos");
    meta.removeMember("moreheader");

    std::map<std::string,int> trackIDs;
    std::map<std::string,HeaderEntryDTSC> trackData;


    long long int nowpack = 0;
    
    std::string currentID;
    int nextFreeID = 0;

    for (JSON::ObjIter it = meta["tracks"].ObjBegin(); it != meta["tracks"].ObjEnd(); it++){
      trackIDs.insert(std::pair<std::string,int>(it->first,it->second["trackid"].asInt()));
      trackData[it->first].type = it->second["type"].asString();
      trackData[it->first].trackID = it->second["trackid"].asInt();
      trackData[it->first].type = it->second["type"].asString();
      if (it->second["trackid"].asInt() >= nextFreeID){
        nextFreeID = it->second["trackid"].asInt() + 1;
      }
      it->second.removeMember("keylen");
      it->second.removeMember("keybpos");
      it->second.removeMember("frags");
      it->second.removeMember("keytime");
      it->second.removeMember("keynum");
      it->second.removeMember("keydata");
      it->second.removeMember("keyparts");
      it->second.removeMember("keys");
    }

    F.parseNext();
    while ( !F.getJSON().isNull()){
      currentID = "";
      if (F.getJSON()["trackid"].asInt() == 0){
        if (F.getJSON()["datatype"].asString() == "video"){
          currentID = "video0";
          if (trackData[currentID].trackID == 0){
            trackData[currentID].trackID = nextFreeID++;
          }
          if (meta.isMember("video")){
            meta["tracks"][currentID] = meta["video"];
            meta.removeMember("video");
          }
          trackData[currentID].type = F.getJSON()["datatype"].asString();
        }else{
          if (F.getJSON()["datatype"].asString() == "audio"){
            currentID = "audio0";
            if (trackData[currentID].trackID == 0){
              trackData[currentID].trackID = nextFreeID++;
            }
            if (meta.isMember("audio")){
              meta["tracks"][currentID] = meta["audio"];
              meta.removeMember("audio");
            }
            trackData[currentID].type = F.getJSON()["datatype"].asString();
          }else{
            fprintf(stderr, "Found an unknown package with packetid 0 and datatype %s\n",F.getJSON()["datatype"].asString().c_str());
            F.parseNext();
            continue;
          }
        }
      }else{
        for( std::map<std::string,int>::iterator it = trackIDs.begin(); it != trackIDs.end(); it++ ) {
          if( it->second == F.getJSON()["trackid"].asInt() ) {
            currentID = it->first;
            break;
          }
        }
        if( currentID == "" ) {
          fprintf(stderr, "Found an unknown v2 packet with id %d\n", F.getJSON()["trackid"].asInt());
          F.parseNext();
          continue;
          //should create new track but this shouldnt be needed...
        }
      }
      if (F.getJSON()["time"].asInt() < trackData[currentID].firstms){
        trackData[currentID].firstms = F.getJSON()["time"].asInt();
      }
      if (F.getJSON()["time"].asInt() >= nowpack){
        nowpack = F.getJSON()["time"].asInt();
      }
      if (trackData[currentID].type == "video"){
        if (F.getJSON().isMember("keyframe")){
          int newNum = meta["tracks"][currentID]["keys"].size();
          meta["tracks"][currentID]["keys"][newNum]["num"] = ++trackData[currentID].keynum;<commit_msg>Improved handling unknown packets in DTSCFix.<commit_after>/// \file dtscfix.cpp
/// Contains the code that will attempt to fix the metadata contained in an DTSC file.

#include <string>
#include <mist/dtsc.h>
#include <mist/json.h>
#include <mist/config.h>

///\brief Holds everything unique to converters.
namespace Converters {
  class HeaderEntryDTSC {
    public:
      HeaderEntryDTSC() : totalSize(0), lastKeyTime(-5001), trackID(0), firstms(0x7FFFFFFF), lastms(0), keynum(0) {}
      long long int totalSize;
      std::vector<long long int> parts;
      long long int lastKeyTime;
      long long int trackID;
      long long int firstms;
      long long int lastms;
      long long int keynum;
      std::string type;
  };

  ///\brief Reads a DTSC file and attempts to fix the metadata in it.
  ///\param conf The current configuration of the program.
  ///\return The return code for the fixed program.
  int DTSCFix(Util::Config & conf){
    DTSC::File F(conf.getString("filename"));
    JSON::Value oriheader = F.getFirstMeta();
    JSON::Value meta = F.getMeta();
    JSON::Value pack;

    if ( !oriheader.isMember("moreheader")){
      std::cerr << "This file is too old to fix - please reconvert." << std::endl;
      return 1;
    }
    if (oriheader["moreheader"].asInt() > 0 && !conf.getBool("force")){
      if (meta.isMember("tracks") && meta["tracks"].size() > 0){
        bool isFixed = true;
        for ( JSON::ObjIter trackIt = meta["tracks"].ObjBegin(); trackIt != meta["tracks"].ObjEnd(); trackIt ++) {
          if ( !trackIt->second.isMember("frags") || !trackIt->second.isMember("keynum")){
            isFixed = false;
          }
        }
        if (isFixed){
          std::cerr << "This file was already fixed or doesn't need fixing - cancelling." << std::endl;
          return 0;
        }
      }
    }
    meta.removeMember("isFixed");
    meta.removeMember("keytime");
    meta.removeMember("keybpos");
    meta.removeMember("moreheader");

    std::map<std::string,int> trackIDs;
    std::map<std::string,HeaderEntryDTSC> trackData;


    long long int nowpack = 0;
    
    std::string currentID;
    int nextFreeID = 0;

    for (JSON::ObjIter it = meta["tracks"].ObjBegin(); it != meta["tracks"].ObjEnd(); it++){
      trackIDs.insert(std::pair<std::string,int>(it->first,it->second["trackid"].asInt()));
      trackData[it->first].type = it->second["type"].asString();
      trackData[it->first].trackID = it->second["trackid"].asInt();
      trackData[it->first].type = it->second["type"].asString();
      if (it->second["trackid"].asInt() >= nextFreeID){
        nextFreeID = it->second["trackid"].asInt() + 1;
      }
      it->second.removeMember("keylen");
      it->second.removeMember("keybpos");
      it->second.removeMember("frags");
      it->second.removeMember("keytime");
      it->second.removeMember("keynum");
      it->second.removeMember("keydata");
      it->second.removeMember("keyparts");
      it->second.removeMember("keys");
    }

    F.parseNext();
    while ( !F.getJSON().isNull()){
      currentID = "";
      if (F.getJSON()["trackid"].asInt() == 0){
        if (F.getJSON()["datatype"].asString() == "video"){
          currentID = "video0";
          if (trackData[currentID].trackID == 0){
            trackData[currentID].trackID = nextFreeID++;
          }
          if (meta.isMember("video")){
            meta["tracks"][currentID] = meta["video"];
            meta.removeMember("video");
          }
          trackData[currentID].type = F.getJSON()["datatype"].asString();
        }else{
          if (F.getJSON()["datatype"].asString() == "audio"){
            currentID = "audio0";
            if (trackData[currentID].trackID == 0){
              trackData[currentID].trackID = nextFreeID++;
            }
            if (meta.isMember("audio")){
              meta["tracks"][currentID] = meta["audio"];
              meta.removeMember("audio");
            }
            trackData[currentID].type = F.getJSON()["datatype"].asString();
          }else{
            //fprintf(stderr, "Found an unknown package with packetid 0 and datatype %s\n",F.getJSON()["datatype"].asString().c_str());
            F.parseNext();
            continue;
          }
        }
      }else{
        for( std::map<std::string,int>::iterator it = trackIDs.begin(); it != trackIDs.end(); it++ ) {
          if( it->second == F.getJSON()["trackid"].asInt() ) {
            currentID = it->first;
            break;
          }
        }
        if( currentID == "" ) {
          //fprintf(stderr, "Found an unknown v2 packet with id %d\n", F.getJSON()["trackid"].asInt());
          F.parseNext();
          continue;
          //should create new track but this shouldnt be needed...
        }
      }
      if (F.getJSON()["time"].asInt() < trackData[currentID].firstms){
        trackData[currentID].firstms = F.getJSON()["time"].asInt();
      }
      if (F.getJSON()["time"].asInt() >= nowpack){
        nowpack = F.getJSON()["time"].asInt();
      }
      if (trackData[currentID].type == "video"){
        if (F.getJSON().isMember("keyframe")){
          int newNum = meta["tracks"][currentID]["keys"].size();
          meta["tracks"][currentID]["keys"][newNum]["num"] = ++trackData[currentID].keynum;
{'fraction_non_alphanumeric': 0.177404028886355, 'fraction_numerical': 0.004561003420752566, 'mean_word_length': 2.576282704723072, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7289943', 'n_tokens_mistral': 3296, 'n_tokens_neox': 3223, 'n_words': 662}