<commit_before>#include "src/tmc/tmc.h"

#include "config.h"
#ifdef ENABLE_TMC

#include <climits>
#include <deque>
#include <fstream>
#include <iostream>
#include <regex>
#include <sstream>
#include <string>
#include <utility>

#include <json/json.h>

#include "src/common.h"
#include "src/tmc/event_list.h"
#include "src/tmc/locationdb.h"
#include "src/util.h"

namespace redsea {

namespace tmc {

namespace {

std::map<uint16_t, Event> g_event_data;
std::map<uint16_t, std::string> g_suppl_data;

uint16_t popBits(std::deque<int>* bit_deque, int len) {
  uint16_t result = 0x00;
  if (static_cast<int>(bit_deque->size()) >= len) {
    for (int i=0; i < len; i++) {
      result = (result << 1) | bit_deque->at(0);
      bit_deque->pop_front();
    }
  }
  return result;
}

uint16_t rotl16(uint16_t value, unsigned int count) {
  const unsigned int mask = (CHAR_BIT*sizeof(value)-1);
  count &= mask;
  return (value << count) | (value >> ( (-count) & mask ));
}

// label, field_data (ISO 14819-1: 5.5)
std::vector<std::pair<uint16_t, uint16_t>>
    getFreeformFields(std::vector<MessagePart> parts) {
  static const std::vector<int> field_size(
      {3, 3, 5, 5, 5, 8, 8, 8, 8, 11, 16, 16, 16, 16, 0, 0});

  uint16_t second_gsi = bits(parts[1].data[0], 12, 2);

  // Concatenate freeform data from used message length (derived from
  // GSI of second group)
  std::deque<int> freeform_data_bits;
  for (size_t i=1; i < parts.size(); i++) {
    if (!parts[i].is_received)
      break;

    if (i == 1 || i >= parts.size() - second_gsi) {
      for (int b=0; b < 12; b++)
        freeform_data_bits.push_back((parts[i].data[0] >> (11-b)) & 0x1);
      for (int b=0; b < 16; b++)
        freeform_data_bits.push_back((parts[i].data[1] >> (15-b)) & 0x1);
    }
  }

  // Separate freeform data into fields
  std::vector<std::pair<uint16_t, uint16_t>> result;
  while (freeform_data_bits.size() > 4) {
    uint16_t label = popBits(&freeform_data_bits, 4);
    if (static_cast<int>(freeform_data_bits.size()) < field_size.at(label))
      break;

    uint16_t field_data = popBits(&freeform_data_bits, field_size.at(label));

    if (label == 0x00 && field_data == 0x00)
      break;

    result.push_back({label, field_data});
  }

  return result;
}

std::string timeString(uint16_t field_data) {
  std::string time_string("");

  if (field_data <= 95) {
    char t[6];
    std::snprintf(t, sizeof(t), "%02d:%02d", field_data/4, 15*(field_data % 4));
    time_string = t;

  } else if (field_data <= 200) {
    int days = (field_data - 96) / 24;
    int hour = (field_data - 96) % 24;
    char t[25];
    if (days == 0)
      std::snprintf(t, sizeof(t), "at %02d:00", hour);
    else if (days == 1)
      std::snprintf(t, sizeof(t), "after 1 day at %02d:00", hour);
    else
      std::snprintf(t, sizeof(t), "after %d days at %02d:00", days, hour);
    time_string = t;

  } else if (field_data <= 231) {
    char t[20];
    std::snprintf(t, sizeof(t), "day %d of the month", field_data-200);
    time_string = t;

  } else {
    int mo = (field_data-232) / 2;
    bool end_mid = (field_data-232) % 2;
    std::vector<std::string> month_names({
        "January", "February", "March", "April", "May",
        "June", "July", "August", "September", "October", "November",
        "December"});
    if (mo < 12) {
      time_string = (end_mid ? "end of " : "mid-") + month_names.at(mo);
    }
  }

  return time_string;
}

std::vector<std::string> getScopeStrings(uint16_t mgs) {
  bool mgs_i = bits(mgs, 3, 1);
  bool mgs_n = bits(mgs, 2, 1);
  bool mgs_r = bits(mgs, 1, 1);
  bool mgs_u = bits(mgs, 0, 1);

  std::vector<std::string> scope;
  if (mgs_i)
    scope.push_back("inter-road");
  if (mgs_n)
    scope.push_back("national");
  if (mgs_r)
    scope.push_back("regional");
  if (mgs_u)
    scope.push_back("urban");

  return scope;
}

uint16_t getQuantifierSize(uint16_t code) {
  if (code <= 5)
    return 5;
  else if (code <= 12)
    return 8;
  else
    return 0;
}

std::string getDescWithQuantifier(const Event& event, uint16_t q_value) {
  std::string q("_");
  std::regex q_re("_");

  if (getQuantifierSize(event.quantifier_type) == 5 && q_value == 0)
    q_value = 32;

  switch (event.quantifier_type) {
    case Q_SMALL_NUMBER: {
      int num = q_value;
      if (num > 28)
        num += (num - 28);
      q = std::to_string(num);
      break;
    }
    case Q_NUMBER: {
      int num;
      if (q_value <= 4)
        num = q_value;
      else if (q_value <= 14)
        num = (q_value - 4) * 10;
      else
        num = (q_value - 12) * 50;
      q = std::to_string(num);
      break;
    }
    case Q_LESS_THAN_METRES: {
      q = "less than " + std::to_string(q_value * 10) + " metres";
      break;
    }
    case Q_PERCENT: {
      q = std::to_string(q_value == 32 ? 0 : q_value * 5) + " %";
      break;
    }
    case Q_UPTO_KMH: {
      q = "of up to " + std::to_string(q_value * 5) + " km/h";
      break;
    }
    case Q_UPTO_TIME: {
      if (q_value <= 10)
        q = "of up to " + std::to_string(q_value * 5) + " minutes";
      else if (q_value <= 22)
        q = "of up to " + std::to_string(q_value - 10) + " hours";
      else
        q = "of up to " + std::to_string((q_value - 20) * 6) + " hours";
      break;
    }
    case Q_DEG_CELSIUS: {
      q = std::to_string(q_value - 51) + " degrees Celsius";
      break;
    }
    case Q_TIME: {
      int m = (q_value - 1) * 10;
      int h = m / 60;
      m = m % 60;

      char t[6];
      std::snprintf(t, sizeof(t), "%02d:%02d", m, h);
      q = t;
      break;
    }
    case Q_TONNES: {
      int decitonnes;
      if (q_value <= 100)
        decitonnes = q_value;
      else
        decitonnes = 100 + (q_value - 100) * 5;

      int whole_tonnes = decitonnes / 10;
      decitonnes = decitonnes % 10;

      q = std::to_string(whole_tonnes) + "." + std::to_string(decitonnes) +
        " tonnes";
      break;
    }
    case Q_METRES: {
      int decimetres;
      if (q_value <= 100)
        decimetres = q_value;
      else
        decimetres = 100 + (q_value - 100) * 5;

      int whole_metres = decimetres / 10;
      decimetres = decimetres % 10;

      q = std::to_string(whole_metres) + "." + std::to_string(decimetres) +
        " metres";
      break;
    }
    case Q_UPTO_MILLIMETRES: {
      q = "of up to " + std::to_string(q_value) + " millimetres";
      break;
    }
    case Q_MHZ: {
      CarrierFrequency freq(q_value);
      q = freq.getString();
      break;
    }
    case Q_KHZ: {
      CarrierFrequency freq(q_value, true);
      q = freq.getString();
      break;
    }
    default: {
      //*stream_ << jsonVal("debug", "q_value =" + std::to_string(q_value) +
      //    ", q_type=" + std::to_string(event.quantifier_type));
      q = "TODO";
    }
  }

  std::string desc = std::regex_replace(event.description_with_quantifier,
      q_re, q);
  return desc;
}

std::string ucfirst(std::string in) {
  if (in.size() > 0)
    in[0] = std::toupper(in[0]);
  return in;
}

void loadEventData() {
  for (std::vector<std::string> fields : readCSV(tmc_data_events, ';')) {
    if (fields.size() != 9)
      continue;

    uint16_t code = std::stoi(fields[0]);
    std::vector<std::string> strings(2);
    std::vector<uint16_t> nums(6);

    for (int col=1; col < 3; col++)
      strings[col-1] = fields[col];

    for (int col=3; col < 9; col++)
      nums[col-3] = std::stoi(fields[col]);

    bool allow_q = (strings[1].size() > 0);

    g_event_data.insert({code, {strings[0], strings[1], nums[0], nums[1],
        nums[2], nums[3], nums[4], nums[5], allow_q}});
  }

  for (std::vector<std::string> fields : readCSV(tmc_data_suppl, ';')) {
    uint16_t code = std::stoi(fields[0]);
    std::string desc = fields[1];

    g_suppl_data.insert({code, desc});
  }
}

std::map<uint16_t, ServiceKey> loadServiceKeyTable() {
  std::map<uint16_t, ServiceKey> result;

  for (std::vector<std::string> fields :
       readCSV("service_key_table.csv", ',')) {
    uint16_t encid;

    std::vector<uint8_t> nums(3);

    try {
      encid = std::stoi(fields.at(0));
      nums[0] = std::stoi(fields.at(1));
      nums[1] = std::stoi(fields.at(2));
      nums[2] = std::stoi(fields.at(3));
    } catch (const std::exception& e) {
      continue;
    }

    result.insert({encid, {nums[0], nums[1], nums[2]}});
  }

  return result;
}

void decodeLocation(const LocationDatabase& db, uint16_t ltn,
                    Json::Value* jsroot) {
  if ((*jsroot)["tmc"]["message"].isMember("location")) {
    uint16_t lcd = (*jsroot)["tmc"]["message"]["location"].asUInt();
    int extent = std::stoi((*jsroot)["tmc"]["message"]["extent"].asString());
    bool is_pos = (extent >= 0);


    printf("db.ltn%d  ltn=%d\n",db.ltn,ltn);
    if (db.points.count(lcd) > 0 && db.ltn == ltn) {

      (*jsroot)["tmc"]["message"].removeMember("location");
      (*jsroot)["tmc"]["message"].removeMember("extent");
      std::vector<Point> pts;
      int points_left = abs(extent) + 1;
      uint16_t this_lcd = lcd;
      while (points_left > 0 && db.points.count(this_lcd) > 0) {
        pts.push_back(db.points.at(this_lcd));
        this_lcd = (is_pos ? db.points.at(this_lcd).pos_off :
                             db.points.at(this_lcd).neg_off);
        points_left--;
      }

      for (int i=0; i < static_cast<int>(pts.size()); i++) {
//        (*jsroot)["tmc"]["message"]["locations"].append(pts[i].lcd);
        (*jsroot)["tmc"]["message"]["coordinates"][i].append(pts[i].lat);
        (*jsroot)["tmc"]["message"]["coordinates"][i].append(pts[i].lon);
      }

      if (pts.size() > 1 && pts.at(0).name1.length() > 0 &&
          pts.at(pts.size()-1).name1.length() > 0) {
        (*jsroot)["tmc"]["message"]["span_from"] = pts.at(0).name1;
        (*jsroot)["tmc"]["message"]["span_to"] = pts.at(pts.size()-1).name1;
      }
      uint16_t roa_lcd = db.points.at(lcd).roa_lcd;<commit_msg>rm debug printf<commit_after>#include "src/tmc/tmc.h"

#include "config.h"
#ifdef ENABLE_TMC

#include <climits>
#include <deque>
#include <fstream>
#include <iostream>
#include <regex>
#include <sstream>
#include <string>
#include <utility>

#include <json/json.h>

#include "src/common.h"
#include "src/tmc/event_list.h"
#include "src/tmc/locationdb.h"
#include "src/util.h"

namespace redsea {

namespace tmc {

namespace {

std::map<uint16_t, Event> g_event_data;
std::map<uint16_t, std::string> g_suppl_data;

uint16_t popBits(std::deque<int>* bit_deque, int len) {
  uint16_t result = 0x00;
  if (static_cast<int>(bit_deque->size()) >= len) {
    for (int i=0; i < len; i++) {
      result = (result << 1) | bit_deque->at(0);
      bit_deque->pop_front();
    }
  }
  return result;
}

uint16_t rotl16(uint16_t value, unsigned int count) {
  const unsigned int mask = (CHAR_BIT*sizeof(value)-1);
  count &= mask;
  return (value << count) | (value >> ( (-count) & mask ));
}

// label, field_data (ISO 14819-1: 5.5)
std::vector<std::pair<uint16_t, uint16_t>>
    getFreeformFields(std::vector<MessagePart> parts) {
  static const std::vector<int> field_size(
      {3, 3, 5, 5, 5, 8, 8, 8, 8, 11, 16, 16, 16, 16, 0, 0});

  uint16_t second_gsi = bits(parts[1].data[0], 12, 2);

  // Concatenate freeform data from used message length (derived from
  // GSI of second group)
  std::deque<int> freeform_data_bits;
  for (size_t i=1; i < parts.size(); i++) {
    if (!parts[i].is_received)
      break;

    if (i == 1 || i >= parts.size() - second_gsi) {
      for (int b=0; b < 12; b++)
        freeform_data_bits.push_back((parts[i].data[0] >> (11-b)) & 0x1);
      for (int b=0; b < 16; b++)
        freeform_data_bits.push_back((parts[i].data[1] >> (15-b)) & 0x1);
    }
  }

  // Separate freeform data into fields
  std::vector<std::pair<uint16_t, uint16_t>> result;
  while (freeform_data_bits.size() > 4) {
    uint16_t label = popBits(&freeform_data_bits, 4);
    if (static_cast<int>(freeform_data_bits.size()) < field_size.at(label))
      break;

    uint16_t field_data = popBits(&freeform_data_bits, field_size.at(label));

    if (label == 0x00 && field_data == 0x00)
      break;

    result.push_back({label, field_data});
  }

  return result;
}

std::string timeString(uint16_t field_data) {
  std::string time_string("");

  if (field_data <= 95) {
    char t[6];
    std::snprintf(t, sizeof(t), "%02d:%02d", field_data/4, 15*(field_data % 4));
    time_string = t;

  } else if (field_data <= 200) {
    int days = (field_data - 96) / 24;
    int hour = (field_data - 96) % 24;
    char t[25];
    if (days == 0)
      std::snprintf(t, sizeof(t), "at %02d:00", hour);
    else if (days == 1)
      std::snprintf(t, sizeof(t), "after 1 day at %02d:00", hour);
    else
      std::snprintf(t, sizeof(t), "after %d days at %02d:00", days, hour);
    time_string = t;

  } else if (field_data <= 231) {
    char t[20];
    std::snprintf(t, sizeof(t), "day %d of the month", field_data-200);
    time_string = t;

  } else {
    int mo = (field_data-232) / 2;
    bool end_mid = (field_data-232) % 2;
    std::vector<std::string> month_names({
        "January", "February", "March", "April", "May",
        "June", "July", "August", "September", "October", "November",
        "December"});
    if (mo < 12) {
      time_string = (end_mid ? "end of " : "mid-") + month_names.at(mo);
    }
  }

  return time_string;
}

std::vector<std::string> getScopeStrings(uint16_t mgs) {
  bool mgs_i = bits(mgs, 3, 1);
  bool mgs_n = bits(mgs, 2, 1);
  bool mgs_r = bits(mgs, 1, 1);
  bool mgs_u = bits(mgs, 0, 1);

  std::vector<std::string> scope;
  if (mgs_i)
    scope.push_back("inter-road");
  if (mgs_n)
    scope.push_back("national");
  if (mgs_r)
    scope.push_back("regional");
  if (mgs_u)
    scope.push_back("urban");

  return scope;
}

uint16_t getQuantifierSize(uint16_t code) {
  if (code <= 5)
    return 5;
  else if (code <= 12)
    return 8;
  else
    return 0;
}

std::string getDescWithQuantifier(const Event& event, uint16_t q_value) {
  std::string q("_");
  std::regex q_re("_");

  if (getQuantifierSize(event.quantifier_type) == 5 && q_value == 0)
    q_value = 32;

  switch (event.quantifier_type) {
    case Q_SMALL_NUMBER: {
      int num = q_value;
      if (num > 28)
        num += (num - 28);
      q = std::to_string(num);
      break;
    }
    case Q_NUMBER: {
      int num;
      if (q_value <= 4)
        num = q_value;
      else if (q_value <= 14)
        num = (q_value - 4) * 10;
      else
        num = (q_value - 12) * 50;
      q = std::to_string(num);
      break;
    }
    case Q_LESS_THAN_METRES: {
      q = "less than " + std::to_string(q_value * 10) + " metres";
      break;
    }
    case Q_PERCENT: {
      q = std::to_string(q_value == 32 ? 0 : q_value * 5) + " %";
      break;
    }
    case Q_UPTO_KMH: {
      q = "of up to " + std::to_string(q_value * 5) + " km/h";
      break;
    }
    case Q_UPTO_TIME: {
      if (q_value <= 10)
        q = "of up to " + std::to_string(q_value * 5) + " minutes";
      else if (q_value <= 22)
        q = "of up to " + std::to_string(q_value - 10) + " hours";
      else
        q = "of up to " + std::to_string((q_value - 20) * 6) + " hours";
      break;
    }
    case Q_DEG_CELSIUS: {
      q = std::to_string(q_value - 51) + " degrees Celsius";
      break;
    }
    case Q_TIME: {
      int m = (q_value - 1) * 10;
      int h = m / 60;
      m = m % 60;

      char t[6];
      std::snprintf(t, sizeof(t), "%02d:%02d", m, h);
      q = t;
      break;
    }
    case Q_TONNES: {
      int decitonnes;
      if (q_value <= 100)
        decitonnes = q_value;
      else
        decitonnes = 100 + (q_value - 100) * 5;

      int whole_tonnes = decitonnes / 10;
      decitonnes = decitonnes % 10;

      q = std::to_string(whole_tonnes) + "." + std::to_string(decitonnes) +
        " tonnes";
      break;
    }
    case Q_METRES: {
      int decimetres;
      if (q_value <= 100)
        decimetres = q_value;
      else
        decimetres = 100 + (q_value - 100) * 5;

      int whole_metres = decimetres / 10;
      decimetres = decimetres % 10;

      q = std::to_string(whole_metres) + "." + std::to_string(decimetres) +
        " metres";
      break;
    }
    case Q_UPTO_MILLIMETRES: {
      q = "of up to " + std::to_string(q_value) + " millimetres";
      break;
    }
    case Q_MHZ: {
      CarrierFrequency freq(q_value);
      q = freq.getString();
      break;
    }
    case Q_KHZ: {
      CarrierFrequency freq(q_value, true);
      q = freq.getString();
      break;
    }
    default: {
      //*stream_ << jsonVal("debug", "q_value =" + std::to_string(q_value) +
      //    ", q_type=" + std::to_string(event.quantifier_type));
      q = "TODO";
    }
  }

  std::string desc = std::regex_replace(event.description_with_quantifier,
      q_re, q);
  return desc;
}

std::string ucfirst(std::string in) {
  if (in.size() > 0)
    in[0] = std::toupper(in[0]);
  return in;
}

void loadEventData() {
  for (std::vector<std::string> fields : readCSV(tmc_data_events, ';')) {
    if (fields.size() != 9)
      continue;

    uint16_t code = std::stoi(fields[0]);
    std::vector<std::string> strings(2);
    std::vector<uint16_t> nums(6);

    for (int col=1; col < 3; col++)
      strings[col-1] = fields[col];

    for (int col=3; col < 9; col++)
      nums[col-3] = std::stoi(fields[col]);

    bool allow_q = (strings[1].size() > 0);

    g_event_data.insert({code, {strings[0], strings[1], nums[0], nums[1],
        nums[2], nums[3], nums[4], nums[5], allow_q}});
  }

  for (std::vector<std::string> fields : readCSV(tmc_data_suppl, ';')) {
    uint16_t code = std::stoi(fields[0]);
    std::string desc = fields[1];

    g_suppl_data.insert({code, desc});
  }
}

std::map<uint16_t, ServiceKey> loadServiceKeyTable() {
  std::map<uint16_t, ServiceKey> result;

  for (std::vector<std::string> fields :
       readCSV("service_key_table.csv", ',')) {
    uint16_t encid;

    std::vector<uint8_t> nums(3);

    try {
      encid = std::stoi(fields.at(0));
      nums[0] = std::stoi(fields.at(1));
      nums[1] = std::stoi(fields.at(2));
      nums[2] = std::stoi(fields.at(3));
    } catch (const std::exception& e) {
      continue;
    }

    result.insert({encid, {nums[0], nums[1], nums[2]}});
  }

  return result;
}

void decodeLocation(const LocationDatabase& db, uint16_t ltn,
                    Json::Value* jsroot) {
  if ((*jsroot)["tmc"]["message"].isMember("location")) {
    uint16_t lcd = (*jsroot)["tmc"]["message"]["location"].asUInt();
    int extent = std::stoi((*jsroot)["tmc"]["message"]["extent"].asString());
    bool is_pos = (extent >= 0);

    if (db.points.count(lcd) > 0 && db.ltn == ltn) {

      (*jsroot)["tmc"]["message"].removeMember("location");
      (*jsroot)["tmc"]["message"].removeMember("extent");
      std::vector<Point> pts;
      int points_left = abs(extent) + 1;
      uint16_t this_lcd = lcd;
      while (points_left > 0 && db.points.count(this_lcd) > 0) {
        pts.push_back(db.points.at(this_lcd));
        this_lcd = (is_pos ? db.points.at(this_lcd).pos_off :
                             db.points.at(this_lcd).neg_off);
        points_left--;
      }

      for (int i=0; i < static_cast<int>(pts.size()); i++) {
//        (*jsroot)["tmc"]["message"]["locations"].append(pts[i].lcd);
        (*jsroot)["tmc"]["message"]["coordinates"][i].append(pts[i].lat);
        (*jsroot)["tmc"]["message"]["coordinates"][i].append(pts[i].lon);
      }

      if (pts.size() > 1 && pts.at(0).name1.length() > 0 &&
          pts.at(pts.size()-1).name1.length() > 0) {
        (*jsroot)["tmc"]["message"]["span_from"] = pts.at(0).name1;
        (*jsroot)["tmc"]["message"]["span_to"] = pts.at(pts.size()-1).name1;
      }
      uint16_t roa_lcd = db.points.at(lcd).roa_lcd;
{'fraction_non_alphanumeric': 0.181657223796034, 'fraction_numerical': 0.03308377175232699, 'mean_word_length': 2.803193535975375, 'pattern_counts': {'":': 0, '<': 123, '<?xml version=': 0, '>': 115, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3572067', 'n_tokens_mistral': 8306, 'n_tokens_neox': 7773, 'n_words': 1585}