<commit_before>//
//  expiry_categorize.cpp
//  See the file "LICENSE.md" for the full license governing this code.
//

#include "compile.h"
#if COMPILE_DMZ

#include "expiry_categorize.h"
#include "cv/image_util.h"
#include "cv/morph.h"
#include "cv/stats.h"
#include <time.h>

#define DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE 0
#define DEBUG_EXPIRY_CATEGORIZATION_RESULTS 1

// digit categorizers
#include "models/expiry/modelc_ab0b6054.hpp"
#include "models/expiry/modelc_d3fc216e.hpp"

#define GROUPED_RECTS_VERTICAL_ALLOWANCE (kTrimmedCharacterImageHeight / 2)
#define GROUPED_RECTS_HORIZONTAL_ALLOWANCE (kTrimmedCharacterImageWidth / 2)

#define kExpiryDecayFactor 0.7f
#define kExpiryMinStability 0.7f

#define digit_to_int(c) ((uint8_t)c - (uint8_t)'0')

typedef Eigen::Matrix<float, 16, 11, Eigen::RowMajor> DigitModelInput;
typedef Eigen::Matrix<float, 1, 10, Eigen::RowMajor> DigitProbabilities;

#pragma mark - image preparation

DMZ_INTERNAL void prepare_image_for_cat(IplImage *image, IplImage *as_float, CharacterRectListIterator rect) {
  // Input image: IPL_DEPTH_8U [0 - 255]
  // Data for models: IPL_DEPTH_32F [0.0 - 1.0]
  
  cvSetImageROI(image, cvRect(rect->left, rect->top, kTrimmedCharacterImageWidth, kTrimmedCharacterImageHeight));
  
  // TODO: optimize this a lot!
  
  // Gradient
  IplImage *filtered_image = cvCreateImage(cvSize(kTrimmedCharacterImageWidth, kTrimmedCharacterImageHeight), IPL_DEPTH_8U, 1);
  //llcv_morph_grad3_2d_cross_u8(image, filtered_image);
  IplConvKernel *kernel = cvCreateStructuringElementEx(3, 3, 1, 1, CV_SHAPE_CROSS, NULL);
  cvMorphologyEx(image, filtered_image, NULL, kernel, CV_MOP_GRADIENT, 1);
  cvReleaseStructuringElement(&kernel);
  
  // Equalize
  llcv_equalize_hist(filtered_image, filtered_image);
  
  // Bilateral filter
  int aperture = 3;
  double space_sigma = (aperture / 2.0 - 1) * 0.3 + 0.8;
  double color_sigma = (aperture - 1) / 3.0;
  IplImage *smoothed_image = cvCreateImage(cvSize(kTrimmedCharacterImageWidth, kTrimmedCharacterImageHeight), IPL_DEPTH_8U, 1);
  cvSmooth(filtered_image, smoothed_image, CV_BILATERAL, aperture, aperture, space_sigma, color_sigma);
  
  // Convert to float
  cvConvertScale(smoothed_image, as_float, 1.0f / 255.0f, 0);
  
  cvReleaseImage(&smoothed_image);
  cvReleaseImage(&filtered_image);
  
  cvResetImageROI(image);

#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  dmz_debug_timer_print("prepare image", 2);
#endif
}

#pragma mark - categorize expiry digits via machine learning

DMZ_INTERNAL inline std::vector<DigitProbabilities> digit_probabilities(IplImage *as_float) {
  std::vector<DigitProbabilities> probabilities;
  assert(as_float->width * sizeof(float) == as_float->widthStep);
  Eigen::Map<DigitModelInput> digit_model_input((float *)as_float->imageData);

  probabilities.push_back(applyc_ab0b6054(digit_model_input));
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  /*suseconds_t interval2 = */ dmz_debug_timer_print("apply model 1", 2);
  //  dmz_debug_print("Faster: %d, %.2f\n", interval1 - interval2, ((float)interval2) / ((float)interval1));
#endif
  
  probabilities.push_back(applyc_d3fc216e(digit_model_input));
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  /*suseconds_t interval2 = */ dmz_debug_timer_print("apply model 2", 2);
  //  dmz_debug_print("Faster: %d, %.2f\n", interval1 - interval2, ((float)interval2) / ((float)interval1));
#endif

  return probabilities;
}


DMZ_INTERNAL inline ExpiryGroupScores categorize_expiry_digits(IplImage *card_y, IplImage *as_float, GroupedRects group, char *expiries_string) {
  std::vector<std::string> expiry_strings;
  std::vector<ExpiryGroupScores> probability_vector;
  
  char positions[256];
  sprintf(positions, " top: %3d, left: %3d character-lefts:", group.top, group.left);
  
  for (int character_index = 0; character_index < 5; character_index++) {
    if (character_index == 2) { // the slash character
      continue;
    }
    
    CharacterRectListIterator rect = group.character_rects.begin() + character_index;
    
    prepare_image_for_cat(card_y, as_float, rect);
    std::vector<DigitProbabilities> probabilities = digit_probabilities(as_float);
    
    char position[32];
    sprintf(position, " %3d", rect->left);
    strcat(positions, position);
    
    if (character_index == 0) {
      ExpiryGroupScores scores;
      for (int model_index = 0; model_index < probabilities.size(); model_index++) {
        char header[64];
        sprintf(header, "**/** Model %d ", model_index);
        expiry_strings.push_back(std::string(header));
        probability_vector.push_back(scores);
      }
      
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
      dmz_debug_timer_print("initialize expiry_strings", 2);
#endif
    }
    
    for (int model_index = 0; model_index < expiry_strings.size(); model_index++) {
      float max_probability = 0.0f;
      int most_probable_digit = -1;
      for (int digit_index = 0; digit_index < 10; digit_index++) {
        float digit_probability = probabilities[model_index](0, digit_index);
        probability_vector[model_index](character_index, digit_index) = digit_probability;
        if (digit_probability > max_probability) {
          max_probability = digit_probability;
          most_probable_digit = digit_index;
        }
      }
      if (most_probable_digit >= 0 && max_probability > 0.7) {
        expiry_strings[model_index][character_index] = (char)(int('0') + most_probable_digit);
      }
    }
  }
  
  expiries_string[0] = '\0';
  for (int model_index = 0; model_index < expiry_strings.size(); model_index++) {
    if (strlen(expiries_string)) {
      strcat(expiries_string, "\n");
    }
    strcat(expiries_string, expiry_strings[model_index].c_str());
    strcat(expiries_string, positions);
    strcat(expiries_string, "\n");
    
    strcat(expiries_string, "        ");
    char label_string[32];
    for (int char_label = 0; char_label < 10; char_label++) {
      sprintf(label_string, "   %d  ", char_label);
      strcat(expiries_string, label_string);
    }
    strcat(expiries_string, "\n");
    
    for (int character_index = 0; character_index < 5; character_index++) {
      if (character_index == 2) { // the slash character
        continue;
      }
      
      char char_pos_string[32];
      sprintf(char_pos_string, "char %d:", character_index);
      strcat(expiries_string, char_pos_string);
      
      for (int digit_index = 0; digit_index < 10; digit_index++) {
        char prob_string[32];
        sprintf(prob_string, " %5.3f", probability_vector[model_index](character_index, digit_index));
        strcat(expiries_string, prob_string);
      }
      strcat(expiries_string, "\n");
    }
  }
  
  // TODO: if we end up using more than one model, return their coalesced results
  return probability_vector[0];
}


DMZ_INTERNAL void expiry_aggregate_grouped_rects(GroupedRectsList &aggregated_groups, GroupedRectsList &new_groups) {
  // Coalesce equivalent groups within new_groups (*** TODO *** IS THIS STEP EVER ACTUALLY NECESSARY? ***)
  for (size_t new_index_1 = 0; new_index_1 < new_groups.size(); new_index_1++) {
    GroupedRects &group1 = new_groups[new_index_1];
    int top1 = group1.top;
    int left1 = group1.left;
    size_t nChars1 = group1.character_rects.size();
    float groups_coalesced_so_far = 1;
    
    for (size_t new_index_2 = new_groups.size() - 1; new_index_2 > new_index_1; new_index_2--) {
      GroupedRects &group2 = new_groups[new_index_2];
      if (abs(group2.top - top1) > GROUPED_RECTS_VERTICAL_ALLOWANCE ||
          abs(group2.left - left1) > GROUPED_RECTS_HORIZONTAL_ALLOWANCE ||
          group2.character_rects.size() != nChars1) {
        continue;
      }

      group1.scores = ((group1.scores * groups_coalesced_so_far) + group2.scores) / (groups_coalesced_so_far + 1);
      groups_coalesced_so_far++;
      new_groups.erase(new_groups.begin() + new_index_2);
      dmz_debug_print("** Yup, coalesced a new group with another! WTF? ***\n");
    }
  }
  
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  dmz_debug_timer_print("coalesce new groups", 2);
#endif<commit_msg>Expiry: weighted average of categorization models<commit_after>//
//  expiry_categorize.cpp
//  See the file "LICENSE.md" for the full license governing this code.
//

#include "compile.h"
#if COMPILE_DMZ

#include "expiry_categorize.h"
#include "cv/image_util.h"
#include "cv/morph.h"
#include "cv/stats.h"
#include <time.h>

#define DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE 0
#define DEBUG_EXPIRY_CATEGORIZATION_RESULTS 1

// digit categorizers
#include "models/expiry/modelc_ab0b6054.hpp"
#include "models/expiry/modelc_d3fc216e.hpp"

#define GROUPED_RECTS_VERTICAL_ALLOWANCE (kTrimmedCharacterImageHeight / 2)
#define GROUPED_RECTS_HORIZONTAL_ALLOWANCE (kTrimmedCharacterImageWidth / 2)

#define kExpiryDecayFactor 0.7f
#define kExpiryMinStability 0.7f

#define digit_to_int(c) ((uint8_t)c - (uint8_t)'0')

typedef Eigen::Matrix<float, 16, 11, Eigen::RowMajor> DigitModelInput;
typedef Eigen::Matrix<float, 1, 10, Eigen::RowMajor> DigitProbabilities;

#pragma mark - image preparation

DMZ_INTERNAL void prepare_image_for_cat(IplImage *image, IplImage *as_float, CharacterRectListIterator rect) {
  // Input image: IPL_DEPTH_8U [0 - 255]
  // Data for models: IPL_DEPTH_32F [0.0 - 1.0]
  
  cvSetImageROI(image, cvRect(rect->left, rect->top, kTrimmedCharacterImageWidth, kTrimmedCharacterImageHeight));
  
  // TODO: optimize this a lot!
  
  // Gradient
  IplImage *filtered_image = cvCreateImage(cvSize(kTrimmedCharacterImageWidth, kTrimmedCharacterImageHeight), IPL_DEPTH_8U, 1);
  //llcv_morph_grad3_2d_cross_u8(image, filtered_image);
  IplConvKernel *kernel = cvCreateStructuringElementEx(3, 3, 1, 1, CV_SHAPE_CROSS, NULL);
  cvMorphologyEx(image, filtered_image, NULL, kernel, CV_MOP_GRADIENT, 1);
  cvReleaseStructuringElement(&kernel);
  
  // Equalize
  llcv_equalize_hist(filtered_image, filtered_image);
  
  // Bilateral filter
  int aperture = 3;
  double space_sigma = (aperture / 2.0 - 1) * 0.3 + 0.8;
  double color_sigma = (aperture - 1) / 3.0;
  IplImage *smoothed_image = cvCreateImage(cvSize(kTrimmedCharacterImageWidth, kTrimmedCharacterImageHeight), IPL_DEPTH_8U, 1);
  cvSmooth(filtered_image, smoothed_image, CV_BILATERAL, aperture, aperture, space_sigma, color_sigma);
  
  // Convert to float
  cvConvertScale(smoothed_image, as_float, 1.0f / 255.0f, 0);
  
  cvReleaseImage(&smoothed_image);
  cvReleaseImage(&filtered_image);
  
  cvResetImageROI(image);

#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  dmz_debug_timer_print("prepare image", 2);
#endif
}

#pragma mark - categorize expiry digits via machine learning

DMZ_INTERNAL inline std::vector<DigitProbabilities> digit_probabilities(IplImage *as_float) {
  std::vector<DigitProbabilities> probabilities;
  assert(as_float->width * sizeof(float) == as_float->widthStep);
  Eigen::Map<DigitModelInput> digit_model_input((float *)as_float->imageData);

  probabilities.push_back(applyc_ab0b6054(digit_model_input));
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  /*suseconds_t interval2 = */ dmz_debug_timer_print("apply model 1", 2);
  //  dmz_debug_print("Faster: %d, %.2f\n", interval1 - interval2, ((float)interval2) / ((float)interval1));
#endif
  
  probabilities.push_back(applyc_d3fc216e(digit_model_input));
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  /*suseconds_t interval2 = */ dmz_debug_timer_print("apply model 2", 2);
  //  dmz_debug_print("Faster: %d, %.2f\n", interval1 - interval2, ((float)interval2) / ((float)interval1));
#endif

  return probabilities;
}


DMZ_INTERNAL inline ExpiryGroupScores categorize_expiry_digits(IplImage *card_y, IplImage *as_float, GroupedRects group, char *expiries_string) {
  std::vector<std::string> expiry_strings;
  std::vector<ExpiryGroupScores> probability_vector;
  
  char positions[256];
  sprintf(positions, " top: %3d, left: %3d character-lefts:", group.top, group.left);
  
  for (int character_index = 0; character_index < 5; character_index++) {
    if (character_index == 2) { // the slash character
      continue;
    }
    
    CharacterRectListIterator rect = group.character_rects.begin() + character_index;
    
    prepare_image_for_cat(card_y, as_float, rect);
    std::vector<DigitProbabilities> probabilities = digit_probabilities(as_float);
    
    char position[32];
    sprintf(position, " %3d", rect->left);
    strcat(positions, position);
    
    if (character_index == 0) {
      ExpiryGroupScores scores;
      for (int model_index = 0; model_index < probabilities.size(); model_index++) {
        char header[64];
        sprintf(header, "**/** Model %d ", model_index);
        expiry_strings.push_back(std::string(header));
        probability_vector.push_back(scores);
      }
      
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
      dmz_debug_timer_print("initialize expiry_strings", 2);
#endif
    }
    
    for (int model_index = 0; model_index < expiry_strings.size(); model_index++) {
      float max_probability = 0.0f;
      int most_probable_digit = -1;
      for (int digit_index = 0; digit_index < 10; digit_index++) {
        float digit_probability = probabilities[model_index](0, digit_index);
        probability_vector[model_index](character_index, digit_index) = digit_probability;
        if (digit_probability > max_probability) {
          max_probability = digit_probability;
          most_probable_digit = digit_index;
        }
      }
      if (most_probable_digit >= 0 && max_probability > 0.7) {
        expiry_strings[model_index][character_index] = (char)(int('0') + most_probable_digit);
      }
    }
  }
  
  expiries_string[0] = '\0';
  for (int model_index = 0; model_index < expiry_strings.size(); model_index++) {
    if (strlen(expiries_string)) {
      strcat(expiries_string, "\n");
    }
    strcat(expiries_string, expiry_strings[model_index].c_str());
    strcat(expiries_string, positions);
    strcat(expiries_string, "\n");
    
    strcat(expiries_string, "        ");
    char label_string[32];
    for (int char_label = 0; char_label < 10; char_label++) {
      sprintf(label_string, "   %d  ", char_label);
      strcat(expiries_string, label_string);
    }
    strcat(expiries_string, "\n");
    
    for (int character_index = 0; character_index < 5; character_index++) {
      if (character_index == 2) { // the slash character
        continue;
      }
      
      char char_pos_string[32];
      sprintf(char_pos_string, "char %d:", character_index);
      strcat(expiries_string, char_pos_string);
      
      for (int digit_index = 0; digit_index < 10; digit_index++) {
        char prob_string[32];
        sprintf(prob_string, " %5.3f", probability_vector[model_index](character_index, digit_index));
        strcat(expiries_string, prob_string);
      }
      strcat(expiries_string, "\n");
    }
  }
  
  // Apply some weightings to the models, on the perhaps-mistaken belief
  // that the newer models perform a bit better than the older ones.
  float model_weighting[] = {0.3f, 0.7f};
  assert(sizeof(model_weighting)/sizeof(float) == probability_vector.size());
  
  ExpiryGroupScores average_vector = ExpiryGroupScores::Zero();
  for (int model_index = 0; model_index < probability_vector.size(); model_index++) {
    average_vector += probability_vector[model_index] * model_weighting[model_index];
  }
  
  return average_vector;
}


DMZ_INTERNAL void expiry_aggregate_grouped_rects(GroupedRectsList &aggregated_groups, GroupedRectsList &new_groups) {
  // Coalesce equivalent groups within new_groups (*** TODO *** IS THIS STEP EVER ACTUALLY NECESSARY? ***)
  for (size_t new_index_1 = 0; new_index_1 < new_groups.size(); new_index_1++) {
    GroupedRects &group1 = new_groups[new_index_1];
    int top1 = group1.top;
    int left1 = group1.left;
    size_t nChars1 = group1.character_rects.size();
    float groups_coalesced_so_far = 1;
    
    for (size_t new_index_2 = new_groups.size() - 1; new_index_2 > new_index_1; new_index_2--) {
      GroupedRects &group2 = new_groups[new_index_2];
      if (abs(group2.top - top1) > GROUPED_RECTS_VERTICAL_ALLOWANCE ||
          abs(group2.left - left1) > GROUPED_RECTS_HORIZONTAL_ALLOWANCE ||
          group2.character_rects.size() != nChars1) {
        continue;
      }

      group1.scores = ((group1.scores * groups_coalesced_so_far) + group2.scores) / (groups_coalesced_so_far + 1);
      groups_coalesced_so_far++;
      new_groups.erase(new_groups.begin() + new_index_2);
      dmz_debug_print("** Yup, coalesced a new group with another! WTF? ***\n");
    }
  }
  
#if DEBUG_EXPIRY_CATEGORIZATION_PERFORMANCE
  dmz_debug_timer_print("coalesce new groups", 2);
#endif
{'fraction_non_alphanumeric': 0.11544227886056972, 'fraction_numerical': 0.021229385307346325, 'mean_word_length': 4.358611825192802, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7430466', 'n_tokens_mistral': 6251, 'n_tokens_neox': 5851, 'n_words': 1101}