<commit_before>/*
 * Copyright (c) 2014 New Designs Unlimited, LLC
 * Opensource Automated License Plate Recognition [http://www.openalpr.com]
 *
 * This file is part of OpenAlpr.
 *
 * OpenAlpr is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License
 * version 3 as published by the Free Software Foundation
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include "charactersegmenter.h"

using namespace cv;
using namespace std;

CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data)
{
  this->pipeline_data = pipeline_data;
  this->config = pipeline_data->config;

  this->confidence = 0;

  if (this->config->debugCharSegmenter)
    cout << "Starting CharacterSegmenter" << endl;

  //CharacterRegion charRegion(img, debug);

  timespec startTime;
  getTime(&startTime);

  pipeline_data->clearThresholds();
  pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config);

  medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3);

  cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl;
  if (pipeline_data->plate_inverted)
    bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray);


  if (this->config->debugCharSegmenter)
  {
    displayImage(config, "CharacterSegmenter  Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3));
  }

//  if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0)
//  {
//    Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U);
//    charAnalysis->bestThreshold.copyTo(img_contours);
//    cvtColor(img_contours, img_contours, CV_GRAY2RGB);
//
//    vector<vector<Point> > allowedContours;
//    for (uint i = 0; i < charAnalysis->bestContours.size(); i++)
//    {
//      if (charAnalysis->bestContours.goodIndices[i])
//        allowedContours.push_back(charAnalysis->bestContours.contours[i]);
//    }
//
//    drawContours(img_contours, charAnalysis->bestContours.contours,
//                 -1, // draw all contours
//                 cv::Scalar(255,0,0), // in blue
//                 1); // with a thickness of 1
//
//    drawContours(img_contours, allowedContours,
//                 -1, // draw all contours
//                 cv::Scalar(0,255,0), // in green
//                 1); // with a thickness of 1
//
//
//    line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1);
//    line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1);
//
//
//    Mat bordered = addLabel(img_contours, "Best Contours");
//    imgDbgGeneral.push_back(bordered);
//  }

  for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++)
  {
    this->top = pipeline_data->textLines[lineidx].topLine;
    this->bottom = pipeline_data->textLines[lineidx].bottomLine;
    

    float avgCharHeight = pipeline_data->textLines[lineidx].lineHeight;
    float height_to_width_ratio = pipeline_data->config->charHeightMM / pipeline_data->config->charWidthMM;
    float avgCharWidth = avgCharHeight / height_to_width_ratio;
    //float avgCharWidth = median(charWidths.data(), charWidths.size());

    //removeSmallContours(pipeline_data->thresholds, charAnalysis->allTextContours, avgCharWidth, avgCharHeight);

    // Do the histogram analysis to figure out char regions

    timespec startTime;
    getTime(&startTime);

    vector<Mat> allHistograms;

    vector<Rect> allBoxes;
    for (uint i = 0; i < pipeline_data->thresholds.size(); i++)
    {
      Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U);

      fillConvexPoly(histogramMask, pipeline_data->textLines[lineidx].linePolygon.data(), pipeline_data->textLines[lineidx].linePolygon.size(), Scalar(255,255,255));

      VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask);

      if (this->config->debugCharSegmenter)
      {
        Mat histoCopy(vertHistogram.histoImg.size(), vertHistogram.histoImg.type());
        //vertHistogram.copyTo(histoCopy);
        cvtColor(vertHistogram.histoImg, histoCopy, CV_GRAY2RGB);

	string label = "threshold: " + toString(i);
        allHistograms.push_back(addLabel(histoCopy, label));
      }

//
      float score = 0;
      vector<Rect> charBoxes = getHistogramBoxes(vertHistogram, avgCharWidth, avgCharHeight, &score);

      if (this->config->debugCharSegmenter)
      {
        for (uint cboxIdx = 0; cboxIdx < charBoxes.size(); cboxIdx++)
        {
          rectangle(allHistograms[i], charBoxes[cboxIdx], Scalar(0, 255, 0));
        }

        Mat histDashboard = drawImageDashboard(allHistograms, allHistograms[0].type(), 1);
        displayImage(config, "Char seg histograms", histDashboard);
      }

      for (uint z = 0; z < charBoxes.size(); z++)
        allBoxes.push_back(charBoxes[z]);
      //drawAndWait(&histogramMask);
    }

    float medianCharWidth = avgCharWidth;
    vector<int> widthValues;
    // Compute largest char width
    for (uint i = 0; i < allBoxes.size(); i++)
    {
      widthValues.push_back(allBoxes[i].width);
    }

    medianCharWidth = median(widthValues.data(), widthValues.size());

    if (config->debugTiming)
    {
      timespec endTime;
      getTime(&endTime);
      cout << "  -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl;
    }

    vector<Rect> candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], allBoxes, medianCharWidth);

    if (this->config->debugCharSegmenter)
    {
      // Setup the dashboard images to show the cleaning filters
      for (uint i = 0; i < pipeline_data->thresholds.size(); i++)
      {
        Mat cleanImg = Mat::zeros(pipeline_data->thresholds[i].size(), pipeline_data->thresholds[i].type());
        Mat boxMask = getCharBoxMask(pipeline_data->thresholds[i], candidateBoxes);
        pipeline_data->thresholds[i].copyTo(cleanImg);
        bitwise_and(cleanImg, boxMask, cleanImg);
        cvtColor(cleanImg, cleanImg, CV_GRAY2BGR);

        for (uint c = 0; c < candidateBoxes.size(); c++)
          rectangle(cleanImg, candidateBoxes[c], Scalar(0, 255, 0), 1);
        imgDbgCleanStages.push_back(cleanImg);
      }
    }

    getTime(&startTime);

    filterEdgeBoxes(pipeline_data->thresholds, candidateBoxes, medianCharWidth, avgCharHeight);

    candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes);

    candidateBoxes = combineCloseBoxes(candidateBoxes, medianCharWidth);

    cleanCharRegions(pipeline_data->thresholds, candidateBoxes);
    cleanMostlyFullBoxes(pipeline_data->thresholds, candidateBoxes);

    //cleanBasedOnColor(thresholds, colorFilter.colorMask, candidateBoxes);

    candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes);
    pipeline_data->charRegions = candidateBoxes;

    if (config->debugTiming)
    {
      timespec endTime;
      getTime(&endTime);
      cout << "  -- Character Segmentation Box cleaning/filtering Time: " << diffclock(startTime, endTime) << "ms." << endl;
    }

    if (this->config->debugCharSegmenter)
    {
      Mat imgDash = drawImageDashboard(pipeline_data->thresholds, CV_8U, 3);
      displayImage(config, "Segmentation after cleaning", imgDash);

      Mat generalDash = drawImageDashboard(this->imgDbgGeneral, this->imgDbgGeneral[0].type(), 2);
      displayImage(config, "Segmentation General", generalDash);

      Mat cleanImgDash = drawImageDashboard(this->imgDbgCleanStages, this->imgDbgCleanStages[0].type(), 3);
      displayImage(config, "Segmentation Clean Filters", cleanImgDash);
    }
  }

  if (config->debugTiming)<commit_msg>Fixed multiline support in character analysis<commit_after>/*
 * Copyright (c) 2014 New Designs Unlimited, LLC
 * Opensource Automated License Plate Recognition [http://www.openalpr.com]
 *
 * This file is part of OpenAlpr.
 *
 * OpenAlpr is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License
 * version 3 as published by the Free Software Foundation
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

#include "charactersegmenter.h"

using namespace cv;
using namespace std;

CharacterSegmenter::CharacterSegmenter(PipelineData* pipeline_data)
{
  this->pipeline_data = pipeline_data;
  this->config = pipeline_data->config;

  this->confidence = 0;

  if (this->config->debugCharSegmenter)
    cout << "Starting CharacterSegmenter" << endl;

  //CharacterRegion charRegion(img, debug);

  timespec startTime;
  getTime(&startTime);

  pipeline_data->clearThresholds();
  pipeline_data->thresholds = produceThresholds(pipeline_data->crop_gray, config);

  medianBlur(pipeline_data->crop_gray, pipeline_data->crop_gray, 3);

  cout << "Segmenter: inverted: " << pipeline_data->plate_inverted << endl;
  if (pipeline_data->plate_inverted)
    bitwise_not(pipeline_data->crop_gray, pipeline_data->crop_gray);


  if (this->config->debugCharSegmenter)
  {
    displayImage(config, "CharacterSegmenter  Thresholds", drawImageDashboard(pipeline_data->thresholds, CV_8U, 3));
  }

//  if (this->config->debugCharSegmenter && pipeline_data->textLines.size() > 0)
//  {
//    Mat img_contours(charAnalysis->bestThreshold.size(), CV_8U);
//    charAnalysis->bestThreshold.copyTo(img_contours);
//    cvtColor(img_contours, img_contours, CV_GRAY2RGB);
//
//    vector<vector<Point> > allowedContours;
//    for (uint i = 0; i < charAnalysis->bestContours.size(); i++)
//    {
//      if (charAnalysis->bestContours.goodIndices[i])
//        allowedContours.push_back(charAnalysis->bestContours.contours[i]);
//    }
//
//    drawContours(img_contours, charAnalysis->bestContours.contours,
//                 -1, // draw all contours
//                 cv::Scalar(255,0,0), // in blue
//                 1); // with a thickness of 1
//
//    drawContours(img_contours, allowedContours,
//                 -1, // draw all contours
//                 cv::Scalar(0,255,0), // in green
//                 1); // with a thickness of 1
//
//
//    line(img_contours, pipeline_data->textLines[0].linePolygon[0], pipeline_data->textLines[0].linePolygon[1], Scalar(255, 0, 255), 1);
//    line(img_contours, pipeline_data->textLines[0].linePolygon[3], pipeline_data->textLines[0].linePolygon[2], Scalar(255, 0, 255), 1);
//
//
//    Mat bordered = addLabel(img_contours, "Best Contours");
//    imgDbgGeneral.push_back(bordered);
//  }

  
  
  for (uint lineidx = 0; lineidx < pipeline_data->textLines.size(); lineidx++)
  {
    this->top = pipeline_data->textLines[lineidx].topLine;
    this->bottom = pipeline_data->textLines[lineidx].bottomLine;
    

    float avgCharHeight = pipeline_data->textLines[lineidx].lineHeight;
    float height_to_width_ratio = pipeline_data->config->charHeightMM / pipeline_data->config->charWidthMM;
    float avgCharWidth = avgCharHeight / height_to_width_ratio;
    //float avgCharWidth = median(charWidths.data(), charWidths.size());

    //removeSmallContours(pipeline_data->thresholds, charAnalysis->allTextContours, avgCharWidth, avgCharHeight);

    // Do the histogram analysis to figure out char regions

    timespec startTime;
    getTime(&startTime);

    vector<Mat> allHistograms;

    vector<Rect> lineBoxes;
    for (uint i = 0; i < pipeline_data->thresholds.size(); i++)
    {
      Mat histogramMask = Mat::zeros(pipeline_data->thresholds[i].size(), CV_8U);

      fillConvexPoly(histogramMask, pipeline_data->textLines[lineidx].linePolygon.data(), pipeline_data->textLines[lineidx].linePolygon.size(), Scalar(255,255,255));

      VerticalHistogram vertHistogram(pipeline_data->thresholds[i], histogramMask);

      if (this->config->debugCharSegmenter)
      {
        Mat histoCopy(vertHistogram.histoImg.size(), vertHistogram.histoImg.type());
        //vertHistogram.copyTo(histoCopy);
        cvtColor(vertHistogram.histoImg, histoCopy, CV_GRAY2RGB);

	string label = "threshold: " + toString(i);
        allHistograms.push_back(addLabel(histoCopy, label));
      }

//
      float score = 0;
      vector<Rect> charBoxes = getHistogramBoxes(vertHistogram, avgCharWidth, avgCharHeight, &score);

      if (this->config->debugCharSegmenter)
      {
        for (uint cboxIdx = 0; cboxIdx < charBoxes.size(); cboxIdx++)
        {
          rectangle(allHistograms[i], charBoxes[cboxIdx], Scalar(0, 255, 0));
        }

        Mat histDashboard = drawImageDashboard(allHistograms, allHistograms[0].type(), 1);
        displayImage(config, "Char seg histograms", histDashboard);
      }

      for (uint z = 0; z < charBoxes.size(); z++)
        lineBoxes.push_back(charBoxes[z]);
      //drawAndWait(&histogramMask);
    }

    float medianCharWidth = avgCharWidth;
    vector<int> widthValues;
    // Compute largest char width
    for (uint i = 0; i < lineBoxes.size(); i++)
    {
      widthValues.push_back(lineBoxes[i].width);
    }

    medianCharWidth = median(widthValues.data(), widthValues.size());

    if (config->debugTiming)
    {
      timespec endTime;
      getTime(&endTime);
      cout << "  -- Character Segmentation Create and Score Histograms Time: " << diffclock(startTime, endTime) << "ms." << endl;
    }

    vector<Rect> candidateBoxes = getBestCharBoxes(pipeline_data->thresholds[0], lineBoxes, medianCharWidth);

    if (this->config->debugCharSegmenter)
    {
      // Setup the dashboard images to show the cleaning filters
      for (uint i = 0; i < pipeline_data->thresholds.size(); i++)
      {
        Mat cleanImg = Mat::zeros(pipeline_data->thresholds[i].size(), pipeline_data->thresholds[i].type());
        Mat boxMask = getCharBoxMask(pipeline_data->thresholds[i], candidateBoxes);
        pipeline_data->thresholds[i].copyTo(cleanImg);
        bitwise_and(cleanImg, boxMask, cleanImg);
        cvtColor(cleanImg, cleanImg, CV_GRAY2BGR);

        for (uint c = 0; c < candidateBoxes.size(); c++)
          rectangle(cleanImg, candidateBoxes[c], Scalar(0, 255, 0), 1);
        imgDbgCleanStages.push_back(cleanImg);
      }
    }

    getTime(&startTime);

    filterEdgeBoxes(pipeline_data->thresholds, candidateBoxes, medianCharWidth, avgCharHeight);
    candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes);
    candidateBoxes = combineCloseBoxes(candidateBoxes, medianCharWidth);
    cleanMostlyFullBoxes(pipeline_data->thresholds, candidateBoxes);

    candidateBoxes = filterMostlyEmptyBoxes(pipeline_data->thresholds, candidateBoxes);
    
    for (uint cbox = 0; cbox < candidateBoxes.size(); cbox++)
      pipeline_data->charRegions.push_back(candidateBoxes[cbox]);

    if (config->debugTiming)
    {
      timespec endTime;
      getTime(&endTime);
      cout << "  -- Character Segmentation Box cleaning/filtering Time: " << diffclock(startTime, endTime) << "ms." << endl;
    }

    if (this->config->debugCharSegmenter)
    {
      Mat imgDash = drawImageDashboard(pipeline_data->thresholds, CV_8U, 3);
      displayImage(config, "Segmentation after cleaning", imgDash);

      Mat generalDash = drawImageDashboard(this->imgDbgGeneral, this->imgDbgGeneral[0].type(), 2);
      displayImage(config, "Segmentation General", generalDash);

      Mat cleanImgDash = drawImageDashboard(this->imgDbgCleanStages, this->imgDbgCleanStages[0].type(), 3);
      displayImage(config, "Segmentation Clean Filters", cleanImgDash);
    }
  }
  
  cleanCharRegions(pipeline_data->thresholds, pipeline_data->charRegions);
    drawAndWait(&pipeline_data->thresholds[0]);

  if (config->debugTiming)
{'fraction_non_alphanumeric': 0.1281815956926089, 'fraction_numerical': 0.011502692119432207, 'mean_word_length': 3.751453488372093, 'pattern_counts': {'":': 0, '<': 88, '<?xml version=': 0, '>': 175, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3947805', 'n_tokens_mistral': 5297, 'n_tokens_neox': 5072, 'n_words': 1040}