<commit_before>#include <iostream>
#include <algorithm>
#include <limits>
#include <assert.h>
#include <fstream>
#include <string.h>
#include <cuda_runtime.h>
#include <sys/stat.h>
#include <map>
#include "debug_macros.hpp"

// #include "../cctag/cmdline.hpp"

#include "frame.h"
#include "debug_image.h"
#include "edge_list.h"
#include "cmp_list.h"
#include "assist.h"

#undef CHATTY_WRITE_DEBUG_PLANE

namespace popart {

using namespace std;

/*************************************************************
 * Frame
 *************************************************************/

void Frame::hostDebugDownload( const cctag::Parameters& params )
{
    delete [] _h_debug_hyst_edges;

    _h_debug_hyst_edges = new unsigned char[ getWidth() * getHeight() ];

    POP_SYNC_CHK;

    POP_CUDA_MEMCPY_2D_ASYNC( _h_debug_hyst_edges, getWidth() * sizeof(uint8_t),
                              _d_hyst_edges.data, _d_hyst_edges.step,
                              _d_hyst_edges.cols * sizeof(uint8_t),
                              _d_hyst_edges.rows,
                              cudaMemcpyDeviceToHost, _stream );

    POP_CUDA_MEMCPY_2D_ASYNC( _h_edges.data, _h_edges.step,
                              _d_edges.data, _d_edges.step,
                              _d_edges.cols * sizeof(uint8_t),
                              _d_edges.rows,
                              cudaMemcpyDeviceToHost, _stream );
    POP_CHK_CALL_IFSYNC;
}

void Frame::hostDebugCompare( unsigned char* pix )
{
#ifndef NDEBUG
#ifdef DEBUG_WRITE_ORIGINAL_AS_PGM
    bool found_mistake = false;
    size_t mistake_ct = 0;

    for( int h=0; h<_d_plane.rows; h++ ) {
        for( int w=0; w<_d_plane.cols; w++ ) {
            size_t pos = h*_d_plane.cols+w;
            if( pix[pos] != _h_plane.data[pos] ) {
                mistake_ct++;
                if( found_mistake == false ) {
                    found_mistake = true;
                    cerr << "Found first error at (" << w << "," << h << "): "
                         << "orig " << pix[pos]
                         << "copy " << _h_plane.data[pos]
                         << endl;
                }
            }
        }
    }
    if( found_mistake ) {
        cerr << "Total errors: " << mistake_ct << endl;
    } else {
        cerr << "Found no difference between original and re-downloaded frame" << endl;
    }
#endif // DEBUG_WRITE_ORIGINAL_AS_PGM
#endif // NDEBUG
}

void Frame::writeHostDebugPlane( string filename, const cctag::Parameters& params )
{
    filename = params._debugDir + filename;

    string s;

#ifdef DEBUG_WRITE_ORIGINAL_AS_PGM
    const cv::cuda::PtrStepSzb& b = _h_plane;
    DebugImage::writePGM( filename + "-01.pgm", b );
#ifdef DEBUG_WRITE_ORIGINAL_AS_ASCII
    DebugImage::writeASCII( filename + "-01-img-ascii.txt", b );
#endif // DEBUG_WRITE_ORIGINAL_AS_ASCII
#endif // WRITE_ORIGINAL_AS_PGM

#ifdef DEBUG_WRITE_DX_AS_PGM
    cv::cuda::PtrStepSz16s dx( getHeight(),
                               getWidth(),
                               _h_dx,
                               getWidth()*sizeof(int16_t) );
    DebugImage::writePGMscaled( filename + "-02-dx.pgm", dx );
#ifdef DEBUG_WRITE_DX_AS_ASCII
    DebugImage::writeASCII( filename + "-02-dx-ascii.txt", dx );
#endif // DEBUG_WRITE_DX_AS_ASCII
#endif // DEBUG_WRITE_DX_AS_PGM

#ifdef DEBUG_WRITE_DY_AS_PGM
    cv::cuda::PtrStepSz16s dy( getHeight(),
                               getWidth(),
                               _h_dy,
                               getWidth()*sizeof(int16_t) );
    DebugImage::writePGMscaled( filename + "-02-dy.pgm", dy );
#ifdef DEBUG_WRITE_DY_AS_ASCII
    DebugImage::writeASCII( filename + "-02-dy-ascii.txt", dy );
#endif // DEBUG_WRITE_DY_AS_ASCII
#endif // DEBUG_WRITE_DY_AS_PGM


#ifdef DEBUG_WRITE_MAG_AS_PGM
    const cv::cuda::PtrStepSz32u& mag = _h_mag;
    DebugImage::writePGMscaled( filename + "-03-mag.pgm", mag );
#ifdef DEBUG_WRITE_MAG_AS_ASCII
    DebugImage::writeASCII( filename + "-03-mag-ascii.txt", mag );
#endif // DEBUG_WRITE_MAG_AS_ASCII
#endif // DEBUG_WRITE_MAG_AS_PGM

#ifdef DEBUG_WRITE_MAP_AS_PGM
    cv::cuda::PtrStepSzb   map( getHeight(),
                                getWidth(),
                                _h_debug_map,
                                getWidth()*sizeof(uint8_t) );
    DebugImage::writePGMscaled( filename + "-03-map.pgm", map );
#ifdef DEBUG_WRITE_MAP_AS_ASCII
    DebugImage::writeASCII( filename + "-03-map-ascii.txt", map );
#endif // DEBUG_WRITE_MAP_AS_ASCII
#endif // DEBUG_WRITE_MAP_AS_PGM

#ifdef DEBUG_WRITE_HYSTEDGES_AS_PGM
    cv::cuda::PtrStepSzb   hystedges( getHeight(),
                                      getWidth(),
                                      _h_debug_hyst_edges,
                                      getWidth()*sizeof(uint8_t) );
    DebugImage::writePGMscaled( filename + "-04-hystedges.pgm", hystedges );
#endif // DEBUG_WRITE_HYSTEDGES_AS_PGM

#ifdef DEBUG_WRITE_EDGES_AS_PGM
    const cv::cuda::PtrStepSzb&  edges = _h_edges;
#endif // DEBUG_WRITE_EDGES_AS_PGM

#ifdef DEBUG_WRITE_EDGES_AS_PGM
    DebugImage::writePGMscaled( filename + "-05-edges.pgm", edges );
#endif // DEBUG_WRITE_EDGES_AS_PGM

#ifndef NDEBUG
#ifdef DEBUG_WRITE_EDGELIST_AS_PPM
    {
        /* Very basic debugging stage:
         * do we really put all of those edge points into the edge list
         * that are also present in the edge image?
         *
         * Confirmed that this works
         */
        vector<int2> out;
        _vote._all_edgecoords.debug_out( EDGE_POINT_MAX, out );

        PtrStepSzbNull edgelistplane( edges.cols, edges.rows );
        DebugImage::plotPoints( out, edgelistplane.e, false, DebugImage::BLUE );
        DebugImage::writePGMscaled( filename + "-05-edgelist.pgm", edgelistplane.e );
#ifdef DEBUG_WRITE_EDGELIST_AS_ASCII
        int2cmp c;
        std::sort( out.begin(), out.end(), c );
        DebugImage::writeASCII( filename + "-05-edgelist.txt", out );
#endif // DEBUG_WRITE_EDGELIST_AS_ASCII
    }
#endif // DEBUG_WRITE_EDGELIST_AS_PPM

#ifdef DEBUG_WRITE_VOTERS_AS_PPM<commit_msg>tolerate a few more warnings in DEBUG mode<commit_after>#include <iostream>
#include <algorithm>
#include <limits>
#include <assert.h>
#include <fstream>
#include <string.h>
#include <cuda_runtime.h>
#include <sys/stat.h>
#include <map>
#include "debug_macros.hpp"

// #include "../cctag/cmdline.hpp"

#include "frame.h"
#include "debug_image.h"
#include "edge_list.h"
#include "cmp_list.h"
#include "assist.h"

#undef CHATTY_WRITE_DEBUG_PLANE

namespace popart {

using namespace std;

/*************************************************************
 * Frame
 *************************************************************/

void Frame::hostDebugDownload( const cctag::Parameters& params )
{
    delete [] _h_debug_hyst_edges;

    _h_debug_hyst_edges = new unsigned char[ getWidth() * getHeight() ];

    POP_SYNC_CHK;

    POP_CUDA_MEMCPY_2D_ASYNC( _h_debug_hyst_edges, getWidth() * sizeof(uint8_t),
                              _d_hyst_edges.data, _d_hyst_edges.step,
                              _d_hyst_edges.cols * sizeof(uint8_t),
                              _d_hyst_edges.rows,
                              cudaMemcpyDeviceToHost, _stream );

    POP_CUDA_MEMCPY_2D_ASYNC( _h_edges.data, _h_edges.step,
                              _d_edges.data, _d_edges.step,
                              _d_edges.cols * sizeof(uint8_t),
                              _d_edges.rows,
                              cudaMemcpyDeviceToHost, _stream );
    POP_CHK_CALL_IFSYNC;
}

void Frame::hostDebugCompare( unsigned char* pix )
{
#ifndef NDEBUG
#ifdef DEBUG_WRITE_ORIGINAL_AS_PGM
    bool found_mistake = false;
    size_t mistake_ct = 0;

    for( int h=0; h<_d_plane.rows; h++ ) {
        for( int w=0; w<_d_plane.cols; w++ ) {
            size_t pos = h*_d_plane.cols+w;
            if( pix[pos] != _h_plane.data[pos] ) {
                mistake_ct++;
                if( found_mistake == false ) {
                    found_mistake = true;
                    cerr << "Found first error at (" << w << "," << h << "): "
                         << "orig " << pix[pos]
                         << "copy " << _h_plane.data[pos]
                         << endl;
                }
            }
        }
    }
    if( found_mistake ) {
        cerr << "Total errors: " << mistake_ct << endl;
    } else {
        cerr << "Found no difference between original and re-downloaded frame" << endl;
    }
#endif // DEBUG_WRITE_ORIGINAL_AS_PGM
#endif // NDEBUG
}

void Frame::writeHostDebugPlane( string filename, const cctag::Parameters& params )
{
    filename = params._debugDir + filename;

    string s;

#ifdef DEBUG_WRITE_ORIGINAL_AS_PGM
    const cv::cuda::PtrStepSzb& b = _h_plane;
    DebugImage::writePGM( filename + "-01.pgm", b );
#ifdef DEBUG_WRITE_ORIGINAL_AS_ASCII
    DebugImage::writeASCII( filename + "-01-img-ascii.txt", b );
#endif // DEBUG_WRITE_ORIGINAL_AS_ASCII
#endif // WRITE_ORIGINAL_AS_PGM

#ifdef DEBUG_WRITE_DX_AS_PGM
    cv::cuda::PtrStepSz16s dx( getHeight(),
                               getWidth(),
                               _h_dx,
                               getWidth()*sizeof(int16_t) );
    DebugImage::writePGMscaled( filename + "-02-dx.pgm", dx );
#ifdef DEBUG_WRITE_DX_AS_ASCII
    DebugImage::writeASCII( filename + "-02-dx-ascii.txt", dx );
#endif // DEBUG_WRITE_DX_AS_ASCII
#endif // DEBUG_WRITE_DX_AS_PGM

#ifdef DEBUG_WRITE_DY_AS_PGM
    cv::cuda::PtrStepSz16s dy( getHeight(),
                               getWidth(),
                               _h_dy,
                               getWidth()*sizeof(int16_t) );
    DebugImage::writePGMscaled( filename + "-02-dy.pgm", dy );
#ifdef DEBUG_WRITE_DY_AS_ASCII
    DebugImage::writeASCII( filename + "-02-dy-ascii.txt", dy );
#endif // DEBUG_WRITE_DY_AS_ASCII
#endif // DEBUG_WRITE_DY_AS_PGM


#ifdef DEBUG_WRITE_MAG_AS_PGM
    const cv::cuda::PtrStepSz32u& mag = _h_mag;
    DebugImage::writePGMscaled( filename + "-03-mag.pgm", mag );
#ifdef DEBUG_WRITE_MAG_AS_ASCII
    DebugImage::writeASCII( filename + "-03-mag-ascii.txt", mag );
#endif // DEBUG_WRITE_MAG_AS_ASCII
#endif // DEBUG_WRITE_MAG_AS_PGM

#ifdef DEBUG_WRITE_MAP_AS_PGM
    cv::cuda::PtrStepSzb   map( getHeight(),
                                getWidth(),
                                _h_debug_map,
                                getWidth()*sizeof(uint8_t) );
    DebugImage::writePGMscaled( filename + "-03-map.pgm", map );
#ifdef DEBUG_WRITE_MAP_AS_ASCII
    DebugImage::writeASCII( filename + "-03-map-ascii.txt", map );
#endif // DEBUG_WRITE_MAP_AS_ASCII
#endif // DEBUG_WRITE_MAP_AS_PGM

#ifdef DEBUG_WRITE_HYSTEDGES_AS_PGM
    cv::cuda::PtrStepSzb   hystedges( getHeight(),
                                      getWidth(),
                                      _h_debug_hyst_edges,
                                      getWidth()*sizeof(uint8_t) );
    DebugImage::writePGMscaled( filename + "-04-hystedges.pgm", hystedges );
#endif // DEBUG_WRITE_HYSTEDGES_AS_PGM

#ifndef NDEBUG
    const cv::cuda::PtrStepSzb&  edges = _h_edges;
#endif // NDEBUG

#ifdef DEBUG_WRITE_EDGES_AS_PGM
    DebugImage::writePGMscaled( filename + "-05-edges.pgm", edges );
#endif // DEBUG_WRITE_EDGES_AS_PGM

#ifndef NDEBUG
#ifdef DEBUG_WRITE_EDGELIST_AS_PPM
    {
        /* Very basic debugging stage:
         * do we really put all of those edge points into the edge list
         * that are also present in the edge image?
         *
         * Confirmed that this works
         */
        vector<int2> out;
        _vote._all_edgecoords.debug_out( EDGE_POINT_MAX, out );

        PtrStepSzbNull edgelistplane( edges.cols, edges.rows );
        DebugImage::plotPoints( out, edgelistplane.e, false, DebugImage::BLUE );
        DebugImage::writePGMscaled( filename + "-05-edgelist.pgm", edgelistplane.e );
#ifdef DEBUG_WRITE_EDGELIST_AS_ASCII
        int2cmp c;
        std::sort( out.begin(), out.end(), c );
        DebugImage::writeASCII( filename + "-05-edgelist.txt", out );
#endif // DEBUG_WRITE_EDGELIST_AS_ASCII
    }
#endif // DEBUG_WRITE_EDGELIST_AS_PPM

#ifdef DEBUG_WRITE_VOTERS_AS_PPM
{'fraction_non_alphanumeric': 0.13046346693056243, 'fraction_numerical': 0.008246742536698005, 'mean_word_length': 2.3893236444941306, 'pattern_counts': {'":': 0, '<': 87, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2230576', 'n_tokens_mistral': 4208, 'n_tokens_neox': 4066, 'n_words': 734}