<commit_before>#include "RichEditSpellChecker.h"
#include <assert.h>
#include <process.h>
#include <iomanip>
#include <iterator>
#include <locale>
#include <sstream>
#include <strstream>
#include "tscSession.h"
#include <richedit.h>
#include "CheckSpellingDlg.h"

using namespace std;
using namespace TolonSpellCheck;

CRichEditSpellChecker::CRichEditSpellChecker(CSession* pSession) : 
    m_hWndRichEdit(NULL),
	m_pSession(pSession),
    m_nState(SpellCheckState_IDLE),
    m_dwCharsDone(0),
    m_dwCharsTotal(0),
    m_bStopFlag(false)
{
    ::InitializeCriticalSection(&m_cs);
}

CRichEditSpellChecker::~CRichEditSpellChecker()
{
    ::DeleteCriticalSection(&m_cs);
}

DWORD CRichEditSpellChecker::WT_RichEditStreamOutCallback(DWORD dwCookie, LPBYTE pbBuff, LONG cb, LONG *pcb)
{
	CRichEditSpellChecker* pThis = reinterpret_cast<CRichEditSpellChecker*>(dwCookie);
	
	if (pThis)
		return pThis->WT_DoCallbackWork(pbBuff, cb, pcb);
	else
		return static_cast<DWORD>(-1);
}

void CRichEditSpellChecker::StartSpellCheck(HWND hwndEditCtrl)
{
    m_hWndRichEdit = hwndEditCtrl;
    m_bStopFlag = false;
    SetState(SpellCheckState_IDLE);

    unsigned int hThread = 0;
    hThread = _beginthreadex(NULL, 0, &WT_CheckSpelling, static_cast<void*>(this), CREATE_SUSPENDED, NULL);
    BOOL b = FALSE;
    b = ::SetThreadPriority((HANDLE)hThread, THREAD_PRIORITY_LOWEST);
    ::ResumeThread((HANDLE)hThread);
    ::CloseHandle((HANDLE)hThread);
}

void CRichEditSpellChecker::StopSpellCheck()
{
    SetStopFlag();
}

unsigned int CRichEditSpellChecker::WT_CheckSpelling(void* p)
{
    CRichEditSpellChecker* pThis = reinterpret_cast<CRichEditSpellChecker*>(p);

    if (pThis)
    {
	    EDITSTREAM es;
	    es.dwCookie = reinterpret_cast<DWORD>(pThis);
	    es.dwError = 0;
	    es.pfnCallback = &CRichEditSpellChecker::WT_RichEditStreamOutCallback;
    	
	    pThis->WT_PreSpellCheck();
        ::SendMessage(pThis->GetRichEditHwnd(), EM_STREAMOUT, SF_UNICODE | SF_TEXT, reinterpret_cast<LPARAM>(&es));
        pThis->WT_PostSpellCheck();
   }

    return 0;
}

void CRichEditSpellChecker::WT_PreSpellCheck()
{
    // Reset word container
	m_sWord.str(L"");

    // Find the total number of characters in the control.
    DWORD dwChars = 0;
    GETTEXTLENGTHEX etlex = {0};
    dwChars = ::SendMessage(GetRichEditHwnd(), EM_GETTEXTLENGTHEX, reinterpret_cast<WPARAM>(&etlex), NULL);
    SetTotalChars(dwChars);

    // Reset the char count<commit_msg>Richedit streaming update in progress...<commit_after>#include "RichEditSpellChecker.h"
#include <assert.h>
#include <process.h>
#include <iomanip>
#include <iterator>
#include <locale>
#include <sstream>
#include <strstream>
#include "tscSession.h"
#include <richedit.h>
#include "CheckSpellingDlg.h"

using namespace std;
using namespace TolonSpellCheck;

CRichEditSpellChecker::CRichEditSpellChecker(CSession* pSession) : 
    m_hWndRichEdit(NULL),
	m_pSession(pSession),
    m_nState(SpellCheckState_IDLE),
    m_dwCharsDone(0),
    m_dwCharsTotal(0),
    m_bStopFlag(false)
{
    ::InitializeCriticalSection(&m_cs);
}

CRichEditSpellChecker::~CRichEditSpellChecker()
{
    ::DeleteCriticalSection(&m_cs);
}

DWORD CRichEditSpellChecker::WT_RichEditStreamOutCallback(DWORD dwCookie, LPBYTE pbBuff, LONG cb, LONG *pcb)
{
	CRichEditSpellChecker* pThis = reinterpret_cast<CRichEditSpellChecker*>(dwCookie);
	
	if (pThis)
		return pThis->WT_DoCallbackWork(pbBuff, cb, pcb);
	else
		return static_cast<DWORD>(-1);
}

void CRichEditSpellChecker::StartSpellCheck(HWND hwndEditCtrl)
{
    m_hWndRichEdit = hwndEditCtrl;
    m_bStopFlag = false;
    SetState(SpellCheckState_IDLE);

    unsigned int hThread = 0;
    hThread = _beginthreadex(NULL, 0, &WT_CheckSpelling, static_cast<void*>(this), CREATE_SUSPENDED, NULL);
    BOOL b = FALSE;
    b = ::SetThreadPriority((HANDLE)hThread, THREAD_PRIORITY_LOWEST);
    ::ResumeThread((HANDLE)hThread);
    ::CloseHandle((HANDLE)hThread);
}

void CRichEditSpellChecker::StopSpellCheck()
{
    SetStopFlag();
}

unsigned int CRichEditSpellChecker::WT_CheckSpelling(void* p)
{
    CRichEditSpellChecker* pThis = reinterpret_cast<CRichEditSpellChecker*>(p);

    if (pThis)
    {
	    EDITSTREAM es;
	    es.dwCookie = reinterpret_cast<DWORD>(pThis);
	    es.dwError = 0;
	    es.pfnCallback = &CRichEditSpellChecker::WT_RichEditStreamOutCallback;
    	
	    pThis->WT_PreSpellCheck();
        //::SendMessage(pThis->GetRichEditHwnd(), EM_STREAMOUT, SF_UNICODE | SF_TEXT, reinterpret_cast<LPARAM>(&es));

        LRESULT lr = 0;
        GETTEXTLENGTHEX gtlex = {0};
        gtlex.codepage = 1200; //CP_UNICODE
        gtlex.flags = GTL_NUMCHARS | GTL_CLOSE | GTL_USECRLF;
        lr = ::SendMessage(pThis->GetRichEditHwnd(), EM_GETTEXTLENGTHEX, reinterpret_cast<WPARAM>(&gtlex), NULL);

        if (lr > 0)
        {

        }

        pThis->WT_PostSpellCheck();
   }

    return 0;
}

void CRichEditSpellChecker::WT_PreSpellCheck()
{
    // Reset word container
	m_sWord.str(L"");

    // Find the total number of characters in the control.
    DWORD dwChars = 0;
    GETTEXTLENGTHEX etlex = {0};
    dwChars = ::SendMessage(GetRichEditHwnd(), EM_GETTEXTLENGTHEX, reinterpret_cast<WPARAM>(&etlex), NULL);
    SetTotalChars(dwChars);

    // Reset the char count
{'fraction_non_alphanumeric': 0.12004573170731707, 'fraction_numerical': 0.004763719512195122, 'mean_word_length': 4.884529147982063, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4022113', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1927, 'n_words': 318}