newBufferLength = (aCount + mLeftOverCount); buffer[newBufferLength] = '\0'; char* last_delimiter = NULL; char* scan = buffer + newBufferLength; while (scan > buffer) { if (strchr(kBayesianFilterTokenDelimiters, *--scan)) { last_delimiter = scan;
if (readCount == 0) { rv = NS_ERROR_UNEXPECTED; NS_WARNING("failed to tokenize");
NS_IMETHODIMP TokenStreamListener::OnDataAvailable(nsIRequest *aRequest, nsISupports *aContext, nsIInputStream *aInputStream, PRUint32 aOffset, PRUint32 aCount){    nsresult rv;    PRUint32 newBufferLength = (aCount + mLeftOverCount);    if (newBufferLength > mBufferSize) {        PRUint32 newBufferSize = newBufferLength * 2;        rv = mBuffer->Grow(newBufferSize);        if (NS_FAILED(rv))            return rv;        mBufferSize = newBufferSize;    }        char* buffer = mBuffer->GetBuffer();    rv = aInputStream->Read(buffer + mLeftOverCount, aCount, &aCount);    /* consume the tokens up to the last legal token delimiter in the buffer. */    newBufferLength = (aCount + mLeftOverCount);    buffer[newBufferLength] = '\0';    char* last_delimiter = NULL;    char* scan = buffer + newBufferLength;    while (scan > buffer) {        if (strchr(kBayesianFilterTokenDelimiters, *--scan)) {            last_delimiter = scan;            break;        }    }        if (last_delimiter) {        *last_delimiter = '\0';        mTokenizer.tokenize(buffer);        PRUint32 consumedCount = 1 + (last_delimiter - buffer);        mLeftOverCount = newBufferLength - consumedCount;        if (mLeftOverCount)            memmove(buffer, buffer + consumedCount, mLeftOverCount);    }    return rv;}