<commit_before>//  IBM_PROLOG_BEGIN_TAG
//  This is an automatically generated prolog.
//
//  $Source: src/usr/trace/trace.C $
//
//  IBM CONFIDENTIAL
//
//  COPYRIGHT International Business Machines Corp. 2011
//
//  p1
//
//  Object Code Only (OCO) source materials
//  Licensed Internal Code Source Materials
//  IBM HostBoot Licensed Internal Code
//
//  The source code for this program is not published or other-
//  wise divested of its trade secrets, irrespective of what has
//  been deposited with the U.S. Copyright Office.
//
//  Origin: 30
//
//  IBM_PROLOG_END

/**
 *  @file trace.C
 *
 *  @brief Implementation of class Trace
 */


/* TODO
 *  - Add support in for debug trace enable/disable
 *  - FORMAT_PRINTF support
 *  - %s support
 *  - Multiple buffer support
 *
 */

/******************************************************************************/
// Includes
/******************************************************************************/
#include <trace/interface.H>
#include <stdarg.h>
#include <stdlib.h>
#include <arch/ppc.H>
#include <kernel/console.H>
#include <kernel/pagemgr.H>
#include <limits.h>
#include <stdlib.h>
#include <sys/task.h>
#include <sys/sync.h>
#include <string_ext.h>
#include <util/align.H>

#include <trace/trace.H>




/******************************************************************************/
// Namespace
/******************************************************************************/
namespace TRACE
{

/******************************************************************************/
// Globals/Constants
/******************************************************************************/

const uint32_t TRAC_TIME_REAL   = 0;  // upper 32 = seconds, lower 32 = microseconds
const uint32_t TRAC_TIME_50MHZ  = 1;
const uint32_t TRAC_TIME_200MHZ = 2;
const uint32_t TRAC_TIME_167MHZ = 3;  // 166666667Hz
const uint32_t COMP_NAME_SIZE   = 16; // NULL terminated string

// Initial implementation is to allocate a fixed 2KB buffer to each
// component on request.
// NOTE: any change to this value will require change to Trace::initBuffer()
const uint64_t TRAC_DEFAULT_BUFFER_SIZE = 0x0800;  //2KB

// NOTE: This constant should only be changed to an even number for now.
// Same reason as above.
const uint64_t TRAC_MAX_NUM_BUFFERS = 24;

const char * const TRAC_DEFAULT_BUFFER_NAME = "DEFAULT";

// Global component trace buffer array.  Initially allow for 24 buffers max.
// Keep global so it can be found in syms file
typedef struct trace_desc_array {
    char comp[COMP_NAME_SIZE];        // the buffer name
    trace_desc_t * td_entry;          // pointer to the buffer
}trace_desc_array_t;

trace_desc_array_t g_desc_array[TRAC_MAX_NUM_BUFFERS];

/******************************************************************************/
// TracInit::TracInit()
/******************************************************************************/
TracInit::TracInit(trace_desc_t **o_td, const char *i_comp,const size_t i_size)
{
    TRAC_INIT_BUFFER(o_td,i_comp,i_size);
}

/******************************************************************************/
// TracInit::~TracInit()
/******************************************************************************/
TracInit::~TracInit()
{
}


/******************************************************************************/
// Trace::getTheInstance
/******************************************************************************/
Trace& Trace::getTheInstance()
{
    return Singleton<Trace>::instance();
}

/******************************************************************************/
// Trace::Trace
/******************************************************************************/
Trace::Trace()
{
    mutex_init(&iv_trac_mutex);

    memset(g_desc_array, 0, sizeof(g_desc_array));
}

/******************************************************************************/
// Trace::~Trace
/******************************************************************************/
Trace::~Trace()
{

}


/******************************************************************************/
// trace_adal_init_buffer
/******************************************************************************/
void Trace::initBuffer(trace_desc_t **o_td, const char* i_comp,
                            const size_t i_size )
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t i = 0;
    char * l_td = NULL;
    char l_comp[COMP_NAME_SIZE] = {'\0'};

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/
    if(*o_td == NULL)
    {
        // Limit component name to 15 characters.
        // Too bad we don't have strncpy(), strncmp()
        if (strlen(i_comp) > (COMP_NAME_SIZE -1))
        {
            memcpy(l_comp, i_comp, COMP_NAME_SIZE - 1);
        }
        else
        {
            strcpy(l_comp, i_comp);
        }

        // make string upper case 
        strupr(l_comp);

        // CRITICAL REGION START
        mutex_lock(&iv_trac_mutex);

        // Search through the descriptor array for the first unallocated buffer.
        // The last buffer is the reserved default buffer for any component
        // which didn't get its own buffer.
        for (i = 0; i < (TRAC_MAX_NUM_BUFFERS - 1); i++)
        {
            if(!strcmp(l_comp, g_desc_array[i].comp))
            {
                //printk("Trace::initBuffer - buffer already allocated %d\n", i);

                // Buffer is already allocated.  Return the buffer.
                *o_td = g_desc_array[i].td_entry;

                break;
            }
            else if (strlen(g_desc_array[i].comp) == 0)
            {
                //printk("Trace::initBuffer - found unallocated buffer %d\n", i);

                // Found the first unallocated buffer; use this one.

                // Set the component name for the buffer
                strcpy(g_desc_array[i].comp, l_comp);

                // Allocate memory if needed
                if (NULL == g_desc_array[i].td_entry)
                {
                    //printk("Trace::initBuffer - allocate memory\n");

                    // Allocate memory
                    // TODO can't handle i_size yet - everything is coded
                    // around TRAC_DEFAULT_BUFFER_SIZE
                    l_td = static_cast<char *>(malloc(TRAC_DEFAULT_BUFFER_SIZE));

                    g_desc_array[i].td_entry =
                        reinterpret_cast<trace_desc_t *>(l_td);

                }

                // Initialize the buffer header
                initValuesBuffer(g_desc_array[i].td_entry,
                                 g_desc_array[i].comp);

                // Return the newly allocated buffer
                *o_td = g_desc_array[i].td_entry;

                break;
            }
        }

        if ((TRAC_MAX_NUM_BUFFERS - 1) == i)
        {
            //printk("Trace::initBuffer - allocate default buffer %d\n", i);

            // We're out of buffers to allocate.
            // Use the default buffer reserved for everyone else.
            // Initialize only once
            if (strlen(g_desc_array[i].comp) == 0)
            {
                // Set the component name for the buffer
                strcpy(g_desc_array[i].comp, TRAC_DEFAULT_BUFFER_NAME);

                // Allocate memory if needed
                // Memory should have already been reserved if
                // TRAC_MAX_NUM_BUFFERS is an even # and we're using
                // PageManager::allocatePage().  Add check just in
                // case TRAC_MAC_NUM_BUFFERS is set to an odd number.
                if (NULL == g_desc_array[i].td_entry)
                {
                    //printk("Trace::initBuffer - allocate memory\n");

                    // Allocate memory for buffer
                    l_td = static_cast<char *>(malloc(TRAC_DEFAULT_BUFFER_SIZE));

                    g_desc_array[i].td_entry =
                        reinterpret_cast<trace_desc_t *>(l_td);
                }

                // Initialize the buffer header
                initValuesBuffer(g_desc_array[i].td_entry,
                                 g_desc_array[i].comp);
            }

            // Return the default buffer
            *o_td = g_desc_array[i].td_entry;
        }

        mutex_unlock(&iv_trac_mutex);
        // CRITICAL REGION END

    }

    return;
}

/******************************************************************************/
// initValuesBuffer
/******************************************************************************/
void Trace::initValuesBuffer(trace_desc_t *o_buf,const char *i_comp)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    // Initialize it to all 0's
    memset(o_buf,0,(size_t)TRAC_DEFAULT_BUFFER_SIZE);

    o_buf->ver = TRACE_BUF_VERSION;
    o_buf->hdr_len = sizeof(trace_buf_head_t);
    o_buf->time_flg = TRAC_TIME_REAL;
    o_buf->endian_flg = 'B';  // Big Endian
    strcpy(o_buf->comp,i_comp);
    o_buf->size = TRAC_DEFAULT_BUFFER_SIZE;
    o_buf->times_wrap = 0;
    o_buf->next_free = sizeof(trace_buf_head_t);

    return;
}

/******************************************************************************/
// trace_adal_write_all
/******************************************************************************/
void Trace::trace_adal_write_all(trace_desc_t *io_td,
                                 const trace_hash_val i_hash,
                                 const char * i_fmt,
                                 const uint32_t i_line,
                                 const int32_t i_type, ...)
{
    va_list args;
    va_start(args, i_type);

    getTheInstance()._trace_adal_write_all(io_td,
                                           i_hash,
                                           i_fmt,
                                           i_line,
                                           i_type, args);

    va_end(args);
}

/******************************************************************************/
// trace_adal_write_all
/******************************************************************************/
void Trace::_trace_adal_write_all(trace_desc_t *io_td,
                                 const trace_hash_val i_hash,
                                 const char * i_fmt,
                                 const uint32_t i_line,
                                 const int32_t i_type, va_list i_args)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t                l_entry_size = 0;
    uint32_t                l_data_size= 0;
    //trace_entire_entry_t    l_entry;
    trace_bin_entry_t       l_entry;
    uint64_t                l_str_map = 0;
    uint64_t                l_char_map = 0;
    uint64_t                l_double_map = 0;


    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    uint32_t num_args = 0;
    uint32_t num_4byte_args = 0; //fsp-trace counts 8-byte args as 2 4-byte args
    const char* _fmt = i_fmt;

    // Save a copy of input args because calling
    // va_arg() on a va_list is a one-shot.
    va_list l_args;
    va_copy (l_args, i_args);


    // Sum the sizes of the items in i_args in order to know how big to
    // allocate the entry.

    for (size_t i = 0; i <= strlen(_fmt); i++)
    {
        if ('%' == _fmt[i])
        {
            i++;

            if ('%' == _fmt[i])
            {
                continue;
            }
            else if ('s' == _fmt[i])
            {
                // Set flag to indicate argument is a string
                l_str_map = l_str_map | (1 << num_args);

                // String counts as one 4-byte arg
                num_args++;
                num_4byte_args++;

                char * l_str = va_arg(i_args, char *);
                size_t l_strLen = strlen(l_str);

                // Add to total size of number of arguments we're tracing
                // and account for word alignment
                l_data_size += l_strLen + 1;
                l_data_size = ALIGN_4(l_data_size);

                //printk("Trace: STRING %s: strlen %d num_args %d l_data_size %d\n",
                //       l_str, static_cast<uint32_t>(l_strLen),
                //       num_args, l_data_size);
                //printk("Trace: l_str_map 0x%16llX\n",
                //     static_cast<long long>(l_str_map));
            }
            else if ('c' == _fmt[i])
            {
                // Set flag to indicate argument is a char
                l_char_map = l_char_map | (1 << num_args);

                // Increment arg counts
                num_args++;
                num_4byte_args++;

                // Retrieve the argument to increment to next one
                uint32_t l_tmpData = va_arg(i_args, uint32_t);

                // Add to total size; data is word aligned
                l_data_size += sizeof(l_tmpData);
            }
            else if (('e' == _fmt[i]) || ('f' == _fmt[i]) || ('g' == _fmt[i]))
            {
                // Set flag to indicate argument is a double
                l_double_map = l_double_map | (1 << num_args);

                // Numbers count as two 4-byte arg
                num_args++;
                num_4byte_args += 2;

                // Retrieve the argument to increment to next one
                double l_tmpData = va_arg(i_args,double);

                // Add to total size; data is word aligned
                l_data_size += sizeof(l_tmpData);
            }
            else
            {
                // Numbers count as two 4-byte arg
                num_args++;
                num_4byte_args += 2;

                // Retrieve the argument to increment to next one
                uint64_t l_tmpData = va_arg(i_args, uint64_t);

                // Add to total size; data is word aligned
                l_data_size += sizeof(l_tmpData);
            }
        }
    }

    va_end( i_args );


    if((num_4byte_args <= TRAC_MAX_ARGS) && (io_td != NULL))
    {
        // Fill in the entry structure
        l_entry.stamp.tid = static_cast<uint32_t>(task_gettid());

        // Length is equal to size of data
        l_entry.head.length = l_data_size;
        //l_entry.head.tag = TRACE_FIELDTRACE;
        l_entry.head.tag = TRACE_COMP_TRACE;
        l_entry.head.hash = i_hash;
        l_entry.head.line = i_line;

        // Time stamp
        convertTime(&l_entry.stamp);

        // Calculate total space needed for the entry, which is a
        // combination of the data size from above, the entry
        // headers, and an overall length field.
        l_entry_size = l_data_size;
        l_entry_size += sizeof(trace_entry_stamp_t);
        l_entry_size += sizeof(trace_entry_head_t);

        // We always add the size of the entry at the end of the trace entry
        // so the parsing tool can easily walk the trace buffer stack so we
        // need to add that on to total size
        l_entry_size += sizeof(uint32_t);

        // Word align the entry
        l_entry_size = ALIGN_4(l_entry_size);

        // Allocate buffer for the arguments we're tracing
        void * l_buffer = malloc(l_data_size);
        memset(l_buffer, 0, l_data_size);
        char * l_ptr = static_cast<char *> (l_buffer);

        // Now copy the arguments to the buffer.


        for (size_t i = 0; i < num_args; i++)
        {
            uint32_t l_strLen = 0;

            if (l_str_map & (1 << i))
            {
                // Save string to buffer
                strcpy(l_ptr, va_arg(l_args, char *));

                //printk("Trace: Saved String %s Arg[%d]\n", l_ptr, static_cast<uint32_t>(i));

                // Length = string length + NULL termination
                l_strLen += (strlen(l_ptr) + 1);

                // Increment pointer to next word alignment
                l_ptr += l_strLen;
                l_ptr = reinterpret_cast<char *>(
                        ALIGN_4(reinterpret_cast<uint64_t>(l_ptr)) );

                //printk("Trace::trace_adal_write_all - l_buffer %p l_ptr %p l_strLen %d\n",
                //       l_buffer, l_ptr, l_strLen);
                //printk("Trace::trace_adal_write_all - num_args %d l_data_size %d l_entry_size %d\n",
                //       num_args, l_data_size, l_entry_size);
            }
            else if (l_char_map & (1 << i))
            {
                // Save char to buffer & increment pointer (no need to align)
                *(reinterpret_cast<uint32_t *>(l_ptr)) = va_arg(l_args, uint32_t);
                l_ptr += sizeof(uint32_t);

            }
            else if (l_double_map & (1 << i))
            {
                // Save number to buffer & increment pointer (no need to align)
                *(reinterpret_cast<double *>(l_ptr)) = va_arg(l_args, double);

                l_ptr += sizeof(double);
            }
            else
            {
                // Save number to buffer & increment pointer (no need to align)
                *(reinterpret_cast<uint64_t *>(l_ptr)) = va_arg(l_args, uint64_t);
                l_ptr += sizeof(uint64_t);
            }
        }

        va_end(l_args);

        // We now have total size and need to reserve a part of the trace
        // buffer for this

        // CRITICAL REGION START
        mutex_lock(&iv_trac_mutex);

        // Update the entry count
        io_td->te_count++;

        // First write the header
        writeData(io_td,
                  static_cast<void *>(&l_entry),
                  sizeof(l_entry));

        // Now write the actual data
        writeData(io_td,
                  l_buffer,
                  l_data_size);

        // Now write the size at the end
        // Note that fsp-trace assumes this to be a 32 bit long word
        writeData(io_td,
                  static_cast<void *>(&l_entry_size),
                  sizeof(l_entry_size));

        mutex_unlock(&iv_trac_mutex);
        // CRITICAL REGION END

        // Free allocated memory
        free(l_buffer);
    }

    return;
}

/******************************************************************************/
// trace_adal_write_bin
/******************************************************************************/
void Trace::trace_adal_write_bin(trace_desc_t *io_td,const trace_hash_val i_hash,
                                 const uint32_t i_line,
                                 const void *i_ptr,
                                 const uint32_t i_size,
                                 const int32_t type)
{
    getTheInstance()._trace_adal_write_bin(io_td, i_hash,
                                           i_line,
                                           i_ptr,
                                           i_size,
                                           type);
}

/******************************************************************************/
// trace_adal_write_bin
/******************************************************************************/
void Trace::_trace_adal_write_bin(trace_desc_t *io_td,const trace_hash_val i_hash,
                                 const uint32_t i_line,
                                 const void *i_ptr,
                                 const uint32_t i_size,
                                 const int32_t type)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t                l_entry_size = 0;
    trace_bin_entry_t       l_entry;

    /*----------------------------------------------------------------------    --*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/
    do
    {

        if((io_td == NULL) || (i_ptr == NULL) || (i_size == 0))
        {
            break;
        }

        // Calculate total space needed
        l_entry_size = sizeof(trace_entry_stamp_t);
        l_entry_size += sizeof(trace_entry_head_t);

        // We always add the size of the entry at the end of the trace entry
        // so the parsing tool can easily walk the trace buffer stack so we
        // need to add that on to total size
        l_entry_size += sizeof(uint32_t);

        // Now add on size for actual size of the binary data
        l_entry_size += i_size;

        // Word align the entry
        l_entry_size = ALIGN_4(l_entry_size);

        // Fill in the entry structure
        l_entry.stamp.tid = static_cast<uint32_t>(task_gettid());

        // Length is equal to size of data
        l_entry.head.length = i_size;
        l_entry.head.tag = TRACE_FIELDBIN;
        l_entry.head.hash = i_hash;
        l_entry.head.line = i_line;

        // We now have total size and need to reserve a part of the trace
        // buffer for this

        // Time stamp
        convertTime(&l_entry.stamp);

        // CRITICAL REGION START
        mutex_lock(&iv_trac_mutex);

        // Increment trace counter
        io_td->te_count++;

        // First write the header
        writeData(io_td,
                  static_cast<void *>(&l_entry),
                  sizeof(l_entry));

        // Now write the actual binary data
        writeData(io_td,
                  i_ptr,
                  i_size);

        // Now write the size at the end
        writeData(io_td,
                  static_cast<void *>(&l_entry_size),
                  sizeof(l_entry_size));

        // CRITICAL REGION END
        mutex_unlock(&iv_trac_mutex);

    }while(0);

    return;
}

/******************************************************************************/
// writeData
/******************************************************************************/
void Trace::writeData(trace_desc_t *io_td,
                    const void *i_ptr,
                    const uint32_t i_size)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t            l_total_size = i_size;
    void                *l_buf_ptr = NULL;
    uint32_t            l_offset = 0;
    uint64_t            l_align = 0;

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    do
    {

        if(i_size > (io_td->size-sizeof(trace_buf_head_t)))
        {
            // unreasonable size, caller is asking to write something
            // that is very nearly the size of the entire buffer
            break;
        }


        if((io_td->next_free + l_total_size) > io_td->size)
        {
            // Does not fit entirely, write what fits, and wrap the buffer.

            // Get the pointer to current location in buffer
            l_buf_ptr = reinterpret_cast<char *>(io_td) + io_td->next_free;
            // Figure out the alignment
            l_align = ALIGN_4(reinterpret_cast<uint64_t>(l_buf_ptr)) -
                      reinterpret_cast<uint64_t>(l_buf_ptr);
            // Add on the alignment
            l_buf_ptr = reinterpret_cast<void *>(reinterpret_cast<uint64_t>
                         (l_buf_ptr) + l_align);
            // Ensure offset accounts for the alignment
            l_offset = io_td->size-io_td->next_free - l_align;
            // Copy in what fits
            memcpy(l_buf_ptr,i_ptr,static_cast<size_t>(l_offset));

            l_total_size -= l_offset;

            // Now adjust the main header of buffer
            io_td->times_wrap++;
            io_td->next_free = io_td->hdr_len;
        }

        // Get the pointer to current location in buffer
        l_buf_ptr = reinterpret_cast<char *>(io_td) + io_td->next_free;
        // Figure out the alignment
        l_align = ALIGN_4(reinterpret_cast<uint64_t>(l_buf_ptr)) -
                  reinterpret_cast<uint64_t>(l_buf_ptr);
        // Add on the alignment
        l_buf_ptr = reinterpret_cast<void *>(reinterpret_cast<uint64_t>
                                 (l_buf_ptr) + l_align);

        memcpy(l_buf_ptr,reinterpret_cast<const char *>(i_ptr) + l_offset,
               l_total_size);

        // Make sure size is correct for word alignment
        // Note that this works with binary trace because only the binary data
        // has the potential to be un-word aligned.  If two parts of the binary
        // trace had this problem then this code would not work.
        // Note that fsp-trace will ignore garbage data in the unaligned areas.
        l_total_size = ALIGN_4(l_total_size);
        io_td->next_free += l_total_size;

    }while(0);

    return;

}

/******************************************************************************/
// convertTime
/******************************************************************************/
void Trace::convertTime(trace_entry_stamp_t *o_entry)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    // TODO - Future Sprint will collect proc frequency and correctly
    //        calculate this.
    uint64_t l_time = getTB();
    //o_entry->tbh = l_time && 0xFFFFFFFF00000000;
    //o_entry->tbl = l_time && 0x00000000FFFFFFFF;

    // This basically supports SIMICS, but will look weird on real hw
    o_entry->tbh = (l_time / 512000000);
    o_entry->tbl = ((l_time - (o_entry->tbh * 512000000)) / 512);

}



/******************************************************************************/
// findTdByName
/******************************************************************************/
trace_desc_t * Trace::findTdByName(const char *i_pName)
{
    trace_desc_t *      l_td = NULL;
    char                l_comp[COMP_NAME_SIZE];


    uint64_t i = strlen(i_pName);
    if ( i  )
    {
        if ( i  > (COMP_NAME_SIZE -1))
        {
            // Limit component name.
            memcpy(l_comp, i_pName, COMP_NAME_SIZE - 1);
            l_comp[ COMP_NAME_SIZE - 1 ] = 0;
        }
        else
        {
            strcpy( l_comp, i_pName );
        }

        // Use upper case.
        strupr( l_comp );

        // Search the buffers array
        for(i=0;
            (i < (TRAC_MAX_NUM_BUFFERS - 1)) &&
            (strlen(g_desc_array[i].comp) != 0);
            i++)
        {
            if(0 == strcmp(l_comp, g_desc_array[i].comp))
            {
                // Return this one.
                l_td = g_desc_array[i].td_entry;
                break;
            }
        }

    }

    return l_td;
}




/*****************************************************************************/
// getBuffer() called by ErrlEntry.collectTrace()
// Return how many bytes copied to output buffer.
// If given a null pointer or zero buffer then return the full size
// of the buffer.
//
// Otherwise return zero on error; perhaps the component name/trace buffer
// name is not found, or maybe the size of buffer given is too small to even
// hold a trace buffer header.

uint64_t Trace::getBuffer( const char * i_pComp,
                           void *       o_data,
                           uint64_t     i_size )
{
    const char *        l_pchEntry = NULL;         // use this to walk the entries
    const char *        l_pchEntryEOL = NULL;      // end of list of entries
    const char *        l_pchTraceBuffer = NULL;   // source buffer, including header
    const char *        l_pchTraceData = NULL;     // source data, just past header
    const char *        l_pchTraceEOB = NULL;      // end of source buffer<commit_msg>RTC4367. Update trace functions to ensure thread-safe implementation.<commit_after>//  IBM_PROLOG_BEGIN_TAG
//  This is an automatically generated prolog.
//
//  $Source: src/usr/trace/trace.C $
//
//  IBM CONFIDENTIAL
//
//  COPYRIGHT International Business Machines Corp. 2011
//
//  p1
//
//  Object Code Only (OCO) source materials
//  Licensed Internal Code Source Materials
//  IBM HostBoot Licensed Internal Code
//
//  The source code for this program is not published or other-
//  wise divested of its trade secrets, irrespective of what has
//  been deposited with the U.S. Copyright Office.
//
//  Origin: 30
//
//  IBM_PROLOG_END

/**
 *  @file trace.C
 *
 *  @brief Implementation of class Trace
 */


/* TODO
 *  - Add support in for debug trace enable/disable
 *  - FORMAT_PRINTF support
 *  - %s support
 *  - Multiple buffer support
 *
 */

/******************************************************************************/
// Includes
/******************************************************************************/
#include <trace/interface.H>
#include <stdarg.h>
#include <stdlib.h>
#include <arch/ppc.H>
#include <kernel/console.H>
#include <kernel/pagemgr.H>
#include <limits.h>
#include <stdlib.h>
#include <sys/task.h>
#include <sys/sync.h>
#include <string_ext.h>
#include <util/align.H>

#include <trace/trace.H>




/******************************************************************************/
// Namespace
/******************************************************************************/
namespace TRACE
{

/******************************************************************************/
// Globals/Constants
/******************************************************************************/

const uint32_t TRAC_TIME_REAL   = 0;  // upper 32 = seconds, lower 32 = microseconds
const uint32_t TRAC_TIME_50MHZ  = 1;
const uint32_t TRAC_TIME_200MHZ = 2;
const uint32_t TRAC_TIME_167MHZ = 3;  // 166666667Hz
const uint32_t COMP_NAME_SIZE   = 16; // NULL terminated string

// Initial implementation is to allocate a fixed 2KB buffer to each
// component on request.
// NOTE: any change to this value will require change to Trace::initBuffer()
const uint64_t TRAC_DEFAULT_BUFFER_SIZE = 0x0800;  //2KB

// NOTE: This constant should only be changed to an even number for now.
// Same reason as above.
const uint64_t TRAC_MAX_NUM_BUFFERS = 24;

const char * const TRAC_DEFAULT_BUFFER_NAME = "DEFAULT";

// Global component trace buffer array.  Initially allow for 24 buffers max.
// Keep global so it can be found in syms file
typedef struct trace_desc_array {
    char comp[COMP_NAME_SIZE];        // the buffer name
    trace_desc_t * td_entry;          // pointer to the buffer
}trace_desc_array_t;

trace_desc_array_t g_desc_array[TRAC_MAX_NUM_BUFFERS];

/******************************************************************************/
// TracInit::TracInit()
/******************************************************************************/
TracInit::TracInit(trace_desc_t **o_td, const char *i_comp,const size_t i_size)
{
    TRAC_INIT_BUFFER(o_td,i_comp,i_size);
}

/******************************************************************************/
// TracInit::~TracInit()
/******************************************************************************/
TracInit::~TracInit()
{
}


/******************************************************************************/
// Trace::getTheInstance
/******************************************************************************/
Trace& Trace::getTheInstance()
{
    return Singleton<Trace>::instance();
}

/******************************************************************************/
// Trace::Trace
/******************************************************************************/
Trace::Trace()
{
    mutex_init(&iv_trac_mutex);

    memset(g_desc_array, 0, sizeof(g_desc_array));
}

/******************************************************************************/
// Trace::~Trace
/******************************************************************************/
Trace::~Trace()
{

}


/******************************************************************************/
// trace_adal_init_buffer
/******************************************************************************/
void Trace::initBuffer(trace_desc_t **o_td, const char* i_comp,
                            const size_t i_size )
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t i = 0;
    char * l_td = NULL;
    char l_comp[COMP_NAME_SIZE] = {'\0'};

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/
    if(*o_td == NULL)
    {
        // Limit component name to 15 characters.
        // Too bad we don't have strncpy(), strncmp()
        if (strlen(i_comp) > (COMP_NAME_SIZE -1))
        {
            memcpy(l_comp, i_comp, COMP_NAME_SIZE - 1);
        }
        else
        {
            strcpy(l_comp, i_comp);
        }

        // make string upper case 
        strupr(l_comp);

        // CRITICAL REGION START
        mutex_lock(&iv_trac_mutex);

        // Search through the descriptor array for the first unallocated buffer.
        // The last buffer is the reserved default buffer for any component
        // which didn't get its own buffer.
        for (i = 0; i < (TRAC_MAX_NUM_BUFFERS - 1); i++)
        {
            if(!strcmp(l_comp, g_desc_array[i].comp))
            {
                //printk("Trace::initBuffer - buffer already allocated %d\n", i);

                // Buffer is already allocated.  Return the buffer.
                *o_td = g_desc_array[i].td_entry;

                break;
            }
            else if (strlen(g_desc_array[i].comp) == 0)
            {
                //printk("Trace::initBuffer - found unallocated buffer %d\n", i);

                // Found the first unallocated buffer; use this one.

                // Set the component name for the buffer
                strcpy(g_desc_array[i].comp, l_comp);

                // Allocate memory if needed
                if (NULL == g_desc_array[i].td_entry)
                {
                    //printk("Trace::initBuffer - allocate memory\n");

                    // Allocate memory
                    // TODO can't handle i_size yet - everything is coded
                    // around TRAC_DEFAULT_BUFFER_SIZE
                    l_td = static_cast<char *>(malloc(TRAC_DEFAULT_BUFFER_SIZE));

                    g_desc_array[i].td_entry =
                        reinterpret_cast<trace_desc_t *>(l_td);

                }

                // Initialize the buffer header
                initValuesBuffer(g_desc_array[i].td_entry,
                                 g_desc_array[i].comp);

                // Return the newly allocated buffer
                *o_td = g_desc_array[i].td_entry;

                break;
            }
        }

        if ((TRAC_MAX_NUM_BUFFERS - 1) == i)
        {
            //printk("Trace::initBuffer - allocate default buffer %d\n", i);

            // We're out of buffers to allocate.
            // Use the default buffer reserved for everyone else.
            // Initialize only once
            if (strlen(g_desc_array[i].comp) == 0)
            {
                // Set the component name for the buffer
                strcpy(g_desc_array[i].comp, TRAC_DEFAULT_BUFFER_NAME);

                // Allocate memory if needed
                // Memory should have already been reserved if
                // TRAC_MAX_NUM_BUFFERS is an even # and we're using
                // PageManager::allocatePage().  Add check just in
                // case TRAC_MAC_NUM_BUFFERS is set to an odd number.
                if (NULL == g_desc_array[i].td_entry)
                {
                    //printk("Trace::initBuffer - allocate memory\n");

                    // Allocate memory for buffer
                    l_td = static_cast<char *>(malloc(TRAC_DEFAULT_BUFFER_SIZE));

                    g_desc_array[i].td_entry =
                        reinterpret_cast<trace_desc_t *>(l_td);
                }

                // Initialize the buffer header
                initValuesBuffer(g_desc_array[i].td_entry,
                                 g_desc_array[i].comp);
            }

            // Return the default buffer
            *o_td = g_desc_array[i].td_entry;
        }

        mutex_unlock(&iv_trac_mutex);
        // CRITICAL REGION END

    }

    return;
}

/******************************************************************************/
// initValuesBuffer
/******************************************************************************/
void Trace::initValuesBuffer(trace_desc_t *o_buf,const char *i_comp)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    // Initialize it to all 0's
    memset(o_buf,0,(size_t)TRAC_DEFAULT_BUFFER_SIZE);

    o_buf->ver = TRACE_BUF_VERSION;
    o_buf->hdr_len = sizeof(trace_buf_head_t);
    o_buf->time_flg = TRAC_TIME_REAL;
    o_buf->endian_flg = 'B';  // Big Endian
    strcpy(o_buf->comp,i_comp);
    o_buf->size = TRAC_DEFAULT_BUFFER_SIZE;
    o_buf->times_wrap = 0;
    o_buf->next_free = sizeof(trace_buf_head_t);

    return;
}

/******************************************************************************/
// trace_adal_write_all
/******************************************************************************/
void Trace::trace_adal_write_all(trace_desc_t *io_td,
                                 const trace_hash_val i_hash,
                                 const char * i_fmt,
                                 const uint32_t i_line,
                                 const int32_t i_type, ...)
{
    va_list args;
    va_start(args, i_type);

    getTheInstance()._trace_adal_write_all(io_td,
                                           i_hash,
                                           i_fmt,
                                           i_line,
                                           i_type, args);

    va_end(args);
}

/******************************************************************************/
// trace_adal_write_all
/******************************************************************************/
void Trace::_trace_adal_write_all(trace_desc_t *io_td,
                                 const trace_hash_val i_hash,
                                 const char * i_fmt,
                                 const uint32_t i_line,
                                 const int32_t i_type, va_list i_args)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t                l_entry_size = 0;
    uint32_t                l_data_size= 0;
    //trace_entire_entry_t    l_entry;
    trace_bin_entry_t       l_entry;
    uint64_t                l_str_map = 0;
    uint64_t                l_char_map = 0;
    uint64_t                l_double_map = 0;


    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    uint32_t num_args = 0;
    uint32_t num_4byte_args = 0; //fsp-trace counts 8-byte args as 2 4-byte args
    const char* _fmt = i_fmt;

    // Save a copy of input args because calling
    // va_arg() on a va_list is a one-shot.
    va_list l_args;
    va_copy (l_args, i_args);


    // Sum the sizes of the items in i_args in order to know how big to
    // allocate the entry.

    for (size_t i = 0; i <= strlen(_fmt); i++)
    {
        if ('%' == _fmt[i])
        {
            i++;

            if ('%' == _fmt[i])
            {
                continue;
            }
            else if ('s' == _fmt[i])
            {
                // Set flag to indicate argument is a string
                l_str_map = l_str_map | (1 << num_args);

                // String counts as one 4-byte arg
                num_args++;
                num_4byte_args++;

                char * l_str = va_arg(i_args, char *);
                size_t l_strLen = strlen(l_str);

                // Add to total size of number of arguments we're tracing
                // and account for word alignment
                l_data_size += l_strLen + 1;
                l_data_size = ALIGN_4(l_data_size);

                //printk("Trace: STRING %s: strlen %d num_args %d l_data_size %d\n",
                //       l_str, static_cast<uint32_t>(l_strLen),
                //       num_args, l_data_size);
                //printk("Trace: l_str_map 0x%16llX\n",
                //     static_cast<long long>(l_str_map));
            }
            else if ('c' == _fmt[i])
            {
                // Set flag to indicate argument is a char
                l_char_map = l_char_map | (1 << num_args);

                // Increment arg counts
                num_args++;
                num_4byte_args++;

                // Retrieve the argument to increment to next one
                uint32_t l_tmpData = va_arg(i_args, uint32_t);

                // Add to total size; data is word aligned
                l_data_size += sizeof(l_tmpData);
            }
            else if (('e' == _fmt[i]) || ('f' == _fmt[i]) || ('g' == _fmt[i]))
            {
                // Set flag to indicate argument is a double
                l_double_map = l_double_map | (1 << num_args);

                // Numbers count as two 4-byte arg
                num_args++;
                num_4byte_args += 2;

                // Retrieve the argument to increment to next one
                double l_tmpData = va_arg(i_args,double);

                // Add to total size; data is word aligned
                l_data_size += sizeof(l_tmpData);
            }
            else
            {
                // Numbers count as two 4-byte arg
                num_args++;
                num_4byte_args += 2;

                // Retrieve the argument to increment to next one
                uint64_t l_tmpData = va_arg(i_args, uint64_t);

                // Add to total size; data is word aligned
                l_data_size += sizeof(l_tmpData);
            }
        }
    }

    va_end( i_args );


    if((num_4byte_args <= TRAC_MAX_ARGS) && (io_td != NULL))
    {
        // Fill in the entry structure
        l_entry.stamp.tid = static_cast<uint32_t>(task_gettid());

        // Length is equal to size of data
        l_entry.head.length = l_data_size;
        //l_entry.head.tag = TRACE_FIELDTRACE;
        l_entry.head.tag = TRACE_COMP_TRACE;
        l_entry.head.hash = i_hash;
        l_entry.head.line = i_line;

        // Time stamp
        convertTime(&l_entry.stamp);

        // Calculate total space needed for the entry, which is a
        // combination of the data size from above, the entry
        // headers, and an overall length field.
        l_entry_size = l_data_size;
        l_entry_size += sizeof(trace_entry_stamp_t);
        l_entry_size += sizeof(trace_entry_head_t);

        // We always add the size of the entry at the end of the trace entry
        // so the parsing tool can easily walk the trace buffer stack so we
        // need to add that on to total size
        l_entry_size += sizeof(uint32_t);

        // Word align the entry
        l_entry_size = ALIGN_4(l_entry_size);

        // Allocate buffer for the arguments we're tracing
        void * l_buffer = malloc(l_data_size);
        memset(l_buffer, 0, l_data_size);
        char * l_ptr = static_cast<char *> (l_buffer);

        // Now copy the arguments to the buffer.


        for (size_t i = 0; i < num_args; i++)
        {
            uint32_t l_strLen = 0;

            if (l_str_map & (1 << i))
            {
                // Save string to buffer
                strcpy(l_ptr, va_arg(l_args, char *));

                //printk("Trace: Saved String %s Arg[%d]\n", l_ptr, static_cast<uint32_t>(i));

                // Length = string length + NULL termination
                l_strLen += (strlen(l_ptr) + 1);

                // Increment pointer to next word alignment
                l_ptr += l_strLen;
                l_ptr = reinterpret_cast<char *>(
                        ALIGN_4(reinterpret_cast<uint64_t>(l_ptr)) );

                //printk("Trace::trace_adal_write_all - l_buffer %p l_ptr %p l_strLen %d\n",
                //       l_buffer, l_ptr, l_strLen);
                //printk("Trace::trace_adal_write_all - num_args %d l_data_size %d l_entry_size %d\n",
                //       num_args, l_data_size, l_entry_size);
            }
            else if (l_char_map & (1 << i))
            {
                // Save char to buffer & increment pointer (no need to align)
                *(reinterpret_cast<uint32_t *>(l_ptr)) = va_arg(l_args, uint32_t);
                l_ptr += sizeof(uint32_t);

            }
            else if (l_double_map & (1 << i))
            {
                // Save number to buffer & increment pointer (no need to align)
                *(reinterpret_cast<double *>(l_ptr)) = va_arg(l_args, double);

                l_ptr += sizeof(double);
            }
            else
            {
                // Save number to buffer & increment pointer (no need to align)
                *(reinterpret_cast<uint64_t *>(l_ptr)) = va_arg(l_args, uint64_t);
                l_ptr += sizeof(uint64_t);
            }
        }

        va_end(l_args);

        // We now have total size and need to reserve a part of the trace
        // buffer for this

        // CRITICAL REGION START
        mutex_lock(&iv_trac_mutex);

        // Update the entry count
        io_td->te_count++;

        // First write the header
        writeData(io_td,
                  static_cast<void *>(&l_entry),
                  sizeof(l_entry));

        // Now write the actual data
        writeData(io_td,
                  l_buffer,
                  l_data_size);

        // Now write the size at the end
        // Note that fsp-trace assumes this to be a 32 bit long word
        writeData(io_td,
                  static_cast<void *>(&l_entry_size),
                  sizeof(l_entry_size));

        mutex_unlock(&iv_trac_mutex);
        // CRITICAL REGION END

        // Free allocated memory
        free(l_buffer);
    }

    return;
}

/******************************************************************************/
// trace_adal_write_bin
/******************************************************************************/
void Trace::trace_adal_write_bin(trace_desc_t *io_td,const trace_hash_val i_hash,
                                 const uint32_t i_line,
                                 const void *i_ptr,
                                 const uint32_t i_size,
                                 const int32_t type)
{
    getTheInstance()._trace_adal_write_bin(io_td, i_hash,
                                           i_line,
                                           i_ptr,
                                           i_size,
                                           type);
}

/******************************************************************************/
// trace_adal_write_bin
/******************************************************************************/
void Trace::_trace_adal_write_bin(trace_desc_t *io_td,const trace_hash_val i_hash,
                                 const uint32_t i_line,
                                 const void *i_ptr,
                                 const uint32_t i_size,
                                 const int32_t type)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t                l_entry_size = 0;
    trace_bin_entry_t       l_entry;

    /*----------------------------------------------------------------------    --*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/
    do
    {

        if((io_td == NULL) || (i_ptr == NULL) || (i_size == 0))
        {
            break;
        }

        // Calculate total space needed
        l_entry_size = sizeof(trace_entry_stamp_t);
        l_entry_size += sizeof(trace_entry_head_t);

        // We always add the size of the entry at the end of the trace entry
        // so the parsing tool can easily walk the trace buffer stack so we
        // need to add that on to total size
        l_entry_size += sizeof(uint32_t);

        // Now add on size for actual size of the binary data
        l_entry_size += i_size;

        // Word align the entry
        l_entry_size = ALIGN_4(l_entry_size);

        // Fill in the entry structure
        l_entry.stamp.tid = static_cast<uint32_t>(task_gettid());

        // Length is equal to size of data
        l_entry.head.length = i_size;
        l_entry.head.tag = TRACE_FIELDBIN;
        l_entry.head.hash = i_hash;
        l_entry.head.line = i_line;

        // We now have total size and need to reserve a part of the trace
        // buffer for this

        // Time stamp
        convertTime(&l_entry.stamp);

        // CRITICAL REGION START
        mutex_lock(&iv_trac_mutex);

        // Increment trace counter
        io_td->te_count++;

        // First write the header
        writeData(io_td,
                  static_cast<void *>(&l_entry),
                  sizeof(l_entry));

        // Now write the actual binary data
        writeData(io_td,
                  i_ptr,
                  i_size);

        // Now write the size at the end
        writeData(io_td,
                  static_cast<void *>(&l_entry_size),
                  sizeof(l_entry_size));

        // CRITICAL REGION END
        mutex_unlock(&iv_trac_mutex);

    }while(0);

    return;
}

/******************************************************************************/
// writeData
/******************************************************************************/
void Trace::writeData(trace_desc_t *io_td,
                    const void *i_ptr,
                    const uint32_t i_size)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/
    uint32_t            l_total_size = i_size;
    void                *l_buf_ptr = NULL;
    uint32_t            l_offset = 0;
    uint64_t            l_align = 0;

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    do
    {

        if(i_size > (io_td->size-sizeof(trace_buf_head_t)))
        {
            // unreasonable size, caller is asking to write something
            // that is very nearly the size of the entire buffer
            break;
        }


        if((io_td->next_free + l_total_size) > io_td->size)
        {
            // Does not fit entirely, write what fits, and wrap the buffer.

            // Get the pointer to current location in buffer
            l_buf_ptr = reinterpret_cast<char *>(io_td) + io_td->next_free;
            // Figure out the alignment
            l_align = ALIGN_4(reinterpret_cast<uint64_t>(l_buf_ptr)) -
                      reinterpret_cast<uint64_t>(l_buf_ptr);
            // Add on the alignment
            l_buf_ptr = reinterpret_cast<void *>(reinterpret_cast<uint64_t>
                         (l_buf_ptr) + l_align);
            // Ensure offset accounts for the alignment
            l_offset = io_td->size-io_td->next_free - l_align;
            // Copy in what fits
            memcpy(l_buf_ptr,i_ptr,static_cast<size_t>(l_offset));

            l_total_size -= l_offset;

            // Now adjust the main header of buffer
            io_td->times_wrap++;
            io_td->next_free = io_td->hdr_len;
        }

        // Get the pointer to current location in buffer
        l_buf_ptr = reinterpret_cast<char *>(io_td) + io_td->next_free;
        // Figure out the alignment
        l_align = ALIGN_4(reinterpret_cast<uint64_t>(l_buf_ptr)) -
                  reinterpret_cast<uint64_t>(l_buf_ptr);
        // Add on the alignment
        l_buf_ptr = reinterpret_cast<void *>(reinterpret_cast<uint64_t>
                                 (l_buf_ptr) + l_align);

        memcpy(l_buf_ptr,reinterpret_cast<const char *>(i_ptr) + l_offset,
               l_total_size);

        // Make sure size is correct for word alignment
        // Note that this works with binary trace because only the binary data
        // has the potential to be un-word aligned.  If two parts of the binary
        // trace had this problem then this code would not work.
        // Note that fsp-trace will ignore garbage data in the unaligned areas.
        l_total_size = ALIGN_4(l_total_size);
        io_td->next_free += l_total_size;

    }while(0);

    return;

}

/******************************************************************************/
// convertTime
/******************************************************************************/
void Trace::convertTime(trace_entry_stamp_t *o_entry)
{
    /*------------------------------------------------------------------------*/
    /*  Local Variables                                                       */
    /*------------------------------------------------------------------------*/

    /*------------------------------------------------------------------------*/
    /*  Code                                                                  */
    /*------------------------------------------------------------------------*/

    // TODO - Future Sprint will collect proc frequency and correctly
    //        calculate this.
    uint64_t l_time = getTB();
    //o_entry->tbh = l_time && 0xFFFFFFFF00000000;
    //o_entry->tbl = l_time && 0x00000000FFFFFFFF;

    // This basically supports SIMICS, but will look weird on real hw
    o_entry->tbh = (l_time / 512000000);
    o_entry->tbl = ((l_time - (o_entry->tbh * 512000000)) / 512);

}



/******************************************************************************/
// findTdByName
/******************************************************************************/
trace_desc_t * Trace::findTdByName(const char *i_pName)
{
    trace_desc_t *      l_td = NULL;
    char                l_comp[COMP_NAME_SIZE];


    uint64_t i = strlen(i_pName);
    if ( i  )
    {
        if ( i  > (COMP_NAME_SIZE -1))
        {
            // Limit component name.
            memcpy(l_comp, i_pName, COMP_NAME_SIZE - 1);
            l_comp[ COMP_NAME_SIZE - 1 ] = 0;
        }
        else
        {
            strcpy( l_comp, i_pName );
        }

        // Use upper case.
        strupr( l_comp );

        // Lock critical section to access g_desc_array
        mutex_lock(&iv_trac_mutex);

        // Search the buffers array
        for(i=0;
            (i < (TRAC_MAX_NUM_BUFFERS - 1)) &&
            (strlen(g_desc_array[i].comp) != 0);
            i++)
        {
            if(0 == strcmp(l_comp, g_desc_array[i].comp))
            {
                // Return this one.
                l_td = g_desc_array[i].td_entry;
                break;
            }
        }

        // Unlock cretical section
        mutex_unlock(&iv_trac_mutex);

    }

    return l_td;
}




/*****************************************************************************/
// getBuffer() called by ErrlEntry.collectTrace()
// Return how many bytes copied to output buffer.
// If given a null pointer or zero buffer then return the full size
// of the buffer.
//
// Otherwise return zero on error; perhaps the component name/trace buffer
// name is not found, or maybe the size of buffer given is too small to even
// hold a trace buffer header.

uint64_t Trace::getBuffer( const char * i_pComp,
                           void *       o_data,
                           uint64_t     i_size )
{
    const char *        l_pchEntry = NULL;         // use this to walk the entries
    const char *        l_pchEntryEOL = NULL;      // end of list of entries
    const char *        l_pchTraceBuffer = NULL;   // source buffer, including header
    const char *        l_pchTraceData = NULL;     // source data, just past header
    const char *        l_pchTraceEOB = NULL;      // end of source buffer
{'fraction_non_alphanumeric': 0.23342477891185442, 'fraction_numerical': 0.009405830933596879, 'mean_word_length': 1.4565927166178316, 'pattern_counts': {'":': 0, '<': 127, '<?xml version=': 0, '>': 153, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 118, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5357334', 'n_tokens_mistral': 15930, 'n_tokens_neox': 14718, 'n_words': 3773}