<commit_before>#include <julia.h>
#include "Values.h"
#include "lvalue.h"

#include <iostream>

using namespace std;

template <typename V,typename E> static shared_ptr<nj::Value> reboxArray(jl_value_t *jlarray)
{
   shared_ptr<nj::Value> value;
   V *p = (V*)jl_array_data(jlarray);
   int ndims = jl_array_ndims(jlarray);
   vector<int> dims;


   for(int dim = 0;dim < ndims;dim++) dims.push_back(jl_array_dim(jlarray,dim));

   nj::Array<V,E> *array = new nj::Array<V,E>(dims);
   
printf("In rebox array: dim = %lu\n",dims.size());

   value.reset(array);
   memcpy(array->ptr(),p,array->size()*sizeof(V));
   return value;
}

static shared_ptr<nj::Value> getArrayValue(jl_value_t *jlarray)
{
   shared_ptr<nj::Value> value;
   jl_value_t *elementType = jl_tparam0(jl_typeof(jlarray));

printf("In getArray Value\n");
  
   if(jl_is_int64(elementType)) value = reboxArray<int64_t,nj::Int64_t>(jlarray);
   else if(jl_is_int32(elementType)) value = reboxArray<int,nj::Int32_t>(jlarray);
   else if(jl_is_float64(elementType)) value = reboxArray<double,nj::Float64_t>(jlarray); 
   else if(jl_is_float32(elementType)) value = reboxArray<float,nj::Float32_t>(jlarray);
   else if(jl_is_uint64(elementType)) value = reboxArray<uint64_t,nj::UInt64_t>(jlarray); 
   else if(jl_is_uint32(elementType)) value = reboxArray<unsigned int,nj::UInt32_t>(jlarray);
   else if(jl_is_int8(elementType)) value = reboxArray<char,nj::Char_t>(jlarray);
   else if(jl_is_int16(elementType)) value = reboxArray<short,nj::Int16_t>(jlarray);
   else if(jl_is_uint8(elementType)) value = reboxArray<unsigned char,nj::UChar_t>(jlarray);
   else if(jl_is_uint16(elementType)) value = reboxArray<unsigned short,nj::UInt16_t>(jlarray);

   return value;
}

vector<shared_ptr<nj::Value>> nj::lvalue(jl_value_t *jlvalue)
{
   vector<shared_ptr<nj::Value>> res;

   if(jl_is_null(jlvalue))
   {
      shared_ptr<nj::Value>  value(new nj::Null);
      res.push_back(value);
   }
   else if(jl_is_array(jlvalue))
   {
      res.push_back(getArrayValue(jlvalue));
   }
   else
   {   
      shared_ptr<nj::Value> value;

      if(jl_is_int64(jlvalue)) value.reset(new nj::Int64(jl_unbox_int64(jlvalue)));
      else if(jl_is_int32(jlvalue)) value.reset(new nj::Int32(jl_unbox_int32(jlvalue)));
      else if(jl_is_float64(jlvalue)) value.reset(new nj::Float64(jl_unbox_float64(jlvalue)));<commit_msg>More debugging Changed check for array element type<commit_after>#include <julia.h>
#include "Values.h"
#include "lvalue.h"

#include <iostream>

using namespace std;

template <typename V,typename E> static shared_ptr<nj::Value> reboxArray(jl_value_t *jlarray)
{
   shared_ptr<nj::Value> value;
   V *p = (V*)jl_array_data(jlarray);
   int ndims = jl_array_ndims(jlarray);
   vector<int> dims;


   for(int dim = 0;dim < ndims;dim++) dims.push_back(jl_array_dim(jlarray,dim));

   nj::Array<V,E> *array = new nj::Array<V,E>(dims);
   
printf("In rebox array: dim = %lu\n",dims.size());

   value.reset(array);
   memcpy(array->ptr(),p,array->size()*sizeof(V));
   return value;
}

static shared_ptr<nj::Value> getArrayValue(jl_value_t *jlarray)
{
   shared_ptr<nj::Value> value;
   jl_value_t *elementType = jl_tparam0(jl_typeof(jlarray));

printf("In getArray Value\n");
  
   if(jl_is_int64(elementType)) value = reboxArray<int64_t,nj::Int64_t>(jlarray);
   else if(jl_is_int32(elementType)) value = reboxArray<int,nj::Int32_t>(jlarray);
   else if(elementType == (jl_value_t*)jl_float64_type) value = reboxArray<double,nj::Float64_t>(jlarray); 
   else if(jl_is_float32(elementType)) value = reboxArray<float,nj::Float32_t>(jlarray);
   else if(jl_is_uint64(elementType)) value = reboxArray<uint64_t,nj::UInt64_t>(jlarray); 
   else if(jl_is_uint32(elementType)) value = reboxArray<unsigned int,nj::UInt32_t>(jlarray);
   else if(jl_is_int8(elementType)) value = reboxArray<char,nj::Char_t>(jlarray);
   else if(jl_is_int16(elementType)) value = reboxArray<short,nj::Int16_t>(jlarray);
   else if(jl_is_uint8(elementType)) value = reboxArray<unsigned char,nj::UChar_t>(jlarray);
   else if(jl_is_uint16(elementType)) value = reboxArray<unsigned short,nj::UInt16_t>(jlarray);

   return value;
}

vector<shared_ptr<nj::Value>> nj::lvalue(jl_value_t *jlvalue)
{
   vector<shared_ptr<nj::Value>> res;

   if(jl_is_null(jlvalue))
   {
      shared_ptr<nj::Value>  value(new nj::Null);
      res.push_back(value);
   }
   else if(jl_is_array(jlvalue))
   {
      res.push_back(getArrayValue(jlvalue));
   }
   else
   {   
      shared_ptr<nj::Value> value;

      if(jl_is_int64(jlvalue)) value.reset(new nj::Int64(jl_unbox_int64(jlvalue)));
      else if(jl_is_int32(jlvalue)) value.reset(new nj::Int32(jl_unbox_int32(jlvalue)));
      else if(jl_is_float64(jlvalue)) value.reset(new nj::Float64(jl_unbox_float64(jlvalue)));
{'fraction_non_alphanumeric': 0.1553701772679875, 'fraction_numerical': 0.02419186652763295, 'mean_word_length': 6.115727002967359, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1483465', 'n_tokens_mistral': 2072, 'n_tokens_neox': 1877, 'n_words': 267}