<commit_before>/*
Copyright (c) by respective owners including Yahoo!, Microsoft, and
individual contributors. All rights reserved.  Released under a BSD (revised)
license as described in the file LICENSE.
*/

#include <msclr\marshal_cppstd.h>
#include "clr_io.h"

using namespace System;
using namespace System::Runtime::InteropServices;

namespace VW
{
clr_io_buf::clr_io_buf(Stream^ stream) : m_stream(stream)
{ if (stream == nullptr)
    throw gcnew ArgumentNullException("stream");

  files.push_back(0);
}

int clr_io_buf::open_file(const char* name, bool stdin_off, int flag)
{ return 0;
}

void clr_io_buf::reset_file(int f)
{ m_stream->Seek(0, SeekOrigin::Begin);
  endloaded = space.begin;
  space.end = space.begin;
}

ssize_t clr_io_buf::read_file(int f, void* buf, size_t nbytes)
{ auto buffer = gcnew array<unsigned char>((int)nbytes);
  int readBytes = m_stream->Read(buffer, 0, (int)nbytes);

  Marshal::Copy(buffer, 0, IntPtr(buf), (int)nbytes);

  return readBytes;
}

size_t clr_io_buf::num_files()
{ return 1;
}

ssize_t clr_io_buf::write_file(int file, const void* buf, size_t nbytes)
{ auto buffer = gcnew array<unsigned char>((int)nbytes);
  Marshal::Copy(IntPtr((void*)buf), buffer, 0, (int)nbytes);

  m_stream->Write(buffer, 0, (int)nbytes);
<commit_msg>removed endloaded from .net interface<commit_after>/*
Copyright (c) by respective owners including Yahoo!, Microsoft, and
individual contributors. All rights reserved.  Released under a BSD (revised)
license as described in the file LICENSE.
*/

#include <msclr\marshal_cppstd.h>
#include "clr_io.h"

using namespace System;
using namespace System::Runtime::InteropServices;

namespace VW
{
clr_io_buf::clr_io_buf(Stream^ stream) : m_stream(stream)
{ if (stream == nullptr)
    throw gcnew ArgumentNullException("stream");

  files.push_back(0);
}

int clr_io_buf::open_file(const char* name, bool stdin_off, int flag)
{ return 0;
}

void clr_io_buf::reset_file(int f)
{ m_stream->Seek(0, SeekOrigin::Begin);
  head = space.begin;
  space.end = space.begin;
}

ssize_t clr_io_buf::read_file(int f, void* buf, size_t nbytes)
{ auto buffer = gcnew array<unsigned char>((int)nbytes);
  int readBytes = m_stream->Read(buffer, 0, (int)nbytes);

  Marshal::Copy(buffer, 0, IntPtr(buf), (int)nbytes);

  return readBytes;
}

size_t clr_io_buf::num_files()
{ return 1;
}

ssize_t clr_io_buf::write_file(int file, const void* buf, size_t nbytes)
{ auto buffer = gcnew array<unsigned char>((int)nbytes);
  Marshal::Copy(IntPtr((void*)buf), buffer, 0, (int)nbytes);

  m_stream->Write(buffer, 0, (int)nbytes);

{'fraction_non_alphanumeric': 0.13532376890267545, 'fraction_numerical': 0.006203955021326095, 'mean_word_length': 5.935483870967742, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '65309', 'n_tokens_mistral': 968, 'n_tokens_neox': 938, 'n_words': 234}