<commit_before>// Copyright (C) 2013 DNAnexus, Inc.
//
// This file is part of dx-toolkit (DNAnexus platform client libraries).
//
//   Licensed under the Apache License, Version 2.0 (the "License"); you may
//   not use this file except in compliance with the License. You may obtain a
//   copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//   License for the specific language governing permissions and limitations
//   under the License.

#include "chunk.h"

#include <stdexcept>
#include <fstream>
#include <sstream>

#include <curl/curl.h>
#include <boost/thread.hpp>
#include <boost/lexical_cast.hpp>

#include <iostream>
#include <stdio.h>

#include "options.h"
#include "dxjson/dxjson.h"
#include "dxcpp/dxcpp.h"
#include "dxcpp/utils.h"

#include "SimpleHttpLib/ignore_sigpipe.h"

extern "C" {
#include "compress.h"
}

#include "dxcpp/dxlog.h"

using namespace std;

/* Initialize the extern variables, decalred in chunk.h */
queue<pair<time_t, int64_t> > instantaneousBytesAndTimestampQueue;
int64_t sumOfInstantaneousBytes = 0;
boost::mutex instantaneousBytesMutex;
// it takes roughly 30sec to reach queue size = 5000 on my computer.
// Unfortunately, standard C++ does not allow time resolution
// smaller than seconds, so we need to set it to some higher value
// (like ~30sec) to mitigate rounding effect.
const size_t MAX_QUEUE_SIZE = 5000;

// Replace contents of "dest" with gzip of empty string 
void get_empty_string_gzip(vector<char> &dest) {
  DXLOG(dx::logINFO) << "Computing gzip of zero length string...";
  int64_t destLen = gzCompressBound(0);
  dest.clear();
  dest.resize(destLen);
  const char *data = "";
  int compressStatus = gzCompress((Bytef *) (&(dest[0])), (uLongf *) &destLen,
                                  (const Bytef *) (data), 0ul, Z_DEFAULT_COMPRESSION);

  if (compressStatus == Z_MEM_ERROR) {
    throw runtime_error("compression failed: not enough memory");
  } else if (compressStatus == Z_BUF_ERROR) {
    throw runtime_error("compression failed: output buffer too small");
  } else if (compressStatus != Z_OK) {
    throw runtime_error("compression failed: " + boost::lexical_cast<string>(compressStatus));
  }
  if (destLen < (int64_t) dest.size()) {
    dest.resize(destLen);
  }
  DXLOG(dx::logINFO) << "Gzip of zero length string computed to be " << dest.size() << "bytes long";
}

void Chunk::read() {
  const int64_t len = end - start;
  data.clear();
  data.resize(len);
  if (len == 0) {
    // For empty file case (empty chunk)
    return;
  }
#ifdef WINDOWS_BUILD
  // For windows we use fseeko64() & fread(): since we
  // compile a 32bit UA version, and standard library functions
  // do not allow to read > 2GB locations in file
  FILE *fp = fopen(localFile.c_str(), "rb");
  if (!fp) {
    ostringstream msg;
    msg << "file('" << localFile.c_str() << "') cannot be opened for reading (errno=" << errno
        << ")... readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
  if(fseeko64(fp, off64_t(start), SEEK_SET) != 0) {
    ostringstream msg;
    msg << "unable to seek to location '" << off64_t(start) << "' in the file '" << localFile.c_str()
        << "' (errno=" << errno << ")... readdata failed on chunk " << (*this);
    fclose(fp);
    throw runtime_error(msg.str());
  }
  fread(&(data[0]), 1, len, fp);
  int errflg = ferror(fp); // get error status before we close the file handler
  fclose(fp);
  if (errflg) {
    ostringstream msg;
    msg << "unable to read '" << len << "' bytes from location '" << off64_t(start) << "' in the file '"
        << localFile.c_str() << "' (errno=" << errno << ")... readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
#else
  ifstream in(localFile.c_str(), ifstream::in | ifstream::binary);
  if (!in) {
    ostringstream msg;
    msg << "file('" << localFile.c_str() << "') cannot be opened for reading..." <<
           "readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
  in.seekg(start);
  if (!in.good()) {
    ostringstream msg;
    msg << "unable to seek to location '" << start << "' in the file '" << localFile.c_str()
        << "' (fail bit = " << in.fail() << ", bad bit = " << in.bad() << ", eofbit = "
        << in.eof() <<")... readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
  in.read(&(data[0]), len);
  if (!in.good()) {
  } else {<commit_msg>Fixing a regression in UA code<commit_after>// Copyright (C) 2013 DNAnexus, Inc.
//
// This file is part of dx-toolkit (DNAnexus platform client libraries).
//
//   Licensed under the Apache License, Version 2.0 (the "License"); you may
//   not use this file except in compliance with the License. You may obtain a
//   copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//   License for the specific language governing permissions and limitations
//   under the License.

#include "chunk.h"

#include <stdexcept>
#include <fstream>
#include <sstream>

#include <curl/curl.h>
#include <boost/thread.hpp>
#include <boost/lexical_cast.hpp>

#include <iostream>
#include <stdio.h>

#include "options.h"
#include "dxjson/dxjson.h"
#include "dxcpp/dxcpp.h"
#include "dxcpp/utils.h"

#include "SimpleHttpLib/ignore_sigpipe.h"

extern "C" {
#include "compress.h"
}

#include "dxcpp/dxlog.h"

using namespace std;

/* Initialize the extern variables, decalred in chunk.h */
queue<pair<time_t, int64_t> > instantaneousBytesAndTimestampQueue;
int64_t sumOfInstantaneousBytes = 0;
boost::mutex instantaneousBytesMutex;
// it takes roughly 30sec to reach queue size = 5000 on my computer.
// Unfortunately, standard C++ does not allow time resolution
// smaller than seconds, so we need to set it to some higher value
// (like ~30sec) to mitigate rounding effect.
const size_t MAX_QUEUE_SIZE = 5000;

// Replace contents of "dest" with gzip of empty string 
void get_empty_string_gzip(vector<char> &dest) {
  DXLOG(dx::logINFO) << "Computing gzip of zero length string...";
  int64_t destLen = gzCompressBound(0);
  dest.clear();
  dest.resize(destLen);
  const char *data = "";
  int compressStatus = gzCompress((Bytef *) (&(dest[0])), (uLongf *) &destLen,
                                  (const Bytef *) (data), 0ul, Z_DEFAULT_COMPRESSION);

  if (compressStatus == Z_MEM_ERROR) {
    throw runtime_error("compression failed: not enough memory");
  } else if (compressStatus == Z_BUF_ERROR) {
    throw runtime_error("compression failed: output buffer too small");
  } else if (compressStatus != Z_OK) {
    throw runtime_error("compression failed: " + boost::lexical_cast<string>(compressStatus));
  }
  if (destLen < (int64_t) dest.size()) {
    dest.resize(destLen);
  }
  DXLOG(dx::logINFO) << "Gzip of zero length string computed to be " << dest.size() << "bytes long";
}

void Chunk::read() {
  const int64_t len = end - start;
  data.clear();
  data.resize(len);
  if (len == 0) {
    // For empty file case (empty chunk)
    return;
  }
#ifdef WINDOWS_BUILD
  // For windows we use fseeko64() & fread(): since we
  // compile a 32bit UA version, and standard library functions
  // do not allow to read > 2GB locations in file
  FILE *fp = fopen(localFile.c_str(), "rb");
  if (!fp) {
    ostringstream msg;
    msg << "file('" << localFile.c_str() << "') cannot be opened for reading (errno=" << errno
        << ")... readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
  if(fseeko64(fp, off64_t(start), SEEK_SET) != 0) {
    ostringstream msg;
    msg << "unable to seek to location '" << off64_t(start) << "' in the file '" << localFile.c_str()
        << "' (errno=" << errno << ")... readdata failed on chunk " << (*this);
    fclose(fp);
    throw runtime_error(msg.str());
  }
  fread(&(data[0]), 1, len, fp);
  int errflg = ferror(fp); // get error status before we close the file handler
  fclose(fp);
  if (errflg) {
    ostringstream msg;
    msg << "unable to read '" << len << "' bytes from location '" << off64_t(start) << "' in the file '"
        << localFile.c_str() << "' (errno=" << errno << ")... readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
#else
  ifstream in(localFile.c_str(), ifstream::in | ifstream::binary);
  if (!in) {
    ostringstream msg;
    msg << "file('" << localFile.c_str() << "') cannot be opened for reading..." <<
           "readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
  in.seekg(start);
  if (!in.good()) {
    ostringstream msg;
    msg << "unable to seek to location '" << start << "' in the file '" << localFile.c_str()
        << "' (fail bit = " << in.fail() << ", bad bit = " << in.bad() << ", eofbit = "
        << in.eof() <<")... readdata failed on chunk " << (*this);
    throw runtime_error(msg.str());
  }
  in.read(&(data[0]), len);
  if (!in.good()) {
{'fraction_non_alphanumeric': 0.15483594252627064, 'fraction_numerical': 0.011151619129315891, 'mean_word_length': 3.418285172903837, 'pattern_counts': {'":': 0, '<': 209, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '62117', 'n_tokens_mistral': 3120, 'n_tokens_neox': 3014, 'n_words': 956}