<commit_before>/*
 * fMBT, free Model Based Testing tool
 * Copyright (c) 2011, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU Lesser General Public License,
 * version 2.1, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */
#include <string>
#include <vector>
#include <cstdio>
#include <cstring>
#include <cstdlib>
#include <algorithm>

#include <boost/regex.hpp>
#include <glib.h>
#include <glib-object.h>
#include <glib/gprintf.h>

#include <string>
#include <vector>
#include <sstream>
#include <fstream>

#ifndef __MINGW32__
#include <dlfcn.h>
#else
#include <windows.h>
extern "C" {

// For Dparse
char *strndup(const char *s, size_t n)
{
  return g_strndup(s,n);
}
}
#endif
#include <fcntl.h>
#include <unistd.h>

#include "helper.hh"
#include "proxy.hh"
Proxy callback_proxy;

#include "log.hh"
#include "writable.hh"

bool human_readable=true;

void escape_free(const char* msg)
{
#ifdef DROI
  delete [] msg;
#else
  g_free((void*)msg);
#endif
}
void *load_lib(const std::string& libname,const std::string& model_filename)
{
#ifndef __MINGW32__
  std::string name_candidate(libname);
  std::string errormessages;

  if (model_filename!="") {
    return dlopen(model_filename.c_str(),RTLD_NOW);
  }

  // clear error..
  dlerror();
  void* handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="./"+name_candidate;
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate=libname+".so";
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="./"+name_candidate;
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="lib"+libname+".so";
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="./"+name_candidate;
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    fprintf(stderr, "%s", errormessages.c_str());
  }
  // clear error..
  dlerror();
  return handle;
#else
  return NULL;
#endif
}

int find(const std::vector<std::string> &v,const std::string s,int def)
{
  for(unsigned i=0;i<v.size();i++) {
    if (v[i]==s) {
      return i;
    }
  }
  return def;
}

int find(const std::vector<std::string*> &v,const std::string s,int def)
{
  for(unsigned i=0;i<v.size();i++) {
    if (*v[i]==s) {
      return i;
    }
  }
  return def;
}

bool isOutputName(const std::string& name)
{
  return (name.c_str()[0]=='o');
}

bool isInputName(const std::string& name)
{
  return (name.c_str()[0]=='i') ||
    (name.c_str()[0]=='d');
}

void clear_whitespace(std::string& s){
  std::string white (" \t\f\v\n\r"); /* more whilespace? */
  size_t pos;

  pos=s.find_last_not_of(white);
  if (pos==std::string::npos) {
    s.clear();
  } else {
    s.erase(pos+1);
    pos=s.find_first_not_of(white);
    if (pos!=std::string::npos) {
      s=s.substr(pos);
    }
  }
}

void clear_coding(std::string& s){
  std::string coding ("\"");
  size_t pos;

  pos=s.find_last_not_of(coding);
  if (pos==std::string::npos) {
    s.clear();
  } else {
    s.erase(pos+1);
    pos=s.find_first_not_of(coding);
    if (pos!=std::string::npos) {
      s=s.substr(pos);
    }
  }
}

std::string removehash(const std::string& s);

std::string filetype(const std::string& _s)
{
  std::string s=removehash(_s);
  size_t found=s.find_last_of(".");
  std::string r=s.substr(found+1);
  found=r.find_last_of("#");
  if (found==r.npos) {
    return r;
  }
  return r.substr(0,found);
}

char* unescape_string(char* msg)
{
  int l=std::strlen(msg);
  int j=0;

  for(int i=0;i<l;i++) {
    if (msg[i]=='%' && i+2<l) {
      char* endp;
      char s[] = {
        msg[i+1],msg[i+2],0 };
      char c=strtol(s,&endp,16);
      if (endp!=s) {
        msg[j]=c;
        i+=2;
      } else {
        msg[j]=msg[i];
      }
    } else {
      msg[j]=msg[i];
    }
    j++;
  }
  msg[j]=0;

  return msg;
}

void unescape_string(std::string& msg)
{
  char* tmp=strdup(msg.c_str());
  unescape_string(tmp);

  msg=tmp;
  free(tmp);
}


char* escape_string(const char* msg)
{
#ifdef DROI
  int len=std::strlen(msg);
  char* endp=new char[3*len+1];
  char* ret=endp;
  endp[0]=0;

  for(int i=0;i<len;i++) {
    char c=msg[i];
    if (isalnum(c)) {
      *endp=c;
      endp++;
    } else {
      endp+=std::sprintf(endp,"%%%X",c);
    }
  }
  *endp=0;

  return ret;
#else
  return g_uri_escape_string(msg,NULL,TRUE);
#endif
}

void escape_string(std::string& msg)
{
  char* s=escape_string(msg.c_str());
  msg=std::string(s);
  escape_free(s);
}

std::string removehash(const std::string& s)
{
  size_t cutpos = s.find_last_of("#");
  if (cutpos == s.npos) {
    return s;
  } else {
    return s.substr(0,cutpos);
  }
  return "";
}

#ifndef DROI
char* readfile(const char* filename,const char* preprocess)
{
  int status;
  char* ret=readfile(filename,preprocess,status);
  if (status) {
    g_free(ret);
    ret=NULL;
  }
  return ret;
}

char* readfile(const char* filename,const char* preprocess,int& status)
{
  char* out=NULL;
  char* err=NULL;

  if (preprocess==NULL) {
    return readfile(filename);
  }

  std::string s(preprocess);

  s=s + " " + filename;

  if (!g_spawn_command_line_sync(s.c_str(),&out,&err,&status,NULL)) {

    if (!status) {
      status=-1;
    }

    if (err) {
      g_free(out);
      out=err;
      err=NULL;
    }

  }
  if (err) {
    free(err);
  }
  return out;
 }
#endif

char* _readfile(const char* filename)
{
  std::string fn(filename);
  size_t cutpos = fn.find_last_of("#");

  if (cutpos == fn.npos) {
    /* read file contents always without preprocessing when glib not
     * available */
    std::ifstream f;
    size_t file_len;
    f.open(filename, std::fstream::in | std::fstream::ate);
    if (!f.is_open())
      return NULL;
    file_len = f.tellg();
    f.seekg(0, std::ios::beg);
    char *contents = (char*)malloc(file_len+1);
    if (!contents) {
      f.close();
      return NULL;
    }
    f.read(contents, file_len);
    f.close();
    contents[file_len] = '\0';

    /* drop all lines that start with # */
    char *cleaned_up_contents = (char*)malloc(file_len+1);
    if (!cleaned_up_contents) {
        free(contents);
        return NULL;
    }
    memset(cleaned_up_contents,0,file_len+1);

    size_t clean_pos = 0; // position in "cleaned_up_contents"
    size_t cont_pos = 0; // position in "contents"
    char prev_char = '\n';
    while (cont_pos < file_len)
    {
        if (contents[cont_pos] == '#' && prev_char == '\n') {
            // skip comment line
            while (cont_pos < file_len && contents[cont_pos] != '\n')
                cont_pos++;
            if (cont_pos < file_len) cont_pos++; // skip new line char
	    cleaned_up_contents[clean_pos] = '\n';
	    clean_pos++;
        } else {
            prev_char = cleaned_up_contents[clean_pos] = contents[cont_pos];
            cont_pos++;
            clean_pos++;
        }
    }
    cleaned_up_contents[clean_pos] = '\0';
    free(contents);
    return cleaned_up_contents;
  } else {
    return unescape_string(strdup(fn.substr(cutpos+1).c_str()));
  }
}

char* readfile(const char* filename) {
  char* ret=_readfile(filename);
  if (!ret) {
    const char* prefix=g_getenv("AAL_INCLUDE_PREFIX");
    if (prefix) {
      gchar* fname = g_build_filename(prefix,filename,NULL);
      ret=_readfile(fname);
      g_free(fname);
    }
  }
  return ret;
}

std::string capsulate(std::string s) {

  if (s=="") {
    return "\"";
  }

  std::ostringstream t(std::ios::out | std::ios::binary);

  if (human_readable) {
    t << ":" << s.length() << ":" << s << "\"";
  } else {
    char* sesc=escape_string(s.c_str());
    t << "#" << sesc << "\"";
    escape_free(sesc);
  }

  return t.str();
}

bool string2vector(Log& log,const char* s,std::vector<int>& a,
		   int min,int max,Writable* w)
{
  int v;
  int i=0;
  char* endp;

  const char* ss=s;
  v=strtol(ss,&endp,10);
  a.resize(0);
  while (endp!=ss) {
    if (v<min||v>max) {
      if (w) {
	w->status=false;
	w->errormsg="out of range";
      }
      return false;
    }
    a.push_back(v);
    ss=endp;
    v=strtol(ss,&endp,10);
    i++;
  }

  if (ss[0]!='\0' && ss[0]!='\n') {
    if (w) {
      w->status=false;
      w->errormsg=std::string("Illegal character \"")+ss[0]+"\"";
    }

    char *escaped_line = escape_string(ss);
    if (escaped_line) {
      static const char* m[] = { "<remote error=\"I/O error: integer expected, got: %s\"/>\n",
				 "Remote expected integer, got \"%s\"\n"};
      log.error(m, escaped_line);
      escape_free(escaped_line);
    }
    return false;
  }

  return true;
}

#ifndef DROI
std::string replace(boost::regex& expression,
                    const char* format_string,
                    std::string::iterator first,
                    std::string::iterator last)
{
  std::ostringstream t(std::ios::out | std::ios::binary);
  std::ostream_iterator<char> oi(t);
  std::string s;
  boost::regex_replace(oi,first,last,expression,format_string,
                       boost::match_default | boost::format_all
                       | boost::format_first_only);

  s=t.str();
  return s;
}

#endif

void print_vectors(int* v,unsigned size,std::vector<std::string>& s,const char* prefix,int add)
{
  for(unsigned i=0;i<size;i++) {
    if (s[v[i]]!="")
      printf("%s%i:%s\n",prefix,i+add,s[v[i]].c_str());
  }
}

void print_vector(std::vector<std::string>& s,const char* prefix,int add)
{
  for(unsigned i=0;i<s.size();i++) {
    if (s[i]!="") {
      printf("%s%i:%s\n",prefix,i+add,s[i].c_str());
    }
  }
}

std::string to_string(const float f)
{
  std::stringstream ss;
  ss << f;
  return ss.str();
}

#include <iomanip>

std::string to_string(const struct timeval&t,bool minutes)
{
  std::stringstream ss;
  int sec=t.tv_sec%60;
  int min=t.tv_sec/60;
  if (min>0 && minutes) {
    ss << min << "min " << sec << "." << std::setfill('0') << std::setw(6) << t.tv_usec << "s";
  } else {
    ss << t.tv_sec << "." << std::setfill('0') << std::setw(6) << t.tv_usec;
  }
  return ss.str();
}

std::string to_string(const int t)
{
  std::stringstream ss;
  ss << t;
  return ss.str();
}

std::string to_string(const int cnt,const int* t,
		      const std::vector<std::string>& st)
{
  std::string ret;

  if (cnt==0) {
    return ret;
  }

  ret=st[t[0]];

  for(int i=1;i<cnt;i++) {
    ret+=" "+st[t[i]];
  }

  return ret;
}

void strvec(std::vector<std::string>& v,std::string& s,
            const std::string& separator)
{
  size_t cutpos;

  while ((cutpos=s.find_first_of(separator))!=s.npos) {
    std::string a=s.substr(0,cutpos);
    v.push_back(a);
    s=s.substr(cutpos+1);
  }

  v.push_back(s);
}

void nonblock(int fd)
{
#ifndef __MINGW32__
  int flags = fcntl(fd, F_GETFL, 0);
  fcntl(fd, F_SETFL, flags | O_NONBLOCK);
#endif
}

void block(int fd)
{
#ifndef __MINGW32__
  int flags = fcntl(fd, F_GETFL, 0);
  fcntl(fd, F_SETFL, flags & (~O_NONBLOCK));
#endif
}

/*
 * nonblock_getline reads lines from the stream. Unlike with normal
 * getline, underlying fd can be non-blocking. The function returns
 * the number of bytes copied to lineptr, or -1 on error. If full line
 * does not fit into internal read buffer (line is longer than
 * MAX_LINE_LENGTH), the contents of the buffer is returned as a
 * line.
 */

ssize_t nonblock_getline(char **lineptr, size_t *n,
			 GIOChannel *stream, char* &read_buf,
			 size_t &read_buf_pos)
{

  gsize si;
#ifdef __MINGW32__
  // Our dear friend windows doesn't support nonblocking read.
  // Let's emulate it with poll
  GPollFD pfd;

  g_io_channel_win32_make_pollfd (stream,
				  (GIOCondition)(G_IO_IN | G_IO_ERR | G_IO_HUP),
				  &pfd);

  gint poll_ret = g_io_channel_win32_poll (&pfd, 1, 1);

  if (!(poll_ret>0 && (pfd.revents&G_IO_IN))) {
    return 0;
  }
#else
  GError *er=NULL;
  g_io_channel_set_flags(stream,(GIOFlags)(G_IO_FLAG_NONBLOCK|(int)g_io_channel_get_flags(stream)),&er);
  if (er) {
    return 0;
  }
#endif
  g_io_channel_set_line_term(stream,NULL,-1);

  if (*lineptr) {
    g_free(*lineptr);
    *lineptr=NULL;
    *n=0;
  }

  GIOStatus st=
    g_io_channel_read_line(stream,lineptr,&si,NULL,NULL);

  if (st==G_IO_STATUS_AGAIN) {
    return 0;
  }

  if (st==G_IO_STATUS_ERROR) {
    return -1;
  }

  if (n)
    *n=si;

  return si;
}

ssize_t getline(char **lineptr, size_t *n, GIOChannel *stream)
{
  gsize si;
  gsize ret;
  GIOStatus status;

  if (*lineptr) {
    g_free(*lineptr);
    *lineptr=NULL;<commit_msg>Fix bug in filename encoding, when using preprocessor<commit_after>/*
 * fMBT, free Model Based Testing tool
 * Copyright (c) 2011, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU Lesser General Public License,
 * version 2.1, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
 *
 */
#include <string>
#include <vector>
#include <cstdio>
#include <cstring>
#include <cstdlib>
#include <algorithm>

#include <boost/regex.hpp>
#include <glib.h>
#include <glib-object.h>
#include <glib/gprintf.h>

#include <string>
#include <vector>
#include <sstream>
#include <fstream>

#ifndef __MINGW32__
#include <dlfcn.h>
#else
#include <windows.h>
extern "C" {

// For Dparse
char *strndup(const char *s, size_t n)
{
  return g_strndup(s,n);
}
}
#endif
#include <fcntl.h>
#include <unistd.h>

#include "helper.hh"
#include "proxy.hh"
Proxy callback_proxy;

#include "log.hh"
#include "writable.hh"

bool human_readable=true;

void escape_free(const char* msg)
{
#ifdef DROI
  delete [] msg;
#else
  g_free((void*)msg);
#endif
}
void *load_lib(const std::string& libname,const std::string& model_filename)
{
#ifndef __MINGW32__
  std::string name_candidate(libname);
  std::string errormessages;

  if (model_filename!="") {
    return dlopen(model_filename.c_str(),RTLD_NOW);
  }

  // clear error..
  dlerror();
  void* handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="./"+name_candidate;
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate=libname+".so";
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="./"+name_candidate;
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="lib"+libname+".so";
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    errormessages += dlerror() + std::string("\n");
    name_candidate="./"+name_candidate;
    handle=dlopen(name_candidate.c_str(),RTLD_NOW);
  }
  if (!handle) {
    fprintf(stderr, "%s", errormessages.c_str());
  }
  // clear error..
  dlerror();
  return handle;
#else
  return NULL;
#endif
}

int find(const std::vector<std::string> &v,const std::string s,int def)
{
  for(unsigned i=0;i<v.size();i++) {
    if (v[i]==s) {
      return i;
    }
  }
  return def;
}

int find(const std::vector<std::string*> &v,const std::string s,int def)
{
  for(unsigned i=0;i<v.size();i++) {
    if (*v[i]==s) {
      return i;
    }
  }
  return def;
}

bool isOutputName(const std::string& name)
{
  return (name.c_str()[0]=='o');
}

bool isInputName(const std::string& name)
{
  return (name.c_str()[0]=='i') ||
    (name.c_str()[0]=='d');
}

void clear_whitespace(std::string& s){
  std::string white (" \t\f\v\n\r"); /* more whilespace? */
  size_t pos;

  pos=s.find_last_not_of(white);
  if (pos==std::string::npos) {
    s.clear();
  } else {
    s.erase(pos+1);
    pos=s.find_first_not_of(white);
    if (pos!=std::string::npos) {
      s=s.substr(pos);
    }
  }
}

void clear_coding(std::string& s){
  std::string coding ("\"");
  size_t pos;

  pos=s.find_last_not_of(coding);
  if (pos==std::string::npos) {
    s.clear();
  } else {
    s.erase(pos+1);
    pos=s.find_first_not_of(coding);
    if (pos!=std::string::npos) {
      s=s.substr(pos);
    }
  }
}

std::string removehash(const std::string& s);

std::string filetype(const std::string& _s)
{
  std::string s=removehash(_s);
  size_t found=s.find_last_of(".");
  std::string r=s.substr(found+1);
  found=r.find_last_of("#");
  if (found==r.npos) {
    return r;
  }
  return r.substr(0,found);
}

char* unescape_string(char* msg)
{
  int l=std::strlen(msg);
  int j=0;

  for(int i=0;i<l;i++) {
    if (msg[i]=='%' && i+2<l) {
      char* endp;
      char s[] = {
        msg[i+1],msg[i+2],0 };
      char c=strtol(s,&endp,16);
      if (endp!=s) {
        msg[j]=c;
        i+=2;
      } else {
        msg[j]=msg[i];
      }
    } else {
      msg[j]=msg[i];
    }
    j++;
  }
  msg[j]=0;

  return msg;
}

void unescape_string(std::string& msg)
{
  char* tmp=strdup(msg.c_str());
  unescape_string(tmp);

  msg=tmp;
  free(tmp);
}


char* escape_string(const char* msg)
{
#ifdef DROI
  int len=std::strlen(msg);
  char* endp=new char[3*len+1];
  char* ret=endp;
  endp[0]=0;

  for(int i=0;i<len;i++) {
    char c=msg[i];
    if (isalnum(c)) {
      *endp=c;
      endp++;
    } else {
      endp+=std::sprintf(endp,"%%%X",c);
    }
  }
  *endp=0;

  return ret;
#else
  return g_uri_escape_string(msg,NULL,TRUE);
#endif
}

void escape_string(std::string& msg)
{
  char* s=escape_string(msg.c_str());
  msg=std::string(s);
  escape_free(s);
}

std::string removehash(const std::string& s)
{
  size_t cutpos = s.find_last_of("#");
  if (cutpos == s.npos) {
    return s;
  } else {
    return s.substr(0,cutpos);
  }
  return "";
}

#ifndef DROI
char* readfile(const char* filename,const char* preprocess)
{
  int status;
  char* ret=readfile(filename,preprocess,status);
  if (status) {
    g_free(ret);
    ret=NULL;
  }
  return ret;
}

char* readfile(const char* filename,const char* preprocess,int& status)
{
  char* out=NULL;
  char* err=NULL;

  if (preprocess==NULL) {
    return readfile(filename);
  }

  std::string s(preprocess);

  gchar* fn = g_shell_quote (filename);

  s=s + " " + fn;

  g_free(fn);

  if (!g_spawn_command_line_sync(s.c_str(),&out,&err,&status,NULL)) {

    if (!status) {
      status=-1;
    }

    if (err) {
      g_free(out);
      out=err;
      err=NULL;
    }

  }
  if (err) {
    free(err);
  }
  return out;
 }
#endif

char* _readfile(const char* filename)
{
  std::string fn(filename);
  size_t cutpos = fn.find_last_of("#");

  if (cutpos == fn.npos) {
    /* read file contents always without preprocessing when glib not
     * available */
    std::ifstream f;
    size_t file_len;
    f.open(filename, std::fstream::in | std::fstream::ate);
    if (!f.is_open())
      return NULL;
    file_len = f.tellg();
    f.seekg(0, std::ios::beg);
    char *contents = (char*)malloc(file_len+1);
    if (!contents) {
      f.close();
      return NULL;
    }
    f.read(contents, file_len);
    file_len=f.gcount();
    f.close();
    contents[file_len] = '\0';

    /* drop all lines that start with # */
    char *cleaned_up_contents = (char*)malloc(file_len+1);
    if (!cleaned_up_contents) {
        free(contents);
        return NULL;
    }
    memset(cleaned_up_contents,0,file_len+1);

    size_t clean_pos = 0; // position in "cleaned_up_contents"
    size_t cont_pos = 0; // position in "contents"
    char prev_char = '\n';
    while (cont_pos < file_len)
    {
        if (contents[cont_pos] == '#' && prev_char == '\n') {
            // skip comment line
            while (cont_pos < file_len && contents[cont_pos] != '\n')
                cont_pos++;
            if (cont_pos < file_len) cont_pos++; // skip new line char
	    cleaned_up_contents[clean_pos] = '\n';
	    clean_pos++;
        } else {
            prev_char = cleaned_up_contents[clean_pos] = contents[cont_pos];
            cont_pos++;
            clean_pos++;
        }
    }
    cleaned_up_contents[clean_pos] = '\0';
    free(contents);
    return cleaned_up_contents;
  } else {
    return unescape_string(strdup(fn.substr(cutpos+1).c_str()));
  }
}

char* readfile(const char* filename) {
  char* ret=_readfile(filename);
  if (!ret) {
    const char* prefix=g_getenv("AAL_INCLUDE_PREFIX");
    if (prefix) {
      gchar* fname = g_build_filename(prefix,filename,NULL);
      ret=_readfile(fname);
      g_free(fname);
    }
  }
  return ret;
}

std::string capsulate(std::string s) {

  if (s=="") {
    return "\"";
  }

  std::ostringstream t(std::ios::out | std::ios::binary);

  if (human_readable) {
    t << ":" << s.length() << ":" << s << "\"";
  } else {
    char* sesc=escape_string(s.c_str());
    t << "#" << sesc << "\"";
    escape_free(sesc);
  }

  return t.str();
}

bool string2vector(Log& log,const char* s,std::vector<int>& a,
		   int min,int max,Writable* w)
{
  int v;
  int i=0;
  char* endp;

  const char* ss=s;
  v=strtol(ss,&endp,10);
  a.resize(0);
  while (endp!=ss) {
    if (v<min||v>max) {
      if (w) {
	w->status=false;
	w->errormsg="out of range";
      }
      return false;
    }
    a.push_back(v);
    ss=endp;
    v=strtol(ss,&endp,10);
    i++;
  }

  if (ss[0]!='\0' && ss[0]!='\n' && ss[0]!='\r' ) {
    if (w) {
      w->status=false;
      w->errormsg=std::string("Illegal character \"")+ss[0]+"\"";
    }

    char *escaped_line = escape_string(ss);
    if (escaped_line) {
      static const char* m[] = { "<remote error=\"I/O error: integer expected, got: %s\"/>\n",
				 "Remote expected integer, got \"%s\"\n"};
      log.error(m, escaped_line);
      escape_free(escaped_line);
    }
    return false;
  }

  return true;
}

#ifndef DROI
std::string replace(boost::regex& expression,
                    const char* format_string,
                    std::string::iterator first,
                    std::string::iterator last)
{
  std::ostringstream t(std::ios::out | std::ios::binary);
  std::ostream_iterator<char> oi(t);
  std::string s;
  boost::regex_replace(oi,first,last,expression,format_string,
                       boost::match_default | boost::format_all
                       | boost::format_first_only);

  s=t.str();
  return s;
}

#endif

void print_vectors(int* v,unsigned size,std::vector<std::string>& s,const char* prefix,int add)
{
  for(unsigned i=0;i<size;i++) {
    if (s[v[i]]!="")
      printf("%s%i:%s\n",prefix,i+add,s[v[i]].c_str());
  }
}

void print_vector(std::vector<std::string>& s,const char* prefix,int add)
{
  for(unsigned i=0;i<s.size();i++) {
    if (s[i]!="") {
      printf("%s%i:%s\n",prefix,i+add,s[i].c_str());
    }
  }
}

std::string to_string(const float f)
{
  std::stringstream ss;
  ss << f;
  return ss.str();
}

#include <iomanip>

std::string to_string(const struct timeval&t,bool minutes)
{
  std::stringstream ss;
  int sec=t.tv_sec%60;
  int min=t.tv_sec/60;
  if (min>0 && minutes) {
    ss << min << "min " << sec << "." << std::setfill('0') << std::setw(6) << t.tv_usec << "s";
  } else {
    ss << t.tv_sec << "." << std::setfill('0') << std::setw(6) << t.tv_usec;
  }
  return ss.str();
}

std::string to_string(const int t)
{
  std::stringstream ss;
  ss << t;
  return ss.str();
}

std::string to_string(const int cnt,const int* t,
		      const std::vector<std::string>& st)
{
  std::string ret;

  if (cnt==0) {
    return ret;
  }

  ret=st[t[0]];

  for(int i=1;i<cnt;i++) {
    ret+=" "+st[t[i]];
  }

  return ret;
}

void strvec(std::vector<std::string>& v,std::string& s,
            const std::string& separator)
{
  size_t cutpos;

  while ((cutpos=s.find_first_of(separator))!=s.npos) {
    std::string a=s.substr(0,cutpos);
    v.push_back(a);
    s=s.substr(cutpos+1);
  }

  v.push_back(s);
}

void nonblock(int fd)
{
#ifndef __MINGW32__
  int flags = fcntl(fd, F_GETFL, 0);
  fcntl(fd, F_SETFL, flags | O_NONBLOCK);
#endif
}

void block(int fd)
{
#ifndef __MINGW32__
  int flags = fcntl(fd, F_GETFL, 0);
  fcntl(fd, F_SETFL, flags & (~O_NONBLOCK));
#endif
}

/*
 * nonblock_getline reads lines from the stream. Unlike with normal
 * getline, underlying fd can be non-blocking. The function returns
 * the number of bytes copied to lineptr, or -1 on error. If full line
 * does not fit into internal read buffer (line is longer than
 * MAX_LINE_LENGTH), the contents of the buffer is returned as a
 * line.
 */

ssize_t nonblock_getline(char **lineptr, size_t *n,
			 GIOChannel *stream, char* &read_buf,
			 size_t &read_buf_pos)
{

  gsize si;
#ifdef __MINGW32__
  // Our dear friend windows doesn't support nonblocking read.
  // Let's emulate it with poll
  GPollFD pfd;

  g_io_channel_win32_make_pollfd (stream,
				  (GIOCondition)(G_IO_IN | G_IO_ERR | G_IO_HUP),
				  &pfd);

  gint poll_ret = g_io_channel_win32_poll (&pfd, 1, 1);

  if (!(poll_ret>0 && (pfd.revents&G_IO_IN))) {
    return 0;
  }
#else
  GError *er=NULL;
  g_io_channel_set_flags(stream,(GIOFlags)(G_IO_FLAG_NONBLOCK|(int)g_io_channel_get_flags(stream)),&er);
  if (er) {
    return 0;
  }
#endif
  g_io_channel_set_line_term(stream,NULL,-1);

  if (*lineptr) {
    g_free(*lineptr);
    *lineptr=NULL;
    *n=0;
  }

  GIOStatus st=
    g_io_channel_read_line(stream,lineptr,&si,NULL,NULL);

  if (st==G_IO_STATUS_AGAIN) {
    return 0;
  }

  if (st==G_IO_STATUS_ERROR) {
    return -1;
  }

  if (n)
    *n=si;

  return si;
}

ssize_t getline(char **lineptr, size_t *n, GIOChannel *stream)
{
  gsize si=0;
  gsize ret=0;
  GIOStatus status;

  if (*lineptr) {
    g_free(*lineptr);
    *lineptr=NULL;
{'fraction_non_alphanumeric': 0.17420805998125585, 'fraction_numerical': 0.008134957825679474, 'mean_word_length': 3.2887459807073953, 'pattern_counts': {'":': 4, '<': 175, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '451916', 'n_tokens_mistral': 10673, 'n_tokens_neox': 10395, 'n_words': 2147}