<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "stdafx.h"

#include <algorithm>
#include <iterator>
#include <set>
#include <sstream>

#include <zorba/config.h>
#include <zorba/internal/cxx_util.h>
#include <zorba/diagnostic_list.h>
#include <zorba/store_consts.h>

#include "runtime/csv/csv.h"
#include "store/api/item_factory.h"
#include "system/globalenv.h"
#include "types/casting.h"
#include "types/root_typemanager.h"
#include "types/typeops.h"
#include "util/ascii_util.h"
#include "util/json_parser.h"
#include "util/stl_util.h"
#include "zorbatypes/decimal.h"
#include "zorbatypes/float.h"
#include "zorbatypes/integer.h"
#include "zorbautils/store_util.h"

#include "csv_util.h"

using namespace std;

namespace zorba {

///////////////////////////////////////////////////////////////////////////////

#define IS_ATOMIC_TYPE(ITEM,TYPE) \
  ( (ITEM)->isAtomic() && TypeOps::is_subtype( (ITEM)->getTypeCode(), store::TYPE ) )

#define IS_JSON_NULL(ITEM) \
  ( (ITEM)->isAtomic() && (ITEM)->getTypeCode() == store::JS_NULL )

static bool is_stringable( store::Item_t const &item ) {
  switch ( item->getKind() ) {
    case store::Item::ATOMIC:
    case store::Item::NODE:
      return true;
    default:
      return false;
  }
}

inline zstring get_string_value( store::Item_t const &item ) {
  return is_stringable( item ) ? item->getStringValue() : "";
}

static bool get_array_opt( store::Item_t const &object,
                           char const *opt_name, store::Item_t *result,
                           QueryLoc const &loc ) {
  if ( get_json_option( object, opt_name, result ) ) {
    if ( (*result)->getKind() != store::Item::ARRAY )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( *result ),
          opt_name,
          ZED( INVALID_OPTION_MustBeArray )
        ),
        ERROR_LOC( loc )
      );
    return true;
  }
  return false;
}

static bool get_bool_opt( store::Item_t const &object,
                          char const *opt_name, bool *result,
                          QueryLoc const &loc ) {
  store::Item_t opt_item;
  if ( get_json_option( object, opt_name, &opt_item ) ) {
    if ( !IS_ATOMIC_TYPE( opt_item, XS_BOOLEAN ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( opt_item ),
          opt_name,
          ZED( INVALID_OPTION_MustBeBoolean )
        ),
        ERROR_LOC( loc )
      );
    *result = opt_item->getBooleanValue();
    return true;
  }
  return false;
}

static bool get_char_opt( store::Item_t const &object,
                          char const *opt_name, char *result,
                          QueryLoc const &loc ) {
  store::Item_t opt_item;
  if ( get_json_option( object, opt_name, &opt_item ) ) {
    if ( !IS_ATOMIC_TYPE( opt_item, XS_STRING ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( opt_item ),
          opt_name,
          ZED( INVALID_OPTION_MustBeASCIIChar )
        ),
        ERROR_LOC( loc )
      );
    zstring const value( opt_item->getStringValue() );
    if ( value.size() != 1 || !ascii::is_ascii( value[0] ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS( value, opt_name, ZED( INVALID_OPTION_MustBeASCIIChar ) ),
        ERROR_LOC( loc )
      );
    *result = value[0];
    return true;
  }
  return false;
}

static bool get_string_opt( store::Item_t const &object,
                            char const *opt_name, zstring *result,
                            QueryLoc const &loc ) {
  store::Item_t opt_item;
  if ( get_json_option( object, opt_name, &opt_item ) ) {
    if ( !IS_ATOMIC_TYPE( opt_item, XS_STRING ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( opt_item ),
          opt_name,
          ZED( INVALID_OPTION_MustBeString )
        ),
        ERROR_LOC( loc )
      );
    opt_item->getStringValue2( *result );
    return true;
  }
  return false;
}

static json::type parse_json( zstring const &s, json::token *ptoken ) {
  mem_streambuf buf( (char*)s.data(), s.size() );
  istringstream iss;
  iss.ios::rdbuf( &buf );
  json::lexer lex( iss );
  if ( !lex.next( ptoken, false ) )
    return json::none;
  json::token::type const tt = ptoken->get_type();
  if ( tt == json::token::number ) {
    //
    // The JSON lexer will stop lex'ing as soon as it finds a valid token.  For
    // strings that start with numbers, e.g., "870 Market St", the lexer will
    // return "870" as a number token but we need to know the type of the whole
    // string 's'; hence we need to do an extra check for numer tokens.
    //
    // If the length(t) < length(s), remove trailing whitespace from 's' and
    // check again.  If length(t) < length(s2), it means that that the token is
    // really a string; if length(t) == length(s2), then the length difference
    // was caused only by whitespace that can be ignored and the token is still
    // a number.
    //
    json::token::value_type const &value = ptoken->get_value();
    if ( value.size() < s.size() ) {<commit_msg>Fixed typo in comment.<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include "stdafx.h"

#include <algorithm>
#include <iterator>
#include <set>
#include <sstream>

#include <zorba/config.h>
#include <zorba/internal/cxx_util.h>
#include <zorba/diagnostic_list.h>
#include <zorba/store_consts.h>

#include "runtime/csv/csv.h"
#include "store/api/item_factory.h"
#include "system/globalenv.h"
#include "types/casting.h"
#include "types/root_typemanager.h"
#include "types/typeops.h"
#include "util/ascii_util.h"
#include "util/json_parser.h"
#include "util/stl_util.h"
#include "zorbatypes/decimal.h"
#include "zorbatypes/float.h"
#include "zorbatypes/integer.h"
#include "zorbautils/store_util.h"

#include "csv_util.h"

using namespace std;

namespace zorba {

///////////////////////////////////////////////////////////////////////////////

#define IS_ATOMIC_TYPE(ITEM,TYPE) \
  ( (ITEM)->isAtomic() && TypeOps::is_subtype( (ITEM)->getTypeCode(), store::TYPE ) )

#define IS_JSON_NULL(ITEM) \
  ( (ITEM)->isAtomic() && (ITEM)->getTypeCode() == store::JS_NULL )

static bool is_stringable( store::Item_t const &item ) {
  switch ( item->getKind() ) {
    case store::Item::ATOMIC:
    case store::Item::NODE:
      return true;
    default:
      return false;
  }
}

inline zstring get_string_value( store::Item_t const &item ) {
  return is_stringable( item ) ? item->getStringValue() : "";
}

static bool get_array_opt( store::Item_t const &object,
                           char const *opt_name, store::Item_t *result,
                           QueryLoc const &loc ) {
  if ( get_json_option( object, opt_name, result ) ) {
    if ( (*result)->getKind() != store::Item::ARRAY )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( *result ),
          opt_name,
          ZED( INVALID_OPTION_MustBeArray )
        ),
        ERROR_LOC( loc )
      );
    return true;
  }
  return false;
}

static bool get_bool_opt( store::Item_t const &object,
                          char const *opt_name, bool *result,
                          QueryLoc const &loc ) {
  store::Item_t opt_item;
  if ( get_json_option( object, opt_name, &opt_item ) ) {
    if ( !IS_ATOMIC_TYPE( opt_item, XS_BOOLEAN ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( opt_item ),
          opt_name,
          ZED( INVALID_OPTION_MustBeBoolean )
        ),
        ERROR_LOC( loc )
      );
    *result = opt_item->getBooleanValue();
    return true;
  }
  return false;
}

static bool get_char_opt( store::Item_t const &object,
                          char const *opt_name, char *result,
                          QueryLoc const &loc ) {
  store::Item_t opt_item;
  if ( get_json_option( object, opt_name, &opt_item ) ) {
    if ( !IS_ATOMIC_TYPE( opt_item, XS_STRING ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( opt_item ),
          opt_name,
          ZED( INVALID_OPTION_MustBeASCIIChar )
        ),
        ERROR_LOC( loc )
      );
    zstring const value( opt_item->getStringValue() );
    if ( value.size() != 1 || !ascii::is_ascii( value[0] ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS( value, opt_name, ZED( INVALID_OPTION_MustBeASCIIChar ) ),
        ERROR_LOC( loc )
      );
    *result = value[0];
    return true;
  }
  return false;
}

static bool get_string_opt( store::Item_t const &object,
                            char const *opt_name, zstring *result,
                            QueryLoc const &loc ) {
  store::Item_t opt_item;
  if ( get_json_option( object, opt_name, &opt_item ) ) {
    if ( !IS_ATOMIC_TYPE( opt_item, XS_STRING ) )
      throw XQUERY_EXCEPTION(
        csv::INVALID_OPTION,
        ERROR_PARAMS(
          get_string_value( opt_item ),
          opt_name,
          ZED( INVALID_OPTION_MustBeString )
        ),
        ERROR_LOC( loc )
      );
    opt_item->getStringValue2( *result );
    return true;
  }
  return false;
}

static json::type parse_json( zstring const &s, json::token *ptoken ) {
  mem_streambuf buf( (char*)s.data(), s.size() );
  istringstream iss;
  iss.ios::rdbuf( &buf );
  json::lexer lex( iss );
  if ( !lex.next( ptoken, false ) )
    return json::none;
  json::token::type const tt = ptoken->get_type();
  if ( tt == json::token::number ) {
    //
    // The JSON lexer will stop lex'ing as soon as it finds a valid token.  For
    // strings that start with numbers, e.g., "870 Market St", the lexer will
    // return "870" as a number token but we need to know the type of the whole
    // string 's'; hence we need to do an extra check for number tokens.
    //
    // If the length(t) < length(s), remove trailing whitespace from 's' and
    // check again.  If length(t) < length(s2), it means that that the token is
    // really a string; if length(t) == length(s2), then the length difference
    // was caused only by whitespace that can be ignored and the token is still
    // a number.
    //
    json::token::value_type const &value = ptoken->get_value();
    if ( value.size() < s.size() ) {
{'fraction_non_alphanumeric': 0.12957819822947406, 'fraction_numerical': 0.004165943412601979, 'mean_word_length': 2.5863678804855277, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6787971', 'n_tokens_mistral': 3767, 'n_tokens_neox': 3732, 'n_words': 999}