<commit_before>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <string>
#include <fstream>
#include <vector>
#include <sstream>

#include "zorbautils/fatal.h"
#include "zorbaerrors/error_manager.h"
#include "zorbatypes/zorbatypesError.h"
#include "zorbatypes/URI.h"
#include "zorbaerrors/error_messages.h"

#include "runtime/sequences/SequencesImpl.h"
#include "runtime/booleans/BooleanImpl.h"
#include "runtime/numerics/NumericsImpl.h"
#include "runtime/api/runtimecb.h"
#include "runtime/core/arithmetic_impl.h"
#include "runtime/util/iterator_impl.h"

#include "system/globalenv.h"

#include "types/casting.h"
#include "types/typeops.h"
#include "store/api/store.h"
#include "store/api/iterator.h"
#include "store/api/item_factory.h"
#include "store/api/pul.h"

#include "context/static_context.h"
#include "context/collation_cache.h"

#include "util/web/web.h"
#include "store/util/hashset_node_handle.h"

#include "runtime/booleans/compare_types.h"
#include "runtime/util/handle_hashset_item_value.h"

using namespace std;
namespace zorba {
  

/*______________________________________________________________________
|  
| 15.1 General Functions and Operators on Sequences
|_______________________________________________________________________*/

static XQPCollator*
getCollator(
    RuntimeCB* aRuntimeCB,
    const QueryLoc& loc,
    PlanState& planState,
    const PlanIterator* iter)
{
  store::Item_t lCollationItem;
  store::Item_t temp;

  if (!PlanIterator::consumeNext(lCollationItem, iter, planState))
      ZORBA_ERROR_LOC_DESC(XPTY0004, loc, "An empty-sequence is not allowed as collation parameter");

  if (PlanIterator::consumeNext(temp, iter, planState))
      ZORBA_ERROR_LOC_DESC(XPTY0004, loc, "A sequence of more then one item is not allowed as collation parameter");
    
  xqtref_t lCollationItemType = planState.theCompilerCB->m_sctx->get_typemanager()->
                                create_value_type (lCollationItem);

  // TODO resolve uri (base-uri)

  return planState.theRuntimeCB->theCollationCache->getCollator(lCollationItem->getStringValue());
}


//15.1.2 op:concatenate 
//---------------------
void
FnConcatIteratorState::init(PlanState& planState) 
{
  PlanIteratorState::init(planState);
  theCurIter = 0;
}

void
FnConcatIteratorState::reset(PlanState& planState) 
{
  PlanIteratorState::reset(planState);
  theCurIter = 0;
}


bool
FnConcatIterator::nextImpl(store::Item_t& result, PlanState& planState) const 
{
  std::auto_ptr<store::PUL> pul;

  FnConcatIteratorState* state;
  DEFAULT_STACK_INIT(FnConcatIteratorState, state, planState);

  if (theIsUpdating)
    pul.reset(GENV_ITEMFACTORY->createPendingUpdateList());

  for (; state->theCurIter < theChildren.size(); ++state->theCurIter) 
  {
    while(consumeNext(result, theChildren[state->theCurIter].getp(), planState))
    {
      if (theIsUpdating)
      {
        ZORBA_FATAL(result->isPul(), "");

        pul->mergeUpdates(result);
      }
      else
      {
        STACK_PUSH (true, state);
      }
    }
  }
  
  if (theIsUpdating) {
    result = pul.release();
    STACK_PUSH(result != NULL, state);
  }

  STACK_END (state);
}

//15.1.3 fn:index-of
bool 
FnIndexOfIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
  store::Item_t lSequenceItem;
  store::Item_t lCollationItem;
  xqtref_t      lCollationItemType;

  FnIndexOfIteratorState* state;
  DEFAULT_STACK_INIT(FnIndexOfIteratorState, state, planState);
  
  if (!consumeNext(state->theSearchItem, theChildren[1].getp(), planState))
  {
    ZORBA_ERROR_LOC_DESC( FORG0006, loc, 
         "An empty sequence is not allowed as search item of fn:index-of");    
  }

  if ( theChildren.size() == 3 )
    state->theCollator = getCollator(planState.theRuntimeCB, loc, planState, theChildren[2].getp());

  while ( consumeNext(lSequenceItem, theChildren[0].getp(), planState))
  {
    // inc the position in the sequence; do it at the beginning of the loop because index-of starts with one
    ++state->theCurrentPos;
    
    if (CompareIterator::valueEqual(planState.theRuntimeCB,
                                    lSequenceItem, state->theSearchItem, state->theCollator) == 0)
      STACK_PUSH(GENV_ITEMFACTORY->createInteger(
        result,
        Integer::parseInt(state->theCurrentPos)), 
        state
      );
  }

  STACK_END (state);
}

void
FnIndexOfIteratorState::init(PlanState& planState) 
{
 PlanIteratorState::init(planState);
 theCurrentPos = 0;
 theSearchItem = NULL;
 theCollator = 0;
}

void
FnIndexOfIteratorState::reset(PlanState& planState) {
 PlanIteratorState::reset(planState);
 theCurrentPos = 0;
 theSearchItem = NULL;
 theCollator = 0;
}


//15.1.4 fn:empty
/*
 * If the value of $arg is the empty sequence, the function returns true; 
 * otherwise, the function returns false.
 */
bool 
FnEmptyIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
  store::Item_t lSequenceItem;

  PlanIteratorState* state;
  DEFAULT_STACK_INIT(PlanIteratorState, state, planState);

  if ( !consumeNext(lSequenceItem, theChildren[0].getp(), planState))
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, true ), state);
  }
  else
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, false ), state);   
  }

  STACK_END (state);
}

//15.1.5 fn:exists
/*
 * If the value of $arg is not the empty sequence, the function returns true; 
 * otherwise, the function returns false.
 */
bool 
FnExistsIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
  store::Item_t lSequenceItem;
  
  PlanIteratorState* state;
  DEFAULT_STACK_INIT(PlanIteratorState, state, planState);
  
  if ( consumeNext(lSequenceItem, theChildren[0].getp(), planState) ) 
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, true ), state);
  }
  else
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, false ), state);   
  }

  STACK_END (state);
}


//15.1.6 fn:distinct-values
/**
 * Returns the sequence that results from removing from arg all but one of a set of
 * values that are eq to one other. 
 * The order in which the sequence of values is returned is ·implementation dependent·.
 * Here, we return the first item that is not a duplicate and throw away the remaining ones
 */
FnDistinctValuesIterator::FnDistinctValuesIterator(const QueryLoc& loc,
                                                   vector<PlanIter_t>& args)
 : NaryBaseIterator<FnDistinctValuesIterator, FnDistinctValuesIteratorState> ( loc, args )
{ }

FnDistinctValuesIterator::~FnDistinctValuesIterator()
{
}

bool 
FnDistinctValuesIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
  store::Item_t lItem;
  xqtref_t lItemType;
  XQPCollator* lCollator;
  
  FnDistinctValuesIteratorState* state;
  DEFAULT_STACK_INIT(FnDistinctValuesIteratorState, state, planState);

  if (theChildren.size() == 2)
  {
    lCollator = getCollator(planState.theRuntimeCB, loc, planState, theChildren[1].getp());

    ValueCollCompareParam* vcp = new ValueCollCompareParam(planState.theRuntimeCB);
    vcp->theCollator = lCollator;
    state->theAlreadySeenMap = 
      new ItemValueCollHandleHashSet(vcp);
  } else {
    ValueCollCompareParam* vcp = new ValueCollCompareParam(planState.theRuntimeCB);
    state->theAlreadySeenMap = new ItemValueCollHandleHashSet(vcp);
  }

  while (consumeNext(result, theChildren[0].getp(), planState)) {
    if (result->isNaN ()) {
      if (! state->theHasNaN) {
        state->theHasNaN = true;
        STACK_PUSH(true, state);
      }
    } else if ( ! state->theAlreadySeenMap->find(result) ) {
      // check if the item is already in the map
      state->theAlreadySeenMap->insert(result);
      STACK_PUSH(true, state);
    }
  }
    
  STACK_END (state);
}

void
FnDistinctValuesIteratorState::init(PlanState& planState) 
{
  PlanIteratorState::init(planState);
}

void
FnDistinctValuesIteratorState::reset(PlanState& planState) {
  PlanIteratorState::reset(planState);
  theHasNaN = false;
  theAlreadySeenMap->clear();
}

FnDistinctValuesIteratorState::FnDistinctValuesIteratorState()
  : theHasNaN (false),
    theAlreadySeenMap(0) {}

FnDistinctValuesIteratorState::~FnDistinctValuesIteratorState() 
{
  if (theAlreadySeenMap) {
    delete theAlreadySeenMap;
    theAlreadySeenMap = 0;
  }
}


//15.1.7 fn:insert-before
bool 
FnInsertBeforeIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
 store::Item_t lInsertItem;
 store::Item_t lPositionItem;
 
 FnInsertBeforeIteratorState* state;
 DEFAULT_STACK_INIT(FnInsertBeforeIteratorState, state, planState);<commit_msg>fixed bug in FcDistinctValuesIterator<commit_after>/*
 * Copyright 2006-2008 The FLWOR Foundation.
 * 
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 * 
 * http://www.apache.org/licenses/LICENSE-2.0
 * 
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
#include <string>
#include <fstream>
#include <vector>
#include <sstream>

#include "zorbautils/fatal.h"
#include "zorbaerrors/error_manager.h"
#include "zorbatypes/zorbatypesError.h"
#include "zorbatypes/URI.h"
#include "zorbaerrors/error_messages.h"

#include "runtime/sequences/SequencesImpl.h"
#include "runtime/booleans/BooleanImpl.h"
#include "runtime/numerics/NumericsImpl.h"
#include "runtime/api/runtimecb.h"
#include "runtime/core/arithmetic_impl.h"
#include "runtime/util/iterator_impl.h"

#include "system/globalenv.h"

#include "types/casting.h"
#include "types/typeops.h"
#include "store/api/store.h"
#include "store/api/iterator.h"
#include "store/api/item_factory.h"
#include "store/api/pul.h"

#include "context/static_context.h"
#include "context/collation_cache.h"

#include "util/web/web.h"
#include "store/util/hashset_node_handle.h"

#include "runtime/booleans/compare_types.h"
#include "runtime/util/handle_hashset_item_value.h"

using namespace std;
namespace zorba {
  

/*______________________________________________________________________
|  
| 15.1 General Functions and Operators on Sequences
|_______________________________________________________________________*/

static XQPCollator*
getCollator(
    RuntimeCB* aRuntimeCB,
    const QueryLoc& loc,
    PlanState& planState,
    const PlanIterator* iter)
{
  store::Item_t lCollationItem;
  store::Item_t temp;

  if (!PlanIterator::consumeNext(lCollationItem, iter, planState))
      ZORBA_ERROR_LOC_DESC(XPTY0004, loc, "An empty-sequence is not allowed as collation parameter");

  if (PlanIterator::consumeNext(temp, iter, planState))
      ZORBA_ERROR_LOC_DESC(XPTY0004, loc, "A sequence of more then one item is not allowed as collation parameter");
    
  xqtref_t lCollationItemType = planState.theCompilerCB->m_sctx->get_typemanager()->
                                create_value_type (lCollationItem);

  // TODO resolve uri (base-uri)

  return planState.theRuntimeCB->theCollationCache->getCollator(lCollationItem->getStringValue());
}


//15.1.2 op:concatenate 
//---------------------
void
FnConcatIteratorState::init(PlanState& planState) 
{
  PlanIteratorState::init(planState);
  theCurIter = 0;
}

void
FnConcatIteratorState::reset(PlanState& planState) 
{
  PlanIteratorState::reset(planState);
  theCurIter = 0;
}


bool
FnConcatIterator::nextImpl(store::Item_t& result, PlanState& planState) const 
{
  std::auto_ptr<store::PUL> pul;

  FnConcatIteratorState* state;
  DEFAULT_STACK_INIT(FnConcatIteratorState, state, planState);

  if (theIsUpdating)
    pul.reset(GENV_ITEMFACTORY->createPendingUpdateList());

  for (; state->theCurIter < theChildren.size(); ++state->theCurIter) 
  {
    while(consumeNext(result, theChildren[state->theCurIter].getp(), planState))
    {
      if (theIsUpdating)
      {
        ZORBA_FATAL(result->isPul(), "");

        pul->mergeUpdates(result);
      }
      else
      {
        STACK_PUSH (true, state);
      }
    }
  }
  
  if (theIsUpdating) {
    result = pul.release();
    STACK_PUSH(result != NULL, state);
  }

  STACK_END (state);
}

//15.1.3 fn:index-of
bool 
FnIndexOfIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
  store::Item_t lSequenceItem;
  store::Item_t lCollationItem;
  xqtref_t      lCollationItemType;

  FnIndexOfIteratorState* state;
  DEFAULT_STACK_INIT(FnIndexOfIteratorState, state, planState);
  
  if (!consumeNext(state->theSearchItem, theChildren[1].getp(), planState))
  {
    ZORBA_ERROR_LOC_DESC( FORG0006, loc, 
         "An empty sequence is not allowed as search item of fn:index-of");    
  }

  if ( theChildren.size() == 3 )
    state->theCollator = getCollator(planState.theRuntimeCB, loc, planState, theChildren[2].getp());

  while ( consumeNext(lSequenceItem, theChildren[0].getp(), planState))
  {
    // inc the position in the sequence; do it at the beginning of the loop because index-of starts with one
    ++state->theCurrentPos;
    
    if (CompareIterator::valueEqual(planState.theRuntimeCB,
                                    lSequenceItem, state->theSearchItem, state->theCollator) == 0)
      STACK_PUSH(GENV_ITEMFACTORY->createInteger(
        result,
        Integer::parseInt(state->theCurrentPos)), 
        state
      );
  }

  STACK_END (state);
}

void
FnIndexOfIteratorState::init(PlanState& planState) 
{
 PlanIteratorState::init(planState);
 theCurrentPos = 0;
 theSearchItem = NULL;
 theCollator = 0;
}

void
FnIndexOfIteratorState::reset(PlanState& planState) {
 PlanIteratorState::reset(planState);
 theCurrentPos = 0;
 theSearchItem = NULL;
 theCollator = 0;
}


//15.1.4 fn:empty
/*
 * If the value of $arg is the empty sequence, the function returns true; 
 * otherwise, the function returns false.
 */
bool 
FnEmptyIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
  store::Item_t lSequenceItem;

  PlanIteratorState* state;
  DEFAULT_STACK_INIT(PlanIteratorState, state, planState);

  if ( !consumeNext(lSequenceItem, theChildren[0].getp(), planState))
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, true ), state);
  }
  else
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, false ), state);   
  }

  STACK_END (state);
}

//15.1.5 fn:exists
/*
 * If the value of $arg is not the empty sequence, the function returns true; 
 * otherwise, the function returns false.
 */
bool 
FnExistsIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
  store::Item_t lSequenceItem;
  
  PlanIteratorState* state;
  DEFAULT_STACK_INIT(PlanIteratorState, state, planState);
  
  if ( consumeNext(lSequenceItem, theChildren[0].getp(), planState) ) 
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, true ), state);
  }
  else
  {
    STACK_PUSH (GENV_ITEMFACTORY->createBoolean ( result, false ), state);   
  }

  STACK_END (state);
}


//15.1.6 fn:distinct-values
/**
 * Returns the sequence that results from removing from arg all but one of a set of
 * values that are eq to one other. 
 * The order in which the sequence of values is returned is ·implementation dependent·.
 * Here, we return the first item that is not a duplicate and throw away the remaining ones
 */
FnDistinctValuesIterator::FnDistinctValuesIterator(const QueryLoc& loc,
                                                   vector<PlanIter_t>& args)
 : NaryBaseIterator<FnDistinctValuesIterator, FnDistinctValuesIteratorState> ( loc, args )
{ }

FnDistinctValuesIterator::~FnDistinctValuesIterator()
{
}


bool 
FnDistinctValuesIterator::nextImpl(store::Item_t& result, PlanState& planState) const 
{
  store::Item_t lItem;
  xqtref_t lItemType;
  XQPCollator* lCollator;
  
  FnDistinctValuesIteratorState* state;
  DEFAULT_STACK_INIT(FnDistinctValuesIteratorState, state, planState);

  if (theChildren.size() == 2)
  {
    lCollator = getCollator(planState.theRuntimeCB, loc, planState, theChildren[1].getp());

    ValueCollCompareParam* vcp = new ValueCollCompareParam(planState.theRuntimeCB);
    vcp->theCollator = lCollator;
    state->theAlreadySeenMap = 
      new ItemValueCollHandleHashSet(vcp);
  } else {
    ValueCollCompareParam* vcp = new ValueCollCompareParam(planState.theRuntimeCB);
    state->theAlreadySeenMap = new ItemValueCollHandleHashSet(vcp);
  }

  while (consumeNext(result, theChildren[0].getp(), planState)) {
    if (result->isNaN ()) {
      if (! state->theHasNaN) {
        state->theHasNaN = true;
        STACK_PUSH(true, state);
      }
    } else if ( ! state->theAlreadySeenMap->find(result) ) {
      // check if the item is already in the map
      state->theAlreadySeenMap->insert(result);
      STACK_PUSH(true, state);
    }
  }
    
  STACK_END (state);
}


FnDistinctValuesIteratorState::FnDistinctValuesIteratorState()
  :
  theHasNaN (false),
  theAlreadySeenMap(0)
{
}


FnDistinctValuesIteratorState::~FnDistinctValuesIteratorState() 
{
  if (theAlreadySeenMap) 
  {
    delete theAlreadySeenMap;
    theAlreadySeenMap = 0;
  }
}


void
FnDistinctValuesIteratorState::init(PlanState& planState) 
{
  PlanIteratorState::init(planState);
}


void
FnDistinctValuesIteratorState::reset(PlanState& planState) 
{
  PlanIteratorState::reset(planState);
  theHasNaN = false;
  if (theAlreadySeenMap)
    theAlreadySeenMap->clear();
}


//15.1.7 fn:insert-before
bool 
FnInsertBeforeIterator::nextImpl(store::Item_t& result, PlanState& planState) const {
 store::Item_t lInsertItem;
 store::Item_t lPositionItem;
 
 FnInsertBeforeIteratorState* state;
 DEFAULT_STACK_INIT(FnInsertBeforeIteratorState, state, planState);
{'fraction_non_alphanumeric': 0.11265919233699793, 'fraction_numerical': 0.007510612822466529, 'mean_word_length': 4.2908148574719265, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2258684', 'n_tokens_mistral': 6061, 'n_tokens_neox': 5882, 'n_words': 1292}