<commit_before>/*
    Copyright 2002-2013 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
/************************************************************************
 *
 * @file       EntityFeatures.h
 * @author     <NAME> (<EMAIL>)
 * @date       Mon Jun 12 2006
 * @version    $Id$
 * copyright   Copyright (C) 2006-2012 by CEA LIST
 * Project     s2lp - Automaton
 * 
 * @brief generic features associated to a recognized entity :
 * normalized information attached to the entity
 * 
 * 
 ***********************************************************************/

#ifndef ENTITYFEATURES_H
#define ENTITYFEATURES_H

#include "AutomatonExport.h"
#include "common/Data/LimaString.h"
#include "boost/any.hpp"

#include <vector>
#include <iostream>

namespace Lima {
namespace LinguisticProcessing {
namespace Automaton {

#define DEFAULT_ATTRIBUTE "value"
#define UNDEFLENGTH std::numeric_limits<uint64_t>::max()
#define UNDEFPOSITION std::numeric_limits<uint64_t>::max()

/** 
 * @brief a generic feature that is defined by a name and a value, which can be
 * of any type (use boost::any type)
 */
class LIMA_AUTOMATON_EXPORT EntityFeature {
public:
  EntityFeature();
  template<typename ValueType> 
    EntityFeature(const std::string& name, 
                  const ValueType& value):
    m_name(name),
    m_value(boost::any(value))
  {}
  ~EntityFeature();
  
  bool operator==(const EntityFeature& f) const;
  // comparison operator on name only
  bool operator==(const std::string& name) const;

  void setName(const std::string& name) { m_name=name; }
  void setValue(const boost::any& val) { m_value=val; }

  const std::string& getName() const { return m_name; }
  const boost::any& getValue() const { return m_value; }
  boost::any getValue() { return m_value; }

  void setPosition(const uint64_t& pos) { m_pos=pos; }
  void setLength(const uint64_t& val) { m_len=val; }
  uint64_t getPosition() const { return m_pos; }
  uint64_t getLength() const { return m_len; }

  // these functions test value type using RTTI: this is not really
  // efficient and should be avoided unless necessary (i.e. if the
  // type of the feature is known)
  std::string getValueString() const;
  LimaString getValueLimaString() const;

private:
  std::string m_name;
  boost::any m_value;
  uint64_t m_pos;
  uint64_t m_len;
};

/** 
 * @brief a list of generic features: each feature is unique (only one
 * feature for a name)
 */
class LIMA_AUTOMATON_EXPORT EntityFeatures: public std::vector<EntityFeature> {
public:
  EntityFeatures();
  ~EntityFeatures();
  bool operator==(const EntityFeatures& f) const;

  template<typename ValueType>
  void addFeature(const std::string& name,
                  const ValueType& value)
    {
      // if feature with same name already exists, overwrite it
      EntityFeatures::iterator it=find(name);
      if (it!=end()) {
//      if( (it!=end()) && (name==DEFAULT_ATTRIBUTE) ){
        (*it).setValue(boost::any(value));
      }
      else {
        //push empy feature and set values to avoid two copies
        //of value (do not know the type: it may be a big class)
        push_back(EntityFeature());
        back().setName(name);
        back().setValue(boost::any(value));
      }
    }
  
  template<typename ValueType>
  void appendFeature(const std::string& name,
                  const ValueType& value);
  

  EntityFeatures::const_iterator find(const std::string& featureName) const;
  EntityFeatures::iterator find(const std::string& featureName);

  friend LIMA_AUTOMATON_EXPORT std::ostream& operator<<(std::ostream& os, const EntityFeatures& f);
};

} // end namespace
} // end namespace
} // end namespace

#endif<commit_msg>add explicit instantiation of template function to be exported to dll<commit_after>/*
    Copyright 2002-2013 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
/************************************************************************
 *
 * @file       EntityFeatures.h
 * @author     <NAME> (<EMAIL>)
 * @date       Mon Jun 12 2006
 * @version    $Id$
 * copyright   Copyright (C) 2006-2012 by CEA LIST
 * Project     s2lp - Automaton
 * 
 * @brief generic features associated to a recognized entity :
 * normalized information attached to the entity
 * 
 * 
 ***********************************************************************/

#ifndef ENTITYFEATURES_H
#define ENTITYFEATURES_H

#include "AutomatonExport.h"
#include "common/Data/LimaString.h"
#include "boost/any.hpp"

#include <vector>
#include <iostream>

namespace Lima {
namespace LinguisticProcessing {
namespace Automaton {

#define DEFAULT_ATTRIBUTE "value"
#define UNDEFLENGTH std::numeric_limits<uint64_t>::max()
#define UNDEFPOSITION std::numeric_limits<uint64_t>::max()

/** 
 * @brief a generic feature that is defined by a name and a value, which can be
 * of any type (use boost::any type)
 */
class LIMA_AUTOMATON_EXPORT EntityFeature {
public:
  EntityFeature();
  template<typename ValueType> 
    EntityFeature(const std::string& name, 
                  const ValueType& value):
    m_name(name),
    m_value(boost::any(value))
  {}
  ~EntityFeature();
  
  bool operator==(const EntityFeature& f) const;
  // comparison operator on name only
  bool operator==(const std::string& name) const;

  void setName(const std::string& name) { m_name=name; }
  void setValue(const boost::any& val) { m_value=val; }

  const std::string& getName() const { return m_name; }
  const boost::any& getValue() const { return m_value; }
  boost::any getValue() { return m_value; }

  void setPosition(const uint64_t& pos) { m_pos=pos; }
  void setLength(const uint64_t& val) { m_len=val; }
  uint64_t getPosition() const { return m_pos; }
  uint64_t getLength() const { return m_len; }

  // these functions test value type using RTTI: this is not really
  // efficient and should be avoided unless necessary (i.e. if the
  // type of the feature is known)
  std::string getValueString() const;
  LimaString getValueLimaString() const;

private:
  std::string m_name;
  boost::any m_value;
  uint64_t m_pos;
  uint64_t m_len;
};

/** 
 * @brief a list of generic features: each feature is unique (only one
 * feature for a name)
 */
class LIMA_AUTOMATON_EXPORT EntityFeatures: public std::vector<EntityFeature> {
public:
  EntityFeatures();
  ~EntityFeatures();
  bool operator==(const EntityFeatures& f) const;

  template<typename ValueType>
  void addFeature(const std::string& name,
                  const ValueType& value)
    {
      // if feature with same name already exists, overwrite it
      EntityFeatures::iterator it=find(name);
      if (it!=end()) {
//      if( (it!=end()) && (name==DEFAULT_ATTRIBUTE) ){
        (*it).setValue(boost::any(value));
      }
      else {
        //push empy feature and set values to avoid two copies
        //of value (do not know the type: it may be a big class)
        push_back(EntityFeature());
        back().setName(name);
        back().setValue(boost::any(value));
      }
    }
  
  template<typename ValueType>
  void appendFeature(const std::string& name,
                  const ValueType& value);

  EntityFeatures::const_iterator find(const std::string& featureName) const;
  EntityFeatures::iterator find(const std::string& featureName);

  friend LIMA_AUTOMATON_EXPORT std::ostream& operator<<(std::ostream& os, const EntityFeatures& f);
};

  extern template LIMA_AUTOMATON_EXPORT void EntityFeatures::appendFeature<int>(const std::string&, const int& );
  extern template LIMA_AUTOMATON_EXPORT void EntityFeatures::appendFeature<double>(const std::string&, const double& );
  extern template LIMA_AUTOMATON_EXPORT void EntityFeatures::appendFeature<QString>(const std::string&, const QString& );
} // end namespace
} // end namespace
} // end namespace

#endif
{'fraction_non_alphanumeric': 0.13114030246164035, 'fraction_numerical': 0.00883099679876366, 'mean_word_length': 3.806366047745358, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1569296', 'n_tokens_mistral': 2708, 'n_tokens_neox': 2596, 'n_words': 895}