<commit_before>/*
 * RTokenizer.cpp
 *
 * Copyright (C) 2009-11 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include <core/r_util/RTokenizer.hpp>

#include <boost/regex.hpp>

#include <iostream>

#include <core/Error.hpp>
#include <core/Log.hpp>
#include <core/StringUtils.hpp>


// TODO: implement "project-level" default encoding setting

// TODO: implement file monitoring

// TODO: some type of cap on number of files or directories? (or just
// have searches be really slow for that case?)

// TODO: once we have file monitoring the source database will only
//       need to track indexes for dirty files

// TODO: consider further optimization of only generating source
//       indexes on-demand for the src db


namespace core {
namespace r_util {

namespace {

class TokenPatterns
{
private:
   friend TokenPatterns& tokenPatterns();
   TokenPatterns()
      : NUMBER(L"[0-9]*(\\.[0-9]*)?([eE][+-]?[0-9]*)?[Li]?"),
        HEX_NUMBER(L"0x[0-9a-fA-F]*L?"),<commit_msg>update todo comments<commit_after>/*
 * RTokenizer.cpp
 *
 * Copyright (C) 2009-11 by RStudio, Inc.
 *
 * This program is licensed to you under the terms of version 3 of the
 * GNU Affero General Public License. This program is distributed WITHOUT
 * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT,
 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the
 * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details.
 *
 */

#include <core/r_util/RTokenizer.hpp>

#include <boost/regex.hpp>

#include <iostream>

#include <core/Error.hpp>
#include <core/Log.hpp>
#include <core/StringUtils.hpp>


// TODO: implement "project-level" default encoding setting

// TODO: deal with project encoding not found on target system

// TODO: implement file monitoring

// TODO: some type of cap on number of files or directories? (or just
// have searches be really slow for that case?)

// TODO: once we have file monitoring the source database will only
//       need to track indexes for dirty files

// TODO: consider further optimization of only generating source
//       indexes on-demand for the src db


namespace core {
namespace r_util {

namespace {

class TokenPatterns
{
private:
   friend TokenPatterns& tokenPatterns();
   TokenPatterns()
      : NUMBER(L"[0-9]*(\\.[0-9]*)?([eE][+-]?[0-9]*)?[Li]?"),
        HEX_NUMBER(L"0x[0-9a-fA-F]*L?"),
{'fraction_non_alphanumeric': 0.11805810959911732, 'fraction_numerical': 0.013240161824200073, 'mean_word_length': 3.665523156089194, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4132886', 'n_tokens_mistral': 867, 'n_tokens_neox': 830, 'n_words': 303}