<commit_before><?php

/**
 * Hoa
 *
 *
 * @license
 *
 * New BSD License
 *
 * Copyright © 2007-2013, <NAME>. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the Hoa nor the names of its contributors may be
 *       used to endorse or promote products derived from this software without
 *       specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

namespace {

from('Hoa')

/**
 * \Hoa\String\Exception
 */
-> import('String.Exception');

}

namespace Hoa\String {

/**
 * Class \Hoa\String.
 *
 * This class represents a UTF-8 string.
 * Please, see:
 *     • http://www.ietf.org/rfc/rfc3454.txt;
 *     • http://unicode.org/reports/tr9/;
 *     • http://www.unicode.org/Public/6.0.0/ucd/UnicodeData.txt.
 *
 * @author     <NAME> <<EMAIL>>
 * @copyright  Copyright © 2007-2013 <NAME>.
 * @license    New BSD License
 */

class String implements \ArrayAccess, \Countable, \IteratorAggregate {

    /**
     * Left-To-Right.
     *
     * @const int
     */
    const LTR              = 0;

    /**
     * Right-To-Left.
     *
     * @const int
     */
    const RTL              = 1;

    /**
     * ZERO WIDTH NON-BREAKING SPACE (ZWNPBSP, aka byte-order mark, BOM).
     *
     * @const int
     */
    const BOM              = 0xfeff;

    /**
     * LEFT-TO-RIGHT MARK.
     *
     * @const int
     */
    const LRM              = 0x200e;

    /**
     * RIGHT-TO-LEFT MARK.
     *
     * @const int
     */
    const RLM              = 0x200f;

    /**
     * LEFT-TO-RIGHT EMBEDDING.
     *
     * @const int
     */
    const LRE              = 0x202a;

    /**
     * RIGHT-TO-LEFT EMBEDDING.
     *
     * @const int
     */
    const RLE              = 0x202b;

    /**
     * POP DIRECTIONAL FORMATTING.
     *
     * @const int
     */
    const PDF              = 0x202c;

    /**
     * LEFT-TO-RIGHT OVERRIDE.
     *
     * @const int
     */
    const LRO              = 0x202d;

    /**
     * RIGHT-TO-LEFT OVERRIDE.
     *
     * @const int
     */
    const RLO              = 0x202e;

    /**
     * Represent the beginning of the string.
     *
     * @const int
     */
    const BEGINNING        = 1;

    /**
     * Represent the end of the string.
     *
     * @const int
     */
    const END              = 2;

    /**
     * Split: non-empty pieces is returned.
     *
     * @const int
     */
    const WITHOUT_EMPTY    = PREG_SPLIT_NO_EMPTY;

    /**
     * Split: parenthesized expression in the delimiter pattern will be captured
     * and returned.
     *
     * @const int
     */
    const WITH_DELIMITERS  = PREG_SPLIT_DELIM_CAPTURE;

    /**
     * Split: offsets of captures will be returned.
     *
     * @const int
     */
    const WITH_OFFSET      = 260; //   PREG_OFFSET_CAPTURE
                                  // | PREG_SPLIT_OFFSET_CAPTURE

    /**
     * Group results by patterns.
     *
     * @const int
     */
    const GROUP_BY_PATTERN = PREG_PATTERN_ORDER;

    /**
     * Group results by tuple (set of patterns).
     *
     * @const int
     */
    const GROUP_BY_TUPLE   = PREG_SET_ORDER;

    /**
     * Current string.
     *
     * @var \Hoa\String string
     */
    protected $_string          = null;

    /**
     * Direction. Please see self::LTR and self::RTL constants.
     *
     * @var \Hoa\String int
     */
    protected $_direction       = null;

    /**
     * Collator.
     *
     * @var \Collator object
     */
    protected static $_collator = null;



    /**
     * Construct a UTF-8 string.
     *
     * @access  public
     * @param   string  $string    String.
     * @return  void
     */
    public function __construct ( $string = null ) {

        if(false === extension_loaded('mbstring'))
            throw new Exception(
                '%s needs the mbstring extension.', 0, get_class($this));

        if(null !== $string)
            $this->append($string);

        return;
    }

    /**
     * Append a substring to the current string, i.e. add to the end.
     *
     * @access  public
     * @param   string  $substring    Substring to append.
     * @return  \Hoa\String
     */
    public function append ( $substring ) {

        $direction = $this->getDirection();
<commit_msg>Remove strong dependency check for mbstring<commit_after><?php

/**
 * Hoa
 *
 *
 * @license
 *
 * New BSD License
 *
 * Copyright © 2007-2013, <NAME>. All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *     * Neither the name of the Hoa nor the names of its contributors may be
 *       used to endorse or promote products derived from this software without
 *       specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS AND CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

namespace {

from('Hoa')

/**
 * \Hoa\String\Exception
 */
-> import('String.Exception');

}

namespace Hoa\String {

/**
 * Class \Hoa\String.
 *
 * This class represents a UTF-8 string.
 * Please, see:
 *     • http://www.ietf.org/rfc/rfc3454.txt;
 *     • http://unicode.org/reports/tr9/;
 *     • http://www.unicode.org/Public/6.0.0/ucd/UnicodeData.txt.
 *
 * @author     <NAME> <<EMAIL>>
 * @copyright  Copyright © 2007-2013 <NAME>.
 * @license    New BSD License
 */

class String implements \ArrayAccess, \Countable, \IteratorAggregate {

    /**
     * Left-To-Right.
     *
     * @const int
     */
    const LTR              = 0;

    /**
     * Right-To-Left.
     *
     * @const int
     */
    const RTL              = 1;

    /**
     * ZERO WIDTH NON-BREAKING SPACE (ZWNPBSP, aka byte-order mark, BOM).
     *
     * @const int
     */
    const BOM              = 0xfeff;

    /**
     * LEFT-TO-RIGHT MARK.
     *
     * @const int
     */
    const LRM              = 0x200e;

    /**
     * RIGHT-TO-LEFT MARK.
     *
     * @const int
     */
    const RLM              = 0x200f;

    /**
     * LEFT-TO-RIGHT EMBEDDING.
     *
     * @const int
     */
    const LRE              = 0x202a;

    /**
     * RIGHT-TO-LEFT EMBEDDING.
     *
     * @const int
     */
    const RLE              = 0x202b;

    /**
     * POP DIRECTIONAL FORMATTING.
     *
     * @const int
     */
    const PDF              = 0x202c;

    /**
     * LEFT-TO-RIGHT OVERRIDE.
     *
     * @const int
     */
    const LRO              = 0x202d;

    /**
     * RIGHT-TO-LEFT OVERRIDE.
     *
     * @const int
     */
    const RLO              = 0x202e;

    /**
     * Represent the beginning of the string.
     *
     * @const int
     */
    const BEGINNING        = 1;

    /**
     * Represent the end of the string.
     *
     * @const int
     */
    const END              = 2;

    /**
     * Split: non-empty pieces is returned.
     *
     * @const int
     */
    const WITHOUT_EMPTY    = PREG_SPLIT_NO_EMPTY;

    /**
     * Split: parenthesized expression in the delimiter pattern will be captured
     * and returned.
     *
     * @const int
     */
    const WITH_DELIMITERS  = PREG_SPLIT_DELIM_CAPTURE;

    /**
     * Split: offsets of captures will be returned.
     *
     * @const int
     */
    const WITH_OFFSET      = 260; //   PREG_OFFSET_CAPTURE
                                  // | PREG_SPLIT_OFFSET_CAPTURE

    /**
     * Group results by patterns.
     *
     * @const int
     */
    const GROUP_BY_PATTERN = PREG_PATTERN_ORDER;

    /**
     * Group results by tuple (set of patterns).
     *
     * @const int
     */
    const GROUP_BY_TUPLE   = PREG_SET_ORDER;

    /**
     * Current string.
     *
     * @var \Hoa\String string
     */
    protected $_string          = null;

    /**
     * Direction. Please see self::LTR and self::RTL constants.
     *
     * @var \Hoa\String int
     */
    protected $_direction       = null;

    /**
     * Collator.
     *
     * @var \Collator object
     */
    protected static $_collator = null;



    /**
     * Construct a UTF-8 string.
     *
     * @access  public
     * @param   string  $string    String.
     * @return  void
     */
    public function __construct ( $string = null ) {

        if(false === function_exists('mb_strlen'))
            throw new Exception(
                '%s needs the mbstring extension.', 0, get_class($this));

        if(null !== $string)
            $this->append($string);

        return;
    }

    /**
     * Append a substring to the current string, i.e. add to the end.
     *
     * @access  public
     * @param   string  $substring    Substring to append.
     * @return  \Hoa\String
     */
    public function append ( $substring ) {

        $direction = $this->getDirection();

{'fraction_non_alphanumeric': 0.10264780558578164, 'fraction_numerical': 0.011425462459194777, 'mean_word_length': 1.793566362715299, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '162294', 'n_tokens_mistral': 3544, 'n_tokens_neox': 3250, 'n_words': 1035}