<commit_before>// This file is part of Grust, GObject introspection bindings for Rust
//
// Copyright (C) 2013, 2014  <NAME> <<EMAIL>>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

use ffi;
use types::{gchar,gsize};
use types::{gpointer};

use libc;
use libc::c_char;
use std::c_str::{CString,ToCStr};
use std::kinds::marker;
use std::mem::transmute;
use std::ptr::copy_nonoverlapping_memory;
use std::slice;
use std::str;
use std::string;

pub struct UTF8Chars<'a> {
    data: *const gchar,
    lifetime: marker::ContravariantLifetime<'a>
}

impl<'a> UTF8Chars<'a> {
    pub unsafe fn wrap(ptr: *const gchar) -> UTF8Chars<'a> {
        UTF8Chars { data: ptr, lifetime: marker::ContravariantLifetime }
    }

    pub fn to_string(&self) -> String {
        unsafe {
            string::raw::from_buf(self.data as *const u8)
        }
    }
}

#[inline]
unsafe fn utf8_cont_bits(p: *const gchar) -> u32 {
    let byte = *(p as *const u8);
    debug_assert!((byte & 0xC0) == 0x80);
    (byte & 0x3F) as u32
}

impl<'a> Iterator<char> for UTF8Chars<'a> {
    fn next(&mut self) -> Option<char> {
        let p = self.data;
        let first_byte = unsafe { *(p as *const u8) };
        if first_byte == 0 {
            return None;
        }
        unsafe {
            let mut wc: u32;
            if first_byte < 0xC0 {
                debug_assert!(first_byte < 0x80);
                wc = first_byte as u32;
                self.data = p.offset(1);
            } else {
                let c1 = utf8_cont_bits(p.offset(1));
                if first_byte < 0xE0 {
                    wc = ((first_byte & 0x1F) as u32 << 6) | c1;
                    debug_assert!(wc >= 0x80);
                    self.data = p.offset(2);
                } else {
                    let c2 = utf8_cont_bits(p.offset(2));
                    if first_byte < 0xF0 {
                        wc = ((first_byte & 0x0F) as u32 << 12)
                             | (c1 << 6) | c2;
                        debug_assert!(wc >= 0x0800);
                        debug_assert!((wc & 0xF800) != 0xD800);
                        self.data = p.offset(3);
                    } else {
                        debug_assert!(first_byte <= 0xF4);
                        let c3 = utf8_cont_bits(p.offset(3));
                        wc = ((first_byte & 0x07) as u32 << 18)
                             | (c1 << 12) | (c2 << 6) | c3;
                        debug_assert!(wc >= 0x010000 && wc <= 0x10FFFF);
                        self.data = p.offset(4);
                    }
                }
            }
            Some(transmute(wc))
        }
    }
}

unsafe fn dup_to_c_str(source: *const c_char, len: uint) -> CString {
    let copy = libc::malloc(len as libc::size_t + 1) as *mut c_char;
    copy_nonoverlapping_memory(copy, source, len + 1);
    CString::new(copy as *const c_char, true)
}

impl<'a> ToCStr for UTF8Chars<'a> {

    #[inline]
    fn to_c_str(&self) -> CString {
        unsafe { self.to_c_str_unchecked() }
    }

    unsafe fn to_c_str_unchecked(&self) -> CString {
        let src = self.data as *const c_char;
        let len = libc::strlen(src) as uint;
        dup_to_c_str(src, len)
    }

    fn with_c_str<T>(&self, f: |*const i8| -> T) -> T {
        f(self.data as *const i8)
    }

    unsafe fn with_c_str_unchecked<T>(&self, f: |*const i8| -> T) -> T {
        f(self.data as *const i8)
    }
}

pub struct UTF8Buf {
    data: *mut gchar,
}

impl UTF8Buf {

    pub unsafe fn wrap(data: *mut gchar) -> UTF8Buf {
        UTF8Buf { data: data }
    }

    pub fn chars<'a>(&'a self) -> UTF8Chars<'a> {
        unsafe { UTF8Chars::wrap(self.data as *const gchar) }
    }

    pub fn into_collection(self) -> UTF8Str {
        unsafe {
            let len = libc::strlen(self.data as *const c_char);
            UTF8Str { buf: transmute(self), len: len as uint }
        }
    }

    #[inline]
    pub fn to_string(&self) -> String { self.chars().to_string() }

    #[inline]
    pub fn to_owned(&self) -> String { self.to_string() }

    #[inline]
    pub fn into_string(self) -> String { self.to_string() }

    #[inline]
    pub fn into_owned(self) -> String { self.to_string() }<commit_msg>utf8: Remove .to_owned() and .into_owned()<commit_after>// This file is part of Grust, GObject introspection bindings for Rust
//
// Copyright (C) 2013, 2014  <NAME> <<EMAIL>>
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or (at your option) any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

use ffi;
use types::{gchar,gsize};
use types::{gpointer};

use libc;
use libc::c_char;
use std::c_str::{CString,ToCStr};
use std::kinds::marker;
use std::mem::transmute;
use std::ptr::copy_nonoverlapping_memory;
use std::slice;
use std::str;
use std::string;

pub struct UTF8Chars<'a> {
    data: *const gchar,
    lifetime: marker::ContravariantLifetime<'a>
}

impl<'a> UTF8Chars<'a> {
    pub unsafe fn wrap(ptr: *const gchar) -> UTF8Chars<'a> {
        UTF8Chars { data: ptr, lifetime: marker::ContravariantLifetime }
    }

    pub fn to_string(&self) -> String {
        unsafe {
            string::raw::from_buf(self.data as *const u8)
        }
    }
}

#[inline]
unsafe fn utf8_cont_bits(p: *const gchar) -> u32 {
    let byte = *(p as *const u8);
    debug_assert!((byte & 0xC0) == 0x80);
    (byte & 0x3F) as u32
}

impl<'a> Iterator<char> for UTF8Chars<'a> {
    fn next(&mut self) -> Option<char> {
        let p = self.data;
        let first_byte = unsafe { *(p as *const u8) };
        if first_byte == 0 {
            return None;
        }
        unsafe {
            let mut wc: u32;
            if first_byte < 0xC0 {
                debug_assert!(first_byte < 0x80);
                wc = first_byte as u32;
                self.data = p.offset(1);
            } else {
                let c1 = utf8_cont_bits(p.offset(1));
                if first_byte < 0xE0 {
                    wc = ((first_byte & 0x1F) as u32 << 6) | c1;
                    debug_assert!(wc >= 0x80);
                    self.data = p.offset(2);
                } else {
                    let c2 = utf8_cont_bits(p.offset(2));
                    if first_byte < 0xF0 {
                        wc = ((first_byte & 0x0F) as u32 << 12)
                             | (c1 << 6) | c2;
                        debug_assert!(wc >= 0x0800);
                        debug_assert!((wc & 0xF800) != 0xD800);
                        self.data = p.offset(3);
                    } else {
                        debug_assert!(first_byte <= 0xF4);
                        let c3 = utf8_cont_bits(p.offset(3));
                        wc = ((first_byte & 0x07) as u32 << 18)
                             | (c1 << 12) | (c2 << 6) | c3;
                        debug_assert!(wc >= 0x010000 && wc <= 0x10FFFF);
                        self.data = p.offset(4);
                    }
                }
            }
            Some(transmute(wc))
        }
    }
}

unsafe fn dup_to_c_str(source: *const c_char, len: uint) -> CString {
    let copy = libc::malloc(len as libc::size_t + 1) as *mut c_char;
    copy_nonoverlapping_memory(copy, source, len + 1);
    CString::new(copy as *const c_char, true)
}

impl<'a> ToCStr for UTF8Chars<'a> {

    #[inline]
    fn to_c_str(&self) -> CString {
        unsafe { self.to_c_str_unchecked() }
    }

    unsafe fn to_c_str_unchecked(&self) -> CString {
        let src = self.data as *const c_char;
        let len = libc::strlen(src) as uint;
        dup_to_c_str(src, len)
    }

    fn with_c_str<T>(&self, f: |*const i8| -> T) -> T {
        f(self.data as *const i8)
    }

    unsafe fn with_c_str_unchecked<T>(&self, f: |*const i8| -> T) -> T {
        f(self.data as *const i8)
    }
}

pub struct UTF8Buf {
    data: *mut gchar,
}

impl UTF8Buf {

    pub unsafe fn wrap(data: *mut gchar) -> UTF8Buf {
        UTF8Buf { data: data }
    }

    pub fn chars<'a>(&'a self) -> UTF8Chars<'a> {
        unsafe { UTF8Chars::wrap(self.data as *const gchar) }
    }

    pub fn into_collection(self) -> UTF8Str {
        unsafe {
            let len = libc::strlen(self.data as *const c_char);
            UTF8Str { buf: transmute(self), len: len as uint }
        }
    }

    #[inline]
    pub fn to_string(&self) -> String { self.chars().to_string() }

    #[inline]
    pub fn into_string(self) -> String { self.to_string() }
{'fraction_non_alphanumeric': 0.12714492454000415, 'fraction_numerical': 0.028840190200537522, 'mean_word_length': 1.9344858962693359, 'pattern_counts': {'":': 0, '<': 75, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6904578', 'n_tokens_mistral': 3399, 'n_tokens_neox': 3158, 'n_words': 910}