<commit_before>use std::libc::{c_ulonglong};
use std::vec::{with_capacity, append};
use std::vec::raw::{to_mut_ptr, to_ptr};

#[doc(hidden)]
pub fn marshal<T>(buf: &[u8],
                  padbefore: uint,
                  bytestodrop: uint,
                  f: &fn(*mut u8, *u8, c_ulonglong) -> T
                 ) -> (~[u8], T) {
    let mut dst = with_capacity(buf.len() + padbefore);
    for _ in range(0, padbefore) {
        dst.push(0);
    }
    dst = append(dst, buf);
    let pdst = to_mut_ptr(dst);
    let psrc = to_ptr(dst);
    let res = f(pdst, psrc, dst.len() as c_ulonglong);
    for _ in range(0, bytestodrop) {
        dst.shift();
    }
    (dst, res)
}<commit_msg>Optimized marshal function<commit_after>use std::libc::{c_ulonglong};
use std::vec::{with_capacity, append};
use std::vec::raw::{to_mut_ptr, to_ptr};

#[doc(hidden)]
pub fn marshal<T>(buf: &[u8],
                  padbefore: uint,
                  bytestodrop: uint,
                  f: &fn(*mut u8, *u8, c_ulonglong) -> T
                 ) -> (~[u8], T) {
    let mut dst = with_capacity(buf.len() + padbefore);
    for _ in range(0, padbefore) {
        dst.push(0);
    }
    dst = append(dst, buf);
    let pdst = to_mut_ptr(dst);
    let psrc = to_ptr(dst);
    let res = f(pdst, psrc, dst.len() as c_ulonglong);
    (dst.move_iter().skip(bytestodrop).collect(), res)
}
{'fraction_non_alphanumeric': 0.17142857142857143, 'fraction_numerical': 0.009523809523809525, 'mean_word_length': 2.44080604534005, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7626159', 'n_tokens_mistral': 519, 'n_tokens_neox': 517, 'n_words': 113}