<commit_before>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use std::ffi::OsString;
use std::fmt::Write;
use std::path::PathBuf;

/// The ByteString type represents values encoded using BSER_BYTESTRING.
/// The purpose of this encoding is to represent bytestrings with an arbitrary
/// encoding.
///
/// In practice, as used by watchman, bytestrings have the filesystem encoding
/// on posix systems (which is usually utf8 on linux, guaranteed utf8 on macos)
/// and when they appear as file names in file results are guaranteed to be utf8
/// on Windows systems.
///
/// When it comes to interoperating with Rust code, ByteString is nominally
/// equivalent to `OsString`/`PathBuf` and is convertible to and from those values.
///
/// It is worth noting that on Windows sytems the conversion from ByteString to OsString is
/// potentially lossy: while convention is that bytestring holds utf8 in the common case in
/// watchman, that isn't enforced by its serializer and we may potentially encounter an error
/// during conversion.  Converting from OsString to ByteString can potentially fail on windows
/// because Watchman and thus ByteString doesn't have a way to represent the poorly formed
/// surrogate pairs that Windows allows in its filenames.

#[derive(Clone, PartialEq, Eq)]
pub struct ByteString(Vec<u8>);

impl std::fmt::Debug for ByteString {
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
        let escaped = self.as_escaped_string();
        write!(fmt, "\"{}\"", escaped.escape_debug())
    }
}

impl std::fmt::Display for ByteString {
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
        let escaped = self.as_escaped_string();
        write!(fmt, "\"{}\"", escaped.escape_default())
    }
}

impl std::ops::Deref for ByteString {
    type Target = [u8];

    fn deref(&self) -> &[u8] {
        &self.0
    }
}

impl std::ops::DerefMut for ByteString {
    fn deref_mut(&mut self) -> &mut [u8] {
        &mut self.0
    }
}

impl ByteString {
    /// Returns the raw bytes as a slice
    pub fn as_bytes(&self) -> &[u8] {
        self.0.as_slice()
    }

    /// Returns a version of the bytestring encoded as a mostly-utf-8
    /// string, with invalid sequences escaped using `\xXX` hex notation.
    /// This is for diagnostic and display purposes.
    pub fn as_escaped_string(&self) -> String {
        let mut input = self.0.as_slice();
        let mut output = String::new();

        loop {
            match ::std::str::from_utf8(input) {
                Ok(valid) => {
                    output.push_str(valid);
                    break;
                }
                Err(error) => {
                    let (valid, after_valid) = input.split_at(error.valid_up_to());
                    unsafe {
                        output.push_str(::std::str::from_utf8_unchecked(valid))
                    }

                    if let Some(invalid_sequence_length) = error.error_len() {
                        for b in &after_valid[..invalid_sequence_length] {
                            write!(output, "\\x{:x}", b).unwrap();
                        }
                        input = &after_valid[invalid_sequence_length..];
                    } else {
                        break;
                    }<commit_msg>Reformat with new rustfmt 7/7<commit_after>/*
 * Copyright (c) Meta Platforms, Inc. and affiliates.
 *
 * This source code is licensed under the MIT license found in the
 * LICENSE file in the root directory of this source tree.
 */

use std::ffi::OsString;
use std::fmt::Write;
use std::path::PathBuf;

/// The ByteString type represents values encoded using BSER_BYTESTRING.
/// The purpose of this encoding is to represent bytestrings with an arbitrary
/// encoding.
///
/// In practice, as used by watchman, bytestrings have the filesystem encoding
/// on posix systems (which is usually utf8 on linux, guaranteed utf8 on macos)
/// and when they appear as file names in file results are guaranteed to be utf8
/// on Windows systems.
///
/// When it comes to interoperating with Rust code, ByteString is nominally
/// equivalent to `OsString`/`PathBuf` and is convertible to and from those values.
///
/// It is worth noting that on Windows sytems the conversion from ByteString to OsString is
/// potentially lossy: while convention is that bytestring holds utf8 in the common case in
/// watchman, that isn't enforced by its serializer and we may potentially encounter an error
/// during conversion.  Converting from OsString to ByteString can potentially fail on windows
/// because Watchman and thus ByteString doesn't have a way to represent the poorly formed
/// surrogate pairs that Windows allows in its filenames.

#[derive(Clone, PartialEq, Eq)]
pub struct ByteString(Vec<u8>);

impl std::fmt::Debug for ByteString {
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
        let escaped = self.as_escaped_string();
        write!(fmt, "\"{}\"", escaped.escape_debug())
    }
}

impl std::fmt::Display for ByteString {
    fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
        let escaped = self.as_escaped_string();
        write!(fmt, "\"{}\"", escaped.escape_default())
    }
}

impl std::ops::Deref for ByteString {
    type Target = [u8];

    fn deref(&self) -> &[u8] {
        &self.0
    }
}

impl std::ops::DerefMut for ByteString {
    fn deref_mut(&mut self) -> &mut [u8] {
        &mut self.0
    }
}

impl ByteString {
    /// Returns the raw bytes as a slice
    pub fn as_bytes(&self) -> &[u8] {
        self.0.as_slice()
    }

    /// Returns a version of the bytestring encoded as a mostly-utf-8
    /// string, with invalid sequences escaped using `\xXX` hex notation.
    /// This is for diagnostic and display purposes.
    pub fn as_escaped_string(&self) -> String {
        let mut input = self.0.as_slice();
        let mut output = String::new();

        loop {
            match ::std::str::from_utf8(input) {
                Ok(valid) => {
                    output.push_str(valid);
                    break;
                }
                Err(error) => {
                    let (valid, after_valid) = input.split_at(error.valid_up_to());
                    unsafe { output.push_str(::std::str::from_utf8_unchecked(valid)) }

                    if let Some(invalid_sequence_length) = error.error_len() {
                        for b in &after_valid[..invalid_sequence_length] {
                            write!(output, "\\x{:x}", b).unwrap();
                        }
                        input = &after_valid[invalid_sequence_length..];
                    } else {
                        break;
                    }
{'fraction_non_alphanumeric': 0.1182842287694974, 'fraction_numerical': 0.004910456383593298, 'mean_word_length': 2.326128722382325, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4141788', 'n_tokens_mistral': 1968, 'n_tokens_neox': 1905, 'n_words': 680}