<commit_before>///! fff-lang
///!
///! codemap/codefile
///! filename and file content owner, yield char and pos from file content

use std::fmt;
use std::path::Path;
use std::path::PathBuf;

use super::Span;
use super::CharPos;
use super::CodeMapError;

pub const EOF_CHAR: char = 0u8 as char;

// UTF8 string iterator, 
// yes, an unchecked utf8 string iterator is like this short

/// Get UTF8 char byte length at index of the bytes
///
/// no check bytes is valid UTF8 squence, no check index is valid index, 
/// also don't know if the panic will happen or how will happen
#[inline]
fn char_len_at_index(bytes: &[u8], index: usize) -> usize {
    match bytes[index] {
        0b_0000_0000...0b_1000_0000 => 1,
        0b_1100_0000...0b_1110_0000 => 2,
        0b_1110_0000...0b_1111_0000 => 3,
        0b_1111_0000...0b_1111_1111 => 4,
        _ => panic!("impossible byte value, I think"),
    }
}

/// Get UTF8 char at index of the bytes
///
/// no check bytes is valid UTF8 sequence, no check index is valid index, 
/// no check bytes length is enough if char length is more than 1, 
/// use interpret cast, no check calculated code point value is valid code point
#[inline]
fn char_at_index(bytes: &[u8], index: usize) -> (usize, char) { // char and its utf8 length at index
    let transmute = |v: u32| -> char { unsafe { ::std::mem::transmute(v) } };
    match char_len_at_index(bytes, index) {
        1 => (1, transmute(bytes[index] as u32)),
        2 => (2, transmute((((bytes[index] as u32) & 0b00011111u32) << 6) + ((bytes[index + 1] as u32) & 0b00111111u32))),
        3 => (3, transmute((((bytes[index] as u32) & 0b00001111u32) << 12) + (((bytes[index + 1] as u32) & 0b00111111u32) << 6) 
                    + (((bytes[index + 2] as u32) & 0b00111111u32)))),
        4 => (4, transmute((((bytes[index] as u32) & 0b00000111u32) << 18) + (((bytes[index + 1] as u32) & 0b00111111u32) << 12)
                    + (((bytes[index + 2] as u32) & 0b00111111u32) << 6) + ((bytes[index + 3] as u32) & 0b00111111u32))),
        _ => panic!("impossible branch"),
    }
}

/// input 2 absolute path, calculate p2's relative path to p1
fn to_relative(path1: &Path, path2: &Path) -> PathBuf {
    use std::path::Component;
    use std::path::Prefix;
    
    let (mut path1_components, mut path2_components) = (path1.components(), path2.components());
    let mut dotdot_count = 0;
    let mut c2_mores = Vec::new();
    // for each component pair (c1, c2)
    //     if c1 == c2, continue,
    //     for first c1 != c2, count dotdot = 0, start recording c2's component in c2_mores
    //     then for next Some(c1)s, add dotdot_count 1, until c1 is none
    //     and for next Some(c2)s, push c2_mores, until c2 is none
    loop {
        match (path1_components.next(), path2_components.next()) {
            (None, None) => break,
            (c1, c2) if c1 == c2 => continue,
            // to make `\\?\C:\` same as `C:\`
            // ... every time handling `\\?\` makes me think about life
            (Some(Component::Prefix(prefix_component1)), Some(Component::Prefix(prefix_component2))) => {
                match (prefix_component1.kind(), prefix_component2.kind()) {
                    (Prefix::VerbatimDisk(volumn_name1), Prefix::Disk(volumn_name2))
                    | (Prefix::Disk(volumn_name1), Prefix::VerbatimDisk(volumn_name2)) if volumn_name1 == volumn_name2 => continue,
                    _ => (),
                }
                return path2.to_owned(); // even prefix is not same, then quick return
            }
            (Some(_), Some(c2)) => { dotdot_count += 1; c2_mores.push(c2); }
            (Some(_), None) => { dotdot_count += 1; }
            (None, Some(c2)) => { c2_mores.push(c2); }
        }
    }
    let mut retval = vec![Component::ParentDir; dotdot_count]; // ... very rare `vec!` usage
    retval.extend(c2_mores);
    retval.into_iter().map(Component::as_os_str).collect()
}

pub struct SourceCodeIter<'a> {
    src: &'a SourceCode,
    index: usize,
}
impl<'a> SourceCodeIter<'a> {

    fn new(src: &'a SourceCode) -> SourceCodeIter<'a> { SourceCodeIter{ src, index: 0 } }

    /// Iterate through source code string
    ///
    /// ignores all bare or not bare CR, 
    /// return EOF_CHAR after EOF, fuse that
    pub fn next(&mut self) -> (char, CharPos) {
        loop {
            if self.index == self.src.src.len() {
                return (EOF_CHAR, CharPos::new(self.src.id, self.index));
            } else if self.src.src[self.index] == b'\r' {
                self.index += 1;
                continue;
            } else {
                let (len, ch) = char_at_index(&self.src.src, self.index);
                let retval = (ch, CharPos::new(self.src.id, self.index));
                self.index += len;
                return retval;
            }
        }
    }
}

/// Represents a source code file
#[derive(Eq, PartialEq)]
pub struct SourceCode {
    id: usize,
    name: PathBuf,            // absolute path
    src: Box<[u8]>,           // source code string, in form of owned byte slice, because no need of string methods, utf8 iterator also implemented here not depend on std
    endl_indexes: Vec<usize>, // line end byte indexes
}
impl fmt::Debug for SourceCode {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "source-code#{} {}", self.id, self.get_relative_path().display())
    }
}
impl SourceCode {

    fn _get_endl_indexes(src: &str) -> Vec<usize> {
        src.char_indices().filter(|indice| indice.1 == '\n').map(|indice| indice.0).collect()
    }

    pub fn with_file_name<T>(id: usize, file_name: T) -> Result<SourceCode, CodeMapError> where T: Into<PathBuf> + Clone { // into for adapt String, Clone for construct error
        use std::io::Read;

        let file_name = file_name.into();
        let file_name = file_name.clone().canonicalize().map_err(|e| CodeMapError::CannotOpenFile(file_name.clone(), e))?;
        let mut src = String::new();
        let mut file = ::std::fs::File::open(&file_name).map_err(|e| CodeMapError::CannotOpenFile(file_name.clone(), e))?;
        let _ = file.read_to_string(&mut src).map_err(|e| CodeMapError::CannotReadFile(file_name.clone(), e))?;

        Ok(SourceCode{ 
            id, 
            name: file_name, 
            endl_indexes: Self::_get_endl_indexes(&src),
            src: src.into_bytes().into_boxed_slice(),
        })
    }

    /// pretend to be a with_file_name'd source code in test
    /// @param file_name: vector of path components start from cwd, use &str for &'static str in test oracle
    pub fn pretend_with_file_name(id: usize, file_name: Vec<&str>, src: &str) -> SourceCode {

        let mut file_path = ::std::env::current_dir().unwrap(); // if current dir fails, just go die
        file_path.extend(file_name);

        SourceCode{
            id,
            name: file_path,
            endl_indexes: Self::_get_endl_indexes(src),
            src: src.to_owned().into_bytes().into_boxed_slice(),
        }
    }
    pub fn with_test_str(id: usize, src: &str) -> SourceCode {
        SourceCode{ 
            id, 
            name: format!("<anon#{}>", id).into(),
            endl_indexes: Self::_get_endl_indexes(src),
            src: src.to_owned().into_bytes().into_boxed_slice(),
        }
    }

    pub fn iter<'a>(&'a self) -> SourceCodeIter<'a> { SourceCodeIter::new(self) }
}
impl SourceCode {

    pub fn get_file_id(&self) -> usize { self.id }
    
    pub fn get_absolute_path(&self) -> &Path { &self.name }
    pub fn get_directory(&self) -> PathBuf { self.name.parent().unwrap().to_owned() /* unwrap because it must be a valid file */ } 
    
    pub fn get_file_stem(&self) -> Option<&str> { 
        // flatten `Option<Option<str>` to `Option<String>`
        match self.name.file_stem() {
            Some(stem) => match stem.to_str() {
                Some(stem) => Some(stem),
                None => None,
            }, 
        //   ^ small story here:
            // in my 2e10e182, I added this method to be used in syntax and semantic's multiple file support,
            // I accidentally forget this comma
            // but it passes compile while I'm using the nightly build nightly-x86_64-pc-windows-msvc-17-8-1
            // in another environment I'm defaulting stable-x86_64-pc-windows-msvc-1.19.0, and this is a compile error, which is, I think should be the correct behaviour
            // which also breaks my travis test
            // digging into rustc's parser's history I found rust-lang/rust#40989, which allows no comma here, after block-like (not only block) expressions
            // this feature is committed in a forked library at 17/4/1, and the pull request is merged at about 17/7/19, while stable-1.19.0 is released at 17/7/17
            // I learned how to blame a code change and how rust's language feature is discussed and implemented and finally released, which is very interesting
            None => None,
        }
    }

    pub fn get_relative_path(&self) -> PathBuf { 
        if &format!("{}", self.name.display()).as_bytes()[..5] == b"<anon" { 
            self.name.clone()
        } else {
            // if getcwd failed, then let it go
            to_relative(&::std::env::current_dir().unwrap(), &self.name) 
        }
    }
}
impl SourceCode {
    /// map byte index to (row, column)
    pub fn map_index(&self, charpos: CharPos) -> (usize, usize) {
        assert_eq!{ self.id, charpos.file_id, "incorrect file id" }
        
        let char_id = charpos.char_id;
        
        // get row number and row start character's byte index
        let (row_num, row_start_id) = if self.endl_indexes.len() == 0 {
            (1, 0)<commit_msg>[codemap] add date of previous fix<commit_after>///! fff-lang
///!
///! codemap/codefile
///! filename and file content owner, yield char and pos from file content

use std::fmt;
use std::path::Path;
use std::path::PathBuf;

use super::Span;
use super::CharPos;
use super::CodeMapError;

pub const EOF_CHAR: char = 0u8 as char;

// UTF8 string iterator, 
// yes, an unchecked utf8 string iterator is like this short

/// Get UTF8 char byte length at index of the bytes
///
/// no check bytes is valid UTF8 squence, no check index is valid index, 
/// also don't know if the panic will happen or how will happen
#[inline]
fn char_len_at_index(bytes: &[u8], index: usize) -> usize {
    match bytes[index] {
        0b_0000_0000...0b_1000_0000 => 1,
        0b_1100_0000...0b_1110_0000 => 2,
        0b_1110_0000...0b_1111_0000 => 3,
        0b_1111_0000...0b_1111_1111 => 4,
        _ => panic!("impossible byte value, I think"),
    }
}

/// Get UTF8 char at index of the bytes
///
/// no check bytes is valid UTF8 sequence, no check index is valid index, 
/// no check bytes length is enough if char length is more than 1, 
/// use interpret cast, no check calculated code point value is valid code point
#[inline]
fn char_at_index(bytes: &[u8], index: usize) -> (usize, char) { // char and its utf8 length at index
    let transmute = |v: u32| -> char { unsafe { ::std::mem::transmute(v) } };
    match char_len_at_index(bytes, index) {
        1 => (1, transmute(bytes[index] as u32)),
        2 => (2, transmute((((bytes[index] as u32) & 0b00011111u32) << 6) + ((bytes[index + 1] as u32) & 0b00111111u32))),
        3 => (3, transmute((((bytes[index] as u32) & 0b00001111u32) << 12) + (((bytes[index + 1] as u32) & 0b00111111u32) << 6) 
                    + (((bytes[index + 2] as u32) & 0b00111111u32)))),
        4 => (4, transmute((((bytes[index] as u32) & 0b00000111u32) << 18) + (((bytes[index + 1] as u32) & 0b00111111u32) << 12)
                    + (((bytes[index + 2] as u32) & 0b00111111u32) << 6) + ((bytes[index + 3] as u32) & 0b00111111u32))),
        _ => panic!("impossible branch"),
    }
}

/// input 2 absolute path, calculate p2's relative path to p1
fn to_relative(path1: &Path, path2: &Path) -> PathBuf {
    use std::path::Component;
    use std::path::Prefix;
    
    let (mut path1_components, mut path2_components) = (path1.components(), path2.components());
    let mut dotdot_count = 0;
    let mut c2_mores = Vec::new();
    // for each component pair (c1, c2)
    //     if c1 == c2, continue,
    //     for first c1 != c2, count dotdot = 0, start recording c2's component in c2_mores
    //     then for next Some(c1)s, add dotdot_count 1, until c1 is none
    //     and for next Some(c2)s, push c2_mores, until c2 is none
    loop {
        match (path1_components.next(), path2_components.next()) {
            (None, None) => break,
            (c1, c2) if c1 == c2 => continue,
            // to make `\\?\C:\` same as `C:\`
            // ... every time handling `\\?\` makes me think about life
            (Some(Component::Prefix(prefix_component1)), Some(Component::Prefix(prefix_component2))) => {
                match (prefix_component1.kind(), prefix_component2.kind()) {
                    (Prefix::VerbatimDisk(volumn_name1), Prefix::Disk(volumn_name2))
                    | (Prefix::Disk(volumn_name1), Prefix::VerbatimDisk(volumn_name2)) if volumn_name1 == volumn_name2 => continue,
                    _ => (),
                }
                return path2.to_owned(); // even prefix is not same, then quick return
            }
            (Some(_), Some(c2)) => { dotdot_count += 1; c2_mores.push(c2); }
            (Some(_), None) => { dotdot_count += 1; }
            (None, Some(c2)) => { c2_mores.push(c2); }
        }
    }
    let mut retval = vec![Component::ParentDir; dotdot_count]; // ... very rare `vec!` usage
    retval.extend(c2_mores);
    retval.into_iter().map(Component::as_os_str).collect()
}

pub struct SourceCodeIter<'a> {
    src: &'a SourceCode,
    index: usize,
}
impl<'a> SourceCodeIter<'a> {

    fn new(src: &'a SourceCode) -> SourceCodeIter<'a> { SourceCodeIter{ src, index: 0 } }

    /// Iterate through source code string
    ///
    /// ignores all bare or not bare CR, 
    /// return EOF_CHAR after EOF, fuse that
    pub fn next(&mut self) -> (char, CharPos) {
        loop {
            if self.index == self.src.src.len() {
                return (EOF_CHAR, CharPos::new(self.src.id, self.index));
            } else if self.src.src[self.index] == b'\r' {
                self.index += 1;
                continue;
            } else {
                let (len, ch) = char_at_index(&self.src.src, self.index);
                let retval = (ch, CharPos::new(self.src.id, self.index));
                self.index += len;
                return retval;
            }
        }
    }
}

/// Represents a source code file
#[derive(Eq, PartialEq)]
pub struct SourceCode {
    id: usize,
    name: PathBuf,            // absolute path
    src: Box<[u8]>,           // source code string, in form of owned byte slice, because no need of string methods, utf8 iterator also implemented here not depend on std
    endl_indexes: Vec<usize>, // line end byte indexes
}
impl fmt::Debug for SourceCode {
    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
        write!(f, "source-code#{} {}", self.id, self.get_relative_path().display())
    }
}
impl SourceCode {

    fn _get_endl_indexes(src: &str) -> Vec<usize> {
        src.char_indices().filter(|indice| indice.1 == '\n').map(|indice| indice.0).collect()
    }

    pub fn with_file_name<T>(id: usize, file_name: T) -> Result<SourceCode, CodeMapError> where T: Into<PathBuf> + Clone { // into for adapt String, Clone for construct error
        use std::io::Read;

        let file_name = file_name.into();
        let file_name = file_name.clone().canonicalize().map_err(|e| CodeMapError::CannotOpenFile(file_name.clone(), e))?;
        let mut src = String::new();
        let mut file = ::std::fs::File::open(&file_name).map_err(|e| CodeMapError::CannotOpenFile(file_name.clone(), e))?;
        let _ = file.read_to_string(&mut src).map_err(|e| CodeMapError::CannotReadFile(file_name.clone(), e))?;

        Ok(SourceCode{ 
            id, 
            name: file_name, 
            endl_indexes: Self::_get_endl_indexes(&src),
            src: src.into_bytes().into_boxed_slice(),
        })
    }

    /// pretend to be a with_file_name'd source code in test
    /// @param file_name: vector of path components start from cwd, use &str for &'static str in test oracle
    pub fn pretend_with_file_name(id: usize, file_name: Vec<&str>, src: &str) -> SourceCode {

        let mut file_path = ::std::env::current_dir().unwrap(); // if current dir fails, just go die
        file_path.extend(file_name);

        SourceCode{
            id,
            name: file_path,
            endl_indexes: Self::_get_endl_indexes(src),
            src: src.to_owned().into_bytes().into_boxed_slice(),
        }
    }
    pub fn with_test_str(id: usize, src: &str) -> SourceCode {
        SourceCode{ 
            id, 
            name: format!("<anon#{}>", id).into(),
            endl_indexes: Self::_get_endl_indexes(src),
            src: src.to_owned().into_bytes().into_boxed_slice(),
        }
    }

    pub fn iter<'a>(&'a self) -> SourceCodeIter<'a> { SourceCodeIter::new(self) }
}
impl SourceCode {

    pub fn get_file_id(&self) -> usize { self.id }
    
    pub fn get_absolute_path(&self) -> &Path { &self.name }
    pub fn get_directory(&self) -> PathBuf { self.name.parent().unwrap().to_owned() /* unwrap because it must be a valid file */ } 
    
    pub fn get_file_stem(&self) -> Option<&str> { 
        // flatten `Option<Option<str>` to `Option<String>`
        match self.name.file_stem() {
            Some(stem) => match stem.to_str() {
                Some(stem) => Some(stem),
                None => None,
            }, // < small story here, at 17/8/10:
            // in my 2e10e182, I added this method to be used in syntax and semantic's multiple file support,
            // I accidentally forget this comma
            // but it passes compile while I'm using the nightly build nightly-x86_64-pc-windows-msvc-17-8-1
            // in another environment I'm defaulting stable-x86_64-pc-windows-msvc-1.19.0, and this is a compile error, which is, I think should be the correct behaviour
            // which also breaks my travis test
            // digging into rustc's parser's history I found rust-lang/rust#40989, which allows no comma here, after block-like (not only block) expressions
            // this feature is committed in a forked library at 17/4/1, and the pull request is merged at about 17/7/19, while stable-1.19.0 is released at 17/7/17
            // I learned how to blame a code change and how rust's language feature is discussed and implemented and finally released, which is very interesting
            None => None,
        }
    }

    pub fn get_relative_path(&self) -> PathBuf { 
        if &format!("{}", self.name.display()).as_bytes()[..5] == b"<anon" { 
            self.name.clone()
        } else {
            // if getcwd failed, then let it go
            to_relative(&::std::env::current_dir().unwrap(), &self.name) 
        }
    }
}
impl SourceCode {
    /// map byte index to (row, column)
    pub fn map_index(&self, charpos: CharPos) -> (usize, usize) {
        assert_eq!{ self.id, charpos.file_id, "incorrect file id" }
        
        let char_id = charpos.char_id;
        
        // get row number and row start character's byte index
        let (row_num, row_start_id) = if self.endl_indexes.len() == 0 {
            (1, 0)
{'fraction_non_alphanumeric': 0.14050308008213552, 'fraction_numerical': 0.03506160164271047, 'mean_word_length': 2.4016064257028114, 'pattern_counts': {'":': 0, '<': 66, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1036948', 'n_tokens_mistral': 6652, 'n_tokens_neox': 6221, 'n_words': 1804}