<commit_before>use definitions::ImageBuilder;
use definitions::error::*;
use definitions::typedef::*;
use mnemonic::Mnemonic;
use regex::Regex;
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::BufReader;
use std::io::prelude::*;
use std::path::{Path, PathBuf};

lazy_static! {
    static ref LABELED_MNEMONIC_RE: Regex = Regex::new(r"^\.(.+?) +?(.+)$").unwrap();
    static ref LABEL_RE: Regex = Regex::new(r"^\.([^\s]+)$").unwrap();
    static ref INCLUDE_RE: Regex = Regex::new(r"^include! +([^\s]+)$").unwrap();
}

#[derive(Default)]
struct BASMCompiler {
    label_addr_map: HashMap<String, Address>,
    mnemonics: Vec<Mnemonic>,
    builder: ImageBuilder,
    last_base_dir: Option<PathBuf>,
    deep: usize,
}

impl BASMCompiler {
    fn add_label(&mut self, label: String) {
        self.label_addr_map.entry(label).or_insert(self.mnemonics.len());
    }

    fn base_path(&mut self, orig_path: &Path) -> Result<PathBuf> {
        let file_name = if let Some(ref file_name) = orig_path.file_name() {
            if let Some(..) = self.last_base_dir {
                PathBuf::from(orig_path)
            } else {
                PathBuf::from(file_name.clone())
            }
        } else {
            bail!("unable to obtain file name");
        };

        let absolute_path = if let Some(ref base_dir) = self.last_base_dir {
            base_dir.clone()
        } else {
            let current_dir = if orig_path.is_relative() {
                env::current_dir().chain_err(|| "unable to get current directory")?
            } else {
                if let Some(ref parent) = orig_path.parent() {
                    parent.to_path_buf().clone()
                } else {
                    bail!("unable to get parent directory")
                }
            };

            let base_file = current_dir.join(orig_path);
            let base_dir = if let Some(ref dir) = base_file.parent() {
                dir.clone()
            } else {
                bail!("unable to get parent directory")
            };

            self.last_base_dir = Some(base_dir.to_path_buf().clone());
            base_dir.to_path_buf()
        };

        Ok(absolute_path.join(file_name))
    }

    fn compile_mnemonics(&mut self, file_name: String) -> Result<()> {
        let orig_path = Path::new(&file_name);

        let padding = (0..self.deep).map(|_| " ").collect::<String>();
        println!("BASM\t{}{}", padding, file_name);

        let path = self.base_path(orig_path)?;

        let file = File::open(path).chain_err(|| "unable to open file")?;

        let reader = BufReader::new(file);

        for line in reader.lines() {
            let line = line.chain_err(|| "unable to read line")?;

            if line.is_empty() {
                continue;
            }

            let mut r_split = line.rsplitn(2, ';');

            if r_split.clone().count() > 1 {
                r_split.next();
            }

            if let Some(first_half) = r_split.next() {
                let first_half = first_half.trim();

                let first_half: String = if LABELED_MNEMONIC_RE.is_match(first_half) {
                    let captures = if let Some(captures) = LABELED_MNEMONIC_RE
                           .captures_iter(first_half)
                           .next() {
                        captures
                    } else {
                        bail!("no label capture found")
                    };

                    let label = captures[1].trim();

                    self.add_label(label.to_owned());

                    captures[2].trim().to_owned()
                } else if LABEL_RE.is_match(first_half) {
                    let captures =
                        if let Some(captures) = LABEL_RE.captures_iter(first_half).next() {
                            captures
                        } else {
                            bail!("no label capture found")
                        };

                    let label = captures[1].trim();

                    self.add_label(label.to_owned());
                    continue;
                } else if INCLUDE_RE.is_match(first_half) {
                    let captures =
                        if let Some(captures) = INCLUDE_RE.captures_iter(first_half).next() {
                            captures
                        } else {
                            bail!("no include capture found")
                        };

                    self.deep += 2;
                    self.compile_mnemonics(captures[1].trim().to_owned() + ".basm")?;
                    self.deep -= 2;<commit_msg>Adjusted BASM output<commit_after>use definitions::ImageBuilder;
use definitions::error::*;
use definitions::typedef::*;
use mnemonic::Mnemonic;
use regex::Regex;
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::BufReader;
use std::io::prelude::*;
use std::path::{Path, PathBuf};

lazy_static! {
    static ref LABELED_MNEMONIC_RE: Regex = Regex::new(r"^\.(.+?) +?(.+)$").unwrap();
    static ref LABEL_RE: Regex = Regex::new(r"^\.([^\s]+)$").unwrap();
    static ref INCLUDE_RE: Regex = Regex::new(r"^include! +([^\s]+)$").unwrap();
}

#[derive(Default)]
struct BASMCompiler {
    label_addr_map: HashMap<String, Address>,
    mnemonics: Vec<Mnemonic>,
    builder: ImageBuilder,
    last_base_dir: Option<PathBuf>,
    deep: usize,
}

impl BASMCompiler {
    fn add_label(&mut self, label: String) {
        self.label_addr_map.entry(label).or_insert(self.mnemonics.len());
    }

    fn base_path(&mut self, orig_path: &Path) -> Result<PathBuf> {
        let file_name = if let Some(ref file_name) = orig_path.file_name() {
            if let Some(..) = self.last_base_dir {
                PathBuf::from(orig_path)
            } else {
                PathBuf::from(file_name.clone())
            }
        } else {
            bail!("unable to obtain file name");
        };

        let absolute_path = if let Some(ref base_dir) = self.last_base_dir {
            base_dir.clone()
        } else {
            let current_dir = if orig_path.is_relative() {
                env::current_dir().chain_err(|| "unable to get current directory")?
            } else {
                if let Some(ref parent) = orig_path.parent() {
                    parent.to_path_buf().clone()
                } else {
                    bail!("unable to get parent directory")
                }
            };

            let base_file = current_dir.join(orig_path);
            let base_dir = if let Some(ref dir) = base_file.parent() {
                dir.clone()
            } else {
                bail!("unable to get parent directory")
            };

            self.last_base_dir = Some(base_dir.to_path_buf().clone());
            base_dir.to_path_buf()
        };

        Ok(absolute_path.join(file_name))
    }

    fn compile_mnemonics(&mut self, file_name: String) -> Result<()> {
        let orig_path = Path::new(&file_name);

        let padding = (0..self.deep).map(|_| "  ").collect::<String>();
        println!("BASM    {}{}", padding, file_name);

        let path = self.base_path(orig_path)?;

        let file = File::open(path).chain_err(|| "unable to open file")?;

        let reader = BufReader::new(file);

        for line in reader.lines() {
            let line = line.chain_err(|| "unable to read line")?;

            if line.is_empty() {
                continue;
            }

            let mut r_split = line.rsplitn(2, ';');

            if r_split.clone().count() > 1 {
                r_split.next();
            }

            if let Some(first_half) = r_split.next() {
                let first_half = first_half.trim();

                let first_half: String = if LABELED_MNEMONIC_RE.is_match(first_half) {
                    let captures = if let Some(captures) = LABELED_MNEMONIC_RE
                           .captures_iter(first_half)
                           .next() {
                        captures
                    } else {
                        bail!("no label capture found")
                    };

                    let label = captures[1].trim();

                    self.add_label(label.to_owned());

                    captures[2].trim().to_owned()
                } else if LABEL_RE.is_match(first_half) {
                    let captures =
                        if let Some(captures) = LABEL_RE.captures_iter(first_half).next() {
                            captures
                        } else {
                            bail!("no label capture found")
                        };

                    let label = captures[1].trim();

                    self.add_label(label.to_owned());
                    continue;
                } else if INCLUDE_RE.is_match(first_half) {
                    let captures =
                        if let Some(captures) = INCLUDE_RE.captures_iter(first_half).next() {
                            captures
                        } else {
                            bail!("no include capture found")
                        };

                    self.deep += 1;
                    self.compile_mnemonics(captures[1].trim().to_owned() + ".basm")?;
                    self.deep -= 1;
{'fraction_non_alphanumeric': 0.1294054054054054, 'fraction_numerical': 0.0019459459459459458, 'mean_word_length': 1.5775982167734746, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1821141', 'n_tokens_mistral': 2788, 'n_tokens_neox': 2648, 'n_words': 531}