<commit_before>/*!
 * Processing the markdown and resolving references.
 */

use std::{
    collections::HashSet,
    fs::File,
    io::{Read, Write},
    path::{Path, PathBuf},
};

use pulldown_cmark::{html, Event, LinkType, Parser, Tag};
use url::Url;

use crate::config::ResolvedConfig;

struct RenderAdapter<'a, 'b, 'c: 'a, I: Iterator<Item = Event<'b>>> {
    ctx: &'a mut ProcessorContext<'a, 'c>,
    iter: I,
}

impl<'a, 'b, 'c: 'a, I: Iterator<Item = Event<'b>>> Iterator for RenderAdapter<'a, 'b, 'c, I> {
    type Item = Event<'b>;

    fn next(&mut self) -> Option<Self::Item> {
        let item = self.iter.next()?;
        let styles = &mut self.ctx.styles;
        let render_stack = &mut *self.ctx.render_stack;
        let finished = self.ctx.finished;
        let out_dir = &self.ctx.config.roots.output;
        let base_dir = &self.ctx.config.roots.source;
        let filename = self.ctx.filename;
        Some(match item {
            Event::Start(tag) => match tag {
                i @ Tag::Image(..) => {
                    styles.insert("image");
                    Event::Start(i)
                }
                p @ Tag::Paragraph => {
                    styles.insert("paragraph");
                    Event::Start(p)
                }
                Tag::Link(ty, url, title) => {
                    styles.insert("link");
                    match ty {
                        LinkType::Inline => {
                            if let Ok(parsed) = Url::parse(&url) {
                                // check if scheme is hyperref, if so add to stack and rewrite url
                                if parsed.scheme() == "hyperref" {
                                    let parsed_path: &Path = parsed.path().as_ref();
                                    let fname: PathBuf = if parsed_path.is_absolute() {
                                        out_dir.join(parsed_path.strip_prefix("/").unwrap())
                                    } else {
                                        filename
                                            .parent()
                                            .unwrap_or("/".as_ref())
                                            .join(parsed.path())
                                    }
                                    .with_extension("md");
                                    #[cfg(target_os = "windows")]
                                    // replace with backslashes so that \\?\ isn't broken
                                    let fname: PathBuf =
                                        fname.to_str().unwrap().replace("/", "\\").into();
                                    if let Ok(fname) = fname.canonicalize() {
                                        let fname_for_url = fname.strip_prefix(&base_dir).unwrap();
                                        #[cfg(target_os = "windows")]
                                    // windows is dumb again
                                    let fname_for_url: PathBuf =
                                        fname_for_url.to_str().unwrap().replace("\\", "/").into();
                                        // figure out new location
                                        let new_location = format!(
                                            "/{}",
                                            fname_for_url.with_extension("html").to_str().unwrap(),
                                        );
                                        let input = RenderingInput::Other(fname);
                                        if !render_stack.contains(&input)
                                            && !finished.contains(&input)
                                        {
                                            match input {
                                                RenderingInput::Other(ref fname) => println!(
                                                    "walk: {}",
                                                    fname.to_str().unwrap_or("unknown")
                                                ),
                                                _ => {}
                                            }
                                            render_stack.push(input);
                                        }
                                        Event::Start(Tag::Link(ty, new_location.into(), title))
                                        // link.url = new_location.into_bytes();
                                    } else {
                                        println!("Couldn't resolve hyperref: {}", url);
                                        Event::Start(Tag::Link(ty, url, title))
                                    }
                                } else {
                                    Event::Start(Tag::Link(ty, url, title))
                                }
                            } else {
                                Event::Start(Tag::Link(ty, url, title))
                            }
                        }
                        _ => Event::Start(Tag::Link(ty, url, title)),
                    }
                }
                t => Event::Start(t),
            },
            ev => ev,
        })
    }
}

/// Processing context for a single file
pub struct ProcessorContext<'a, 'b: 'a> {
    styles: &'a mut HashSet<&'b str>,
    filename: &'a Path,
    config: &'a ResolvedConfig,
    finished: &'a HashSet<RenderingInput>,
    render_stack: &'a mut Vec<RenderingInput>,
}

/// Rendering input
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
pub enum RenderingInput {
    Index,
    Keep,
    Other(PathBuf),
}

/// Processes files
pub struct Processor {
    /// Stuff is derived from this
    config: ResolvedConfig,
    // next items to render
    render_stack: Vec<RenderingInput>,
    // items that have already been rendered
    finished: HashSet<RenderingInput>,
}

impl Processor {
    pub fn new(config: ResolvedConfig) -> Self {
        Self {
            config,
            render_stack: Default::default(),
            finished: Default::default(),
        }
    }

    pub fn render_toplevel(&mut self, force: bool) -> anyhow::Result<()> {
        self.render_stack.extend(std::array::IntoIter::new([
            RenderingInput::Index,
            RenderingInput::Keep,
        ]));
        self.render_all(force)?;
        Ok(())
    }

    fn render_all(&mut self, force: bool) -> anyhow::Result<()> {
        while !self.render_stack.is_empty() {
            let stack = std::mem::take(&mut self.render_stack);
            for input in stack {
                println!("RENDER: {:?}", input);
                self.render(input, force)?;
            }
        }
        Ok(())
    }

    pub fn render(&mut self, input: RenderingInput, force: bool) -> anyhow::Result<()> {
        let out_dir = &self.config.roots.output;
        let base_dir = &self.config.roots.source;
        let style_chunks_root = &self.config.lib.styles.chunks_root;
        let prelude_html = &self.config.lib.prelude_location;
        let filename = match input {
            RenderingInput::Index => &self.config.inputs.index,
            RenderingInput::Keep => &self.config.inputs.keep,
            RenderingInput::Other(ref o) => o,
        };

        if !filename.exists() {
            println!(
                "nonexistent: {}, nothing to do",
                filename.to_str().unwrap_or("unknown")
            );
            return Ok(());
        }

        // create out dir if doesn't exist
        if !out_dir.exists() {<commit_msg>Use VecDeque, allows dedup to work properly<commit_after>/*!
 * Processing the markdown and resolving references.
 */

use std::{
    collections::{HashSet, VecDeque},
    fs::File,
    io::{Read, Write},
    path::{Path, PathBuf},
};

use pulldown_cmark::{html, Event, LinkType, Parser, Tag};
use url::Url;

use crate::config::ResolvedConfig;

struct RenderAdapter<'a, 'b, 'c: 'a, I: Iterator<Item = Event<'b>>> {
    ctx: &'a mut ProcessorContext<'a, 'c>,
    iter: I,
}

impl<'a, 'b, 'c: 'a, I: Iterator<Item = Event<'b>>> Iterator for RenderAdapter<'a, 'b, 'c, I> {
    type Item = Event<'b>;

    fn next(&mut self) -> Option<Self::Item> {
        let item = self.iter.next()?;
        let styles = &mut self.ctx.styles;
        let render_stack = &mut *self.ctx.render_stack;
        let finished = self.ctx.finished;
        let out_dir = &self.ctx.config.roots.output;
        let base_dir = &self.ctx.config.roots.source;
        let filename = self.ctx.filename;
        Some(match item {
            Event::Start(tag) => match tag {
                i @ Tag::Image(..) => {
                    styles.insert("image");
                    Event::Start(i)
                }
                p @ Tag::Paragraph => {
                    styles.insert("paragraph");
                    Event::Start(p)
                }
                Tag::Link(ty, url, title) => {
                    styles.insert("link");
                    match ty {
                        LinkType::Inline => {
                            if let Ok(parsed) = Url::parse(&url) {
                                // check if scheme is hyperref, if so add to stack and rewrite url
                                if parsed.scheme() == "hyperref" {
                                    let parsed_path: &Path = parsed.path().as_ref();
                                    let fname: PathBuf = if parsed_path.is_absolute() {
                                        out_dir.join(parsed_path.strip_prefix("/").unwrap())
                                    } else {
                                        filename
                                            .parent()
                                            .unwrap_or("/".as_ref())
                                            .join(parsed.path())
                                    }
                                    .with_extension("md");
                                    #[cfg(target_os = "windows")]
                                    // replace with backslashes so that \\?\ isn't broken
                                    let fname: PathBuf =
                                        fname.to_str().unwrap().replace("/", "\\").into();
                                    if let Ok(fname) = fname.canonicalize() {
                                        let fname_for_url = fname.strip_prefix(&base_dir).unwrap();
                                        #[cfg(target_os = "windows")]
                                    // windows is dumb again
                                    let fname_for_url: PathBuf =
                                        fname_for_url.to_str().unwrap().replace("\\", "/").into();
                                        // figure out new location
                                        let new_location = format!(
                                            "/{}",
                                            fname_for_url.with_extension("html").to_str().unwrap(),
                                        );
                                        let input = RenderingInput::Other(fname);
                                        if !render_stack.contains(&input)
                                            && !finished.contains(&input)
                                        {
                                            match input {
                                                RenderingInput::Other(ref fname) => println!(
                                                    "walk: {}",
                                                    fname.to_str().unwrap_or("unknown")
                                                ),
                                                _ => {}
                                            }
                                            render_stack.push_back(input);
                                        }
                                        Event::Start(Tag::Link(ty, new_location.into(), title))
                                        // link.url = new_location.into_bytes();
                                    } else {
                                        println!("Couldn't resolve hyperref: {}", url);
                                        Event::Start(Tag::Link(ty, url, title))
                                    }
                                } else {
                                    Event::Start(Tag::Link(ty, url, title))
                                }
                            } else {
                                Event::Start(Tag::Link(ty, url, title))
                            }
                        }
                        _ => Event::Start(Tag::Link(ty, url, title)),
                    }
                }
                t => Event::Start(t),
            },
            ev => ev,
        })
    }
}

/// Processing context for a single file
pub struct ProcessorContext<'a, 'b: 'a> {
    styles: &'a mut HashSet<&'b str>,
    filename: &'a Path,
    config: &'a ResolvedConfig,
    finished: &'a HashSet<RenderingInput>,
    render_stack: &'a mut VecDeque<RenderingInput>,
}

/// Rendering input
#[derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)]
pub enum RenderingInput {
    Index,
    Keep,
    Other(PathBuf),
}

/// Processes files
pub struct Processor {
    /// Stuff is derived from this
    config: ResolvedConfig,
    // next items to render
    render_stack: VecDeque<RenderingInput>,
    // items that have already been rendered
    finished: HashSet<RenderingInput>,
}

impl Processor {
    pub fn new(config: ResolvedConfig) -> Self {
        Self {
            config,
            render_stack: Default::default(),
            finished: Default::default(),
        }
    }

    pub fn render_toplevel(&mut self, force: bool) -> anyhow::Result<()> {
        self.render_stack.extend(std::array::IntoIter::new([
            RenderingInput::Index,
            RenderingInput::Keep,
        ]));
        self.render_all(force)?;
        Ok(())
    }

    fn render_all(&mut self, force: bool) -> anyhow::Result<()> {
        while let Some(input) = self.render_stack.pop_front() {
            println!("RENDER: {:?}", input);
            self.render(input, force)?;
        }
        Ok(())
    }

    pub fn render(&mut self, input: RenderingInput, force: bool) -> anyhow::Result<()> {
        let out_dir = &self.config.roots.output;
        let base_dir = &self.config.roots.source;
        let style_chunks_root = &self.config.lib.styles.chunks_root;
        let prelude_html = &self.config.lib.prelude_location;
        let filename = match input {
            RenderingInput::Index => &self.config.inputs.index,
            RenderingInput::Keep => &self.config.inputs.keep,
            RenderingInput::Other(ref o) => o,
        };

        if !filename.exists() {
            println!(
                "nonexistent: {}, nothing to do",
                filename.to_str().unwrap_or("unknown")
            );
            return Ok(());
        }

        // create out dir if doesn't exist
        if !out_dir.exists() {
{'fraction_non_alphanumeric': 0.11743630573248408, 'fraction_numerical': 0.0, 'mean_word_length': 1.0549420586230402, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 138, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1266464', 'n_tokens_mistral': 3610, 'n_tokens_neox': 3548, 'n_words': 775}