<jupyter_start><jupyter_text>Publish .md's from Brain (Obsidian database)<jupyter_code>%pip install python-frontmatter
from pathlib import Path

brain_dir = Path.home() / "Brain"
to_publish = []
for check_dir in (
    brain_dir / "Stream",
):
    assert check_dir.exists(), check_dir
    to_publish.extend(check_dir.rglob("*.md"))

to_publish
import re
import shutil
import frontmatter

blog_dir = Path.cwd()
stream_post_dir = blog_dir / "_posts/stream"
files_dir = blog_dir / "files"
wikilink_re = re.compile(r"!?\[\[([^\]]+)\]\]")
brain_name_re = re.compile(r"([0-9]{4})([0-9]{2})([0-9]{2})([0-9]{2})([0-9]{2})[\s-]+(.*)")
file_copies = []

class FileToCopy:
    def __init__(self, from_fp, to_fp, is_note=False):
        self.from_fp = from_fp
        self.to_fp = to_fp.relative_to(blog_dir)
        self.is_note = is_note
        self.replacements = {}
    
    def __repr__(self):
        return f"{self.from_fp} -> {self.to_fp}; is_note={self.is_note}; {len(self.replacements)} replacements"

    def add_replacement(self, replace_target, replace_dest):
        self.replacements[replace_target] = replace_dest
    
    def do_copy(self):
        if self.is_note:
            from_text = self.from_fp.read_text()
            to_text = wikilink_re.sub(lambda m: f"![{m.group(1)}]({self.replacements[m.group(1)]})", from_text)

            to_text_parsed = frontmatter.loads(to_text)
            if "title" not in to_text_parsed.metadata.keys():
                title = brain_name_re.match(self.from_fp.stem).group(6)
                to_text_parsed.metadata["title"] = title
            to_text = frontmatter.dumps(to_text_parsed)

            self.to_fp.write_text(to_text)
        else:
            shutil.copyfile(self.from_fp, self.to_fp)
            pass

def copy_note(note_fp, post_fp, title):
    note_ftc = FileToCopy(note_fp, post_fp, is_note=True)
    file_copies.append(note_ftc)
    note_text = note_fp.read_text()
    # register wikilinks (links in this format: [[some-other-file.png]])
    wikilink_paths = wikilink_re.findall(note_text)
    for wikilink_url in wikilink_paths:
        wikilink_fp = brain_dir / wikilink_url
        post_wikilink_fp = files_dir / (post_fp.name + "-" + wikilink_fp.name)
        file_copies.append(FileToCopy(wikilink_fp, post_wikilink_fp))
        note_ftc.add_replacement(wikilink_url, f"/{str(post_wikilink_fp.relative_to(blog_dir))}")

def newer(l, r):
    if not l.exists() or not r.exists():
        return False
    return l.lstat().st_mtime > r.lstat().st_mtime

for brain_fp in to_publish:
    m = brain_name_re.match(brain_fp.name)
    if m is None:
        continue
    year, month, day, hour, minute, title = m.groups()
    post_fp = stream_post_dir / (f"{year}-{month}-{day}-{title}")
    print(year, month, day, hour, minute, title, post_fp, post_fp.exists(), newer(brain_fp, post_fp))
    if not post_fp.exists() or newer(brain_fp, post_fp):
        copy_note(brain_fp, post_fp, title)

file_copies
for ftc in file_copies:
    ftc.do_copy()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13791968400263332, 'fraction_numerical': 0.005924950625411455, 'mean_word_length': 3.4043478260869566, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '72309', 'license': 'MIT', 'n_tokens_mistral': 1162, 'n_tokens_neox': 1144, 'n_words': 197, 'path': 'publish_brain.ipynb', 'repo_name': 'bstee615/bstee615.github.io', 'size': '3038'}