<commit_before>import datetime
import os

import dotenv
from getenv import env
import markdown2
import pytz

root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
dotenv.read_dotenv(os.path.join(root_path, '.env'))

# See https://github.com/trentm/python-markdown2/wiki/Extras
MARKDOWN_EXTRAS = [
    'code-friendly',
    'fenced-code-blocks',
    'smarty-pants',
    'tables',
]


def prune_note_files(note_files):
    def is_valid_note(note_file):
        if '~' in note_file:
            return False
        if note_file[0] == '.':
            return False
        return True
    files = [note_file for note_file in note_files if is_valid_note(note_file)]
    return files


def get_note_files():
    current_directory = os.path.dirname(os.path.realpath(__file__))
    notes_directory = os.path.join(current_directory, 'notes')
    files = os.listdir(notes_directory)
    files.sort(reverse=True)
    files = prune_note_files(files)
    files = [os.path.join(notes_directory, note_file) for note_file in files]
    return files


def get_note(note_file, timezone):
    with open(note_file) as note_handle:
        note = note_handle.readlines()
    note = [line.strip() for line in note]
    if len(note) < 3 or not note[1].isdigit():
        return None
    timestamp = int(note[1])
    note_parsed = {}
    note_parsed['title'] = note[0]
    note_parsed['time'] = datetime.datetime.fromtimestamp(
        timestamp, timezone)
    note_parsed['note'] = markdown2.markdown(
        "\n".join(note[2:]),
        extras=MARKDOWN_EXTRAS,
    )
    return note_parsed


def get_notes():
    note_files = get_note_files()<commit_msg>Update utils to accept slug field<commit_after>import datetime
import os

import dotenv
from getenv import env
import markdown2
import pytz

root_path = os.path.dirname(os.path.realpath(__file__)) + '/../'
dotenv.read_dotenv(os.path.join(root_path, '.env'))

# See https://github.com/trentm/python-markdown2/wiki/Extras
MARKDOWN_EXTRAS = [
    'code-friendly',
    'fenced-code-blocks',
    'smarty-pants',
    'tables',
]


def prune_note_files(note_files):
    def is_valid_note(note_file):
        if '~' in note_file:
            return False
        if note_file[0] == '.':
            return False
        return True
    files = [note_file for note_file in note_files if is_valid_note(note_file)]
    return files


def get_note_files():
    current_directory = os.path.dirname(os.path.realpath(__file__))
    notes_directory = os.path.join(current_directory, 'notes')
    files = os.listdir(notes_directory)
    files.sort(reverse=True)
    files = prune_note_files(files)
    files = [os.path.join(notes_directory, note_file) for note_file in files]
    return files


def get_note(note_file, timezone):
    with open(note_file) as note_handle:
        note = note_handle.readlines()
    note = [line.strip() for line in note]
    if len(note) < 4 or not note[2].isdigit():
        return None
    timestamp = int(note[2])
    note_parsed = {}
    note_parsed['title'] = note[0]
    note_parsed['slug'] = note[1]
    note_parsed['time'] = datetime.datetime.fromtimestamp(
        timestamp, timezone)
    note_parsed['note'] = markdown2.markdown(
        "\n".join(note[3:]),
        extras=MARKDOWN_EXTRAS,
    )
    return note_parsed


def get_notes():
    note_files = get_note_files()
{'fraction_non_alphanumeric': 0.12058647516457212, 'fraction_numerical': 0.005685218432076601, 'mean_word_length': 3.313548387096774, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '47471', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1137, 'n_words': 232}