<commit_before>import datetime
import os

import markdown2
import pytz
from varsnap import varsnap

from app.util import cached_function


# See https://github.com/trentm/python-markdown2/wiki/Extras
MARKDOWN_EXTRAS = [
    'code-friendly',
    'fenced-code-blocks',
    'smarty-pants',
    'tables',
]


class Note(object):
    def __init__(self):
        self.note_file = ''
        self.title = ''
        self.slug = ''
        self.time = ''
        self.note = ''
        self.markdown = ''

    @staticmethod
    def get_note_file_data(note_file, timezone):
        with open(note_file) as note_handle:
            lines = note_handle.readlines()
        lines = [line.strip("\n") for line in lines]
        if len(lines) < 4 or not lines[4].isdigit():
            return None
        note_parsed = Note()
        note_parsed.note_file = note_file
        note_parsed.title = lines[0]
        note_parsed.slug = lines[2]
        note_parsed.parse_time(lines[4], timezone)
        note_parsed.parse_markdown(lines[6:])
        note_parsed.markdown = '\n'.join(lines[6:])
        return note_parsed

    @varsnap
    def parse_time(self, timestamp, timezone):
        timestamp = int(timestamp)
        self.time = datetime.datetime.fromtimestamp(timestamp, timezone)
        return self.time

    @varsnap
    def parse_markdown(self, note):
        self.note = markdown2.markdown(
            "\n".join(note),
            extras=MARKDOWN_EXTRAS,
        )
        return self.note


@varsnap
def prune_note_files(note_files):
    def is_valid_note(note_file):
        if '~' in note_file:
            return False
        if note_file[0] == '.':
            return False
        return True
    files = [note_file for note_file in note_files if is_valid_note(note_file)]
    return files


def get_note_files():
    current_directory = os.path.dirname(os.path.realpath(__file__))
    notes_directory = os.path.join(current_directory, 'notes')
    files = os.listdir(notes_directory)
    files.sort(reverse=True)
    files = prune_note_files(files)<commit_msg>Refactor Note parse_time and parse_markdown<commit_after>import datetime
import os

import markdown2
import pytz
from varsnap import varsnap

from app.util import cached_function


# See https://github.com/trentm/python-markdown2/wiki/Extras
MARKDOWN_EXTRAS = [
    'code-friendly',
    'fenced-code-blocks',
    'smarty-pants',
    'tables',
]


class Note(object):
    def __init__(self):
        self.note_file = ''
        self.title = ''
        self.slug = ''
        self.time = ''
        self.note = ''
        self.markdown = ''

    @staticmethod
    def get_note_file_data(note_file, timezone):
        with open(note_file) as note_handle:
            lines = note_handle.readlines()
        lines = [line.strip("\n") for line in lines]
        if len(lines) < 4 or not lines[4].isdigit():
            return None
        note_parsed = Note()
        note_parsed.note_file = note_file
        note_parsed.title = lines[0]
        note_parsed.slug = lines[2]
        note_parsed.time = Note.parse_time(lines[4], timezone)
        note_parsed.note = Note.parse_markdown(lines[6:])
        note_parsed.markdown = '\n'.join(lines[6:])
        return note_parsed

    @staticmethod
    @varsnap
    def parse_time(timestamp, timezone):
        timestamp = int(timestamp)
        time = datetime.datetime.fromtimestamp(timestamp, timezone)
        return time

    @staticmethod
    @varsnap
    def parse_markdown(markdown):
        note = markdown2.markdown(
            "\n".join(markdown),
            extras=MARKDOWN_EXTRAS,
        )
        return note


@varsnap
def prune_note_files(note_files):
    def is_valid_note(note_file):
        if '~' in note_file:
            return False
        if note_file[0] == '.':
            return False
        return True
    files = [note_file for note_file in note_files if is_valid_note(note_file)]
    return files


def get_note_files():
    current_directory = os.path.dirname(os.path.realpath(__file__))
    notes_directory = os.path.join(current_directory, 'notes')
    files = os.listdir(notes_directory)
    files.sort(reverse=True)
    files = prune_note_files(files)
{'fraction_non_alphanumeric': 0.10528823249166269, 'fraction_numerical': 0.005240590757503573, 'mean_word_length': 2.5434599156118143, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1018146', 'n_tokens_mistral': 1339, 'n_tokens_neox': 1326, 'n_words': 269}