<commit_before>#!/usr/bin/python3

import os
import re
import sys
import json
import arrow
import markdown
import unicodedata
from collections import namedtuple
from bs4 import BeautifulSoup


def get_properties(properties_file="properties.json"):
    if os.path.exists(properties_file):
        with open(properties_file) as f:
            return json.loads(f.read())


def generate_pages(properties):
    proj_root = properties.get('root', '/')
    homepage = properties.get('homepage', 'index.html')
    searchpage = properties.get('searchpage', 'search.html')
    site_title = properties.get('site_title', 'PyQuo')
    ts_frmt = properties.get('timestamp_format', 'dddd DD MMMM, YYYY')
    directory = properties.get('source_directory', 'markdown')
    css = properties.get('css')
    entries_to_show = properties.get('entries_to_show', 10)

    all_pages = []
    categories = []
    count = 0

    for root, dirs, files in os.walk(directory):
        valid_files = [file for file in files if not
                       (file.endswith('~') or file.startswith('.'))]

        for valid_file in valid_files:
            with open(os.path.join(root, valid_file), 'r') as input_file:
                md = markdown.Markdown(extensions=['markdown.extensions.meta'])
                markdown_text = input_file.read()
                html = md.convert(markdown_text)
                text = ''.join(BeautifulSoup(html).findAll(text=True))
            if md.Meta == {}:
                continue

            meta = namedtuple('meta', [])
            meta.title = md.Meta['title'][0]
            meta.slug = slugify(meta.title)
            meta.categories = [
                category for category in set([cat.replace(" ", "").lower()
                for cat in md.Meta['category'][0].split(',')])]
            meta.authors = md.Meta['authors'][0]
            meta.date = md.Meta['date'][0]
            meta.tags = md.Meta['tags'][0].split(',')
            meta.header_image = md.Meta['headerimage'][0]
            meta.publish = True if md.Meta['publish'][0].lower() in [
                'true', 'yes'] else False
            meta.content = html
            meta.index = re.findall("\w+", text.lower())
            categories.extend(meta.categories)

            if meta.publish is True:
                generate_static_page(site_title, homepage, searchpage, meta,
                                     css, ts_frmt, proj_root)
                all_pages.append(meta)
                count += 1
        all_cats = [category for category in set(categories)]
        print('{} pages generated with the following categories: {}.'
              .format(count, ', '.join(all_cats)))
        all_cats, tags, chronology, index, word_cloud =\
            extract_site_wide_metadata(all_pages)
        generate_front_and_category_pages(site_title, homepage, searchpage, all_cats, tags,
                            chronology, word_cloud, proj_root, css, ts_frmt,
                            entries_to_show)
        generate_search_page(site_title, homepage, searchpage, index, css,
                             proj_root)


def extract_site_wide_metadata(all_pages, key='index_and_tags',
                               ignore_threshold_percentage=40):
    """Get site-wide metadata.

    - Arrange titles by inverse date order, with link to location.
    - Generate tag cloud (dict of tag and number of times used).
    - Generate calendar so can click on a date and get link.
    """

    categories = []
    tags = {}
    chronology = {}
    index = {}
    for page in all_pages:
        for cat in page.categories:
            category = cat.lower()
            if category not in categories:
                categories.append(category)

            for tag in page.tags:
                if tag not in tags:
                    tags[tag] = 1
                else:
                    tags[tag] += 1


            if page.date not in chronology:
                link = "{}/{}.html".format(category, page.slug)

                if page.date != '':
                    timestamp = arrow.get(page.date)
                else:<commit_msg>tags should be lowercase<commit_after>#!/usr/bin/python3

import os
import re
import sys
import json
import arrow
import markdown
import unicodedata
from collections import namedtuple
from bs4 import BeautifulSoup


def get_properties(properties_file="properties.json"):
    if os.path.exists(properties_file):
        with open(properties_file) as f:
            return json.loads(f.read())


def generate_pages(properties):
    proj_root = properties.get('root', '/')
    homepage = properties.get('homepage', 'index.html')
    searchpage = properties.get('searchpage', 'search.html')
    site_title = properties.get('site_title', 'PyQuo')
    ts_frmt = properties.get('timestamp_format', 'dddd DD MMMM, YYYY')
    directory = properties.get('source_directory', 'markdown')
    css = properties.get('css')
    entries_to_show = properties.get('entries_to_show', 10)

    all_pages = []
    categories = []
    count = 0

    for root, dirs, files in os.walk(directory):
        valid_files = [file for file in files if not
                       (file.endswith('~') or file.startswith('.'))]

        for valid_file in valid_files:
            with open(os.path.join(root, valid_file), 'r') as input_file:
                md = markdown.Markdown(extensions=['markdown.extensions.meta'])
                markdown_text = input_file.read()
                html = md.convert(markdown_text)
                text = ''.join(BeautifulSoup(html).findAll(text=True))
            if md.Meta == {}:
                continue

            meta = namedtuple('meta', [])
            meta.title = md.Meta['title'][0]
            meta.slug = slugify(meta.title)
            meta.categories = [
                category for category in set([cat.replace(" ", "").lower()
                for cat in md.Meta['category'][0].split(',')])]
            meta.authors = md.Meta['authors'][0]
            meta.date = md.Meta['date'][0]
            meta.tags = md.Meta['tags'][0].split(',')
            meta.header_image = md.Meta['headerimage'][0]
            meta.publish = True if md.Meta['publish'][0].lower() in [
                'true', 'yes'] else False
            meta.content = html
            meta.index = re.findall("\w+", text.lower())
            categories.extend(meta.categories)

            if meta.publish is True:
                generate_static_page(site_title, homepage, searchpage, meta,
                                     css, ts_frmt, proj_root)
                all_pages.append(meta)
                count += 1
        all_cats = [category for category in set(categories)]
        print('{} pages generated with the following categories: {}.'
              .format(count, ', '.join(all_cats)))
        all_cats, tags, chronology, index, word_cloud =\
            extract_site_wide_metadata(all_pages)
        generate_front_and_category_pages(site_title, homepage, searchpage, all_cats, tags,
                            chronology, word_cloud, proj_root, css, ts_frmt,
                            entries_to_show)
        generate_search_page(site_title, homepage, searchpage, index, css,
                             proj_root)


def extract_site_wide_metadata(all_pages, key='index_and_tags',
                               ignore_threshold_percentage=40):
    """Get site-wide metadata.

    - Arrange titles by inverse date order, with link to location.
    - Generate tag cloud (dict of tag and number of times used).
    - Generate calendar so can click on a date and get link.
    """

    categories = []
    tags = {}
    chronology = {}
    index = {}
    for page in all_pages:
        for cat in page.categories:
            category = cat.lower()
            if category not in categories:
                categories.append(category)

            for _tag in page.tags:
                tag = _tag.strip().lower()
                if tag not in tags:
                    tags[tag] = 1
                else:
                    tags[tag] += 1


            if page.date not in chronology:
                link = "{}/{}.html".format(category, page.slug)

                if page.date != '':
                    timestamp = arrow.get(page.date)
                else:
{'fraction_non_alphanumeric': 0.11440626883664859, 'fraction_numerical': 0.004098854731766124, 'mean_word_length': 1.97454284689853, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6583905', 'n_tokens_mistral': 2368, 'n_tokens_neox': 2260, 'n_words': 564}