<commit_before>#! /usr/bin/env python3

import argparse
import json
import os
from string import Template
import sys
import urllib

import colorama
import requests
import markdown
import rfc6266

import utilities
from trello import Trello


SETTINGS = json.load(open("settings.json.template", "r"))


def get_setting(setting):
    global SETTINGS
    return SETTINGS[setting]


def load_settings():
    global SETTINGS
    try:
        SETTINGS = json.load(open("settings.json", "r"))
    except:
        pass


def get_attachment(url):
    url = utilities.fix_google_drive_download_url(url)
    url = utilities.fix_dropbox_download_url(url)

    response = requests.get(url)
    mimetype = response.headers.get("content-type").lower().split(";")[0].strip()
    filename = rfc6266.parse_requests_response(response).filename_unsafe

    if not mimetype.startswith("text/") and not mimetype.startswith("image/"):
        raise Exception("Unhandled file type {0}, {1}, {2}".format(url, mimetype, filename))

    if mimetype.startswith("text/"):
        response.encoding = "utf-8"

    attachment = {
        "name": filename,
        "mime": mimetype,
        "url": response.url
    }

    if mimetype.startswith("text/"):
        attachment["content"] = response.text
    else:
        attachment["content"] = response.content

    return attachment


def generate(trello_list):
    articles = []

    for number, trello_card in enumerate(trello_list, start=1):

        print("\n" + str(number) + ".", trello_card["name"] + ":")

        article = {
            "title": trello_card["name"],
            "content": "",
            "images": [],
            "labels": []
        }

        # Add content
        for trello_attachment in trello_card["attachments"]:

            attachment = get_attachment(trello_attachment["url"])

            if attachment["mime"].startswith("text/"):
                article["content"] += attachment["content"]
                print(" - Markdown from", attachment["url"])
            else:
                article["images"].append(attachment)
                print(" - Image from file", attachment["url"])

        # Fallback on description
        if article["content"] == "":
            article["content"] = trello_card["desc"]

        if article["content"] == "":
            print(" - Status:", colorama.Fore.RED + "Warning (no content)", colorama.Fore.RESET)
            continue

        # Add labels
        for label in trello_card["labels"]:
            article["labels"].append(label["name"] if label["name"] != "" else "color-" + label["color"])

        print(" - Labels:", colorama.Fore.MAGENTA + ", ".join(article["labels"]), colorama.Fore.RESET)
        print(" - Status:", colorama.Fore.GREEN + "Ok", colorama.Fore.RESET)

        articles.append(article)

    # Create output folder
    if not os.path.exists(get_setting("folder")):
        os.makedirs(get_setting("folder"))

    # Generate HTML from Markdown
    markdown_text = ""
    for article in articles:
        classes = ""
        if get_setting("features")["labels"]:
            classes = " ".join(article["labels"])
        markdown_text += '<section class="' + classes + '" markdown="1">\n'
        markdown_text += article["content"] + "\n"
        markdown_text += "</section>\n\n"

        if get_setting("features")["lines"] and "noline" not in article["labels"]:
            markdown_text += "------------------------------------\n\n"
<commit_msg>Fix: no line after last section<commit_after>#! /usr/bin/env python3

import argparse
import json
import os
from string import Template
import sys
import urllib

import colorama
import requests
import markdown
import rfc6266

import utilities
from trello import Trello


SETTINGS = json.load(open("settings.json.template", "r"))


def get_setting(setting):
    global SETTINGS
    return SETTINGS[setting]


def load_settings():
    global SETTINGS
    try:
        SETTINGS = json.load(open("settings.json", "r"))
    except:
        pass


def get_attachment(url):
    url = utilities.fix_google_drive_download_url(url)
    url = utilities.fix_dropbox_download_url(url)

    response = requests.get(url)
    mimetype = response.headers.get("content-type").lower().split(";")[0].strip()
    filename = rfc6266.parse_requests_response(response).filename_unsafe

    if not mimetype.startswith("text/") and not mimetype.startswith("image/"):
        raise Exception("Unhandled file type {0}, {1}, {2}".format(url, mimetype, filename))

    if mimetype.startswith("text/"):
        response.encoding = "utf-8"

    attachment = {
        "name": filename,
        "mime": mimetype,
        "url": response.url
    }

    if mimetype.startswith("text/"):
        attachment["content"] = response.text
    else:
        attachment["content"] = response.content

    return attachment


def generate(trello_list):
    articles = []

    for number, trello_card in enumerate(trello_list, start=1):

        print("\n" + str(number) + ".", trello_card["name"] + ":")

        article = {
            "title": trello_card["name"],
            "content": "",
            "images": [],
            "labels": []
        }

        # Add content
        for trello_attachment in trello_card["attachments"]:

            attachment = get_attachment(trello_attachment["url"])

            if attachment["mime"].startswith("text/"):
                article["content"] += attachment["content"]
                print(" - Markdown from", attachment["url"])
            else:
                article["images"].append(attachment)
                print(" - Image from file", attachment["url"])

        # Fallback on description
        if article["content"] == "":
            article["content"] = trello_card["desc"]

        if article["content"] == "":
            print(" - Status:", colorama.Fore.RED + "Warning (no content)", colorama.Fore.RESET)
            continue

        # Add labels
        for label in trello_card["labels"]:
            article["labels"].append(label["name"] if label["name"] != "" else "color-" + label["color"])

        print(" - Labels:", colorama.Fore.MAGENTA + ", ".join(article["labels"]), colorama.Fore.RESET)
        print(" - Status:", colorama.Fore.GREEN + "Ok", colorama.Fore.RESET)

        articles.append(article)

    # Create output folder
    if not os.path.exists(get_setting("folder")):
        os.makedirs(get_setting("folder"))

    # Generate HTML from Markdown
    markdown_text = ""
    for index, article in enumerate(articles):
        classes = ""
        if get_setting("features")["labels"]:
            classes = " ".join(article["labels"])
        markdown_text += '<section class="' + classes + '" markdown="1">\n'
        markdown_text += article["content"] + "\n"
        markdown_text += "</section>\n\n"

        if get_setting("features")["lines"] and "noline" not in article["labels"] and index != len(articles) - 1:
            markdown_text += "------------------------------------\n\n"

{'fraction_non_alphanumeric': 0.1555682304292029, 'fraction_numerical': 0.004705546841579923, 'mean_word_length': 2.5932377049180326, 'pattern_counts': {'":': 20, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1954131', 'n_tokens_mistral': 2090, 'n_tokens_neox': 1957, 'n_words': 446}