<commit_before># Builder

load modules

    {readFileSync, writeFileSync} = require 'fs'
    {renderFile} = require 'jade'
    {safeLoad} = require 'js-yaml'
    marked = require 'marked'

render html

    options = safeLoad readFileSync process.argv[3], 'utf8'
    options.markdown = marked
    html = renderFile process.argv[2], options

write the output

    if process.argv[4]?<commit_msg>custom nbsp parsing in markdown<commit_after># Builder

### Load modules

    {readFileSync, writeFileSync} = require 'fs'
    {renderFile} = require 'jade'
    {safeLoad} = require 'js-yaml'
    marked = require 'marked'

### Add custom `nbsp` support

Extend the default `renderer.paragraph` method
by first replacing all `~` characters
(which I don't use in paragraphs) with `&nbsp;`.
If I want to add a `~` character, I can write `&#126;`.

    renderer = new marked.Renderer()
    {paragraph} = renderer
    renderer.paragraph = (text) ->
        return paragraph text.replace /~/g, '&nbsp;'

Override the `marked` function by a new one with the custom renderer. 

    marked = do (marked) ->
        return (text) ->
            return marked text,
                renderer: renderer

### Render html

    options = safeLoad readFileSync process.argv[3], 'utf8'
    options.markdown = marked
    html = renderFile process.argv[2], options

### Write the output

    if process.argv[4]?
{'fraction_non_alphanumeric': 0.11416184971098266, 'fraction_numerical': 0.007947976878612716, 'mean_word_length': 3.1842900302114803, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6405143', 'n_tokens_mistral': 433, 'n_tokens_neox': 412, 'n_words': 140}