<commit_before>const _ = require('lodash');
const juice = require('juice');
const template = require('./template');
const labsTemplate = require('./template-labs');
const config = require('../../../shared/config');
const settingsCache = require('../../services/settings/cache');
const urlUtils = require('../../../shared/url-utils');
const moment = require('moment-timezone');
const cheerio = require('cheerio');
const api = require('../../api');
const {URL} = require('url');
const mobiledocLib = require('../../lib/mobiledoc');
const htmlToText = require('html-to-text');
const {isUnsplashImage, isLocalContentImage} = require('@tryghost/kg-default-cards/lib/utils');
const logging = require('../../../shared/logging');

const ALLOWED_REPLACEMENTS = ['first_name'];

const getSite = () => {
    const publicSettings = settingsCache.getPublic();
    return Object.assign({}, publicSettings, {
        url: urlUtils.urlFor('home', true),
        iconUrl: publicSettings.icon ? urlUtils.urlFor('image', {image: publicSettings.icon}, true) : null
    });
};

/**
 * createUnsubscribeUrl
 *
 * Takes a member uuid and returns the url that should be used to unsubscribe
 * In case of no member uuid, generates the preview unsubscribe url - `?preview=1`
 *
 * @param {string} uuid
 */
const createUnsubscribeUrl = (uuid) => {
    const siteUrl = urlUtils.getSiteUrl();
    const unsubscribeUrl = new URL(siteUrl);
    unsubscribeUrl.pathname = `${unsubscribeUrl.pathname}/unsubscribe/`.replace('//', '/');
    if (uuid) {
        unsubscribeUrl.searchParams.set('uuid', uuid);
    } else {
        unsubscribeUrl.searchParams.set('preview', '1');
    }

    return unsubscribeUrl.href;
};

// NOTE: serialization is needed to make sure we are using current API and do post transformations
//       such as image URL transformation from relative to absolute
const serializePostModel = async (model, apiVersion = 'v4') => {
    // fetch mobiledoc rather than html and plaintext so we can render email-specific contents
    const frame = {options: {context: {user: true}, formats: 'mobiledoc'}};
    const docName = 'posts';

    await api.shared
        .serializers
        .handle
        .output(model, {docName: docName, method: 'read'}, api[apiVersion].serializers.output, frame);

    return frame.response[docName][0];
};

// removes %% wrappers from unknown replacement strings in email content
const normalizeReplacementStrings = (email) => {
    // we don't want to modify the email object in-place
    const emailContent = _.pick(email, ['html', 'plaintext']);

    const EMAIL_REPLACEMENT_REGEX = /%%(\{.*?\})%%/g;
    const REPLACEMENT_STRING_REGEX = /\{(?<recipientProperty>\w*?)(?:,? *(?:"|&quot;)(?<fallback>.*?)(?:"|&quot;))?\}/;

    ['html', 'plaintext'].forEach((format) => {
        emailContent[format] = emailContent[format].replace(EMAIL_REPLACEMENT_REGEX, (replacementMatch, replacementStr) => {
            const match = replacementStr.match(REPLACEMENT_STRING_REGEX);

            if (match) {
                const {recipientProperty} = match.groups;

                if (ALLOWED_REPLACEMENTS.includes(recipientProperty)) {
                    // keeps wrapping %% for later replacement with real data
                    return replacementMatch;
                }
            }

            // removes %% so output matches user supplied content
            return replacementStr;
        });
    });

    return emailContent;
};

// parses email content and extracts an array of replacements with desired fallbacks
const parseReplacements = (email) => {
    const EMAIL_REPLACEMENT_REGEX = /%%(\{.*?\})%%/g;
    const REPLACEMENT_STRING_REGEX = /\{(?<recipientProperty>\w*?)(?:,? *(?:"|&quot;)(?<fallback>.*?)(?:"|&quot;))?\}/;

    const replacements = [];

    ['html', 'plaintext'].forEach((format) => {
        let result;
        while ((result = EMAIL_REPLACEMENT_REGEX.exec(email[format])) !== null) {
            const [replacementMatch, replacementStr] = result;
            const match = replacementStr.match(REPLACEMENT_STRING_REGEX);

            if (match) {
                const {recipientProperty, fallback} = match.groups;

                if (ALLOWED_REPLACEMENTS.includes(recipientProperty)) {
                    const id = `replacement_${replacements.length + 1}`;

                    replacements.push({
                        format,
                        id,
                        match: replacementMatch,
                        recipientProperty: `member_${recipientProperty}`,
                        fallback
                    });
                }
            }
        }
    });

    return replacements;
};

const getTemplateSettings = async () => {
    return {
        showSiteHeader: settingsCache.get('newsletter_show_header'),
        bodyFontCategory: settingsCache.get('newsletter_body_font_category'),
        showBadge: settingsCache.get('newsletter_show_badge'),
        footerContent: settingsCache.get('newsletter_footer_content'),
        accentColor: settingsCache.get('accent_color')
    };
};

const getLabsTemplateSettings = async () => {
    const templateSettings = {
        headerImage: settingsCache.get('newsletter_header_image'),
        showHeaderIcon: settingsCache.get('newsletter_show_header_icon'),
        showHeaderTitle: settingsCache.get('newsletter_show_header_title'),
        showFeatureImage: settingsCache.get('newsletter_show_feature_image'),
        titleFontCategory: settingsCache.get('newsletter_title_font_category'),
        titleAlignment: settingsCache.get('newsletter_title_alignment'),
        bodyFontCategory: settingsCache.get('newsletter_body_font_category'),
        showBadge: settingsCache.get('newsletter_show_badge'),
        footerContent: settingsCache.get('newsletter_footer_content'),
        accentColor: settingsCache.get('accent_color')
    };

    if (templateSettings.headerImage) {
        if (isUnsplashImage(templateSettings.headerImage)) {<commit_msg>Fixed email header divider showing when title+icon are disabled but no publication icon is set<commit_after>const _ = require('lodash');
const juice = require('juice');
const template = require('./template');
const labsTemplate = require('./template-labs');
const config = require('../../../shared/config');
const settingsCache = require('../../services/settings/cache');
const urlUtils = require('../../../shared/url-utils');
const moment = require('moment-timezone');
const cheerio = require('cheerio');
const api = require('../../api');
const {URL} = require('url');
const mobiledocLib = require('../../lib/mobiledoc');
const htmlToText = require('html-to-text');
const {isUnsplashImage, isLocalContentImage} = require('@tryghost/kg-default-cards/lib/utils');
const logging = require('../../../shared/logging');

const ALLOWED_REPLACEMENTS = ['first_name'];

const getSite = () => {
    const publicSettings = settingsCache.getPublic();
    return Object.assign({}, publicSettings, {
        url: urlUtils.urlFor('home', true),
        iconUrl: publicSettings.icon ? urlUtils.urlFor('image', {image: publicSettings.icon}, true) : null
    });
};

/**
 * createUnsubscribeUrl
 *
 * Takes a member uuid and returns the url that should be used to unsubscribe
 * In case of no member uuid, generates the preview unsubscribe url - `?preview=1`
 *
 * @param {string} uuid
 */
const createUnsubscribeUrl = (uuid) => {
    const siteUrl = urlUtils.getSiteUrl();
    const unsubscribeUrl = new URL(siteUrl);
    unsubscribeUrl.pathname = `${unsubscribeUrl.pathname}/unsubscribe/`.replace('//', '/');
    if (uuid) {
        unsubscribeUrl.searchParams.set('uuid', uuid);
    } else {
        unsubscribeUrl.searchParams.set('preview', '1');
    }

    return unsubscribeUrl.href;
};

// NOTE: serialization is needed to make sure we are using current API and do post transformations
//       such as image URL transformation from relative to absolute
const serializePostModel = async (model, apiVersion = 'v4') => {
    // fetch mobiledoc rather than html and plaintext so we can render email-specific contents
    const frame = {options: {context: {user: true}, formats: 'mobiledoc'}};
    const docName = 'posts';

    await api.shared
        .serializers
        .handle
        .output(model, {docName: docName, method: 'read'}, api[apiVersion].serializers.output, frame);

    return frame.response[docName][0];
};

// removes %% wrappers from unknown replacement strings in email content
const normalizeReplacementStrings = (email) => {
    // we don't want to modify the email object in-place
    const emailContent = _.pick(email, ['html', 'plaintext']);

    const EMAIL_REPLACEMENT_REGEX = /%%(\{.*?\})%%/g;
    const REPLACEMENT_STRING_REGEX = /\{(?<recipientProperty>\w*?)(?:,? *(?:"|&quot;)(?<fallback>.*?)(?:"|&quot;))?\}/;

    ['html', 'plaintext'].forEach((format) => {
        emailContent[format] = emailContent[format].replace(EMAIL_REPLACEMENT_REGEX, (replacementMatch, replacementStr) => {
            const match = replacementStr.match(REPLACEMENT_STRING_REGEX);

            if (match) {
                const {recipientProperty} = match.groups;

                if (ALLOWED_REPLACEMENTS.includes(recipientProperty)) {
                    // keeps wrapping %% for later replacement with real data
                    return replacementMatch;
                }
            }

            // removes %% so output matches user supplied content
            return replacementStr;
        });
    });

    return emailContent;
};

// parses email content and extracts an array of replacements with desired fallbacks
const parseReplacements = (email) => {
    const EMAIL_REPLACEMENT_REGEX = /%%(\{.*?\})%%/g;
    const REPLACEMENT_STRING_REGEX = /\{(?<recipientProperty>\w*?)(?:,? *(?:"|&quot;)(?<fallback>.*?)(?:"|&quot;))?\}/;

    const replacements = [];

    ['html', 'plaintext'].forEach((format) => {
        let result;
        while ((result = EMAIL_REPLACEMENT_REGEX.exec(email[format])) !== null) {
            const [replacementMatch, replacementStr] = result;
            const match = replacementStr.match(REPLACEMENT_STRING_REGEX);

            if (match) {
                const {recipientProperty, fallback} = match.groups;

                if (ALLOWED_REPLACEMENTS.includes(recipientProperty)) {
                    const id = `replacement_${replacements.length + 1}`;

                    replacements.push({
                        format,
                        id,
                        match: replacementMatch,
                        recipientProperty: `member_${recipientProperty}`,
                        fallback
                    });
                }
            }
        }
    });

    return replacements;
};

const getTemplateSettings = async () => {
    return {
        showSiteHeader: settingsCache.get('newsletter_show_header'),
        bodyFontCategory: settingsCache.get('newsletter_body_font_category'),
        showBadge: settingsCache.get('newsletter_show_badge'),
        footerContent: settingsCache.get('newsletter_footer_content'),
        accentColor: settingsCache.get('accent_color')
    };
};

const getLabsTemplateSettings = async () => {
    const templateSettings = {
        headerImage: settingsCache.get('newsletter_header_image'),
        showHeaderIcon: settingsCache.get('newsletter_show_header_icon') && settingsCache.get('icon'),
        showHeaderTitle: settingsCache.get('newsletter_show_header_title'),
        showFeatureImage: settingsCache.get('newsletter_show_feature_image'),
        titleFontCategory: settingsCache.get('newsletter_title_font_category'),
        titleAlignment: settingsCache.get('newsletter_title_alignment'),
        bodyFontCategory: settingsCache.get('newsletter_body_font_category'),
        showBadge: settingsCache.get('newsletter_show_badge'),
        footerContent: settingsCache.get('newsletter_footer_content'),
        accentColor: settingsCache.get('accent_color')
    };

    if (templateSettings.headerImage) {
        if (isUnsplashImage(templateSettings.headerImage)) {
{'fraction_non_alphanumeric': 0.137628353759523, 'fraction_numerical': 0.0008280887711162637, 'mean_word_length': 3.0911246612466123, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3132462', 'n_tokens_mistral': 3410, 'n_tokens_neox': 3309, 'n_words': 733}