<commit_before>const _ = require('lodash');
const debug = require('debug')('W2:portal:extractPageView');
const fs = require('fs');
const path = require('path');
const Promise = require('bluebird');
const RoutesInfo = require('@quoin/expressjs-routes-info');
const urlTemplate = require('url-template');

const basicTypes = require('./../../../../lib/core/basic-types');
const convertCustomLinks = require('./../convert-custom-links');
const createObjResource = require('./create-obj-resource');
const embed = require('./embed');
const sortItems = require('./sort-items');

const IMAGE_PATH = urlTemplate.parse('/public/iic_images/{ImageURL}'); // FIXME: Hard-coded path.
const CONTENT_LINK_RE = require('./../../../../lib/core/content-link-re');

function parseLinks(overviews) {
    if (overviews) {
        return overviews.map((overview) => {
            if (overview && overview.Content && overview.Content.match(CONTENT_LINK_RE)) {
                const contentBasicProperty = _.filter(overview.basicProperties, (prop) => prop.name === 'Content'); // FIXME: Hard-coded
                overview.containsHTML = contentBasicProperty.length && contentBasicProperty[0].propertyType === basicTypes.Text;
                if (overview.containsHTML) {
                    overview.Content = convertCustomLinks(overview.Content);
                }
            }
            return overview;
        });
    }
    return overviews;
}

function extractRelationship(req, resource, persistence, hsEntity, entity) {
    const relationship = hsEntity.getRelationship();

    return Promise.resolve()
        .then(() => relationship.getDocuments(persistence, entity))
        .then((references) => {
            const onlyExistingReferences = references.filter((reference) => reference.id && reference.type);
            return Promise.map(onlyExistingReferences, (reference) => {
                const referenceResource = createObjResource(reference, true);
                resource.styles = {
                    CSV: (hsEntity.style === 'CSV'),
                    Preview: (hsEntity.style === 'Preview'),
                    Table: (hsEntity.style === 'Table'),
                    Tile: (hsEntity.style === 'Tile')
                };
                if (hsEntity.style === 'Preview') {
                    const referenceEntity = hsEntity.getRelationship().getTargetEntity();
                    return referenceEntity.getOverview(persistence, reference)
                        .then(parseLinks)
                        .then((overviews) => {
                            if (overviews && overviews.length) {
                                return overviews[0];
                            }
                            return [];
                        })
                        .then(convertToResource.bind(null, req))
                        .then((overview) => {
                            if (!_.isArray(overview)) {
                                referenceResource.embed('overview', overview);
                            }
                            return referenceResource;
                        })
                        .catch((err) => {
                            // The entity does not offer a relationship named
                            // 'Overview'?
                            debug(`Could not find overview for: reference=`, reference);
                            debug(`Could not find overview for: referenceEntity=`, referenceEntity);
                            debug(`Could not find overview: Error=`, err);
                            return referenceResource;
                        });
                }
                return referenceResource;
            });
        })
        .then((references) => {
            const relationshipResource = createObjResource(relationship, true);
            relationshipResource.embed('references', references);
            resource.embed('relationships', relationshipResource);
            return relationshipResource;
        });
}

function imagePath(req, image) {
    const publicFolder = req.app.get('public-folder');
    const publicFolderPath = req.app.get('public-folder-path');

    let filePath;

    if (image && image.ImageURL && image.ImageURL.indexOf(publicFolderPath) === 0) {
        filePath = path.join(publicFolder, image.ImageURL.substring(publicFolderPath.length));
    } else {
        filePath = path.join(publicFolder, 'iic_images', image.ImageURL || '');
        image.ImageURL = IMAGE_PATH.expand(image);
    }

    try {
        const stats = fs.lstatSync(filePath);
        if (!stats.isFile() && !/^(http|\?)/.test(image.ImageURL)) {
            image.ImageURL = '';
        }
    } catch (e) {
        image.ImageURL = '';
    }
}

function customResource(req, key, item) {
    const resource = convertToResource(req, item);
    if (key === 'Target') {
        resource.link('preview', RoutesInfo.expand('W2:portal:preview', {
            type: item.type,
            id: item.id
        }));
    }
    return resource;
}

function convertToResource(req, data) {
    if (!data) {
        return null;
    }

    if (_.isArray(data)) {
        return data.map(convertToResource.bind(null, req));
    }

    if (data.type === 'Image') {
        imagePath(req, data);
    }

    const basicProperties = _.reduce(
        data,
        (memo, value, key) => {
            if (!_.isArray(value)) {
                return _.extend(memo, {
                    [key]: value
                });
            }
            return memo;
        },
        {}
    );

    let propsToPick = null;
    if (data.basicProperties && data.basicProperties.length) {
        propsToPick = data.basicProperties.reduce((memo, value) => {
            memo.push(value.name);
            return memo;
        }, []);
    }

    const resource = createObjResource(basicProperties, true, propsToPick);

    _.forEach(data, (value, key) => {
        if (_.isArray(value)) {
            resource.embed(key, value.map(customResource.bind(null, req, key)));
        }
    });

    if (resource.type === 'ImageArea') {
        if (resource.HRef) {
            resource.link('target', resource.HRef);
        } else if (resource._embedded && resource._embedded.Targets && resource._embedded.Targets.length) {
            resource.link('target', resource._embedded.Targets[0]._links.self.href);
            resource.link('preview', resource._embedded.Targets[0]._links.preview.href);
        }
    }

    return resource;
}

function createOverviewPanel(req, persistence, hsCurrentEntity, currentInstance) {
    return Promise.resolve()
        // Find the overview.
        .then(() => hsCurrentEntity.getOverview(persistence, currentInstance))
        .then(parseLinks)
        .then(convertToResource.bind(null, req))
        .then((overviews) => {
            const resource = createObjResource({
                alternatingColors: false,
                type: 'Overview'<commit_msg>Fixed ordering of overview paragraphs. Fixes #61<commit_after>const _ = require('lodash');
const debug = require('debug')('W2:portal:extractPageView');
const fs = require('fs');
const path = require('path');
const Promise = require('bluebird');
const RoutesInfo = require('@quoin/expressjs-routes-info');
const urlTemplate = require('url-template');
const warpjsUtils = require('@warp-works/warpjs-utils');

const basicTypes = require('./../../../../lib/core/basic-types');
const convertCustomLinks = require('./../convert-custom-links');
const createObjResource = require('./create-obj-resource');
const embed = require('./embed');
const sortItems = require('./sort-items');

const IMAGE_PATH = urlTemplate.parse('/public/iic_images/{ImageURL}'); // FIXME: Hard-coded path.
const CONTENT_LINK_RE = require('./../../../../lib/core/content-link-re');

function parseLinks(overviews) {
    if (overviews) {
        return overviews.map((overview) => {
            if (overview && overview.Content && overview.Content.match(CONTENT_LINK_RE)) {
                const contentBasicProperty = _.filter(overview.basicProperties, (prop) => prop.name === 'Content'); // FIXME: Hard-coded
                overview.containsHTML = contentBasicProperty.length && contentBasicProperty[0].propertyType === basicTypes.Text;
                if (overview.containsHTML) {
                    overview.Content = convertCustomLinks(overview.Content);
                }
            }
            return overview;
        });
    }
    return overviews;
}

function extractRelationship(req, resource, persistence, hsEntity, entity) {
    const relationship = hsEntity.getRelationship();

    return Promise.resolve()
        .then(() => relationship.getDocuments(persistence, entity))
        .then((references) => {
            const onlyExistingReferences = references.filter((reference) => reference.id && reference.type);
            return Promise.map(onlyExistingReferences, (reference) => {
                const referenceResource = createObjResource(reference, true);
                resource.styles = {
                    CSV: (hsEntity.style === 'CSV'),
                    Preview: (hsEntity.style === 'Preview'),
                    Table: (hsEntity.style === 'Table'),
                    Tile: (hsEntity.style === 'Tile')
                };
                if (hsEntity.style === 'Preview') {
                    const referenceEntity = hsEntity.getRelationship().getTargetEntity();
                    return referenceEntity.getOverview(persistence, reference)
                        .then(parseLinks)
                        .then((overviews) => {
                            if (overviews && overviews.length) {
                                return overviews[0];
                            }
                            return [];
                        })
                        .then(convertToResource.bind(null, req))
                        .then((overview) => {
                            if (!_.isArray(overview)) {
                                referenceResource.embed('overview', overview);
                            }
                            return referenceResource;
                        })
                        .catch((err) => {
                            // The entity does not offer a relationship named
                            // 'Overview'?
                            debug(`Could not find overview for: reference=`, reference);
                            debug(`Could not find overview for: referenceEntity=`, referenceEntity);
                            debug(`Could not find overview: Error=`, err);
                            return referenceResource;
                        });
                }
                return referenceResource;
            });
        })
        .then((references) => {
            const relationshipResource = createObjResource(relationship, true);
            relationshipResource.embed('references', references);
            resource.embed('relationships', relationshipResource);
            return relationshipResource;
        });
}

function imagePath(req, image) {
    const publicFolder = req.app.get('public-folder');
    const publicFolderPath = req.app.get('public-folder-path');

    let filePath;

    if (image && image.ImageURL && image.ImageURL.indexOf(publicFolderPath) === 0) {
        filePath = path.join(publicFolder, image.ImageURL.substring(publicFolderPath.length));
    } else {
        filePath = path.join(publicFolder, 'iic_images', image.ImageURL || '');
        image.ImageURL = IMAGE_PATH.expand(image);
    }

    try {
        const stats = fs.lstatSync(filePath);
        if (!stats.isFile() && !/^(http|\?)/.test(image.ImageURL)) {
            image.ImageURL = '';
        }
    } catch (e) {
        image.ImageURL = '';
    }
}

function customResource(req, key, item) {
    const resource = convertToResource(req, item);
    if (key === 'Target') {
        resource.link('preview', RoutesInfo.expand('W2:portal:preview', {
            type: item.type,
            id: item.id
        }));
    }
    return resource;
}

function convertToResource(req, data) {
    if (!data) {
        return null;
    }

    if (_.isArray(data)) {
        return data.map(convertToResource.bind(null, req));
    }

    if (data.type === 'Image') {
        imagePath(req, data);
    }

    const basicProperties = _.reduce(
        data,
        (memo, value, key) => {
            if (!_.isArray(value)) {
                return _.extend(memo, {
                    [key]: value
                });
            }
            return memo;
        },
        {}
    );

    let propsToPick = null;
    if (data.basicProperties && data.basicProperties.length) {
        propsToPick = data.basicProperties.reduce((memo, value) => {
            memo.push(value.name);
            return memo;
        }, []);
    }

    const resource = createObjResource(basicProperties, true, propsToPick);

    _.forEach(data, (value, key) => {
        if (_.isArray(value)) {
            resource.embed(key, value.map(customResource.bind(null, req, key)));
        }
    });

    if (resource.type === 'ImageArea') {
        if (resource.HRef) {
            resource.link('target', resource.HRef);
        } else if (resource._embedded && resource._embedded.Targets && resource._embedded.Targets.length) {
            resource.link('target', resource._embedded.Targets[0]._links.self.href);
            resource.link('preview', resource._embedded.Targets[0]._links.preview.href);
        }
    }

    return resource;
}

function createOverviewPanel(req, persistence, hsCurrentEntity, currentInstance) {
    return Promise.resolve()
        // Find the overview.
        .then(() => hsCurrentEntity.getOverview(persistence, currentInstance))
        .then(parseLinks)
        .then(convertToResource.bind(null, req))
        .then((overviews) => overviews.sort(warpjsUtils.byPositionThenName))
        .then((overviews) => {
            const resource = createObjResource({
                alternatingColors: false,
                type: 'Overview'
{'fraction_non_alphanumeric': 0.13248561483270582, 'fraction_numerical': 0.0011366058108972083, 'mean_word_length': 1.9153033754400497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2760257', 'n_tokens_mistral': 3647, 'n_tokens_neox': 3501, 'n_words': 692}