<commit_before>/* eslint-disable no-console */
/*
 * module to provide a contentful like SDK to our local cache
 */
"use strict";

const path = require("path");
const fs = require("fs");
const glob = require("glob");

let clientResolveLinks, defaultLocale, localCachePath, locales, currentSpace;

const resolveLinksLimitMax = 10;

/**
 * sdk relies heavily on sys metadata
 * so we cannot omit the sys property on sdk level
 */
function normalizeSelect (query) {
    if (query.select && !/sys/i.test(query.select)) {
        query.select += ",sys";
    }
}


/**
 * fieldWithLocaleFallback
 */
function fieldWithLocaleFallback (field, value, preferredLocaleCode) {

    let newValue, locale;

    // get the locale record from the current metadata
    locale = locales.filter(l => l.code === preferredLocaleCode).shift();

    if (!locale) {
        throw new Error("Unknown locale: " + preferredLocaleCode);
    }

    if (locale.code in value) {
        newValue = value[locale.code];
    } else if (locale.fallbackCode) {
        newValue = fieldWithLocaleFallback(field, value, locale.fallbackCode);
    }

    return newValue;

}

/**
 * filter record by locale
 */
// TODO fallback locale list
function filterLocale(record, preferredLocaleCode) {

    let clone, field, locale;

    // get the locale record from the current metadata
    locale = locales.filter(l => l.code === preferredLocaleCode).shift();

    if (!locale) {
        throw new Error("Unknown locale: " + preferredLocaleCode);
    }

    // clone record
    clone = JSON.parse(JSON.stringify(record));

    if (clone.fields) {

        for (field in clone.fields) {
            if (clone.fields.hasOwnProperty(field)) {
                clone.fields[field] = fieldWithLocaleFallback(field, clone.fields[field], locale.code);
            }
        }

    }

    return clone;

}


/**
 * resolveLinks
 */
function resolveLinks (record, query) {
    return new Promise((resolve, reject) => {

        let fieldsWithLinks, promisesForRecords;

        // to avoid recursive loops, resolving links has a limit
        if (query.include > 0) {

            // find links in record fields
            if (record.fields) {

                // get a list of fields that are links or arrays of links
                fieldsWithLinks = Object.keys(record.fields).filter(key => {

                    if (Array.isArray(record.fields[key])) {
                        if (record.fields[key][0] && record.fields[key][0].sys) {
                            return record.fields[key][0].sys.type === "Link";
                        }
                    } else {
                        if (record.fields[key] && record.fields[key].sys) {
                            return record.fields[key].sys.type === "Link";
                        }
                    }

                });

                // get list of promises for the linked records
                promisesForRecords = fieldsWithLinks.map(key => {

                    let p, sys;

                    // switch on link type
                    sys = record.fields[key].sys;
                    switch(sys.linkType) {
                    case "Entry":
                        p = getEntry(sys.id, { "include": query.include - 1 });
                        break;
                    case "Asset":
                        p = getAsset(sys.id, { "include": query.include - 1 });
                        break;
                    default:
                        p = Promise.reject(
                            new Error("Unknown link type: %s", sys.linkType)
                        );
                        break;
                    }

                    return p;

                });<commit_msg>include content_type when getting Entry of known type<commit_after>/* eslint-disable no-console */
/*
 * module to provide a contentful like SDK to our local cache
 */
"use strict";

const path = require("path");
const fs = require("fs");
const glob = require("glob");

let clientResolveLinks, defaultLocale, localCachePath, locales, currentSpace;

const resolveLinksLimitMax = 10;

/**
 * sdk relies heavily on sys metadata
 * so we cannot omit the sys property on sdk level
 */
function normalizeSelect (query) {
    if (query.select && !/sys/i.test(query.select)) {
        query.select += ",sys";
    }
}


/**
 * fieldWithLocaleFallback
 */
function fieldWithLocaleFallback (field, value, preferredLocaleCode) {

    let newValue, locale;

    // get the locale record from the current metadata
    locale = locales.filter(l => l.code === preferredLocaleCode).shift();

    if (!locale) {
        throw new Error("Unknown locale: " + preferredLocaleCode);
    }

    if (locale.code in value) {
        newValue = value[locale.code];
    } else if (locale.fallbackCode) {
        newValue = fieldWithLocaleFallback(field, value, locale.fallbackCode);
    }

    return newValue;

}

/**
 * filter record by locale
 */
// TODO fallback locale list
function filterLocale(record, preferredLocaleCode) {

    let clone, field, locale;

    // get the locale record from the current metadata
    locale = locales.filter(l => l.code === preferredLocaleCode).shift();

    if (!locale) {
        throw new Error("Unknown locale: " + preferredLocaleCode);
    }

    // clone record
    clone = JSON.parse(JSON.stringify(record));

    if (clone.fields) {

        for (field in clone.fields) {
            if (clone.fields.hasOwnProperty(field)) {
                clone.fields[field] = fieldWithLocaleFallback(field, clone.fields[field], locale.code);
            }
        }

    }

    return clone;

}


/**
 * resolveLinks
 */
function resolveLinks (record, query) {
    return new Promise((resolve, reject) => {

        let fieldsWithLinks, promisesForRecords;

        // to avoid recursive loops, resolving links has a limit
        if (query.include > 0) {

            // find links in record fields
            if (record.fields) {

                // get a list of fields that are links or arrays of links
                fieldsWithLinks = Object.keys(record.fields).filter(key => {

                    if (Array.isArray(record.fields[key])) {
                        if (record.fields[key][0] && record.fields[key][0].sys) {
                            return record.fields[key][0].sys.type === "Link";
                        }
                    } else {
                        if (record.fields[key] && record.fields[key].sys) {
                            return record.fields[key].sys.type === "Link";
                        }
                    }

                });

                // get list of promises for the linked records
                promisesForRecords = fieldsWithLinks.map(key => {

                    let p, sys;

                    // switch on link type
                    sys = record.fields[key].sys;
                    switch(sys.linkType) {
                    case "Entry":
                        // TODO the key may not always match the content type
                        // and getting an entry without the content type means searching ALL the entry files, which gets slow
                        p = getEntry(sys.id, {
                            "content_type": key,
                            "include": query.include - 1
                        });
                        break;
                    case "Asset":
                        p = getAsset(sys.id, { "include": query.include - 1 });
                        break;
                    default:
                        p = Promise.reject(
                            new Error("Unknown link type: %s", sys.linkType)
                        );
                        break;
                    }

                    return p;

                });
{'fraction_non_alphanumeric': 0.10774367535636317, 'fraction_numerical': 0.0020547065622190832, 'mean_word_length': 1.5916805324459236, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3350010', 'n_tokens_mistral': 2038, 'n_tokens_neox': 1908, 'n_words': 560}