<commit_before>/* eslint comma-dangle: [2, "always-multiline"] */
const _ = require('lodash');
const _s = require('underscore.string');
const iso19139helpers = require('./iso19139');
const parseUrl = require('url').parse;

function getFileNameFromHref(href) {
    const fullPath = parseUrl(href).pathname.split('/');
    return fullPath[fullPath.length - 1];
}

function getUniformArray(src) {
    return _.uniq(_.compact(_.flatten(src)));
}

function getLicenseFromLinks(links) {
    let license;
    links.forEach(link => {
        const lcName = link.name.toLowerCase();
        if (lcName.includes('licence') && lcName.includes('ouverte')) {
            license = 'fr-lo'; return;
        }
        if (lcName.includes('odbl')) {
            license = 'odc-odbl'; return;
        }
    });
    return license;
}

const openDataKeywords = [
    'donnee-ouverte',
    'donnees-ouvertes',<commit_msg>Fix type error in dataset conversion<commit_after>/* eslint comma-dangle: [2, "always-multiline"] */
const _ = require('lodash');
const _s = require('underscore.string');
const iso19139helpers = require('./iso19139');
const parseUrl = require('url').parse;

function getFileNameFromHref(href) {
    const pathname = parseUrl(href).pathname;
    if (!pathname) return null;
    const fullPath = parseUrl(href).pathname.split('/');
    return fullPath[fullPath.length - 1];
}

function getUniformArray(src) {
    return _.uniq(_.compact(_.flatten(src)));
}

function getLicenseFromLinks(links) {
    let license;
    links.forEach(link => {
        const lcName = (link.name || '').toLowerCase();
        if (lcName.includes('licence') && lcName.includes('ouverte')) {
            license = 'fr-lo'; return;
        }
        if (lcName.includes('odbl')) {
            license = 'odc-odbl'; return;
        }
    });
    return license;
}

const openDataKeywords = [
    'donnee-ouverte',
    'donnees-ouvertes',
{'fraction_non_alphanumeric': 0.15855572998430142, 'fraction_numerical': 0.012558869701726845, 'mean_word_length': 3.3160270880361176, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6587233', 'n_tokens_mistral': 633, 'n_tokens_neox': 608, 'n_words': 118}