<commit_before>/*global module, require*/
var Path = require('path'),
    _ = require('underscore'),
    passError = require('../util/passError'),
    uniqueId = require('../util/uniqueId'),
    memoizeAsyncAccessor = require('../util/memoizeAsyncAccessor');

function Base(config) {
    _.extend(this, config);
    this.id = uniqueId();
    if (this.url) {
        this.originalExtension = Path.extname(this.url.replace(/[?#].*$/, ''));
    }
}

Base.prototype = {
    isAsset: true, // Avoid instanceof checks

    contentType: 'application/octet-stream',

    defaultExtension: '',

    getExtension: function () {
        if (this.url) {
            return Path.extname(this.url.replace(/[?#].*$/, ''));
        } else if ('originalExtension' in this) {
            return this.originalExtension;
        } else {
            return this.defaultExtension;
        }
    },

    // Side effect: Sets the 'originalEncoding' property if the proxy provides it in the metadata (eg. from HTTP headers)
    getRawSrc: memoizeAsyncAccessor('rawSrc', function (cb) {
        var that = this;
        if (that.rawSrcProxy) {
            that.rawSrcProxy(passError(cb, function (rawSrc, metadata) {
                if (metadata && metadata.originalEncoding) {
                    that.originalEncoding = metadata.originalEncoding;
                }
                cb(null, rawSrc);
            }));
        } else {
            cb(new Error("Don't know how to get the raw asset src: " + that.toString()));
        }
    }),

    hasRawSrc: function () {
        return 'rawSrcProxy' in this || 'rawSrc' in this;
    },

    // Override in subclass if it supports relations:
    getOriginalRelations: memoizeAsyncAccessor('originalRelations', function (cb) {
        process.nextTick(function () {
            cb(null, []);<commit_msg>assets.Base: Don't attempt to extract an extension from an url ending in a slash.<commit_after>/*global module, require*/
var Path = require('path'),
    _ = require('underscore'),
    passError = require('../util/passError'),
    uniqueId = require('../util/uniqueId'),
    memoizeAsyncAccessor = require('../util/memoizeAsyncAccessor');

function Base(config) {
    _.extend(this, config);
    this.id = uniqueId();
    if (this.url && !/\/(?:[?#].*)?$/.test(this.url)) {
        this.originalExtension = Path.extname(this.url.replace(/[?#].*$/, ''));
    }
}

Base.prototype = {
    isAsset: true, // Avoid instanceof checks

    contentType: 'application/octet-stream',

    defaultExtension: '',

    getExtension: function () {
        if (this.url && !/\/(?:[?#].*)?$/.test(this.url)) {
            return Path.extname(this.url.replace(/[?#].*$/, ''));
        } else if ('originalExtension' in this) {
            return this.originalExtension;
        } else {
            return this.defaultExtension;
        }
    },

    // Side effect: Sets the 'originalEncoding' property if the proxy provides it in the metadata (eg. from HTTP headers)
    getRawSrc: memoizeAsyncAccessor('rawSrc', function (cb) {
        var that = this;
        if (that.rawSrcProxy) {
            that.rawSrcProxy(passError(cb, function (rawSrc, metadata) {
                if (metadata && metadata.originalEncoding) {
                    that.originalEncoding = metadata.originalEncoding;
                }
                cb(null, rawSrc);
            }));
        } else {
            cb(new Error("Don't know how to get the raw asset src: " + that.toString()));
        }
    }),

    hasRawSrc: function () {
        return 'rawSrcProxy' in this || 'rawSrc' in this;
    },

    // Override in subclass if it supports relations:
    getOriginalRelations: memoizeAsyncAccessor('originalRelations', function (cb) {
        process.nextTick(function () {
            cb(null, []);
{'fraction_non_alphanumeric': 0.1469968387776607, 'fraction_numerical': 0.0, 'mean_word_length': 2.354240282685512, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1258939', 'n_tokens_mistral': 1082, 'n_tokens_neox': 1053, 'n_words': 255}