<commit_before>/* jslint node: true */
'use strict';

//	ENiGMA½
const Config			= require('./config.js').config;
const Errors			= require('./enig_error.js').Errors;
const sortAreasOrConfs	= require('./conf_area_util.js').sortAreasOrConfs;
const FileEntry			= require('./file_entry.js');
const FileDb			= require('./database.js').dbs.file;
const ArchiveUtil		= require('./archive_util.js');
const CRC32				= require('./crc.js').CRC32;
const Log				= require('./logger.js').log;
const resolveMimeType	= require('./mime_util.js').resolveMimeType;
const stringFormat		= require('./string_format.js');
const wordWrapText		= require('./word_wrap.js').wordWrapText;
const StatLog			= require('./stat_log.js');

//	deps
const _				= require('lodash');
const async			= require('async');
const fs			= require('graceful-fs');
const crypto		= require('crypto');
const paths			= require('path');
const temptmp		= require('temptmp').createTrackedSession('file_area');
const iconv			= require('iconv-lite');
const execFile		= require('child_process').execFile;
const moment		= require('moment');

exports.isInternalArea					= isInternalArea;
exports.getAvailableFileAreas			= getAvailableFileAreas;
exports.getSortedAvailableFileAreas		= getSortedAvailableFileAreas;
exports.isValidStorageTag				= isValidStorageTag;
exports.getAreaStorageDirectoryByTag	= getAreaStorageDirectoryByTag;
exports.getAreaDefaultStorageDirectory	= getAreaDefaultStorageDirectory;
exports.getAreaStorageLocations			= getAreaStorageLocations;
exports.getDefaultFileAreaTag			= getDefaultFileAreaTag;
exports.getFileAreaByTag				= getFileAreaByTag;
exports.getFileEntryPath				= getFileEntryPath;
exports.changeFileAreaWithOptions		= changeFileAreaWithOptions;
exports.scanFile						= scanFile;
exports.scanFileAreaForChanges			= scanFileAreaForChanges;
exports.getDescFromFileName				= getDescFromFileName;
exports.getAreaStats					= getAreaStats;

//	for scheduler:
exports.updateAreaStatsScheduledEvent	= updateAreaStatsScheduledEvent;

const WellKnownAreaTags					= exports.WellKnownAreaTags = {
	Invalid				: '',
	MessageAreaAttach	: 'system_message_attachment',
};

function isInternalArea(areaTag) {
	return areaTag === WellKnownAreaTags.MessageAreaAttach;
}

function getAvailableFileAreas(client, options) {
	options = options || { };

	//	perform ACS check per conf & omit internal if desired
	const allAreas = _.map(Config.fileBase.areas, (areaInfo, areaTag) => Object.assign(areaInfo, { areaTag : areaTag } ));
	
	return _.omitBy(allAreas, areaInfo => {        
		if(!options.includeSystemInternal && isInternalArea(areaInfo.areaTag)) {
			return true;
		}

		if(options.skipAcsCheck) {
			return false;	//	no ACS checks (below)
		}

		if(options.writeAcs && !client.acs.hasFileAreaWrite(areaInfo)) {
			return true;	//	omit
		}

		return !client.acs.hasFileAreaRead(areaInfo);
	});
}

function getSortedAvailableFileAreas(client, options) {
	const areas = _.map(getAvailableFileAreas(client, options), v => v);
	sortAreasOrConfs(areas);
	return areas;
}

function getDefaultFileAreaTag(client, disableAcsCheck) {
	let defaultArea = _.findKey(Config.fileBase, o => o.default);
	if(defaultArea) {
		const area = Config.fileBase.areas[defaultArea];
		if(true === disableAcsCheck || client.acs.hasFileAreaRead(area)) {
			return defaultArea;
		}
	}

	//  just use anything we can
	defaultArea = _.findKey(Config.fileBase.areas, (area, areaTag) => {
		return WellKnownAreaTags.MessageAreaAttach !== areaTag && (true === disableAcsCheck || client.acs.hasFileAreaRead(area));
	});
    
	return defaultArea;
}

function getFileAreaByTag(areaTag) {
	const areaInfo = Config.fileBase.areas[areaTag];
	if(areaInfo) {
		areaInfo.areaTag	= areaTag;	//	convienence!
		areaInfo.storage	= getAreaStorageLocations(areaInfo); 
		return areaInfo;<commit_msg>* Add getAvailableFileAreaTags() * Properly check area tags for system internal<commit_after>/* jslint node: true */
'use strict';

//	ENiGMA½
const Config			= require('./config.js').config;
const Errors			= require('./enig_error.js').Errors;
const sortAreasOrConfs	= require('./conf_area_util.js').sortAreasOrConfs;
const FileEntry			= require('./file_entry.js');
const FileDb			= require('./database.js').dbs.file;
const ArchiveUtil		= require('./archive_util.js');
const CRC32				= require('./crc.js').CRC32;
const Log				= require('./logger.js').log;
const resolveMimeType	= require('./mime_util.js').resolveMimeType;
const stringFormat		= require('./string_format.js');
const wordWrapText		= require('./word_wrap.js').wordWrapText;
const StatLog			= require('./stat_log.js');

//	deps
const _				= require('lodash');
const async			= require('async');
const fs			= require('graceful-fs');
const crypto		= require('crypto');
const paths			= require('path');
const temptmp		= require('temptmp').createTrackedSession('file_area');
const iconv			= require('iconv-lite');
const execFile		= require('child_process').execFile;
const moment		= require('moment');

exports.isInternalArea					= isInternalArea;
exports.getAvailableFileAreas			= getAvailableFileAreas;
exports.getAvailableFileAreaTags		= getAvailableFileAreaTags;
exports.getSortedAvailableFileAreas		= getSortedAvailableFileAreas;
exports.isValidStorageTag				= isValidStorageTag;
exports.getAreaStorageDirectoryByTag	= getAreaStorageDirectoryByTag;
exports.getAreaDefaultStorageDirectory	= getAreaDefaultStorageDirectory;
exports.getAreaStorageLocations			= getAreaStorageLocations;
exports.getDefaultFileAreaTag			= getDefaultFileAreaTag;
exports.getFileAreaByTag				= getFileAreaByTag;
exports.getFileEntryPath				= getFileEntryPath;
exports.changeFileAreaWithOptions		= changeFileAreaWithOptions;
exports.scanFile						= scanFile;
exports.scanFileAreaForChanges			= scanFileAreaForChanges;
exports.getDescFromFileName				= getDescFromFileName;
exports.getAreaStats					= getAreaStats;

//	for scheduler:
exports.updateAreaStatsScheduledEvent	= updateAreaStatsScheduledEvent;

const WellKnownAreaTags					= exports.WellKnownAreaTags = {
	Invalid				: '',
	MessageAreaAttach	: 'system_message_attachment',
	TempDownloads		: 'system_temporary_download',
};

function isInternalArea(areaTag) {
	return [ WellKnownAreaTags.MessageAreaAttach, WellKnownAreaTags.TempDownloads ].includes(areaTag);
}

function getAvailableFileAreas(client, options) {
	options = options || { };

	//	perform ACS check per conf & omit internal if desired
	const allAreas = _.map(Config.fileBase.areas, (areaInfo, areaTag) => Object.assign(areaInfo, { areaTag : areaTag } ));
	
	return _.omitBy(allAreas, areaInfo => {        
		if(!options.includeSystemInternal && isInternalArea(areaInfo.areaTag)) {
			return true;
		}

		if(options.skipAcsCheck) {
			return false;	//	no ACS checks (below)
		}

		if(options.writeAcs && !client.acs.hasFileAreaWrite(areaInfo)) {
			return true;	//	omit
		}

		return !client.acs.hasFileAreaRead(areaInfo);
	});
}

function getAvailableFileAreaTags(client, options) {
	return _.map(getAvailableFileAreas(client, options), area => area.areaTag);
}

function getSortedAvailableFileAreas(client, options) {
	const areas = _.map(getAvailableFileAreas(client, options), v => v);
	sortAreasOrConfs(areas);
	return areas;
}

function getDefaultFileAreaTag(client, disableAcsCheck) {
	let defaultArea = _.findKey(Config.fileBase, o => o.default);
	if(defaultArea) {
		const area = Config.fileBase.areas[defaultArea];
		if(true === disableAcsCheck || client.acs.hasFileAreaRead(area)) {
			return defaultArea;
		}
	}

	//  just use anything we can
	defaultArea = _.findKey(Config.fileBase.areas, (area, areaTag) => {
		return WellKnownAreaTags.MessageAreaAttach !== areaTag && (true === disableAcsCheck || client.acs.hasFileAreaRead(area));
	});
    
	return defaultArea;
}

function getFileAreaByTag(areaTag) {
	const areaInfo = Config.fileBase.areas[areaTag];
	if(areaInfo) {
		areaInfo.areaTag	= areaTag;	//	convienence!
		areaInfo.storage	= getAreaStorageLocations(areaInfo); 
		return areaInfo;
{'fraction_non_alphanumeric': 0.12247712172495925, 'fraction_numerical': 0.0010028832894571893, 'mean_word_length': 5.346857597454256, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5287272', 'n_tokens_mistral': 2726, 'n_tokens_neox': 2547, 'n_words': 455}