<commit_before>import {Injectable, OnDestroy} from '@angular/core';
import {BehaviorSubject} from 'rxjs/BehaviorSubject';
import {Observable} from 'rxjs/Observable';

import {KalturaClient, KalturaMultiRequest} from 'kaltura-ngx-client';
import {KalturaSourceType} from 'kaltura-ngx-client/api/types/KalturaSourceType';
import {KalturaLiveStreamBitrate} from 'kaltura-ngx-client/api/types/KalturaLiveStreamBitrate';
import {KalturaRecordStatus} from 'kaltura-ngx-client/api/types/KalturaRecordStatus';
import {KalturaLiveStreamEntry} from 'kaltura-ngx-client/api/types/KalturaLiveStreamEntry';
import {KalturaDVRStatus} from 'kaltura-ngx-client/api/types/KalturaDVRStatus';
import {KalturaMediaEntry} from 'kaltura-ngx-client/api/types/KalturaMediaEntry';
import {LiveStreamRegenerateStreamTokenAction} from 'kaltura-ngx-client/api/types/LiveStreamRegenerateStreamTokenAction';
import {AppLocalization} from '@kaltura-ng/kaltura-common';
import {AppAuthentication, BrowserService} from 'app-shared/kmc-shell';

import {EntryWidgetKeys} from '../entry-widget-keys';
import {LiveXMLExporter} from './live-xml-exporter';
import {AVAIL_BITRATES} from './bitrates';
import {EntryWidget} from '../entry-widget';
import {serverConfig} from 'config/server';
import {ConversionProfileListAction} from 'kaltura-ngx-client/api/types/ConversionProfileListAction';
import {KalturaConversionProfileFilter} from 'kaltura-ngx-client/api/types/KalturaConversionProfileFilter';
import {KalturaFilterPager} from 'kaltura-ngx-client/api/types/KalturaFilterPager';
import {KalturaConversionProfileType} from 'kaltura-ngx-client/api/types/KalturaConversionProfileType';
import {KalturaNullableBoolean} from 'kaltura-ngx-client/api/types/KalturaNullableBoolean';
import {AreaBlockerMessage} from '@kaltura-ng/kaltura-ui';
import {BaseEntryGetAction} from 'kaltura-ngx-client/api/types/BaseEntryGetAction';

export interface bitrate {
	enabled: boolean,
	bitrate: number,
	width: number,
	height: number,
	errors: string
}

@Injectable()
export class EntryLiveWidget extends EntryWidget implements OnDestroy {

	public _liveType: string = "";
	private dirty: boolean;

	private _conversionProfiles: BehaviorSubject<{ items: any[]}> = new BehaviorSubject<{ items: any[]}>({items: []});
	public _conversionProfiles$ = this._conversionProfiles.asObservable();

	public _recordStatus: string = "";
	public _DVRStatus: string = "";
	public _showDVRWindow: boolean = false;
	public _dvrWindowAvailable: boolean = false;
	public _explicitLive: boolean = true;
	public _liveDashboardEnabled: boolean = false;

	public _selectedConversionProfile: number;
	public _manualStreamsConfiguration = [];
	public _bitrates: bitrate[] = [];
	public _availableBitrates = AVAIL_BITRATES;

	public _autoStartOptions = [
		{label: this._appLocalization.get('applications.content.entryDetails.live.disabled'), value: true},
		{label: this._appLocalization.get('applications.content.entryDetails.live.enabled'), value: false}
	];

	constructor(private _kalturaServerClient: KalturaClient, private _appAuthentication: AppAuthentication, private _appLocalization: AppLocalization, private _browserService: BrowserService) {
		super(EntryWidgetKeys.Live);
	}

	protected onReset() {
		this._DVRStatus = "";
		this._showDVRWindow = false;
		this._dvrWindowAvailable = false;
		this._selectedConversionProfile = null;
		this._explicitLive = true;
		this._manualStreamsConfiguration = [];
		this._bitrates = [];
		this.dirty = false;
	}

	protected onDataSaving(data: KalturaMediaEntry, request: KalturaMultiRequest) {
		if (this._liveType === "universal") {
			// create bitrate array for saving
			let bitrates: KalturaLiveStreamBitrate[] = [];
			this._bitrates.forEach((br: bitrate) => {
				if (br.enabled) {
					bitrates.push(new KalturaLiveStreamBitrate({
						bitrate: br.bitrate,
						width: br.width,
						height: br.height
					}));
				}
			});
			(data as KalturaLiveStreamEntry).bitrates = bitrates;
		}
		if (this._liveType === "kaltura") {
			(data as KalturaLiveStreamEntry).explicitLive = this._explicitLive ? KalturaNullableBoolean.trueValue : KalturaNullableBoolean.falseValue;
			(data as KalturaLiveStreamEntry).conversionProfileId = this._selectedConversionProfile;
		}
	}

	protected onValidate(wasActivated: boolean): Observable<{ isValid: boolean}> {
		return Observable.create(observer => {
			const isValid = this._liveType === "universal" ? this._validateBitrates({updateDirtyMode: false}) : true;
			observer.next({isValid});
			observer.complete()
		});
	}

	protected onActivate(firstTimeActivating : boolean) {
		// set live type and load data accordingly
		switch (this.data.sourceType.toString()) {
			case KalturaSourceType.liveStream.toString():
				this._liveType = "kaltura";
				this._liveDashboardEnabled = serverConfig.externalApps.liveDashboard.enabled;
				this._setRecordStatus();
				this._setDVRStatus();
				super._showLoader();
				this._conversionProfiles.next({items: []});

				return this._kalturaServerClient.request(new ConversionProfileListAction({
						filter: new KalturaConversionProfileFilter({
							typeEqual: KalturaConversionProfileType.liveStream
						}),
						pager: new KalturaFilterPager({
							pageIndex: 1,
							pageSize: 500
						})
					}))
					.cancelOnDestroy(this, this.widgetReset$)
					.monitor('get conversion profiles')

					.catch((error, caught) =>
					{
						super._hideLoader();
						super._showActivationError();
						this._conversionProfiles.next({items: []});
						return Observable.throw(error);
					})
					.do(response => {
						if (response.objects && response.objects.length) {
							// set the default profile first in the array
							response.objects.sort(function (a, b) {
								if (a.isDefault > b.isDefault)
									return -1;
								if (a.isDefault < b.isDefault)
									return 1;
								return 0;
							});
							// create drop down options array
							let conversionProfiles = [];
							response.objects.forEach(profile => {
								conversionProfiles.push({label: profile.name, value: profile.id});
								if (this.data.conversionProfileId === profile.id) {
									this._selectedConversionProfile = profile.id; // preselect this profile in the profiles drop-down
								}
							});
							this._conversionProfiles.next({items: conversionProfiles});
							super._hideLoader();
						}
					});
			case KalturaSourceType.akamaiUniversalLive.toString():
				this._liveType = "universal";
				this._showDVRWindow = true;
				this._setDVRStatus();
				this._setBitrates();
				break;
			case KalturaSourceType.manualLiveStream.toString():
				this._liveType = "manual";
				this._setManualStreams();
				break;
		}
	}

	public _exportXML() {
		const entry = this.data as KalturaLiveStreamEntry;
		const xml: string = LiveXMLExporter.exportXML(entry, this._liveType, this._bitrates);
		this._browserService.download(xml, "export_" + entry.id + ".xml", "text/xml");
	}

	private _setDVRStatus(): void {
		let entry = this.data as KalturaLiveStreamEntry;
		if (!entry.dvrStatus || entry.dvrStatus.toString() === KalturaDVRStatus.disabled.toString()) {
			this._DVRStatus = this._appLocalization.get('app.common.off');
		} else if (entry.dvrStatus.toString() == KalturaDVRStatus.enabled.toString()) {
			this._DVRStatus = this._appLocalization.get('app.common.on');
			if (this._liveType === "kaltura") {
				this._showDVRWindow = true;
				this._dvrWindowAvailable = !isNaN(entry.dvrWindow);
			}
		}
		this._explicitLive = entry.explicitLive === KalturaNullableBoolean.trueValue ? true : false;
	}

	private _setRecordStatus(): void {
		let entry = this.data as KalturaLiveStreamEntry;
		if (!entry.recordStatus || entry.recordStatus.toString() === KalturaRecordStatus.disabled.toString()) {
			this._recordStatus = this._appLocalization.get('app.common.off');
		} else if (entry.recordStatus.toString() === KalturaRecordStatus.appended.toString() || entry.recordStatus.toString() === KalturaRecordStatus.perSession.toString()) {
			this._recordStatus = this._appLocalization.get('app.common.on');
		}
	}

	private _setManualStreams(): void {
		let entry: KalturaLiveStreamEntry = this.data as KalturaLiveStreamEntry;
		if (entry.liveStreamConfigurations) {
			entry.liveStreamConfigurations.forEach(streamConfig => {
				let protocol = streamConfig.protocol.toString();
				let postfix = this._appLocalization.get('applications.content.entryDetails.live.streamUrl');
				this._manualStreamsConfiguration.push({label: protocol + " " + postfix, url: streamConfig.url});
			});
		}
	}

	private _setBitrates(): void {
		this._bitrates = [];
		let entry: KalturaLiveStreamEntry = this.data as KalturaLiveStreamEntry;
		if (entry.bitrates) {
			entry.bitrates.forEach((br: KalturaLiveStreamBitrate) => {
				let bitRate: bitrate = {
					enabled: true,
					bitrate: br.bitrate,
					width: br.width,
					height: br.height,
					errors: ""
				};
				this._bitrates.push(bitRate);
			});
			// prepare empty bitrate slots for missing bitrates
			while (this._bitrates.length < 3) {
				this._bitrates.push({enabled: false, bitrate: 0, width: 0, height: 0, errors: ""});
			}
		}
	}
	public _validateBitrates({updateDirtyMode} : {updateDirtyMode: boolean}): boolean {
		let valid = true;
		this._bitrates.forEach((br: bitrate) => {
			br.errors = "";
			if (br.enabled) {
				if (br.bitrate > 0) {
					if (br.width === 0 || br.height === 0) {
						valid = false;
						br.errors = this._appLocalization.get('applications.content.entryDetails.live.dimensionsError');
					}
				} else {
					valid = false;
					br.errors = this._appLocalization.get('applications.content.entryDetails.live.bitrateError');
				}
			}
		});
		const newStatus: any = {isValid: valid};

		if (updateDirtyMode)
		{
			newStatus.isDirty = true;
		}

		super.updateState(newStatus);

		return valid;
	}

	public setDirty():void{
		super.updateState({isValid: true, isDirty: true});
	}

	public regenerateStreamToken(): void {
		this.sectionBlockerMessage = null;

		const multiRequest = new KalturaMultiRequest(
			new LiveStreamRegenerateStreamTokenAction({entryId: this.data.id}),
			new BaseEntryGetAction({entryId: this.data.id})
		);


		this._kalturaServerClient.multiRequest(multiRequest)
			.cancelOnDestroy(this, this.widgetReset$)
			.tag('block-shell')
			.monitor('regenerate stream token')
			.subscribe(
				response => {
					if (response.hasErrors()) {
						this._showBlockerMessage(new AreaBlockerMessage(
							{
								message: this._appLocalization.get('applications.content.entryDetails.live.regenerateFailure'),
								buttons: [
									{
										label: this._appLocalization.get('app.common.dismiss'),
										action: () => {
											this.sectionBlockerMessage = null;
										}
									},
									{
										label: this._appLocalization.get('app.common.retry'),
										action: () => {
											this.regenerateStreamToken();
										}
									}
								]
							}
						), false);
					}else {
						let entry: KalturaLiveStreamEntry = this.data as KalturaLiveStreamEntry;
						entry.primaryBroadcastingUrl = response[1].result.primaryBroadcastingUrl;
						entry.primaryRtspBroadcastingUrl =  response[1].result.primaryRtspBroadcastingUrl;
						entry.secondaryBroadcastingUrl =  response[1].result.secondaryBroadcastingUrl;
						entry.secondaryRtspBroadcastingUrl =  response[1].result.secondaryRtspBroadcastingUrl;
					}
				}
			);
	}

	ngOnDestroy()
    {

    }
}
<commit_msg>feat: implement permission ANALYTICS_BASE (#539) KMCNG-1595<commit_after>import {Injectable, OnDestroy} from '@angular/core';
import {BehaviorSubject} from 'rxjs/BehaviorSubject';
import {Observable} from 'rxjs/Observable';

import {KalturaClient, KalturaMultiRequest} from 'kaltura-ngx-client';
import {KalturaSourceType} from 'kaltura-ngx-client/api/types/KalturaSourceType';
import {KalturaLiveStreamBitrate} from 'kaltura-ngx-client/api/types/KalturaLiveStreamBitrate';
import {KalturaRecordStatus} from 'kaltura-ngx-client/api/types/KalturaRecordStatus';
import {KalturaLiveStreamEntry} from 'kaltura-ngx-client/api/types/KalturaLiveStreamEntry';
import {KalturaDVRStatus} from 'kaltura-ngx-client/api/types/KalturaDVRStatus';
import {KalturaMediaEntry} from 'kaltura-ngx-client/api/types/KalturaMediaEntry';
import {LiveStreamRegenerateStreamTokenAction} from 'kaltura-ngx-client/api/types/LiveStreamRegenerateStreamTokenAction';
import {AppLocalization} from '@kaltura-ng/kaltura-common';
import {AppAuthentication, BrowserService} from 'app-shared/kmc-shell';

import {EntryWidgetKeys} from '../entry-widget-keys';
import {LiveXMLExporter} from './live-xml-exporter';
import {AVAIL_BITRATES} from './bitrates';
import {EntryWidget} from '../entry-widget';
import {serverConfig} from 'config/server';
import {ConversionProfileListAction} from 'kaltura-ngx-client/api/types/ConversionProfileListAction';
import {KalturaConversionProfileFilter} from 'kaltura-ngx-client/api/types/KalturaConversionProfileFilter';
import {KalturaFilterPager} from 'kaltura-ngx-client/api/types/KalturaFilterPager';
import {KalturaConversionProfileType} from 'kaltura-ngx-client/api/types/KalturaConversionProfileType';
import {KalturaNullableBoolean} from 'kaltura-ngx-client/api/types/KalturaNullableBoolean';
import {AreaBlockerMessage} from '@kaltura-ng/kaltura-ui';
import {BaseEntryGetAction} from 'kaltura-ngx-client/api/types/BaseEntryGetAction';
import { KMCPermissions, KMCPermissionsService } from 'app-shared/kmc-shared/kmc-permissions';

export interface bitrate {
	enabled: boolean,
	bitrate: number,
	width: number,
	height: number,
	errors: string
}

@Injectable()
export class EntryLiveWidget extends EntryWidget implements OnDestroy {

	public _liveType: string = "";
	private dirty: boolean;

	private _conversionProfiles: BehaviorSubject<{ items: any[]}> = new BehaviorSubject<{ items: any[]}>({items: []});
	public _conversionProfiles$ = this._conversionProfiles.asObservable();

	public _recordStatus: string = "";
	public _DVRStatus: string = "";
	public _showDVRWindow: boolean = false;
	public _dvrWindowAvailable: boolean = false;
	public _explicitLive: boolean = true;
	public _liveDashboardEnabled: boolean = false;

	public _selectedConversionProfile: number;
	public _manualStreamsConfiguration = [];
	public _bitrates: bitrate[] = [];
	public _availableBitrates = AVAIL_BITRATES;

	public _autoStartOptions = [
		{label: this._appLocalization.get('applications.content.entryDetails.live.disabled'), value: true},
		{label: this._appLocalization.get('applications.content.entryDetails.live.enabled'), value: false}
	];

	constructor(private _kalturaServerClient: KalturaClient,
              private _appAuthentication: AppAuthentication,
              private _appLocalization: AppLocalization,
              private _permissionsService: KMCPermissionsService,
              private _browserService: BrowserService) {
		super(EntryWidgetKeys.Live);
	}

	protected onReset() {
		this._DVRStatus = "";
		this._showDVRWindow = false;
		this._dvrWindowAvailable = false;
		this._selectedConversionProfile = null;
		this._explicitLive = true;
		this._manualStreamsConfiguration = [];
		this._bitrates = [];
		this.dirty = false;
	}

	protected onDataSaving(data: KalturaMediaEntry, request: KalturaMultiRequest) {
		if (this._liveType === "universal") {
			// create bitrate array for saving
			let bitrates: KalturaLiveStreamBitrate[] = [];
			this._bitrates.forEach((br: bitrate) => {
				if (br.enabled) {
					bitrates.push(new KalturaLiveStreamBitrate({
						bitrate: br.bitrate,
						width: br.width,
						height: br.height
					}));
				}
			});
			(data as KalturaLiveStreamEntry).bitrates = bitrates;
		}
		if (this._liveType === "kaltura") {
			(data as KalturaLiveStreamEntry).explicitLive = this._explicitLive ? KalturaNullableBoolean.trueValue : KalturaNullableBoolean.falseValue;
			(data as KalturaLiveStreamEntry).conversionProfileId = this._selectedConversionProfile;
		}
	}

	protected onValidate(wasActivated: boolean): Observable<{ isValid: boolean}> {
		return Observable.create(observer => {
			const isValid = this._liveType === "universal" ? this._validateBitrates({updateDirtyMode: false}) : true;
			observer.next({isValid});
			observer.complete()
		});
	}

	protected onActivate(firstTimeActivating : boolean) {
		// set live type and load data accordingly
		switch (this.data.sourceType.toString()) {
      case KalturaSourceType.liveStream.toString():
				this._liveType = "kaltura";
        this._liveDashboardEnabled = serverConfig.externalApps.liveDashboard.enabled
          && this._permissionsService.hasPermission(KMCPermissions.ANALYTICS_BASE);
				this._setRecordStatus();
				this._setDVRStatus();
				super._showLoader();
				this._conversionProfiles.next({items: []});

				return this._kalturaServerClient.request(new ConversionProfileListAction({
						filter: new KalturaConversionProfileFilter({
							typeEqual: KalturaConversionProfileType.liveStream
						}),
						pager: new KalturaFilterPager({
							pageIndex: 1,
							pageSize: 500
						})
					}))
					.cancelOnDestroy(this, this.widgetReset$)
					.monitor('get conversion profiles')

					.catch((error, caught) =>
					{
						super._hideLoader();
						super._showActivationError();
						this._conversionProfiles.next({items: []});
						return Observable.throw(error);
					})
					.do(response => {
						if (response.objects && response.objects.length) {
							// set the default profile first in the array
							response.objects.sort(function (a, b) {
								if (a.isDefault > b.isDefault)
									return -1;
								if (a.isDefault < b.isDefault)
									return 1;
								return 0;
							});
							// create drop down options array
							let conversionProfiles = [];
							response.objects.forEach(profile => {
								conversionProfiles.push({label: profile.name, value: profile.id});
								if (this.data.conversionProfileId === profile.id) {
									this._selectedConversionProfile = profile.id; // preselect this profile in the profiles drop-down
								}
							});
							this._conversionProfiles.next({items: conversionProfiles});
							super._hideLoader();
						}
					});
			case KalturaSourceType.akamaiUniversalLive.toString():
				this._liveType = "universal";
				this._showDVRWindow = true;
				this._setDVRStatus();
				this._setBitrates();
				break;
			case KalturaSourceType.manualLiveStream.toString():
				this._liveType = "manual";
				this._setManualStreams();
				break;
		}
	}

	public _exportXML() {
		const entry = this.data as KalturaLiveStreamEntry;
		const xml: string = LiveXMLExporter.exportXML(entry, this._liveType, this._bitrates);
		this._browserService.download(xml, "export_" + entry.id + ".xml", "text/xml");
	}

	private _setDVRStatus(): void {
		let entry = this.data as KalturaLiveStreamEntry;
		if (!entry.dvrStatus || entry.dvrStatus.toString() === KalturaDVRStatus.disabled.toString()) {
			this._DVRStatus = this._appLocalization.get('app.common.off');
		} else if (entry.dvrStatus.toString() == KalturaDVRStatus.enabled.toString()) {
			this._DVRStatus = this._appLocalization.get('app.common.on');
			if (this._liveType === "kaltura") {
				this._showDVRWindow = true;
				this._dvrWindowAvailable = !isNaN(entry.dvrWindow);
			}
		}
		this._explicitLive = entry.explicitLive === KalturaNullableBoolean.trueValue ? true : false;
	}

	private _setRecordStatus(): void {
		let entry = this.data as KalturaLiveStreamEntry;
		if (!entry.recordStatus || entry.recordStatus.toString() === KalturaRecordStatus.disabled.toString()) {
			this._recordStatus = this._appLocalization.get('app.common.off');
		} else if (entry.recordStatus.toString() === KalturaRecordStatus.appended.toString() || entry.recordStatus.toString() === KalturaRecordStatus.perSession.toString()) {
			this._recordStatus = this._appLocalization.get('app.common.on');
		}
	}

	private _setManualStreams(): void {
		let entry: KalturaLiveStreamEntry = this.data as KalturaLiveStreamEntry;
		if (entry.liveStreamConfigurations) {
			entry.liveStreamConfigurations.forEach(streamConfig => {
				let protocol = streamConfig.protocol.toString();
				let postfix = this._appLocalization.get('applications.content.entryDetails.live.streamUrl');
				this._manualStreamsConfiguration.push({label: protocol + " " + postfix, url: streamConfig.url});
			});
		}
	}

	private _setBitrates(): void {
		this._bitrates = [];
		let entry: KalturaLiveStreamEntry = this.data as KalturaLiveStreamEntry;
		if (entry.bitrates) {
			entry.bitrates.forEach((br: KalturaLiveStreamBitrate) => {
				let bitRate: bitrate = {
					enabled: true,
					bitrate: br.bitrate,
					width: br.width,
					height: br.height,
					errors: ""
				};
				this._bitrates.push(bitRate);
			});
			// prepare empty bitrate slots for missing bitrates
			while (this._bitrates.length < 3) {
				this._bitrates.push({enabled: false, bitrate: 0, width: 0, height: 0, errors: ""});
			}
		}
	}
	public _validateBitrates({updateDirtyMode} : {updateDirtyMode: boolean}): boolean {
		let valid = true;
		this._bitrates.forEach((br: bitrate) => {
			br.errors = "";
			if (br.enabled) {
				if (br.bitrate > 0) {
					if (br.width === 0 || br.height === 0) {
						valid = false;
						br.errors = this._appLocalization.get('applications.content.entryDetails.live.dimensionsError');
					}
				} else {
					valid = false;
					br.errors = this._appLocalization.get('applications.content.entryDetails.live.bitrateError');
				}
			}
		});
		const newStatus: any = {isValid: valid};

		if (updateDirtyMode)
		{
			newStatus.isDirty = true;
		}

		super.updateState(newStatus);

		return valid;
	}

	public setDirty():void{
		super.updateState({isValid: true, isDirty: true});
	}

	public regenerateStreamToken(): void {
		this.sectionBlockerMessage = null;

		const multiRequest = new KalturaMultiRequest(
			new LiveStreamRegenerateStreamTokenAction({entryId: this.data.id}),
			new BaseEntryGetAction({entryId: this.data.id})
		);


		this._kalturaServerClient.multiRequest(multiRequest)
			.cancelOnDestroy(this, this.widgetReset$)
			.tag('block-shell')
			.monitor('regenerate stream token')
			.subscribe(
				response => {
					if (response.hasErrors()) {
						this._showBlockerMessage(new AreaBlockerMessage(
							{
								message: this._appLocalization.get('applications.content.entryDetails.live.regenerateFailure'),
								buttons: [
									{
										label: this._appLocalization.get('app.common.dismiss'),
										action: () => {
											this.sectionBlockerMessage = null;
										}
									},
									{
										label: this._appLocalization.get('app.common.retry'),
										action: () => {
											this.regenerateStreamToken();
										}
									}
								]
							}
						), false);
					}else {
						let entry: KalturaLiveStreamEntry = this.data as KalturaLiveStreamEntry;
						entry.primaryBroadcastingUrl = response[1].result.primaryBroadcastingUrl;
						entry.primaryRtspBroadcastingUrl =  response[1].result.primaryRtspBroadcastingUrl;
						entry.secondaryBroadcastingUrl =  response[1].result.secondaryBroadcastingUrl;
						entry.secondaryRtspBroadcastingUrl =  response[1].result.secondaryRtspBroadcastingUrl;
					}
				}
			);
	}

	ngOnDestroy()
    {

    }
}

{'fraction_non_alphanumeric': 0.12347214478706522, 'fraction_numerical': 0.0018441480464896857, 'mean_word_length': 4.0691304347826085, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '379817', 'n_tokens_mistral': 8675, 'n_tokens_neox': 7132, 'n_words': 1292}