<commit_before>import { Injectable } 		from '@angular/core';
import { Observable } 		from 'rxjs/Observable';
import { Observer } 		from 'rxjs/Observer';

import { BackendService, CallData } 	from '../backend.service';
 
@Injectable()
export class IngestService {

    constructor(private backendService: BackendService) {};

    /*
     * ingestFile
     *
     * Uploads and processes a data file in backend service.
     */
    ingestFile(file: File): Observable<string> {
	let request = new Observable((observer: Observer<string>) => {
	    this.backendService.uploadFile(file)
		.subscribe(
		    (value: CallData) => {
			if (value.state == 4) {
			    if (value.status == 200) {
				this.backendService.processFile(value.data)
				    .subscribe(
					(value: string) => observer.next(value),
					(error: string) => observer.error(error),
					() => observer.complete()
				    );
			    }
			    else {
				observer.error(value.data);
			    }
			}
			else {
			    observer.next(value.data);
    			}
		    }
		);
	});

	return request;
    }

    /*
     * uploadFile
     *
     * Uploads a file to server.
     */
    /*
    uploadFile(file: File): Observable<any> {
        let request = new Observable((observer: Observer<any>) => {
            let formData = new FormData();
            let xhr = new XMLHttpRequest();

            formData.append('upload', file, file.name);

            xhr.upload.onprogress = (event: ProgressEvent) => {
                if (event.lengthComputable) {
                    let completed = event.loaded / event.total * 100;
                    observer.next(completed);
                }
                else {
                    observer.next("Uploading...");
                }
            };

            xhr.onreadystatechange = () => {
                if (xhr.readyState == 4) {
                    if (xhr.status == 200) {
                        let name = xhr.response; // Uploaded file name on server returned.

                        let evtSource = new EventSource(this.apiUrl+'ingest/'+name);
                        
                        evtSource.onmessage = (e: any) => {
                                observer.next(e.data);
                        }

                        evtSource.addEventListener("end", (e: any) => {
                            observer.complete();
                            evtSource.close();
                        }, false);

                        evtSource.addEventListener("error", (e: any) => {
                            observer.error(e.data);
                            evtSource.close();
                        }, false);
                    }
                    else {
                        observer.error(xhr.response);
                    }
                }
            };

            xhr.open('POST', this.apiUrl+'ingest', true);
            xhr.send(formData);
        });

        return request;
    }
    */
}

<commit_msg>Code tweaked to not show upload file backend hash in progress bar.<commit_after>import { Injectable } 		from '@angular/core';
import { Observable } 		from 'rxjs/Observable';
import { Observer } 		from 'rxjs/Observer';

import { BackendService, CallData } 	from '../backend.service';
 
@Injectable()
export class IngestService {

    constructor(private backendService: BackendService) {};

    /*
     * ingestFile
     *
     * Uploads and processes a data file in backend service.
     */
    ingestFile(file: File): Observable<string> {
	let request = new Observable((observer: Observer<string>) => {
	    this.backendService.uploadFile(file)
		.subscribe(
		    (value: CallData) => {
			switch(value.state) {
			    case 1:
			    	observer.next(value.data);
				break;
			    case 4:
			        if (value.status == 200) {
				    this.backendService.processFile(value.data)
				        .subscribe(
					    (value: string) => observer.next(value),
					    (error: string) => observer.error(error),
					    () => observer.complete()
				        );
			        }
			        else {
				    observer.error(value.data);
			        }
				break;
				default:
			
    			}
		    }
		);
	});

	return request;
    }

    /*
     * uploadFile
     *
     * Uploads a file to server.
     */
    /*
    uploadFile(file: File): Observable<any> {
        let request = new Observable((observer: Observer<any>) => {
            let formData = new FormData();
            let xhr = new XMLHttpRequest();

            formData.append('upload', file, file.name);

            xhr.upload.onprogress = (event: ProgressEvent) => {
                if (event.lengthComputable) {
                    let completed = event.loaded / event.total * 100;
                    observer.next(completed);
                }
                else {
                    observer.next("Uploading...");
                }
            };

            xhr.onreadystatechange = () => {
                if (xhr.readyState == 4) {
                    if (xhr.status == 200) {
                        let name = xhr.response; // Uploaded file name on server returned.

                        let evtSource = new EventSource(this.apiUrl+'ingest/'+name);
                        
                        evtSource.onmessage = (e: any) => {
                                observer.next(e.data);
                        }

                        evtSource.addEventListener("end", (e: any) => {
                            observer.complete();
                            evtSource.close();
                        }, false);

                        evtSource.addEventListener("error", (e: any) => {
                            observer.error(e.data);
                            evtSource.close();
                        }, false);
                    }
                    else {
                        observer.error(xhr.response);
                    }
                }
            };

            xhr.open('POST', this.apiUrl+'ingest', true);
            xhr.send(formData);
        });

        return request;
    }
    */
}


{'fraction_non_alphanumeric': 0.11978821972203839, 'fraction_numerical': 0.0038054268696227664, 'mean_word_length': 1.2941176470588236, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4311280', 'n_tokens_mistral': 1780, 'n_tokens_neox': 1548, 'n_words': 313}