<commit_before>import { Injectable } from '@angular/core';
import { HttpClient, HttpParams } from '@angular/common/http';
import { BehaviorSubject } from 'rxjs/BehaviorSubject';
import { Observable } from 'rxjs/Observable';
import { Subscription } from 'rxjs/Subscription';
import { map, catchError } from 'rxjs/operators';
import { _throw } from 'rxjs/observable/throw';
import { environment } from '../environments/environment';
import { Fetch } from './data-types';


@Injectable()
export class GeneralDataService {

  public apiUrl = environment.API_URL;
  private _quickLoaded = false;
  private _orgData : {[key: string]: any} = {};
  private _recordCounts : {[key: string]: number} = {};
  private _currentResultSubj = new BehaviorSubject<Fetch.BaseResult<any>>(null);
  private _loaderSub: Subscription = null;
  private _defaultTopicType = 'registration';

  constructor(private _http: HttpClient) {
  }

  getRequestUrl(path: string) : string {
    if(typeof path === 'string' && path.match(/^(\/\/|\w+:\/\/)\w/)) {
      // absolute URL
      return path;
    }
    let root = (<any>window).testApiUrl || this.apiUrl;

    if(root) {
      if(! root.endsWith('/')) root += '/';
      return root + path;
    }
  }

  get defaultTopicType(): string {
    return this._defaultTopicType;
  }

  loadJson(url, params?: HttpParams) : Observable<Object> {
    return this._http.get(url, {params: params})
      .pipe(catchError(error => {
        console.error("JSON load error", error);
        return _throw(error);
      }));
  }

  loadFromApi(path: string, params?: HttpParams) : Observable<Object> {
    let url = this.getRequestUrl(path);
    if(url) {
      return this.loadJson(url, params);
    }
  }

  quickLoad(force?) {
    return new Promise((resolve, reject) => {
      if(this._quickLoaded && !force) {
        resolve(1);
        return;
      }
      let baseurl = this.getRequestUrl('');
      console.log('base url: ' + baseurl);
      if(! baseurl) {
        reject("Base URL not defined");
        return;
      }
      let req = this._http.get(baseurl + 'quickload')
        .pipe(catchError(error => {
          console.error(error);
          return _throw(error);
        }));
      req.subscribe((data: any) => {
        console.log('quickload', data);
        if(data.counts) {
          for (let k in data.counts) {
            this._recordCounts[k] = parseInt(data.counts[k]);
          }
        }
        if(data.records) {
          for (let k in data.records) {
            this._orgData[k] = data.records[k];
          }
        }
        this._quickLoaded = true;
        resolve(1);
      }, err => {
        reject(err);
      });
    });
  }

  getRecordCount (type) {
    return this._recordCounts[type] || 0;
  }

  autocomplete (term) : Observable<Object> {
    if(term === '' || typeof(term) !== 'string') {
      return Observable.of([]);
    }
    let params = new HttpParams().set('q', term);
    return this.loadFromApi('search/autocomplete', params)
      .pipe(map(response => {
        let ret = [];
        for(let row of response['results']) {
          let found = null;
          for(let name of row.names) {
            if(~ name.text.toLowerCase().indexOf(term.toLowerCase())) {
              found = name.text;
              break;
            } else if(found === null) {
              found = name.text;
            }
          }
          if(found !== null) {
            ret.push({id: row.id, term: found});
          }
        }
        console.log(ret);
        return ret;
      }));<commit_msg>remove debug logging<commit_after>import { Injectable } from '@angular/core';
import { HttpClient, HttpParams } from '@angular/common/http';
import { BehaviorSubject } from 'rxjs/BehaviorSubject';
import { Observable } from 'rxjs/Observable';
import { Subscription } from 'rxjs/Subscription';
import { map, catchError } from 'rxjs/operators';
import { _throw } from 'rxjs/observable/throw';
import { environment } from '../environments/environment';
import { Fetch } from './data-types';


@Injectable()
export class GeneralDataService {

  public apiUrl = environment.API_URL;
  private _quickLoaded = false;
  private _orgData : {[key: string]: any} = {};
  private _recordCounts : {[key: string]: number} = {};
  private _currentResultSubj = new BehaviorSubject<Fetch.BaseResult<any>>(null);
  private _loaderSub: Subscription = null;
  private _defaultTopicType = 'registration';

  constructor(private _http: HttpClient) {
  }

  getRequestUrl(path: string) : string {
    if(typeof path === 'string' && path.match(/^(\/\/|\w+:\/\/)\w/)) {
      // absolute URL
      return path;
    }
    let root = (<any>window).testApiUrl || this.apiUrl;

    if(root) {
      if(! root.endsWith('/')) root += '/';
      return root + path;
    }
  }

  get defaultTopicType(): string {
    return this._defaultTopicType;
  }

  loadJson(url, params?: HttpParams) : Observable<Object> {
    return this._http.get(url, {params: params})
      .pipe(catchError(error => {
        console.error("JSON load error", error);
        return _throw(error);
      }));
  }

  loadFromApi(path: string, params?: HttpParams) : Observable<Object> {
    let url = this.getRequestUrl(path);
    if(url) {
      return this.loadJson(url, params);
    }
  }

  quickLoad(force?) {
    return new Promise((resolve, reject) => {
      if(this._quickLoaded && !force) {
        resolve(1);
        return;
      }
      let baseurl = this.getRequestUrl('');
      console.log('base url: ' + baseurl);
      if(! baseurl) {
        reject("Base URL not defined");
        return;
      }
      let req = this._http.get(baseurl + 'quickload')
        .pipe(catchError(error => {
          console.error(error);
          return _throw(error);
        }));
      req.subscribe((data: any) => {
        if(data.counts) {
          for (let k in data.counts) {
            this._recordCounts[k] = parseInt(data.counts[k]);
          }
        }
        if(data.records) {
          for (let k in data.records) {
            this._orgData[k] = data.records[k];
          }
        }
        this._quickLoaded = true;
        resolve(1);
      }, err => {
        reject(err);
      });
    });
  }

  getRecordCount (type) {
    return this._recordCounts[type] || 0;
  }

  autocomplete (term) : Observable<Object> {
    if(term === '' || typeof(term) !== 'string') {
      return Observable.of([]);
    }
    let params = new HttpParams().set('q', term);
    return this.loadFromApi('search/autocomplete', params)
      .pipe(map(response => {
        let ret = [];
        for(let row of response['results']) {
          let found = null;
          for(let name of row.names) {
            if(~ name.text.toLowerCase().indexOf(term.toLowerCase())) {
              found = name.text;
              break;
            } else if(found === null) {
              found = name.text;
            }
          }
          if(found !== null) {
            ret.push({id: row.id, term: found});
          }
        }
        return ret;
      }));
{'fraction_non_alphanumeric': 0.1532041307115575, 'fraction_numerical': 0.0008487763474324516, 'mean_word_length': 2.309925093632959, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2387137', 'n_tokens_mistral': 2152, 'n_tokens_neox': 2135, 'n_words': 469}