<commit_before>import {DataSource, areEquals, VersionedObject, VersionedObjectManager, Invocation, Identifier, ControlCenter, DataSourceInternal, AComponent, Aspect, ImmutableMap} from './core';
import {Reporter, Diagnostic} from '@openmicrostep/msbuildsystem.shared';
import ObjectSet = DataSourceInternal.ObjectSet;
declare var console: any;

export type MemoryDataSourceTransaction = { tr: InMemoryDataSource.DataStoreTransaction };
export class InMemoryDataSource extends DataSource 
{
  constructor(manager: VersionedObjectManager<InMemoryDataSource>, private ds: InMemoryDataSource.DataStore) {
    super(manager);
  }
  static parent = DataSource;
  static definition = {
    is: "class",
    name: "InMemoryDataSource",
    version: 0,
    aspects: DataSource.definition.aspects
  };
  static installAspect(on: ControlCenter, name: 'client'): { new(): DataSource.Aspects.client };
  static installAspect(on: ControlCenter, name: 'server'): { new(ds?: InMemoryDataSource.DataStore): DataSource.Aspects.server };
  static installAspect(on: ControlCenter, name:string): any {
    return on.cache().createAspect(on, name, this);
  }

  implQuery({ tr, sets }: { tr?: InMemoryDataSource.DataStoreTransaction, sets: ObjectSet[] }): { [k: string]: VersionedObject[] } {
    let ds = tr || this.ds;
    let ret = {};
    let cc = this.controlCenter();
    let component = {};
    cc.registerComponent(component);
    let res = DataSourceInternal.applySets(sets, ds.objectsAsArray(), true, {
      aspect: (vo: InMemoryDataSource.DataStoreObject) => cc.aspect(vo.is)!.aspect,
      has: (vo: InMemoryDataSource.DataStoreObject, attribute: string) => vo.has(attribute),
      get: (vo: InMemoryDataSource.DataStoreObject, attribute: string) => {
        if (attribute === "_id")
          return ds.get(vo.id);
        return ds.fixValue(vo.get(attribute));
      },
      todb: (vo: InMemoryDataSource.DataStoreObject, attribute: string, value) => {
        if (attribute === "_id" && typeof value !== "object")
          return ds.get(this.ds.toDSId(value));
        return ds.toDSValue(value);
      }
    });
    res.forEach((objs, set) => {
      ret[set.name] = objs.map(dObject => {
        let cstor = cc.aspect(dObject.is)!;
        let lId = this.ds.fromDSId(dObject.id);
        let lObject = cc.registeredObject(lId) || new cstor();
        let remoteAttributes = new Map<keyof VersionedObject, any>();
        let lManager = lObject.manager();
        cc.registerObjects(component, [lObject]);
        if (set.scope) for (let k of set.scope as (keyof VersionedObject)[])
          remoteAttributes.set(k, ds.fromDSValue(cc, component, dObject.get(k)));
        lManager.setId(lId);
        lManager.mergeWithRemoteAttributes(remoteAttributes, dObject.version);
        return lObject;
      });
    });
    cc.unregisterComponent(component);
    return ret;
  }

  implLoad({tr, objects, scope} : {
    tr?: InMemoryDataSource.DataStoreTransaction;
    objects: VersionedObject[];
    scope?: string[];
  }): VersionedObject[] {
    let ds = tr || this.ds;
    let cc = this.controlCenter();
    let component = {};
    let ret: VersionedObject[] = [];
    cc.registerComponent(component);
    if (objects) {
      for (let lObject of objects) {
        let dbId = this.ds.toDSId(lObject.id());
        let dObject = ds.get(dbId);
        if (dObject) {
          let lManager = lObject.manager();
          let remoteAttributes = new Map<keyof VersionedObject, any>();
          cc.registerObjects(component, [lObject]);
          if (scope) for (let k of scope as (keyof VersionedObject)[])
            remoteAttributes.set(k, ds.fromDSValue(cc, component, dObject.get(k)));
          lManager.mergeWithRemoteAttributes(remoteAttributes, dObject.version);
          ret.push(lObject);
        }
      }
    }
    cc.unregisterComponent(component);
    return ret;
  }

  implBeginTransaction(): InMemoryDataSource.DataStoreTransaction {
    return this.ds.beginTransaction();
  }<commit_msg>core: memory dataSource is now compliant to the scope spec<commit_after>import {DataSource, areEquals, VersionedObject, VersionedObjectManager, Invocation, Identifier, ControlCenter, DataSourceInternal, AComponent, Aspect, ImmutableMap} from './core';
import {Reporter, Diagnostic} from '@openmicrostep/msbuildsystem.shared';
import ObjectSet = DataSourceInternal.ObjectSet;
declare var console: any;

function load(
  cc: ControlCenter, component: AComponent, ds: InMemoryDataSource.DataStoreCRUD, 
  unused: Set<string>, subs: Set<InMemoryDataSource.DataStoreObject>, lObject: VersionedObject, dObject: InMemoryDataSource.DataStoreObject
) {
  let lManager = lObject.manager();
  let remoteAttributes = new Map<keyof VersionedObject, any>();
  cc.registerObjects(component, [lObject]);
  for (let k of unused) {
    let a = lManager.aspect().attributes.get(k);
    if (a) {
      let v = dObject.get(k);
      if (v instanceof InMemoryDataSource.DataStoreObject)
        subs.add(v);
      remoteAttributes.set(k as keyof VersionedObject, ds.fromDSValue(cc, component, v));
      unused.delete(k);
    }
  }
  lManager.mergeWithRemoteAttributes(remoteAttributes, dObject.version);
}

export type MemoryDataSourceTransaction = { tr: InMemoryDataSource.DataStoreTransaction };
export class InMemoryDataSource extends DataSource 
{
  constructor(manager: VersionedObjectManager<InMemoryDataSource>, private ds: InMemoryDataSource.DataStore) {
    super(manager);
  }
  static parent = DataSource;
  static definition = {
    is: "class",
    name: "InMemoryDataSource",
    version: 0,
    aspects: DataSource.definition.aspects
  };
  static installAspect(on: ControlCenter, name: 'client'): { new(): DataSource.Aspects.client };
  static installAspect(on: ControlCenter, name: 'server'): { new(ds?: InMemoryDataSource.DataStore): DataSource.Aspects.server };
  static installAspect(on: ControlCenter, name:string): any {
    return on.cache().createAspect(on, name, this);
  }

  private _loads(
    cc: ControlCenter, component: AComponent, ds: InMemoryDataSource.DataStoreCRUD, 
    scope: string[], lObject: VersionedObject, dObject: InMemoryDataSource.DataStoreObject
  ) {
    
    let unused = new Set(scope as (keyof VersionedObject)[]);
    let subs = new Set<InMemoryDataSource.DataStoreObject>();
    load(cc, component, ds, unused, subs, lObject, dObject);
    while (unused.size > 0 && subs.size > 0) {
      let previousSubs = subs;
      subs = new Set<InMemoryDataSource.DataStoreObject>();
      for (let dObject of previousSubs) {
        let lId = this.ds.fromDSId(dObject.id);
        let lObject = cc.registeredObject(lId, dObject.is);
        load(cc, component, ds, unused, subs, lObject, dObject);
      }
    }
  }

  implQuery({ tr, sets }: { tr?: InMemoryDataSource.DataStoreTransaction, sets: ObjectSet[] }): { [k: string]: VersionedObject[] } {
    let ds = tr || this.ds;
    let ret = {};
    let cc = this.controlCenter();
    let component = {};
    cc.registerComponent(component);
    let res = DataSourceInternal.applySets(sets, ds.objectsAsArray(), true, {
      aspect: (vo: InMemoryDataSource.DataStoreObject) => cc.aspect(vo.is)!.aspect,
      has: (vo: InMemoryDataSource.DataStoreObject, attribute: string) => vo.has(attribute),
      get: (vo: InMemoryDataSource.DataStoreObject, attribute: string) => {
        if (attribute === "_id")
          return ds.get(vo.id);
        return ds.fixValue(vo.get(attribute));
      },
      todb: (vo: InMemoryDataSource.DataStoreObject, attribute: string, value) => {
        if (attribute === "_id" && typeof value !== "object")
          return ds.get(this.ds.toDSId(value));
        return ds.toDSValue(value);
      }
    });
    res.forEach((objs, set) => {
      ret[set.name] = objs.map(dObject => {
        let lId = this.ds.fromDSId(dObject.id);
        let lObject = cc.registeredObject(lId, dObject.is);
        if (set.scope)
          this._loads(cc, component, ds, set.scope, lObject, dObject);
        return lObject;
      });
    });
    cc.unregisterComponent(component);
    return ret;
  }

  implLoad({tr, objects, scope} : {
    tr?: InMemoryDataSource.DataStoreTransaction;
    objects: VersionedObject[];
    scope: string[];
  }): VersionedObject[] {
    let ds = tr || this.ds;
    let cc = this.controlCenter();
    let component = {};
    let ret: VersionedObject[] = [];
    cc.registerComponent(component);
    if (objects) {
      for (let lObject of objects) {
        let dbId = this.ds.toDSId(lObject.id());
        let dObject = ds.get(dbId);
        if (dObject) {
          this._loads(cc, component, ds, scope, lObject, dObject);
          ret.push(lObject);
        }
      }
    }
    cc.unregisterComponent(component);
    return ret;
  }

  implBeginTransaction(): InMemoryDataSource.DataStoreTransaction {
    return this.ds.beginTransaction();
  }
{'fraction_non_alphanumeric': 0.13019484176145962, 'fraction_numerical': 0.00045050118256560425, 'mean_word_length': 3.5702521873391664, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '810818', 'n_tokens_mistral': 2642, 'n_tokens_neox': 2637, 'n_words': 611}