<commit_before>import * as _ from "lodash";

export type Context = { [key: string]: Frame; };
export interface IKeyValuePair extends ReadonlyArray<string | Frame > { 0: string; 1: Frame; }
export const Void: Context = {};

export class Frame {
  public static readonly kOUT = ">>";
  public static readonly kEND = "$$";
  public static readonly BEGIN_EXPR = "(";
  public static readonly END_EXPR = ")";
  public static readonly nil = new Frame(Void, true);
  public static readonly missing: Frame = new Frame({
    missing: Frame.nil,
  });
  public static globals = Frame.missing;

  public up: Frame;
  public callme: boolean;
  constructor(private meta = Void, isNil = false) {
    this.up = Frame.missing;
    this.callme = false;
    if (isNil) {
      this.called_by = (context: Frame, parameter: Frame) => {
        return context;
      };
    }
  }

  public string_open() { return Frame.BEGIN_EXPR; };
  public string_close() { return Frame.END_EXPR; };

  public get_here(key: string, origin: Frame = this): Frame {
    let result = this.meta[key];
    if (result != null) { return result; };
    return Frame.missing;
  }

  public get(key: string, origin: Frame = this): Frame {
    let result = this.get_here(key, origin);
    if (result !== Frame.missing) { return result; };

    let source = this.up || Frame.globals;
    if (source === Frame.missing) {
      if (Frame.globals === Frame.missing) { return Frame.missing; };
      source = Frame.globals;
    }
    return source.get(key, origin);
  }

  public set(key: string, value: Frame): Frame {
    if (this.meta === Void) {
      this.meta = {};
    }
    this.meta[key] = value;
    return this;
  }

  public at(index: number) {
    return Frame.nil;
  }

  public in(contexts = [Frame.nil]): Frame {
    return this;
  }

  public apply(argument: Frame, parameter: Frame) {
    return argument;
  }

  public called_by(context: Frame, parameter: Frame) {
    return context.apply(this, parameter);
  }

  public call(argument: Frame, parameter = Frame.nil) {
    return argument.called_by(this, parameter);
  }

  public meta_copy(): Context {
    return _.clone(this.meta);
  }

  public meta_keys() {
    return _.keys(this.meta);
  }

  public meta_length() {
    return this.meta_keys().length;
  }

  public meta_pairs(): Array<IKeyValuePair> {
    return _.map(this.meta, (value, key): IKeyValuePair => {
      return [key, value];
    });
  }

  public meta_string() {
    return this.meta_pairs().map(([key, value]) => {
      return `.${key} ${value};`;
    }).join(" ");
  }

  public toString() {
    return this.string_open() + this.meta_string() + this.string_close();
  }

  public asArray(): Array<Frame> {
    return _.castArray(this);
  }
};

export class FrameAtom extends Frame {
  public string_prefix() { return ""; };
  public string_suffix() { return ""; };

  public toStringData(): string {
    return this.string_prefix() + this.toData().toString() + this.string_suffix();<commit_msg>Top level nil and void checks<commit_after>import * as _ from "lodash";

export type Context = { [key: string]: Frame; };
export interface IKeyValuePair extends ReadonlyArray<string | Frame > { 0: string; 1: Frame; }
export const NilContext: Context = {};

export class Frame {
  public static readonly kOUT = ">>";
  public static readonly kEND = "$$";
  public static readonly BEGIN_EXPR = "(";
  public static readonly END_EXPR = ")";
  public static readonly nil = new Frame(NilContext, true);
  public static readonly missing: Frame = new Frame({
    missing: Frame.nil,
  });
  public static globals = Frame.missing;

  public up: Frame;
  public callme: boolean;
  constructor(private meta = NilContext, isNil = false) {
    this.up = Frame.missing;
    this.callme = false;
    if (isNil) {
      this.called_by = (context: Frame, parameter: Frame) => {
        return context;
      };
    }
  }

  public string_open() { return Frame.BEGIN_EXPR; };
  public string_close() { return Frame.END_EXPR; };

  public get_here(key: string, origin: Frame = this): Frame {
    let result = this.meta[key];
    if (result != null) { return result; };
    return Frame.missing;
  }

  public get(key: string, origin: Frame = this): Frame {
    let result = this.get_here(key, origin);
    if (result !== Frame.missing) { return result; };

    let source = this.up || Frame.globals;
    if (source === Frame.missing) {
      if (Frame.globals === Frame.missing) { return Frame.missing; };
      source = Frame.globals;
    }
    return source.get(key, origin);
  }

  public set(key: string, value: Frame): Frame {
    if (this.meta === NilContext) {
      this.meta = {};
    }
    this.meta[key] = value;
    return this;
  }

  public at(index: number) {
    return Frame.nil;
  }

  public in(contexts = [Frame.nil]): Frame {
    return this;
  }

  public apply(argument: Frame, parameter: Frame) {
    return argument;
  }

  public called_by(context: Frame, parameter: Frame) {
    return context.apply(this, parameter);
  }

  public call(argument: Frame, parameter = Frame.nil) {
    return argument.called_by(this, parameter);
  }

  public meta_copy(): Context {
    return _.clone(this.meta);
  }

  public meta_keys() {
    return _.keys(this.meta);
  }

  public meta_length() {
    return this.meta_keys().length;
  }

  public meta_pairs(): Array<IKeyValuePair> {
    return _.map(this.meta, (value, key): IKeyValuePair => {
      return [key, value];
    });
  }

  public meta_string() {
    return this.meta_pairs().map(([key, value]) => {
      return `.${key} ${value};`;
    }).join(" ");
  }

  public toString() {
    return this.string_open() + this.meta_string() + this.string_close();
  }

  public asArray(): Array<Frame> {
    return _.castArray(this);
  }

  public is_nil() {
    return this === Frame.nil;
  }

  public is_void() {
    return false;
  }
};

export class FrameAtom extends Frame {
  public string_prefix() { return ""; };
  public string_suffix() { return ""; };

  public toStringData(): string {
    return this.string_prefix() + this.toData().toString() + this.string_suffix();
{'fraction_non_alphanumeric': 0.14425907752698724, 'fraction_numerical': 0.0006542361792607131, 'mean_word_length': 3.157036029911625, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4781636', 'n_tokens_mistral': 1968, 'n_tokens_neox': 1912, 'n_words': 523}