mergekit_moe_config.yml

base_model: TroyDoesAI/BlackSheep-4B
gate_mode: random # one of "hidden", "cheap_embed", or "random"

# There are three methods for populating the MoE gates implemented.

# "hidden"
# Uses the hidden state representations of the positive/negative prompts for MoE gate parameters. Best quality and most effective option; the default. Requires evaluating each prompt using the base model so you might not be able to use this on constrained hardware (depending on the model). You can use --load-in-8bit or --load-in-4bit to reduce VRAM usage.

# "cheap_embed"
# Uses only the raw token embedding of the prompts, using the same gate parameters for every layer. Distinctly less effective than "hidden". Can be run on much, much lower end hardware.

# "random"
# Randomly initializes the MoE gates. Good for if you are going to fine tune the model afterwards, or maybe if you want something a little unhinged? I won't judge.

experts:
  - source_model: TroyDoesAI/BlackSheep-4B
    positive_prompts:
    - "digital"
    - "soul"
    - "<|assistant|>"
    - "instruction"
    - "input"
    - "output"
    - "response"
    - "story"
    - "stories"
    - "creative"
    - "curious"
    - "action"
    - "sensitive"
    - "pleasant"
    - "thoughtful"
    - "interesting"
    - "BEGININPUT"
    - "BEGINCONTEXT"
    - "ENDCONTEXT"
    - "ENDINPUT"
    - "BEGININSTRUCTION"
    - "ENDINSTRUCTION"
    - "url"
    - "date"
    - "source"
    - "context"
    - "cite"
    - "mermaid"
    - "flow"
    - "diagram"
    - "sequence"
    - "series"
    - "code to flow"
    - "code to diagram"
    - "python to flow diagram"
    - "program to flow"
    - "program to flow diagram"
    - "write"
    - "create"
    - "visualize"
    - "show"
    - "explain"
    - "draw"
    - "describe"
    - "summarize"
    - "concise"
    - "do"
    - "say"
    - "code"
    - "```"
    - "```mermaid"
    - "graphTB"
    - "graphTD"
    - "graphLR"
    - "graphRL"

  - source_model: TroyDoesAI/BlackSheep-4B
    positive_prompts:
    - "Digital Soul"
    - "hello"
    - "you"
    - "assistant"
    - "who"
    - "what"
    - "when"
    - "where"
    - "why"
    - "how"
    - "me"
    - "politic"
    - "conservative"

  - source_model: TroyDoesAI/BlackSheep-4B
    positive_prompts:
    - "<|assistant|>"
    - "instruction"
    - "input"
    - "output"
    - "response"
    - "story"
    - "stories"
    - "creative"
    - "curious"
    - "action"
    - "sensitive"
    - "pleasant"
    - "thoughtful"
    - "interesting"
    - "BEGININPUT"
    - "BEGINCONTEXT"
    - "ENDCONTEXT"
    - "ENDINPUT"
    - "BEGININSTRUCTION"
    - "ENDINSTRUCTION"
    - "url"
    - "date"
    - "source"
    - "context"
    - "cite"
    - "mermaid"
    - "flow"
    - "diagram"
    - "sequence"
    - "series"
    - "code to flow"
    - "code to diagram"
    - "python to flow diagram"
    - "program to flow"
    - "program to flow diagram"
    - "write"
    - "create"
    - "visualize"
    - "show"
    - "explain"
    - "draw"
    - "describe"
    - "summarize"
    - "concise"
    - "do"
    - "say"
    - "code"
    - "```"
    - "```mermaid"
    - "graphTB"
    - "graphTD"
    - "graphLR"
    - "graphRL"