Llama-3-DARE-v2-8B

This is a merge of pre-trained language models created using mergekit.

Merge Details

Merge Method

This model was merged using the DARE TIES merge method using meta-llama/Meta-Llama-3-8B as a base.

Models Merged

The following models were included in the merge:

• meta-llama/Meta-Llama-3-8B-Instruct

Configuration

The following YAML configuration was used to produce this model:

base_model:
  model:
    path: meta-llama/Meta-Llama-3-8B
dtype: bfloat16
merge_method: dare_ties
parameters:
  int8_mask: 1.0
  normalize: 0.0
slices:
- sources:
  - layer_range: [0, 32]
    model:
      model:
        path: meta-llama/Meta-Llama-3-8B-Instruct
    parameters:
      density:
      - filter: embed_token
        value: 0.1312973174759711
      - filter: model.norm
        value: 0.9822573068643683
      - filter: lm_head
        value: 0.9822573068643683
      - filter: layers.0.
        value: 0.7585980531146381
      - filter: layers.1.
        value: 0.561884676687423
      - filter: layers.2.
        value: 0.9813332103529979
      - filter: layers.3.
        value: 0.21930890320643737
      - filter: layers.4.
        value: 0.8438595439416953
      - filter: layers.5.
        value: 0.7505829508553786
      - filter: layers.6.
        value: 0.25806652766624183
      - filter: layers.7.
        value: 0.09440963517101368
      - filter: layers.8.
        value: 0.7755855137764502
      - filter: layers.9.
        value: 0.980153628060705
      - filter: layers.10.
        value: 0.19005899180649255
      - filter: layers.11.
        value: 0.8607970083829785
      - filter: layers.12.
        value: 0.397155695730673
      - filter: layers.13.
        value: 0.13006504372382097
      - filter: layers.14.
        value: 0.09190339054787032
      - filter: layers.15.
        value: 0.14507208971010788
      - filter: layers.16.
        value: 0.8783567570968138
      - filter: layers.17.
        value: 0.12105889524706802
      - filter: layers.18.
        value: 0.8613122534393256
      - filter: layers.19.
        value: 0.04810799757747608
      - filter: layers.20.
        value: 0.5576780215726541
      - filter: layers.21.
        value: 0.027665563208104402
      - filter: layers.22.
        value: 0.7853866990645277
      - filter: layers.23.
        value: 0.8674754718436442
      - filter: layers.24.
        value: 0.915444005146838
      - filter: layers.25.
        value: 0.8408012482479134
      - filter: layers.26.
        value: 0.021388326989086454
      - filter: layers.27.
        value: 0.8382317625625855
      - filter: layers.28.
        value: 0.0418890647830244
      - filter: layers.29.
        value: 0.16347844552378538
      - filter: layers.30.
        value: 0.3152357116732385
      - filter: layers.31.
        value: 0.24560833074353305
      weight:
      - filter: embed_token
        value: 0.07729925635324639
      - filter: model.norm.
        value: 0.8178953444711945
      - filter: lm_head
        value: 0.8178953444711945
      - filter: layers.0.
        value: 0.7744866238001816
      - filter: layers.1.
        value: 0.9434402042495054
      - filter: layers.2.
        value: 0.7277347873270702
      - filter: layers.3.
        value: 0.7344542377948238
      - filter: layers.4.
        value: 0.7789550773026275
      - filter: layers.5.
        value: 0.9977070050825179
      - filter: layers.6.
        value: 0.9790331299458723
      - filter: layers.7.
        value: 0.936694814017198
      - filter: layers.8.
        value: 0.8022245662093622
      - filter: layers.9.
        value: 0.9207818987772527
      - filter: layers.10.
        value: 0.9177346250309115
      - filter: layers.11.
        value: 0.9705518504378707
      - filter: layers.12.
        value: 0.6962192295710468
      - filter: layers.13.
        value: 0.6601913931204128
      - filter: layers.14.
        value: 0.7700302171860424
      - filter: layers.15.
        value: 0.8764914331693985
      - filter: layers.16.
        value: 0.1680474521036102
      - filter: layers.17.
        value: 0.7252502118522168
      - filter: layers.18.
        value: 0.974742254778171
      - filter: layers.19.
        value: 0.6195931122778885
      - filter: layers.20.
        value: 0.8808789662609544
      - filter: layers.21.
        value: 0.10257515634421632
      - filter: layers.22.
        value: 0.04739908873803297
      - filter: layers.23.
        value: 0.9609916339993614
      - filter: layers.24.
        value: 0.919873783591387
      - filter: layers.25.
        value: 0.8920927224651306
      - filter: layers.26.
        value: 0.3163445951176943
      - filter: layers.27.
        value: 0.9183620719063076
      - filter: layers.28.
        value: 0.7696116230696886
      - filter: layers.29.
        value: 0.8092141535971384
      - filter: layers.30.
        value: 0.9831761728133218
      - filter: layers.31.
        value: 0.8368550180785097
  - layer_range: [0, 32]
    model:
      model:
        path: meta-llama/Meta-Llama-3-8B
    parameters:
      density:
      - filter: embed_token
        value: 0.17966590577034688
      - filter: model.norm
        value: 0.21103453748756532
      - filter: lm_head
        value: 0.21103453748756532
      - filter: layers.0.
        value: 0.1258111935267289
      - filter: layers.1.
        value: 0.14712518700082466
      - filter: layers.2.
        value: 0.15119918492525058
      - filter: layers.3.
        value: 0.2786737466113946
      - filter: layers.4.
        value: 0.7941421047966343
      - filter: layers.5.
        value: 0.14996187005944175
      - filter: layers.6.
        value: 0.21464495779281623
      - filter: layers.7.
        value: 0.9544388237818261
      - filter: layers.8.
        value: 0.6823542810137563
      - filter: layers.9.
        value: 0.12285240879708
      - filter: layers.10.
        value: 0.9560979622228943
      - filter: layers.11.
        value: 0.18774045206190226
      - filter: layers.12.
        value: 0.040667440606417166
      - filter: layers.13.
        value: 0.06577308115912563
      - filter: layers.14.
        value: 0.10898066770591856
      - filter: layers.15.
        value: 0.7988665902205875
      - filter: layers.16.
        value: 0.6902648842556923
      - filter: layers.17.
        value: 0.21929644905849668
      - filter: layers.18.
        value: 0.031818815520317414
      - filter: layers.19.
        value: 0.09373597067550557
      - filter: layers.20.
        value: 0.05663841107031176
      - filter: layers.21.
        value: 0.573566646668988
      - filter: layers.22.
        value: 0.2100277505470118
      - filter: layers.23.
        value: 0.14741793214499152
      - filter: layers.24.
        value: 0.13421247708500247
      - filter: layers.25.
        value: 0.3923811614341299
      - filter: layers.26.
        value: 0.7128080834275775
      - filter: layers.27.
        value: 0.7173419548629656
      - filter: layers.28.
        value: 0.25148835909796347
      - filter: layers.29.
        value: 0.01041679056502079
      - filter: layers.30.
        value: 0.04502964289907793
      - filter: layers.31.
        value: 0.7933910983298705
      weight:
      - filter: embed_token
        value: 0.1379551165348899
      - filter: model.norm.
        value: 0.1145495237372664
      - filter: lm_head
        value: 0.1145495237372664
      - filter: layers.0.
        value: 0.058088970620673676
      - filter: layers.1.
        value: 0.9812897914515345
      - filter: layers.2.
        value: 0.7981160133946364
      - filter: layers.3.
        value: 0.7634141926997863
      - filter: layers.4.
        value: 0.16065704042361334
      - filter: layers.5.
        value: 0.11985625529888592
      - filter: layers.6.
        value: 0.12337476562239155
      - filter: layers.7.
        value: 0.24780333206355964
      - filter: layers.8.
        value: 0.031049663247481193
      - filter: layers.9.
        value: 0.9399411230471199
      - filter: layers.10.
        value: 0.059148520666240975
      - filter: layers.11.
        value: 0.7231307420560733
      - filter: layers.12.
        value: 0.8604431238597543
      - filter: layers.13.
        value: 0.7874665970878372
      - filter: layers.14.
        value: 0.18061875009463604
      - filter: layers.15.
        value: 0.09518784175500823
      - filter: layers.16.
        value: 0.27798323894923355
      - filter: layers.17.
        value: 0.9413932817490506
      - filter: layers.18.
        value: 0.2512810899514295
      - filter: layers.19.
        value: 0.11709350712163674
      - filter: layers.20.
        value: 0.6739674929530162
      - filter: layers.21.
        value: 0.21208528089924455
      - filter: layers.22.
        value: 0.811536887881144
      - filter: layers.23.
        value: 0.6200821391879456
      - filter: layers.24.
        value: 0.7663192062414923
      - filter: layers.25.
        value: 0.8657672217068108
      - filter: layers.26.
        value: 0.10630047742154969
      - filter: layers.27.
        value: 0.9641883915469491
      - filter: layers.28.
        value: 0.9807033575417018
      - filter: layers.29.
        value: 0.04810198321663159
      - filter: layers.30.
        value: 0.8249574979326807
      - filter: layers.31.
        value: 0.01564957234872716