ashwinram472/alpaca-cleaned-lora-7b

This repo contains a low-rank adapter for Llama-7b finetuned on the Cleaned Alpaca version of the Dataset.

This version was finetuned using:

python finetune.py \
    --base_model 'llama_7b_hf' \
    --data_path 'yahma/alpaca-cleaned' \
    --output_dir 'ashwinram472/lora-alpaca-cleaned' \
    --batch_size 128 \
    --micro_batch_size 4 \
    --num_epochs 10 \
    --learning_rate 1e-4 \
    --cutoff_len 512 \
    --val_set_size 2000 \
    --lora_r 16 \
    --lora_alpha 16 \
    --lora_dropout 0.05 \
    --lora_target_modules '[q_proj,k_proj,v_proj,o_proj]' \
    --train_on_inputs \
    --group_by_length \
    --wandb_project 'alpaca-lora-cleaned'
    --wandb_run_name 'alpaca-lora-10epoch'

For Training logs visit W&B report: here.

model = LlamaForCausalLM.from_pretrained(
        '/llama_7b_hf',
        load_in_8bit=True,
        torch_dtype=torch.float16,
        device_map='auto',
    )

lora_weights = 'ashwinram472/alpaca-cleaned-lora-7b'

model = PeftModel.from_pretrained(
    model,
    lora_weights,
    torch_dtype=torch.float16,
)

tokenizer = LlamaTokenizer.from_pretrained("../models/llama_7b_hf")

def generate_prompt(instruction: str, input_ctxt: str = None) -> str:
    if input_ctxt:
        return f"""Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.

### Instruction:
{instruction}

### Input:
{input_ctxt}

### Response:"""
    else:
        return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request.

### Instruction:
{instruction}

### Response:"""

generation_config = GenerationConfig(
    temperature=0.1,
    top_p=0.75,
    top_k=40,
    num_beams=4,
    max_new_tokens=128,
)

model.eval()


instruction = "Count up from 1 to 500."

input_ctxt = None  # For some tasks, you can provide an input context to help the model generate a better response.

prompt = generate_prompt(instruction, input_ctxt)
input_ids = tokenizer(prompt, return_tensors="pt").input_ids
input_ids = input_ids.to(model.device)

with torch.no_grad():
    outputs = model.generate(
        input_ids=input_ids,
        generation_config=generation_config,
        return_dict_in_generate=True,
        output_scores=True,
    )

response = tokenizer.decode(outputs.sequences[0], skip_special_tokens=True)

print(response.split("### Response:")[1].strip().split("### Instruction")[0])