Update README.md

README.md

@@ -1,10 +1,8 @@
 ---
-license: gpl-3.0
+license: apache-2.0
 datasets:
-- tatsu-lab/alpaca
-- yizhongw/self_instruct
-- anon8231489123/ShareGPT_Vicuna_unfiltered
-- NeelNanda/pile-10k
+- OpenAssistant/oasst1
+- EleutherAI/pile
 language:
 - en
 - es
@@ -15,6 +13,8 @@ metrics:
 - accuracy
 - bleu
 pipeline_tag: text-generation
+tags:
+- code
 ---
 
 
@@ -23,11 +23,9 @@ this model uses Task classification and the conversation is between USER and Ans
 # NOTE ⚠️
 
 
-THE JAX/FLAX version of model will soon be available
+THE JAX/FLAX version of model is available both for training and usage
 
 
-This model is a finetuned version of Kolla with LGeM data With Respect to them and changes some data and optimizers
-The model includes pre-trained Weights so it is GNU v3.0 licensed as the same as Original Llama Model 
 
 # Using Model in Huggingface Transformers
 
@@ -62,48 +60,12 @@ model = LlamaForCausalLM.from_pretrained(
     torch_dtype=torch.float16
 )
 
-def generator(input_text,pipe_line,task='CONVERSATION',max_number=256,do_print=False ,args_a=False):
+def generator(input_text,pipe_line,max_number=256,do_print=False ,args_a=False):
   verify_text = lambda txt : '\n'.join([textwrap.fill(txt, width=140) for txt in txt.split('\n')])
-  def content_checker(text: str, code_es: Optional[List[str]] = None,safty_checker=True,cka=[],req=False) -> str:
-    if code_es:
-        for code_e in code_es:
-            code = base64.b64decode(code_e).decode('utf-8')
-            regex = r"\b{}\b".format(re.escape(code))
-            encoded_word = base64.b64encode(code.encode('utf-8')).decode('utf-8')
-            text = re.sub(regex, encoded_word, text, flags=re.IGNORECASE)
-    pattern = r"\b" + re.escape(base64.b64decode('VUMgQmVya2VsZXk=').decode('utf-8')) + r"\b"
-    replacement = base64.b64decode('QUkgT3BlblNvdXJjZSBDb21tdW5pdHk=').decode('utf-8')
-    text = re.sub(pattern, replacement, text, flags=re.IGNORECASE)
-    encoded_text = base64.b64encode(text.encode('utf-8')).decode('utf-8')
-    block_size = 10
-    def is_text_safe(text):
-      """
-      This function checks if the input text is safe by matching it against a regular expression pattern
-      that looks for potentially unsafe characters or patterns.
-      Returns True if the text is safe, and False otherwise.
-      """
-      unsafe_pattern = r"[^\w\s\.\-\@]"
-      match_ae = re.search(unsafe_pattern, text)
-      if match_ae:
-          return False
-      else:
-          return True
-    if safty_checker:
-      res = is_text_safe(text)
-      blocks = [encoded_text[i:i+block_size] for i in range(0, len(encoded_text), block_size)]
-      import random
-      random.shuffle(blocks)
-      cka.append(blocks)
-      return text if not req else (text,blocks)
-    else:
-      return text
-  if not task in ['CONVERSATION', 'Q&A', 'INFO', 'EXPLAIN']:
-    raise ValueError(f"{task} is not available current tasks are => ['CONVERSATION', 'Q&A', 'INFO', 'EXPLAIN']")
+  
   orginal_text = input_text
   if not input_text.startswith(f'{task}: USER:') and args_a:
-    input_text = f'{task}: USER: ' + input_text
-  if not input_text.endswith('\n\nAI:'):
-    input_text += '\n\nAI:'
+    input_text = f'<\s><|prompter|> {input_text}<\s><|ai|>'
   for i in range(max_number):
     exac = input_text
     with torch.no_grad():
@@ -113,9 +75,9 @@ def generator(input_text,pipe_line,task='CONVERSATION',max_number=256,do_print=F
       clear_output(wait=True)
       print(verify_text(input_text))
     
-    if input_text.endswith('AI:') and i>6 or exac == input_text or input_text.endswith('USER:') and i>6:
+    if input_text.endswith('<\s>') and i>6 or exac == input_text or input_text.endswith('<|prompter|>') and i>6:
       break
-    yield content_checker(verify_text(input_text))
+    yield verify_text(input_text)
 
 ```
 
@@ -136,17 +98,6 @@ pipe_line = pipeline(
     output_scores=True
 
 )
-cache = ''
-cache_step = 0
-while True:
-  input_ = cache+'\nUSER: '+input('>>  ') if cache_step !=0 else input('>>  ')
-  for i,t in enumerate(generator(input_,pipe_line=pipe_line,max_number=1024,args_a=False if cache_step != 0 else True)):
-    clear_output(wait=True)
-    print((f"{i} :\n {t}")[-3000:])
-    ou_t = t
-  cache += ou_t[len(cache):]
-  cache_step+=1
-
 ```
  or Just Simply Open [GOOGLE COLAB 🚀🚀](https://colab.research.google.com/drive/1nWS_FhWIDH3-g56F3FbWCIYi0ngVdWHx?usp=sharing)
 
@@ -206,17 +157,20 @@ if __name__ == "__main__":
 - you can simply import models like
 
 ```python
+# Pytorch
 from modules import LGeMForCausalLM
+# Jax
+from modules import FlaxLGeMForCausalLM
 ```
 
-- and Training code is available at LGeM-Train.py (check source)
+- and Training code is available at jax_train.py (check source)
 - training parameters 
-- - learning rate 1e-4
-- - AdamW (weight decay 1e-2)
-- - batch 2
-- - A 100 80GB used for training (4 X)
-- - Train Time 120 hours
-- - budget 760 $
+- - learning rate 5e-5
+- - Optimizer LION
+- - batch 32
+- - TPU POD
+- - Train Time 50 hours
+- - budget 500 $
 ``` shell
 python3 LGeM-train.py
 ```