<commit_before>import sys
import re
import os

def walk_on_files(dirpath):
  for root, dirs, files in os.walk(dirpath):
    for file in files:
      if file.endswith(".md"):
        full_path= os.path.join(root, file)
        f=open(full_path,"rU")
        print "checking file: " + full_path
        textFile=f.read()
        f.close()
        findStartCode(textFile,full_path)
  return

def findStartCode(textFile,full_path):
  match = re.findall(r'@startDocuBlock\s*(\w+)', textFile)
  if match:      
    for find in match:
      textToReplace=getTextFromSourceFile(find, full_path)

def getTextFromSourceFile(searchText, full_path):
  f=open("allComments.txt", 'rU')
  s=f.read()
  match = re.search(r'@startDocuBlock\s+'+ searchText + "(?:\s+|$)" +'(.+?)@endDocuBlock', s,re.DOTALL)
  if match:
    textExtracted = match.group(1)
    textExtracted = textExtracted.replace("<br />","\n")
    replaceText(textExtracted, full_path, searchText)

def replaceText(text, pathOfFile, searchText):
  f=open(pathOfFile,"rU")
  if f:
    s=f.read()
  f.close()
  f=open(pathOfFile,'w')

  replaced = re.sub("@startDocuBlock\s+"+ searchText + "(?:\s+|$)",text,s)

  # HTTP API changing code
  replaced = replaced.replace("@brief","")
  replaced = re.sub(r"@RESTHEADER{([\s\w\/-]*),([\s\w]*)}", r"###\g<2>\n `\g<1>", replaced)
  replaced = replaced.replace("@RESTDESCRIPTION","")
  replaced = replaced.replace("@RESTURLPARAMS","*URL Parameters*\n")
  replaced = replaced.replace("@RESTQUERYPARAMS","*Query Parameters*\n")
  replaced = replaced.replace("@RESTHEADERPARAMS","*Header Parameters*\n")
  replaced = replaced.replace("@RESTBODYPARAMS","*Body Parameters*\n")
  replaced = replaced.replace("@RESTRETURNCODES","*Return Codes*")
  replaced = re.sub(r"@RESTPARAM{([\s\w]*),([\s\w]*),\s*(\w+)}", r"* *\g<1>* (\g<3>):", replaced)
  #  replaced = re.sub(r"@RESTPARAM{(\s*\w+\-*\w+),(\s*\w+),\s*(\w+)}", r"* *\g<1>* (\g<3>):", replaced)<commit_msg>Small Fixes for API function design<commit_after>import sys
import re
import os

def walk_on_files(dirpath):
  for root, dirs, files in os.walk(dirpath):
    for file in files:
      if file.endswith(".md"):
        full_path= os.path.join(root, file)
        f=open(full_path,"rU")
        print "checking file: " + full_path
        textFile=f.read()
        f.close()
        findStartCode(textFile,full_path)
  return

def findStartCode(textFile,full_path):
  match = re.findall(r'@startDocuBlock\s*(\w+)', textFile)
  if match:      
    for find in match:
      textToReplace=getTextFromSourceFile(find, full_path)

def getTextFromSourceFile(searchText, full_path):
  f=open("allComments.txt", 'rU')
  s=f.read()
  match = re.search(r'@startDocuBlock\s+'+ searchText + "(?:\s+|$)" +'(.+?)@endDocuBlock', s,re.DOTALL)
  if match:
    textExtracted = match.group(1)
    textExtracted = textExtracted.replace("<br />","\n")
    replaceText(textExtracted, full_path, searchText)

def replaceText(text, pathOfFile, searchText):
  f=open(pathOfFile,"rU")
  if f:
    s=f.read()
  f.close()
  f=open(pathOfFile,'w')

  replaced = re.sub("@startDocuBlock\s+"+ searchText + "(?:\s+|$)",text,s)

  # HTTP API changing code
  replaced = replaced.replace("@brief","")
  replaced = re.sub(r"@RESTHEADER{([\s\w\/\_-]*),([\s\w]*)}", r"###\g<2>\n `\g<1>`", replaced)
  replaced = replaced.replace("@RESTDESCRIPTION","")
  replaced = replaced.replace("@RESTURLPARAMS","*URL Parameters*\n")
  replaced = replaced.replace("@RESTQUERYPARAMS","*Query Parameters*\n")
  replaced = replaced.replace("@RESTHEADERPARAMS","*Header Parameters*\n")
  replaced = replaced.replace("@RESTBODYPARAMS","*Body Parameters*\n")
  replaced = replaced.replace("@RESTRETURNCODES","*Return Codes*")
  replaced = re.sub(r"@RESTPARAM{([\s\w\-]*),([\s\w\_\|-]*),\s[optional]}", r"* *\g<1>* (\g<3>):", replaced)
  replaced = re.sub(r"@RESTPARAM{([\s\w-]*),([\s\w\_\|-]*),\s*(\w+)}", r"* *\g<1>*:", replaced)
  #  replaced = re.sub(r"@RESTPARAM{(\s*\w+\-*\w+),(\s*\w+),\s*(\w+)}", r"* *\g<1>* (\g<3>):", replaced)
{'fraction_non_alphanumeric': 0.2196215139442231, 'fraction_numerical': 0.0037350597609561755, 'mean_word_length': 4.421052631578948, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3595159', 'n_tokens_mistral': 1556, 'n_tokens_neox': 1535, 'n_words': 251}