<commit_before>#!/usr/bin/python
import sys, os

# TODO: TEST THIS BEAST

def main(argv):
  folder = argv[1]
  project = argv[2]

  # for each file in specified directory
  for filename in os.listdir(folder):
    # open a file and create a temp file to hold our alternation 
    oldFile = open(filename)
    tmpFile = open("conversionFile.tmp","w+")

    # allow us to keep track of the old front matter we don't want to copy
    startFrontMatter = false;
    endFrontMatter = false;

    # saves relevant font matter from the old file to be used in the new file
    # title is generally the filename, this key is not being used but may be implemented for use later
    frontMatter = {"title" = "", "description" = "", "version" = "", "weight" = ""}

    # for each line in the old file
    for line in oldFile:
      # only write if we have established we're past the front matter 
      if (startFrontMatter && endFrontMatter):
        tmpFile.write(line)
      # save the previous front matter to a dictionary
      elif (startFrontMatter):
        if (line == "---"):
          endFrontMatter = true
          if (line.find("title: ")):
            tmp = line.split(": ")
            frontMatter["title"] = tmp[1]
          elif (line.find("description: "):
            tmp = line.split(": ")
            frontMatter["description"] = tmp[1]
          elif (line.find("version: "):
            tmp = line.split(": ")
            frontMatter["version"] = tmp[1]
          elif (line.find("weight: "):
            tmp = line.split(": ")
            frontMatter["weight"] = tmp[1]
      # check the first line of the file
      elif (line == "---"):
        startFrontMatter = true

    # close the files since we're done copying the content
    oldFile.close()
    tmpFile.close()

    # reset this to use in our hugo file
    startFrontMatter = false

    # TODO: /filename won't suffice, need to keep track of subdirectories as well
    # may need some splitting or a loop to append the full new path
    os.system("hugo new " + project + "/" frontMatter["version"] + "/" + filename)
    hugoFile = open(content + "/" + frontMatter["version"] + "/" + filename, "r+b")

    for line in hugoFile:
      if (startFrontMatter)
        # break if we've reached the end of the front matter
        if (line == "---"):
          break
        elif (line.find("description: ")):
          line = line + "\"" + frontMatter["description"] + "\""
        elif (line.find("version: ")):
          line = line + "\"" + frontMatter["version"] + "\""
        elif (line.find("weight: ")):
          line = line + "\"" + frontMatter["weight"] + "\""
        elif (line.find("menu: ")):
          menu = project + "-" + frontMatter["version"]
          line = line + menu

    # close our hugo file since we're done editing the front matter
    hugoFile.close()

    # append our temp file to this new hugo file
    os.system("cat conversionFile.tmp >> " + "content/" + project + "/" fontMatter["version"] + filename)

    # delete temp file and close files
    os.system("rm conversionFile.tmp")

if __name__ == "__main__":
  if (len(sys.argv) == 3):
    main(sys.argv)
  else:
    print "Please supply the following arg(s):"
    print "1: The folder which you'd like to migrate"
    print "2: The name of the project it belongs to"
    sys.stdout.flush()
<commit_msg>Fix bugs so script can run<commit_after>#!/usr/bin/python
import sys, os

# TODO: TEST THIS BEAST
# TODO: test for file existence, etc.

def main(argv):
  folder = argv[1]
  project = argv[2]

  # for each file in specified directory
  for filename in os.listdir(folder):
    # open a file and create a temp file to hold our alternation 
    oldFile = open(filename)

    print "Starting file: " + filename

    # allow us to keep track of the old front matter we don't want to copy
    startFrontMatter = false;
    endFrontMatter = false;

    # saves relevant font matter from the old file to be used in the new file
    # title is generally the filename, this key is not being used but may be implemented for use later
    frontMatter = {"title": "", "description": "", "version": "", "weight": ""}

    # for each line in the old file
    for line in oldFile:
      # we have established we're past the front matter 
      if (startFrontMatter and endFrontMatter):
        print "Finished front matter"
        break;
      # save the old front matter to a dictionary
      elif (startFrontMatter):
        if (line == "---"):
          endFrontMatter = true
        for attribute in frontMatter:
          if (line.find(attribute + ": ")):
            tmp = line.split(": ")
            print "Saving attribute: " + attribute + " with value: " + tmp[1]
            frontMatter[attribute] = tmp[1]
      # check the first line of the file
      elif (line == "---"):
        print "Starting front matter"
        startFrontMatter = true

    # reset this to use in our hugo file
    startFrontMatter = false

    # TODO: /filename won't suffice, need to keep track of subdirectories as well
    # may need some splitting or a loop to append the full new path
    newPath = folder + "/" + filename
    newPath = hugopath.split(frontMatter["version"] + "/")
    hugoPath = hugoPath[1]

    os.system("hugo new " + project + "/" + frontMatter["version"] + "/" + hugoPath + "/" + filename)
    hugoFile = open(content + "/" + frontMatter["version"] + "/" + "/" + hugoPath + "/" + filename, "r+b")

    print "Created hugo file"

    for line in hugoFile:
      if (startFrontMatter):
        # break if we've reached the end of the front matter
        if (line == "---"):
          break
        for attribute in frontMatter:
          if (line.find("menu:")):
            print "Writing attribute: " + attribute
            menu = " \"" + project + "-" + frontMatter["version"] + "\""
            line = line + menu
          elif (line.find(attribute + ":")):
            print "Writing attribute: " + attribute
            line = line + " \"" + frontMatter[attribute] + "\""
      if (line == "---"):
        startFrontMatter = true

    # close our hugo file since we're done editing the front matter
    hugoFile.close()
    oldFile.close()

    # append our temp file to this new hugo file
    os.system("cat " + filename + " >> content/" + project + "/" + frontMatter["version"] + "/" + hugoPath + "/" + filename)

    print "Appended to hugo file"

if __name__ == "__main__":
  if (len(sys.argv) == 3):
    main(sys.argv)
  else:
    print "Please supply the following arg(s):"
    print "1: The folder which you'd like to migrate"
    print "2: The name of the project it belongs to, eg. 'sensu-core'"
    sys.stdout.flush()

{'fraction_non_alphanumeric': 0.12554177253026452, 'fraction_numerical': 0.0025407263488267822, 'mean_word_length': 2.4300358790363914, 'pattern_counts': {'":': 13, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6095955', 'n_tokens_mistral': 1978, 'n_tokens_neox': 1935, 'n_words': 699}