<commit_before>import os, sys, glob

inFilePath = sys.argv[1]
outFilePath = sys.argv[2]

def buildTable(filePath):
    checkFileExists(filePath)
    out = ""

    tableFile = open(filePath)
    headerItems = tableFile.readline().rstrip().split("\t")

    out += "| " + " | ".join(headerItems) + " |\n"
    out += "|" + "|".join(["------" for x in headerItems]) + "|\n"

    lineCount = 0
    for line in tableFile:
        lineCount += 1
        lineItems = line.rstrip().split("\t")
        out += "|" + "|".join(lineItems) + "|\n"

#        if lineCount == 100:
#            break

    return out

def buildFigure(filePath):
    checkFileExists(filePath)

    newFilePath = filePath
    if filePath.endswith(".pdf"):
        newFilePath = "/tmp/" + os.path.basename(inFilePath) + "__" + os.path.basename(filePath).replace(".pdf", ".png")
        os.system("./scripts/convert_pdf_to_png %s %s" % (filePath, newFilePath))

    return "![](%s)\n" % newFilePath

def checkFileExists(filePath):
    if not os.path.exists(filePath):
        print "No file exists at %s" % filePath
        exit(1)

inFile = open(inFilePath)
outFile = open(outFilePath, 'w')

for line in inFile:
    if line.strip().startswith("@"):
        continue

    if line.startswith("Table:"):
        tableFilePath = line.rstrip().replace("Table:", "")

        if tableFilePath.endswith(".txt"):
            outFile.write(buildTable(tableFilePath))
    elif line.startswith("Figure:"):
        figureFilePath = line.rstrip().replace("Figure:", "")
        outFile.write(buildFigure(figureFilePath))
    else:
        outFile.write(line)

outFile.close()
inFile.close()
<commit_msg>Mon Aug  4 15:29:23 MDT 2014<commit_after>import os, sys, glob

inFilePath = sys.argv[1]
outFilePath = sys.argv[2]

def buildTable(filePath):
    checkFileExists(filePath)
    out = ""

    tableFile = open(filePath)
    headerItems = tableFile.readline().rstrip().split("\t")

    out += "| " + " | ".join(headerItems) + " |\n"
    out += "|" + "|".join(["------" for x in headerItems]) + "|\n"

    lineCount = 0
    for line in tableFile:
        lineCount += 1
        lineItems = line.rstrip().split("\t")
        out += "|" + "|".join(lineItems) + "|\n"

#        if lineCount == 100:
#            break

    return out

def buildFigure(filePath):
    checkFileExists(filePath)

    newFilePath = filePath
    if filePath.endswith(".pdf"):
        newFilePath = "Notebooks/" + os.path.basename(inFilePath) + "__" + os.path.basename(filePath).replace(".pdf", ".png")
        os.system("./scripts/convert_pdf_to_png %s %s" % (filePath, newFilePath))

    return "![](%s)\n" % newFilePath

def checkFileExists(filePath):
    if not os.path.exists(filePath):
        print "No file exists at %s" % filePath
        exit(1)

inFile = open(inFilePath)
outFile = open(outFilePath, 'w')

for line in inFile:
    if line.strip().startswith("@"):
        continue

    if line.startswith("Table:"):
        tableFilePath = line.rstrip().replace("Table:", "")

        if tableFilePath.endswith(".txt"):
            outFile.write(buildTable(tableFilePath))
    elif line.startswith("Figure:"):
        figureFilePath = line.rstrip().replace("Figure:", "")
        outFile.write(buildFigure(figureFilePath))
    else:
        outFile.write(line)

outFile.close()
inFile.close()

{'fraction_non_alphanumeric': 0.17057761732851986, 'fraction_numerical': 0.008122743682310469, 'mean_word_length': 2.8843457943925235, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6110055', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1082, 'n_words': 217}