<commit_before>import sys

if __name__ == "__main__":

    # Make sure we are importing a module cloned into this directory,
    # which should be cloned right before calling this script in order
    # to make sure we are dealing with the most current version.
    sys.path = [p for p in sys.path if not 'usr/local' in p]
    sys.path.append('cclib/src/')
    import cclib

    data_doc = cclib.parser.data.ccData.__doc__
    print(data_doc)<commit_msg>Add full script that generates attributes list<commit_after>import os
import sys


if __name__ == "__main__":

    import cclib

    # Make sure we are importing code from a subdirectory, which should exist
    # and should have been updated just before running this script. Note that
    # this script does not assume any version in the module and just takes
    # what it finds... so an appropriate checkout should be done first.
    if cclib.__file__[:len(os.getcwd())] != os.getcwd():
        print("Do not seem to be importing from current directory")
        sys.exit(1)

    # Need to parse the ccData docstring, since only that currently
    # contains all the information needed for this table.
    data_doc = cclib.parser.data.ccData.__doc__
    attributes = [line for line in data_doc.split('\n') if line[:8].strip() == '']
    attributes = [line for line in attributes if "--" in line]

    # These are the widths of the columns in the table
    wattr = 20
    wdesc = 65
    wunit = 28
    wtype = 32

    dashes = "    "
    for w in [wattr, wdesc, wunit, wtype]:
        dashes += "="*(w-1) + " "
    header = "    "
    header += "Name".ljust(wattr)
    header += "Description".ljust(wdesc)
    header += "Units".ljust(wunit)
    header += "Data type".ljust(wtype)
    print(dashes)
    print(header)
    print(dashes)

    names = []
    for line in attributes:

        # There is always a double dash after the name.
        attr, desc = line.strip().split(' -- ')
        names.append(attr)

        # The type and unit are in parentheses, but these
        # are not always the only parentheses on the line.
        other = desc.split('(')[-1]
        desc = desc[:-len(other)-1].strip()
        other = other.split(')')[0]

        # Furthermore, the unit is not always there.
        if "," in other:
            atype, aunit = other.split(", ")
        else:
            atype = other
            aunit = ''
        
        attr = ("`%s`_" % attr).ljust(20)
        desc = desc.ljust(65)
        aunit = aunit.ljust(28)
        for i in range(1,4):
            atype = atype.replace('[%i]' % i, ' of rank %i' % i)
        print("    " + attr + desc + aunit + atype)

    print(dashes)
    print()

    for n in names:
        print(".. _`%s`: data_notes.html#%s" % (n, n))
{'fraction_non_alphanumeric': 0.11563636363636363, 'fraction_numerical': 0.008, 'mean_word_length': 2.322463768115942, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6774257', 'n_tokens_mistral': 877, 'n_tokens_neox': 838, 'n_words': 301}