<commit_before>#! /usr/bin/env python
"""Collects & diffs public symobls in libcyclus.so. Used to ensure stability 
between versions. Now with 100% fewer vowels!

The following tasks may be useful:

    # update the database to the most recent release tag
    $ ./smbchk.py --update -t 1.X.Y

    # check that the existing database is stable
    $ ./smbchk.py --check

    # check if HEAD is stable, a diff will be printed if it is not
    $ ./smbchk.py --update -t HEAD --no-save --check

"""
from __future__ import print_function
import os
import io
import sys
import argparse
import difflib
import subprocess
import pprint
try:
    import simplejson as json
except ImportError:
    import json

def load(ns):
    """Loads a database of symbols or returns an empty list."""
    if not os.path.isfile(ns.filename):
        return []
    with io.open(ns.filename, 'rt') as f:
        db = json.load(f)
    return db

def save(db, ns):
    """Saves a database of symbols."""
    with io.open(ns.filename, 'wb') as f:
        json.dump(db, f, indent=1, separators=(',', ': '))

def nm(ns):
    """Obtains the latest symbols as a sorted list by running and parsing the 
    posix nm utility. Note that this is *not* compatible with Darwin's nm 
    utility because Apple is unfathomably 'special.'
    """
    # in the nm command, the following option are for:
    #   -g: external only, ie this prints only public symbols
    #   -p: no-sort, doesn't sort the output first, since we will be sorting
    #       later this makes things a tad faster
    #   -C: demangles the C++ symbols.
    #   -fbsd: formats output in default 'bsd' style
    # note that we are only going to pick up symbols in the cyclus namespace
    # because everything else should be external to the cares of cyclus stability
    lib = os.path.abspath(os.path.join(ns.prefix, 'lib', 'libcyclus.so'))
    stdout = subprocess.check_output(['nm', '-g', '-p', '-C', '-fbsd', lib])
    types = set()
    names = set()
    ok_types = {'B', 'b', 'D', 'd', 'R', 'r', 'S', 's', 'T', 't', 'W', 'w', 'u'}
    for line in stdout.splitlines():
        line = line.strip()
        if len(line) == 0 or not line[0].isdigit():
            continue
        val, typ, name = line.split(None, 2)
        types.add(typ)
        if not name.startswith('cyclus::'):
            continue
        if ' ' in name:
            # handle funny private pointer cases
            pre, post = name.split(' ', 1)
            if pre.endswith('*') or post.startswith('std::__'):
                continue
        if typ in ok_types:
            names.add(name)
    return sorted(names)

def git_log():
    """Returns git SHA, date, and timestamp from log."""
    stdout = subprocess.check_output(['git', 'log', '--pretty=format:%H/%ci/%ct', '-n1'])
    return stdout.strip().split('/')

def core_version():
    stdout = subprocess.check_output(['cyclus', '--version'])
    return stdout.splitlines()[0].strip()

def update(db, ns):
    """Updates a symbol database with the latest values."""
    if ns.tag is None:
        sys.exit("a tag for the version must be given when updating, eg '--tag 1.1'")
    symbols = nm(ns)
    sha, d, t = git_log()
    entry = {'symbols': symbols, 'sha': sha, 'date': d, 'timestamp': t, 
             'tag': ns.tag, 'version': core_version(),}
    db.append(entry)

def diff(db, i, j):
    """Diffs two database indices, returns string unified diff."""
    x = db[i]
    y = db[j]<commit_msg>added private naming convention trick<commit_after>#! /usr/bin/env python
"""Collects & diffs public symobls in libcyclus.so. Used to ensure stability 
between versions. Now with 100% fewer vowels!

The following tasks may be useful:

    # update the database to the most recent release tag
    $ ./smbchk.py --update -t 1.X.Y

    # check that the existing database is stable
    $ ./smbchk.py --check

    # check if HEAD is stable, a diff will be printed if it is not
    $ ./smbchk.py --update -t HEAD --no-save --check

"""
from __future__ import print_function
import os
import io
import re
import sys
import argparse
import difflib
import subprocess
import pprint
try:
    import simplejson as json
except ImportError:
    import json

NAME_RE = re.compile('([A-Za-z~_:]+)')

def load(ns):
    """Loads a database of symbols or returns an empty list."""
    if not os.path.isfile(ns.filename):
        return []
    with io.open(ns.filename, 'rt') as f:
        db = json.load(f)
    return db

def save(db, ns):
    """Saves a database of symbols."""
    with io.open(ns.filename, 'wb') as f:
        json.dump(db, f, indent=1, separators=(',', ': '))

def nm(ns):
    """Obtains the latest symbols as a sorted list by running and parsing the 
    posix nm utility. Note that this is *not* compatible with Darwin's nm 
    utility because Apple is unfathomably 'special.'
    """
    # in the nm command, the following option are for:
    #   -g: external only, ie this prints only public symbols
    #   -p: no-sort, doesn't sort the output first, since we will be sorting
    #       later this makes things a tad faster
    #   -C: demangles the C++ symbols.
    #   -fbsd: formats output in default 'bsd' style
    # note that we are only going to pick up symbols in the cyclus namespace
    # because everything else should be external to the cares of cyclus stability
    lib = os.path.abspath(os.path.join(ns.prefix, 'lib', 'libcyclus.so'))
    stdout = subprocess.check_output(['nm', '-g', '-p', '-C', '-fbsd', lib])
    names = set()
    ok_types = {'B', 'b', 'D', 'd', 'R', 'r', 'S', 's', 'T', 't', 'W', 'w', 'u'}
    for line in stdout.splitlines():
        line = line.strip()
        if len(line) == 0 or not line[0].isdigit():
            continue
        val, typ, name = line.split(None, 2)
        if not name.startswith('cyclus::'):
            continue
        if typ not in ok_types:
            continue
        if ' ' in name:
            # handle funny private pointer cases
            pre, post = name.split(' ', 1)
            if pre.endswith('*') or post.startswith('std::__'):
                continue
        # use trailing underscore naming convention to skip private variables
        m = NAME_RE.match(name)
        if m is None or m.group(1).endswith('_'):
            continue
        names.add(name)
    return sorted(names)

def git_log():
    """Returns git SHA, date, and timestamp from log."""
    stdout = subprocess.check_output(['git', 'log', '--pretty=format:%H/%ci/%ct', '-n1'])
    return stdout.strip().split('/')

def core_version():
    stdout = subprocess.check_output(['cyclus', '--version'])
    return stdout.splitlines()[0].strip()

def update(db, ns):
    """Updates a symbol database with the latest values."""
    if ns.tag is None:
        sys.exit("a tag for the version must be given when updating, eg '--tag 1.1'")
    symbols = nm(ns)
    sha, d, t = git_log()
    entry = {'symbols': symbols, 'sha': sha, 'date': d, 'timestamp': t, 
             'tag': ns.tag, 'version': core_version(),}
    db.append(entry)

def diff(db, i, j):
    """Diffs two database indices, returns string unified diff."""
    x = db[i]
    y = db[j]
{'fraction_non_alphanumeric': 0.1365097107726701, 'fraction_numerical': 0.0037725303898281403, 'mean_word_length': 2.6408952187182098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1022086', 'n_tokens_mistral': 2288, 'n_tokens_neox': 2213, 'n_words': 842}