<commit_before>#!/usr/bin/env python
#
# justify_me.py
#
# A very short script to summarise time use from my personal lab book.
#
# My lab book is in LaTeX, and I record my work under \section{} and
# \subsection{} headers. The contents of these headers are in the format:
#
# SUBJECT: HHMM-HHMM; HHMM-HHMM;...
#
# The lab book source is stored in a hierarchical directory structure:
#
# YEAR/MM_month/YYYY-MM-DD/YYYY-MM-DD.tex
#
# This script takes a directory as input argument, searches all subdirectories
# for .tex source files, and scans them for \section{} and \subsection{}
# lines, parsing the content to calculate how much time was spent under each
# heading. It reports some summary statistics.
#
# TODO: Turn output into JSON/graphical output
#
# (c) L.Pritchard 2014

###
# IMPORTS

from argparse import ArgumentParser
from collections import defaultdict

import logging
import logging.handlers

import os
import re
import sys
import traceback

###
# FUNCTIONS


# Parse command-line
def parse_cmdline(args):
    """ Parse command-line arguments
    """
    parser = ArgumentParser(prog="justify_me.py")
    parser.add_argument("-o", "--outfile", dest="outfilename",
                        action="store", default=None,
                        help="Output file")
    parser.add_argument("-i", "--indir", dest="indirname",
                        action="store", default='.',
                        help="Input tab-separated plaintext table")
    parser.add_argument("-v", "--verbose", dest="verbose",
                        action="store_true",
                        help="Give verbose output")
    return parser.parse_args()


# Traverse subdirectories, collecting .tex files and processing the headers
def process_labbooks():
    """ Starting from the input directory, traverse all subdirectories,
        finding .tex files. Process each .tex file to find time spent under
        each heading, and collate.
    """
    # Traverse subdirectories and get list of lab book locations
    texfiles = []
    for root, dirs, files in os.walk(args.indirname):
        texfiles.extend([os.path.join(root, f) for f in files if 
                         os.path.splitext(f)[-1] == '.tex'])

    # Process each book, returning a dictionary of time spent, keyed by
    # section/subsection header
    return [(os.path.split(texfile)[-1], scrape_time(texfile)) for 
            texfile in texfiles]
    

# Report time spent by lab book day
def report_by_day(times, outstream):
    """ Report time spent by lab book day
    """
    for filename, tlist in sorted(times):
        outstream.write("\n%s:\n" % filename)
        total = 0
        for topic, t in sorted(tlist):
            if t:
                outstream.write("\t%30s:\t%.2fh\n" % (topic, t/60.))
                total += t
        outstream.write("Total time spent: %.2fh\n" % (total/60.))


# Report total time recorded in lab boo
def report_total_time(times, outstream):
    """ Report time spent across all lab books
    """
    totals = defaultdict(int)
    days = 0
    outstream.write("\nTotal time spent:\n")
    for filename, tlist in sorted(times):
        days += 1
        for topic, t in sorted(tlist):
            if t:
                totals[topic] += t
    total = sum(totals.values())
    for topic, t in sorted(totals.items()):
        outstream.write("\t%30s:\t%.2fh\t(%.2f%%)\n" % (topic, t/60., 100.*t/total))
    outstream.write("Total time spent: %.2fh\n" % (total/60.))
    outstream.write("Total time spent per lab book: %.2fh\n" % 
                    (total/60./days))


# Takes an iterable of .tex files and processes \section and \subsection
# headers to scrape times spent under the header
def scrape_time(filename):
    """ Loops over a .tex file and scrapes the
        time spent (in format HHMM-HHMM) from each \section and \subsection
        header.
    """
    section_re = r"((?<=\\section\{).*(?=\})|(?<=\\subsection\{).*(?=\}))"
    logger.info("Scraping %s" % filename)
    with open(filename, 'rU') as fh:
        data = fh.read()
        # Get \section{} and \subsection{} elements
        matches = [m for m in re.findall(section_re, data) if
                   len(m.strip()) and ':' in m]
        # Process matches into subject, time values
        times = [process_match(m) for m in matches]
    return times

<commit_msg>Minor change to text output, and unification of topic names<commit_after>#!/usr/bin/env python
#
# justify_me.py
#
# A very short script to summarise time use from my personal lab book.
#
# My lab book is in LaTeX, and I record my work under \section{} and
# \subsection{} headers. The contents of these headers are in the format:
#
# SUBJECT: HHMM-HHMM; HHMM-HHMM;...
#
# The lab book source is stored in a hierarchical directory structure:
#
# YEAR/MM_month/YYYY-MM-DD/YYYY-MM-DD.tex
#
# This script takes a directory as input argument, searches all subdirectories
# for .tex source files, and scans them for \section{} and \subsection{}
# lines, parsing the content to calculate how much time was spent under each
# heading. It reports some summary statistics.
#
# TODO: Turn output into JSON/graphical output
#
# (c) L.Pritchard 2014

###
# IMPORTS

from argparse import ArgumentParser
from collections import defaultdict

import logging
import logging.handlers

import os
import re
import sys
import traceback

###
# FUNCTIONS


# Parse command-line
def parse_cmdline(args):
    """ Parse command-line arguments
    """
    parser = ArgumentParser(prog="justify_me.py")
    parser.add_argument("-o", "--outfile", dest="outfilename",
                        action="store", default=None,
                        help="Output file")
    parser.add_argument("-i", "--indir", dest="indirname",
                        action="store", default='.',
                        help="Input tab-separated plaintext table")
    parser.add_argument("-v", "--verbose", dest="verbose",
                        action="store_true",
                        help="Give verbose output")
    return parser.parse_args()


# Traverse subdirectories, collecting .tex files and processing the headers
def process_labbooks():
    """ Starting from the input directory, traverse all subdirectories,
        finding .tex files. Process each .tex file to find time spent under
        each heading, and collate.
    """
    # Traverse subdirectories and get list of lab book locations
    texfiles = []
    for root, dirs, files in os.walk(args.indirname):
        texfiles.extend([os.path.join(root, f) for f in files if 
                         os.path.splitext(f)[-1] == '.tex'])

    # Process each book, returning a dictionary of time spent, keyed by
    # section/subsection header
    return [(os.path.split(texfile)[-1], scrape_time(texfile)) for 
            texfile in texfiles]
    

# Report time spent by lab book day
def report_by_day(times, outstream):
    """ Report time spent by lab book day
    """
    for filename, tlist in sorted(times):
        outstream.write("\n%s:\n" % filename)
        total = 0
        for topic, t in sorted(tlist):
            if t:
                outstream.write("\t%30s:\t%.2fh\n" % (topic, t/60.))
                total += t
        outstream.write("Total time recorded: %.2fh\n" % (total/60.))


# Report total time recorded in lab boo
def report_total_time(times, outstream):
    """ Report time recorded across all lab books
    """
    totals = defaultdict(int)
    days = 0
    outstream.write("\nTotal time recorded:\n")
    for filename, tlist in sorted(times):
        days += 1
        for topic, t in sorted(tlist):
            if t:
                totals[topic.upper()] += t
    total = sum(totals.values())
    for topic, t in sorted(totals.items()):
        outstream.write("\t%30s:\t%.2fh\t(%.2f%%)\n" %
                        (topic, t/60., 100.*t/total))
    outstream.write("Total time recorded: %.2fh\n" % (total/60.))
    outstream.write("Total time recorded per lab book: %.2fh\n" % 
                    (total/60./days))


# Takes an iterable of .tex files and processes \section and \subsection
# headers to scrape times spent under the header
def scrape_time(filename):
    """ Loops over a .tex file and scrapes the
        time spent (in format HHMM-HHMM) from each \section and \subsection
        header.
    """
    section_re = r"((?<=\\section\{).*(?=\})|(?<=\\subsection\{).*(?=\}))"
    logger.info("Scraping %s" % filename)
    with open(filename, 'rU') as fh:
        data = fh.read()
        # Get \section{} and \subsection{} elements
        matches = [m for m in re.findall(section_re, data) if
                   len(m.strip()) and ':' in m]
        # Process matches into subject, time values
        times = [process_match(m) for m in matches]
    return times


{'fraction_non_alphanumeric': 0.12239316239316239, 'fraction_numerical': 0.007293447293447293, 'mean_word_length': 2.542995559144126, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1502939', 'n_tokens_mistral': 2660, 'n_tokens_neox': 2552, 'n_words': 873}