<commit_before>#!/usr/bin/env python

import re

class InvalidYearError(SyntaxError): pass
class InvalidMonthError(SyntaxError): pass
class InvalidDayError(SyntaxError): pass
class InvalidProjectError(SyntaxError): pass

def handle_year(line, line_number):
    """
    Handle year
    """
    m = re.match(r"^\*{1,1} (\d*)$", line)
    if m:
        return int(m.group(1))

    #raise InvalidYearError("%s %d" % (line, line_number))    

def handle_month(line, line_number):
    """
    Handle month
    """
    m = re.match(r"^\*{2,2} (\w*)$", line)
    if m:
        return m.group(1)

    #raise InvalidMonthError("%s %d" % (line, line_number))    

def handle_day(line, line_number):
    """
    Handle day.
    """
    m = re.match(r"^\*{3,3} (\d*)$", line)
    if m:
        return int(m.group(1))

    #raise InvalidDayError("%s %d" % (line, line_number))    

def handle_project(line, line_number):
    """
    Handle project.
    """
    m = re.match(r"^\*{4,4} (.*)$", line)
    if m:
        return m.group(1)

def handle_work(line, line_number):
    """
    Handle work.
    """
    m = re.match(r"^\*{5,5} ([-+]?\d*\.\d+|\d+) (?:hours|hour) (non-billable|billable) to (.*)\. (.*)$", line)
    if m:
        print m.groups()
        return {
            'length' : m.group(1),
            'billable': m.group(2),
            'owner' : m.group(3),
            'comment': m.group(4)}

    #raise InvalidProjectError("%s %d" % (line, line_number))    

year = None
month = None
day = None
project = None
work = None

data = dict()

line_number = 1
with open("task.org") as f:
    for line in f:
        line = line.strip()
        print "line: %s" % line

        y = handle_year(line, line_number)
        if y:
            year = y

        m = handle_month(line, line_number)
        if m:
            month = m

        d = handle_day(line, line_number)
        if d:
            day = d

        p = handle_project(line, line_number)
        if p:
            project = p

        w = handle_work(line, line_number)
        if w:
            work = w
        
        if year:
            if year not in data:
                data[year] = {}

            if month:
                if month not in data[year]:
                    data[year][month] = {}

                if day:
                    if day not in data[year][month]:
                        data[year][month][day] = dict()

                    if project:
                        if project not in data[year][month][day]:
                            data[year][month][day][project] = list()

                        if work:
                            data[year][month][day][project].append(work)
                            

        line_number += 1

#print "DATA ", data

# from pyparsing import *

# month = Group(oneOf('January February March April May June July August September October November December'))
# month.setName('months')

# year = Group(Word(nums))
# year.setName("year")

# day = Group(Word(nums))
# day.setName("year")

# owner = Group(Word(alphas))
# owner.setName("owner")

# weekend = Literal('WEEKEND')
# weekend.setName('weekend')
# holiday = Literal('HOLIDAY')
# holiday.setName('holiday')
# vacation = Literal('VACATION')
# vacation.setName('vacation')

# stars = dict({
#     'one': Regex(r"\*{1,1}"),
#     'two': Regex(r"\*{2,2}"),
#     'three': Regex(r"\*{3,3}"),
#     'four': Regex(r"\*{4,4}"),
#     'five': Regex(r"\*{5,5}"),2
# })

# for key in stars.viewkeys():
#     stars[key].setName( key + '_star' )

# comment = Group(ZeroOrMore(Word(printables)))
# comment.setName("comment")
# hours = Group(Regex(r"\d+(\.\d*)?").setParseAction(lambda t: float(t[0])))
# hours.setName("hours")

# task_entry = hours + Group(oneOf('hour', 'hours'))  + Group(oneOf("non-billable", "billable")) + Literal("to") + comment
# task_entry.setName("task_entry")

# task_entries = ZeroOrMore(stars['five'] + task_entry)
# owner_entries = ZeroOrMore(stars['four'] + owner + task_entries)
# day_entries = ZeroOrMore(stars['three'] + day + owner_entries)
# month_entries = ZeroOrMore(stars['two'] + month + day_entries)
# year_entries = ZeroOrMore(stars['one'] + year + month_entries)

# #year_entries.setDebug()

# with open("task.org") as f:
#     data = f.read()
#     print data
#     try:
#         rv = year_entries.parseString(data)
#         print rv
#     except ParseException, pe:
#         print "Error: ", pe<commit_msg>Added creation of sqlite database of org-mode data using sqlobject<commit_after>#!/usr/bin/env python

import re
import os.path
from sqlobject import *
import datetime

def handle_year(line):
    """
    Handle year
    """
    m = re.match(r"^\*{1,1} (\d*)$", line)
    if m:
        return int(m.group(1))

def handle_month(line):
    """
    Handle month
    """
    m = re.match(r"^\*{2,2} (\w*)$", line)
    if m:
        return m.group(1)

def handle_day(line):
    """
    Handle day.
    """
    m = re.match(r"^\*{3,3} (\d*)$", line)
    if m:
        return int(m.group(1))

def handle_project(line):
    """
    Handle project.
    """
    m = re.match(r"^\*{4,4} (.*)$", line)
    if m:
        return m.group(1)

def handle_work(line):
    """
    Handle work.
    """
    m = re.match(r"^\*{5,5} ([-+]?\d*\.\d+|\d+) (?:hours|hour) (non-billable|billable) to (.*)\. (.*)$", line)
    if m:
        print m.groups()
        return {
            'length' : m.group(1),
            'billable': m.group(2),
            'owner' : m.group(3),
            'comment': m.group(4)}

year = None
month = None
day = None
project = None
work = None

db_filename = os.path.abspath("task-output.sql") 
if os.path.exists(db_filename):
    os.unlink(db_filename)
connection_string = "sqlite:" + db_filename
connection_string += '?debug=True'
connection = connectionForURI(connection_string)
sqlhub.processConnection = connection

class TaskEntry(SQLObject):
    date = DateCol()
    billable = BoolCol()
    project = StringCol()
    length = FloatCol()
    comment = StringCol()

TaskEntry.createTable()

with open("task.org") as f:
    for line in f:
        line = line.strip()
        print line
        y = handle_year(line)
        if y:
            year = y

        m = handle_month(line)
        if m:
            month = m

        d = handle_day(line)
        if d:
            day = d

        p = handle_project(line)
        if p:
            project = p

        w = handle_work(line)
        if w:
            work = w

        if year and month and day and project and work:
            billable = work['billable'] == 'billable'
            length = float(work['length'])
            comment = work['comment']
            owner = work['owner']
            print work
            date = datetime.datetime.strptime("%d %s %d" % (year, month, day), "%Y %B %d")
            TaskEntry(date = date,
                      billable = billable,
                      length = length,
                      project = owner,
                      comment = comment)

for x in TaskEntry.select(""" date between '2012-07-1' and '2012-07-31' AND billable = 1""", orderBy=['date']):
    print x.date, x.comment, "\n"
{'fraction_non_alphanumeric': 0.16482285394202492, 'fraction_numerical': 0.009242403024786445, 'mean_word_length': 2.1490299823633157, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6925599', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2370, 'n_words': 554}