<commit_before>
lazy = require('lazy')

parse_rich_core = (rcorefile, cb) ->
    console.log "parsing rich core: #{rcorefile.path}"

    lines = new lazy(fs.createReadStream rcorefile.path).lines

    groups = group_lines(lines)

    core = null

    groups.forEach (g) ->
        if not core?
            core =
                cmdline: g.lines[0]
        else
            switch g.name
                when 'date' then core.date = parse_core_date g.lines[0]

    groups.on 'end', () -> cb? core


date_regexp = /(\w+)\s(\w+)\s(\d+)\s([0-9:]+)\s([A-Z]+)\s(\d+)/
parse_core_date = (d) ->
    match = date_regexp.exec(d)
    [_, weekday, month, day, time, timezone, year] = match
    return new Date("#{weekday} #{month} #{day} #{year} #{time} (#{timezone})")


group_lines = (lines) ->
    parser = new lazy()

    group =
        name: undefined
        lines: []

    header = /[---rich-core: (.*)---]/

    lines.forEach (x) ->
        match = header.exec(x)
        if match?<commit_msg>updated rich core parsing to handle lzo files<commit_after>
lazy = require('lazy')
temp = require('temp')

extract_rich_core = (rcorefile, cb) ->
    fn = temp.path(suffix: '.core')
    cmd = "lzop -d #{rcorefile.path} -o#{fn}"
    exec cmd, (err, stdout, stderr) ->
        cb?(err, {path: fn})

parse_rich_core = (rcorefile, cb) ->
    console.log "parsing rich core: #{rcorefile.path}"
    path = rcorefile.path

    if /\.lzo$/.test(path)
        extract_rich_core rcorefile, (err, corefile) ->
            return cb? err if err?
            parse_rich_core corefile, (err, core) ->
               fs.unlink(corefile.path)
               cb?(err, core)
        return

    lines = new lazy(fs.createReadStream path).lines

    groups = group_lines(lines)

    core = null

    groups.forEach (g) ->
        if not core?
            core =
                cmdline: g.lines[0]
        else
            switch g.name
                when 'date' then core.date = parse_core_date g.lines[0]

    groups.on 'end', () -> cb? null, core


date_regexp = /(\w+)\s(\w+)\s(\d+)\s([0-9:]+)\s([A-Z]+)\s(\d+)/
parse_core_date = (d) ->
    match = date_regexp.exec(d)
    [_, weekday, month, day, time, timezone, year] = match
    return new Date("#{weekday} #{month} #{day} #{year} #{time} (#{timezone})")


group_lines = (lines) ->
    parser = new lazy()

    group =
        name: undefined
        lines: []

    header = /[---rich-core: (.*)---]/

    lines.forEach (x) ->
        match = header.exec(x)
        if match?
{'fraction_non_alphanumeric': 0.17358339984038307, 'fraction_numerical': 0.0031923383878691143, 'mean_word_length': 2.110421836228288, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7424080', 'n_tokens_mistral': 895, 'n_tokens_neox': 867, 'n_words': 205}