<commit_before>#!/usr/bin/env perl -w

use strict;
use warnings;

# 
# This script helps to automate the ScanSnap software on a Mac. This is
# the software that shipped with my Fujitu ScanSnap S510M. The software
# won't automatically OCR scanned documents thru the included "ABBYY
# FineReader" app, and it won't let you drop more than one file onto it
# manually at once. It's pretty hateful, really.  This script undoes
# some (some!) of that hate.
# 
# So this just
# 
# 1.  looks for PDFs in some directory,
# 2.  filters out those that were not created by 'ScanSnap Manager',
# 3.  filters out ones that already have a companion OCRed file, and 
# 4.  for what's left, invokes the OCR app, wait until the companion
#     exists, and then continue with the next file. Will timeout any
#     individual file that takes longer than 5 minutes to complete.
# 
# TODOs:
# 
# -   Make this easier to configure (using getopt or something)
# -   Test this with Folder Actions and describe how to set them up
# -   Really could use some automated test coverage, but this is a
#     personal project and I don't care if it breaks
# 


# Look for files with this suffix to determine if they've been 
# OCRed
our $suffix = ' processed by FineReader.pdf';

# Only try to OCR files with these file creator metadata values
our %spotlight_creators = map { $_ => 1 } 
    ('ScanSnap Manager','None');

# Look for PDFs in the following dir to send to OCR
our $source_dir = '/Users/rtimmons/Dropbox/Inbox';

# Set to 1 to enable debugging
our $debug = 1;


# ==============================
# = Don't Edit Below This Line =
# ==============================

##
# Point of entry
sub main {
    my ($dir) = @_;
    
    # Look at all PDFs
    my @all = all_pdfs_in($dir);
    
    # Find out who made them
    my %creators = 
        map { $_ => spotlight_creator($_) }
        @all;

    # Grab only those created by $spotlight_creator
    my @todo = 
        grep { $spotlight_creators{$creators{$_}} }
        @all;

    # Log the ones that we won't do
    my @wont_do = 
        grep { !$spotlight_creators{$creators{$_}} }
        @all;
    mylog(
        "Won't try to OCR following since weren't created by [%spotlight_creators]:",
        "",
        (map { "    [$_] (created by $creators{$_})" } @wont_do),
        ""
    );
    
    # do nothing unless there's stuff to do
    return unless @todo;
    
    # do things in sorted order
    @todo = sort @todo;
    
    # Run OCR on each file
    map { do_ocr($_, $dir); } @todo;
}


# Stupid logger
sub mylog {
    return unless $debug;
    use Data::Dumper;
    $| ||= 1;
    
    if ( ref($_[0]) ) {
        print Dumper(@_);
    }<commit_msg>Don't ocr when creator='None'; cleanup logging<commit_after>#!/usr/bin/env perl -w

use strict;
use warnings;

# 
# This script helps to automate the ScanSnap software on a Mac. This is
# the software that shipped with my Fujitu ScanSnap S510M. The software
# won't automatically OCR scanned documents thru the included "ABBYY
# FineReader" app, and it won't let you drop more than one file onto it
# manually at once. It's pretty hateful, really.  This script undoes
# some (some!) of that hate.
# 
# So this just
# 
# 1.  looks for PDFs in some directory,
# 2.  filters out those that were not created by 'ScanSnap Manager',
# 3.  filters out ones that already have a companion OCRed file, and 
# 4.  for what's left, invokes the OCR app, wait until the companion
#     exists, and then continue with the next file. Will timeout any
#     individual file that takes longer than 5 minutes to complete.
# 
# TODOs:
# 
# -   Make this easier to configure (using getopt or something)
# -   Test this with Folder Actions and describe how to set them up
# -   Really could use some automated test coverage, but this is a
#     personal project and I don't care if it breaks
# 


# Look for files with this suffix to determine if they've been 
# OCRed
our $suffix = ' processed by FineReader.pdf';

# Only try to OCR files with these file creator metadata values
our %spotlight_creators = map { $_ => 1 } 
    ('ScanSnap Manager');

our @spotlight_creators = keys %spotlight_creators;

# Look for PDFs in the following dir to send to OCR
our $source_dir = '/Users/rtimmons/Dropbox/Inbox';

# Set to 1 to enable debugging
our $debug = 1;


# ==============================
# = Don't Edit Below This Line =
# ==============================

##
# Point of entry
sub main {
    my ($dir) = @_;
    
    # Look at all PDFs
    my @all = all_pdfs_in($dir);
    
    # Find out who made them
    my %creators = 
        map { $_ => spotlight_creator($_) }
        @all;

    # Grab only those created by $spotlight_creator
    my @todo = 
        grep { $spotlight_creators{$creators{$_}} }
        @all;

    mylog(
        "Will do following based on creators:",
        "",
        (map { "    [$_] (created by $creators{$_})" } @todo),
        ""
    ) if @todo;

    # Log the ones that we won't do
    my @wont_do = 
        grep { !$spotlight_creators{$creators{$_}} }
        @all;

    mylog(
        "Won't try to OCR following since weren't created by [@spotlight_creators]:",
        "",
        (map { "    [$_] (created by $creators{$_})" } @wont_do),
        ""
    ) if @wont_do;
    
    # do nothing unless there's stuff to do
    return unless @todo;
    
    # do things in sorted order
    @todo = sort @todo;
    
    # Run OCR on each file
    map { do_ocr($_, $dir); } @todo;
}


# Stupid logger
sub mylog {
    return unless $debug;
    use Data::Dumper;
    $| ||= 1;
    
    if ( ref($_[0]) ) {
        print Dumper(@_);
    }
{'fraction_non_alphanumeric': 0.1202486678507993, 'fraction_numerical': 0.004618117229129663, 'mean_word_length': 2.7143799472295513, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1633837', 'n_tokens_mistral': 1848, 'n_tokens_neox': 1761, 'n_words': 711}