<commit_before>#!/usr/bin/env python
#
# <NAME> <<EMAIL>>
#
# Scans PDF file text for keywords and moves file to appropriate directory.
# Takes text / dir names from an ini file
#

import ConfigParser
import glob
import string
import shutil
import re
import os
import sys
import logging
from pprint import pprint, pformat
from argparse import ArgumentParser

try:
    import yaml
except ImportError:
    print "You must install PyYAML to use this script."
    print "Try pip install PyYAML"
    sys.exit(1)

try:
    from PyPDF2 import PdfFileReader
except ImportError:
    print "Requires pyPDF library"
    print "http://pybrary.net/pyPdf/"
    print "Try pip install PyPDF2"
    sys.exit(1)
    
    
def pdf2text(filename):
    """Open PDF file and extract all text as a giant string for each page in a list.
    """
    pagetext = []
    pdf = PdfFileReader(file(filename, "rb"))
    numpages = pdf.getNumPages()
    for pn in range(numpages):
        pagetext.append(pdf.getPage(pn).extractText())
    return pagetext


def read_ini_file(filename):
    """Reads INI file to get match/dest pairs
    The file format expected is one [Main] section followed by a series of 
    value assignments which map keywords to directories:
    John Doe = ./john_doe_dir
    
    one special entry is used:
    nomatch = ~/MyDocuments/test_nomatch
    It must be the first item in the file.
    
    This is used to move files which don't match anything or have no text.
    It is assumed "nomatch" won't occur in any target file.
    
    """
    config = ConfigParser.ConfigParser()
    config.read(filename)
    return config.items('Main')


def look_for_match(text, matchstr):
     """Looks for matchstr in text, returns True if found
     """
     if string.find(text, matchstr) != -1:
         return True
     else:
         return False

     
def move_file(dryrun, oldfile, newfilename, status=""):
     """Moves oldfile to newfilename
     """
     logging.info("move_file() " + oldfile + ' -> ' + \
                  str(newfilename) + ' ' +  str(status))
     if not dryrun:
         shutil.move(oldfile, os.path.expanduser(newfilename))

         
def get_date(text):
    """Returns year and month numeral from file contents
    This doesn't work very well, as all the files are different.
    """
    yearmatch = re.search("(20[0-9][0-9])", text)
    #Take first match we find
    if yearmatch:
        year = yearmatch.group(0)
    else:
        year = "YYYY"
    
    monthmatch = re.search('(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]', text, re.I)
    if monthmatch:
        month = monthmatch.group(0)
    else:
        month = "MM"
    
    import pdb; pdb.set_trace()
    return year, month


def main(args):

    with open(args.yaml_fn, 'r') as f:
        conf = yaml.load(f)
    logging.debug(pformat(conf))

    logging.info("Looking for PDFs in " + conf['watch_folder'])
    pdf_fns = glob.glob(conf['watch_folder'] + '/*.pdf')

    logging.info("Processing files: " + str(pdf_fns))
    
    for filename in pdf_fns:
        pagetext = pdf2text(filename)
        searchtext = '\n'.join(pagetext).lower()
        if pagetext[0] == u'':
            logging.info(filename + " has no OCRed text in it.")
            move_file(args.dryrun, filename, conf['default_folder'], "No Text")
        else:
            match_found = False
            for vector in conf['folders']:
                for matchstr in conf['folders'][vector]:
                    if look_for_match(searchtext, matchstr):
                        # Prepend the vector name onto the filename and put it in the vector folder.
                        newfilename = conf['target_folder'] + '/' + \
                          vector + '/' + vector + '_' + \
                          os.path.basename(filename)<commit_msg>Now creating directory if it doesn't already exist for the copy.<commit_after>#!/usr/bin/env python
#
# <NAME> <<EMAIL>>
#
# Scans PDF file text for keywords and moves file to appropriate directory.
# Takes text / dir names from an ini file
#

import ConfigParser
import glob
import string
import shutil
import re
import os
import sys
import logging
from pprint import pprint, pformat
from argparse import ArgumentParser

try:
    import yaml
except ImportError:
    print "You must install PyYAML to use this script."
    print "Try pip install PyYAML"
    sys.exit(1)

try:
    from PyPDF2 import PdfFileReader
except ImportError:
    print "Requires pyPDF library"
    print "http://pybrary.net/pyPdf/"
    print "Try pip install PyPDF2"
    sys.exit(1)
    
    
def pdf2text(filename):
    """Open PDF file and extract all text as a giant string for each page in a list.
    """
    pagetext = []
    pdf = PdfFileReader(file(filename, "rb"))
    numpages = pdf.getNumPages()
    for pn in range(numpages):
        pagetext.append(pdf.getPage(pn).extractText())
    return pagetext


def read_ini_file(filename):
    """Reads INI file to get match/dest pairs
    The file format expected is one [Main] section followed by a series of 
    value assignments which map keywords to directories:
    <NAME> = ./john_doe_dir
    
    one special entry is used:
    nomatch = ~/MyDocuments/test_nomatch
    It must be the first item in the file.
    
    This is used to move files which don't match anything or have no text.
    It is assumed "nomatch" won't occur in any target file.
    
    """
    config = ConfigParser.ConfigParser()
    config.read(filename)
    return config.items('Main')


def look_for_match(text, matchstr):
     """Looks for matchstr in text, returns True if found
     """
     if string.find(text, matchstr) != -1:
         return True
     else:
         return False

     
def move_file(dryrun, oldfile, newfilename, status=""):
     """Moves oldfile to newfilename
     """
     logging.info("move_file() " + oldfile + ' -> ' + \
                  str(newfilename) + ' ' +  str(status))
     logging.info("Dry_run: " + str(dryrun))
     if not dryrun:
         shutil.move(oldfile, os.path.expanduser(newfilename))

         
def get_date(text):
    """Returns year and month numeral from file contents
    This doesn't work very well, as all the files are different.
    """
    yearmatch = re.search("(20[0-9][0-9])", text)
    #Take first match we find
    if yearmatch:
        year = yearmatch.group(0)
    else:
        year = "YYYY"
    
    monthmatch = re.search('(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) [0-9]', text, re.I)
    if monthmatch:
        month = monthmatch.group(0)
    else:
        month = "MM"
    
    import pdb; pdb.set_trace()
    return year, month


def main(args):

    with open(args.yaml_fn, 'r') as f:
        conf = yaml.load(f)
    logging.debug(pformat(conf))

    logging.info("Looking for PDFs in " + conf['watch_folder'])
    pdf_fns = glob.glob(conf['watch_folder'] + '/*.pdf')

    logging.info("Processing files: " + str(pdf_fns))
    
    for filename in pdf_fns:
        pagetext = pdf2text(filename)
        searchtext = '\n'.join(pagetext).lower()
        if pagetext[0] == u'':
            logging.info(filename + " has no OCRed text in it.")
            move_file(args.dryrun, filename, conf['default_folder'], "No Text")
        else:
            match_found = False
            for vector in conf['folders']:
                for matchstr in conf['folders'][vector]:
                    if look_for_match(searchtext, matchstr):
                        # Prepend the vector name onto the filename and put it in the vector folder.
                        newdir = conf['target_folder'] + '/' + \
                          vector + '/'
                        newfilename = newdir + vector + '_' + \
                          os.path.basename(filename)
                        if not os.path.exists(newdir):
                            logging.debug('Making directory ' + newdir)
                            os.makedirs(newdir)  
{'fraction_non_alphanumeric': 0.10616567898121296, 'fraction_numerical': 0.004539150170218132, 'mean_word_length': 2.3652948663555367, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2520352', 'n_tokens_mistral': 2462, 'n_tokens_neox': 2386, 'n_words': 753}