<commit_before>import os
import re
import datetime
import subprocess

vendorid = 85838187 # <NAME>

# Find all reports in the current directory
reports = []                                                # list of (vendorid, YYYYMMDD), both strings
for filename in os.listdir('.'):
    # NOTE: Download filename format changed on 2011-11-03
    m = re.match(r'S_D_([0-9]+)_([0-9]{8})(_[^.]*)?\.txt(\.gz)?', filename)
    if m is None:
        continue
    reports.append((m.group(1), m.group(2)))
if len(reports) == 0:
    exit('No reports found in the current directory.')

# Find all report dates for the vendor of interest
dates = [x[1] for x in reports if x[0] == str(vendorid)]    # list of YYYYMMDD
if len(reports) == 0:
    exit('No reports in the current directory match the vendor ID ' + str(vendorID) + '.')

# Determine reports available for download
downloadableDates = []                                      # list of YYYYMMDD
now = datetime.datetime.now()
for i in xrange(30):
    downloadableDates.append((now - datetime.timedelta(days = i+1)).strftime('%Y%m%d'))

# Determine reports available for download that haven't already been downloaded
missingDates = list(set(downloadableDates) - set(dates))    # list of YYYYMMDD
missingDates.sort()
if len(missingDates) == 0:
    print 'All reports have been downloaded already.'
    exit(0)

# Download all missing reports, recording any errors
downloadErrors = []                                         # list of (YYYYMMDD, stdoutdata, stderrdata)
for curDate in missingDates:
    downloader = subprocess.Popen(['java', 'Autoingestion', 'autoingestion.properties', str(vendorid), 'Sales', 'Daily', 'Summary', curDate], stdout=subprocess.PIPE)
    out, err = downloader.communicate()
    if 'File Downloaded Successfully' in out:
        continue
    if 'There are no reports available to download for this selection.' in out:<commit_msg>Update to recognize new "no reports available" status message.<commit_after>import os
import re
import datetime
import subprocess

vendorid = 85838187 # <NAME>

# Find all reports in the current directory
reports = []                                                # list of (vendorid, YYYYMMDD), both strings
for filename in os.listdir('.'):
    # NOTE: Download filename format changed on 2011-11-03
    m = re.match(r'S_D_([0-9]+)_([0-9]{8})(_[^.]*)?\.txt(\.gz)?', filename)
    if m is None:
        continue
    reports.append((m.group(1), m.group(2)))
if len(reports) == 0:
    exit('No reports found in the current directory.')

# Find all report dates for the vendor of interest
dates = [x[1] for x in reports if x[0] == str(vendorid)]    # list of YYYYMMDD
if len(reports) == 0:
    exit('No reports in the current directory match the vendor ID ' + str(vendorID) + '.')

# Determine reports available for download
downloadableDates = []                                      # list of YYYYMMDD
now = datetime.datetime.now()
for i in xrange(30):
    downloadableDates.append((now - datetime.timedelta(days = i+1)).strftime('%Y%m%d'))

# Determine reports available for download that haven't already been downloaded
missingDates = list(set(downloadableDates) - set(dates))    # list of YYYYMMDD
missingDates.sort()
if len(missingDates) == 0:
    print 'All reports have been downloaded already.'
    exit(0)

# Download all missing reports, recording any errors
downloadErrors = []                                         # list of (YYYYMMDD, stdoutdata, stderrdata)
for curDate in missingDates:
    downloader = subprocess.Popen(['java', 'Autoingestion', 'autoingestion.properties', str(vendorid), 'Sales', 'Daily', 'Summary', curDate], stdout=subprocess.PIPE)
    out, err = downloader.communicate()
    if 'File Downloaded Successfully' in out:
        continue
    # NOTE: Status message changed format on 2014-06-20
    if ('There are no reports available to download for this selection.' in out or 
        'There is no report available to download, for the selected period'):
{'fraction_non_alphanumeric': 0.11531712208573577, 'fraction_numerical': 0.018049636500376033, 'mean_word_length': 3.1605839416058394, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 41, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '569002', 'n_tokens_mistral': 1217, 'n_tokens_neox': 1141, 'n_words': 400}