<commit_before>#!/usr/bin/env python3

import ftplib
import gzip
import os
import sys

host = 'ftp.ncdc.noaa.gov'
base = '/pub/data/noaa'
retries = 3

ftp = ftplib.FTP(host)
ftp.login()
for line in sys.stdin:
    (year, filename) = line.strip().split()
    for i in range(retries):
        sys.stderr.write('reporter:status:Processing file %s/%s (FTP attempt %d of %d)\n' % (year,
            filename, i + 1, retries))
        try:
            ftp.retrbinary('RETR %s/%s/%s' % (base, year, filename), open(filename, 'wb').write)
        except ftplib.all_errors as error:
            sys.stderr.write('%s\n' % error)
            continue
        count = 0
        for record in gzip.open(filename, 'rb'):
            print('%s\t%s' % (year, record.decode('ISO-8859-1').strip()))
            count += 1
        sys.stderr.write('reporter:counter:NCDC Download,%s,%d\n' % (year, count))
        os.remove(filename)
        break
    else:
        ftp.quit()
        sys.exit(1)
ftp.quit()<commit_msg>Remove downloaded file before updating counter<commit_after>#!/usr/bin/env python3

import ftplib
import gzip
import os
import sys

host = 'ftp.ncdc.noaa.gov'
base = '/pub/data/noaa'
retries = 3

ftp = ftplib.FTP(host)
ftp.login()
for line in sys.stdin:
    (year, filename) = line.strip().split()
    for i in range(retries):
        sys.stderr.write('reporter:status:Processing file %s/%s (FTP attempt %d of %d)\n' % (year,
            filename, i + 1, retries))
        try:
            ftp.retrbinary('RETR %s/%s/%s' % (base, year, filename), open(filename, 'wb').write)
        except ftplib.all_errors as error:
            sys.stderr.write('%s\n' % error)
            continue
        count = 0
        for record in gzip.open(filename, 'rb'):
            print('%s\t%s' % (year, record.decode('ISO-8859-1').strip()))
            count += 1
        os.remove(filename)
        sys.stderr.write('reporter:counter:NCDC Download,%s,%d\n' % (year, count))
        break
    else:
        ftp.quit()
        sys.exit(1)
ftp.quit()
{'fraction_non_alphanumeric': 0.1585425898572132, 'fraction_numerical': 0.010832102412604629, 'mean_word_length': 2.3754152823920265, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '648706', 'n_tokens_mistral': 739, 'n_tokens_neox': 699, 'n_words': 164}