<commit_before>from logging import debug, error, info
from cgi import parse_qs, FieldStorage
from google.appengine.api.labs import taskqueue as tq
from google.appengine.api import memcache as mc
from trackon import tracker

new_queue = tq.Queue('new-trackers')

def main():
    args = FieldStorage()
    if 'tracker-address' in args and args['tracker-address'].value:
        addr = args['tracker-address'].value 
        (r, url) = tracker.check(addr)
        debug("Fetching %s"%url)
        if 'error' not in r:
            debug("Added tracker: %s"%addr)
            tracker.add(addr, r)
        else:
            info("Initial tracker check for %s failed: %s" % (url, r['error']))
            attp = int(args['attempts'].value)+1
            info("Attempt %d scheduled in %d seconds."%(attp, attp*20))
            if attp > 4: # TODO: We should raise the number of attempts?
                return # We give up XXX log this somewhere
            params = {'tracker-address': addr, 'attempts': attp }
            task = tq.Task(params=params, countdown=attp*30)
            new_queue.add(task)


if __name__ == '__main__':
    main()<commit_msg>Log it when we give up on a new tracker, also avoid giving confusing and incorrect messages.<commit_after>from logging import debug, error, info
from cgi import parse_qs, FieldStorage
from google.appengine.api.labs import taskqueue as tq
from google.appengine.api import memcache as mc
from trackon import tracker

new_queue = tq.Queue('new-trackers')

def main():
    args = FieldStorage()
    if 'tracker-address' in args and args['tracker-address'].value:
        addr = args['tracker-address'].value 
        (r, url) = tracker.check(addr)
        debug("Fetching %s"%url)
        if 'error' not in r:
            debug("Added tracker: %s"%addr)
            tracker.add(addr, r)
        else:
            info("Initial tracker check for %s failed: %s" % (url, r['error']))
            attp = int(args['attempts'].value)+1
            if attp > 4: # TODO: We should raise the number of attempts?
                info("Giving up after %d attempts to contact: %s" % (attp, addr))
                return 
            info("Attempt %d scheduled in %d seconds." % (attp, attp*20))
            params = {'tracker-address': addr, 'attempts': attp }
            task = tq.Task(params=params, countdown=attp*30)
            new_queue.add(task)


if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.12036273701566365, 'fraction_numerical': 0.00494641384995878, 'mean_word_length': 2.579646017699115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4496667', 'n_tokens_mistral': 751, 'n_tokens_neox': 726, 'n_words': 233}