<commit_before># user's mapper.
def map(key, value):
    from lfm.data.parse import web

    try: log = web.Log(value)
    except ValueError: return
    ua = web.UserAgent()

    ts = log.timestamp.ymd()
    dimensions = {'country'   : log.country(),
                  'domain'    : log.domain,
                  'useragent' : ua.classify(log.agent),
                  'usertype'  : ("user", "anon")[log.userid == None]
                  }
    values = {'pageviews' : 1}
    
    yield ts, dimensions, values<commit_msg>temporarily wrapping things inside a try-block to get rid of the AttributeErrors I've been seeing.<commit_after># user's mapper.
def map(key, value):
    import sys
    from lfm.data.parse import web

    try: log = web.Log(value)
    except: return
    ua = web.UserAgent()

    try: # been seeing too many AttributeErrors lately..
        ts = log.timestamp.ymd()
        dimensions = {'country'   : log.country(),
                      'domain'    : log.domain,
                      'useragent' : ua.classify(log.agent),
                      'usertype'  : ("user", "anon")[log.userid == None]
                      }
        values = {'pageviews' : 1}

    except AttributeError:
        sys.stderr.write("AttributeError!\n")
        return
    
    yield ts, dimensions, values
{'fraction_non_alphanumeric': 0.12432847275518036, 'fraction_numerical': 0.0015349194167306216, 'mean_word_length': 1.9107142857142858, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1955693', 'n_tokens_mistral': 398, 'n_tokens_neox': 379, 'n_words': 101}