<commit_before>#!/usr/bin/env python

from operator import itemgetter
import datetime
import json
import sys
import argparse
import usb.core
import string


class UsbDevice(object):
    DATA_ENDPOINT = 0x81

    def __init__(self, vendorId=0x04d8, endpoint=0x81):
        self.vendorId = vendorId
        self.endpoint = endpoint
        self.message_buffer = ""
        self.messages_received = 0
        self.good_messages = 0

        self.device = usb.core.find(idVendor=vendorId)
        if not self.device:
            print "Couldn't find a USB device from vendor %s" % self.vendorId
            sys.exit()
        self.device.set_configuration()

    def parse_message(self):
        if "\r\n" in self.message_buffer:
            message,_,remainder= self.message_buffer.partition("\r\n")
            try:
                message = json.loads(message)
            except ValueError:
                pass
            else:
                self.good_messages += 1
                return message
            finally:
                self.message_buffer = remainder
                self.messages_received += 1


    def read(self):
        while True:
            self.message_buffer += self.device.read(self.endpoint,
                    128).tostring()
            parsed_message = self.parse_message()
            if parsed_message is not None:
                return parsed_message

            if (self.messages_received > 0 and
                    self.messages_received % 1000 == 0):
                print "Received %d messages so far (%d%% valid)..." % (
                        self.messages_received,
                        float(self.good_messages) / self.messages_received
                        * 100)

    def run(self):
        message = self.read()
        while message is not None:
            message = self.read()
            self.validate(message)

        print
        print "Finished receiving."

    def validate(self, message):
        assert "name" in message
        assert "value" in message
<commit_msg>Check message count even if all valid.<commit_after>#!/usr/bin/env python

from operator import itemgetter
import datetime
import json
import sys
import argparse
import usb.core
import string


class UsbDevice(object):
    DATA_ENDPOINT = 0x81

    def __init__(self, vendorId=0x04d8, endpoint=0x81):
        self.vendorId = vendorId
        self.endpoint = endpoint
        self.message_buffer = ""
        self.messages_received = 0
        self.good_messages = 0

        self.device = usb.core.find(idVendor=vendorId)
        if not self.device:
            print "Couldn't find a USB device from vendor %s" % self.vendorId
            sys.exit()
        self.device.set_configuration()

    def parse_message(self):
        if "\r\n" in self.message_buffer:
            message,_,remainder= self.message_buffer.partition("\r\n")
            try:
                message = json.loads(message)
            except ValueError:
                pass
            else:
                self.good_messages += 1
                return message
            finally:
                self.message_buffer = remainder
                self.messages_received += 1


    def read(self):
        while True:
            self.message_buffer += self.device.read(self.endpoint,
                    128).tostring()
            parsed_message = self.parse_message()

            if (self.messages_received > 0 and
                    self.messages_received % 1000 == 0):
                print "Received %d messages so far (%d%% valid)..." % (
                        self.messages_received,
                        float(self.good_messages) / self.messages_received
                        * 100)

            if parsed_message is not None:
                return parsed_message


    def run(self):
        message = self.read()
        while message is not None:
            message = self.read()
            self.validate(message)

        print
        print "Finished receiving."

    def validate(self, message):
        assert "name" in message
        assert "value" in message

{'fraction_non_alphanumeric': 0.08209500609013398, 'fraction_numerical': 0.01266747868453106, 'mean_word_length': 1.652454780361757, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6828668', 'n_tokens_mistral': 1135, 'n_tokens_neox': 1040, 'n_words': 282}