<commit_before>var serial = require('serialport');
var fs = require('fs');
var stream = require('binary-stream');
var sets = require('simplesets');

var _ = require("underscore");

var commons = require('./commons');
var parsers = require('./parsing');
var CronJob = require('cron').CronJob;
var attachments = {};

function ChillhubDevice(ttyPath, receive, announce) {
   var self = this;

   var ESC = 0xfe;
   var STX = 0xff;
   self.deviceType = '';
   self.UUID = '';
   self.subscriptions = new sets.Set([]);
   self.buf = [];
   self.msgBody = [];
   self.cronJobs = {};
   self.resources = {};
   self.registered = false;
   self.ignoreMsgCount = 10;

   self.uid = ttyPath;
   self.tty = new serial.SerialPort('/dev/'+ttyPath, { 
      baudrate: 115200, 
      disconnectedCallback: function(err) {
         if (err) {
            console.log('error in disconnectedCallback');
            console.log(err);
            console.log('UUID is ' + self.UUID);
         }
         if (self.resources.hasOwnProperty("setStatus")) {
            self.resources.setStatus("disconnected");
         } else {
            console.log("Could not set status, property setStatus not found.");
         }
      }
   });

   self.tty.open(function(err) {
      if (err) {
         console.log('error opening serial port');
         console.log(err);
         return;
      }

      self.tty.flush(function(err) {
         if (err) {
            console.log("Error flushing the serial port input buffer.");
            console.log(err);
            return;
         } 

         console.log("Serial port input buffer flushed.");

         self.scanForStx = function() {
            while (self.buf.length > 0) {
               var c = self.buf.shift();
               if (c == STX) {
                  self.commHandler = self.waitForLength;
                  self.waitForLength();
                  break;
               }
            }
         }

         self.waitForLength = function() {
            if (self.buf.length > 1) {
               if (self.buf[0] == ESC) {
                 self.buf.shift();
               }
               self.msgLen = self.buf.shift();
               self.bufIndex = 0;
               self.msgBody.length = 0;
               self.commHandler = self.waitForMessage;
               self.waitForMessage();
            }
         }

         self.waitForMessage = function() {
            while (self.buf.length > self.bufIndex) {
               if (self.buf[self.bufIndex] == ESC) {
                  if ((self.buf.length - self.bufIndex) > 1) {
                     self.bufIndex++;
                  } else {
                     return;
                  }
               }
               self.msgBody.push(self.buf[self.bufIndex++]);
               // wait for message plus sent checksum
               if (self.msgBody.length >= self.msgLen+2) {
                  self.checkMessage();
               }
            }
         }

         self.checkMessage= function() {
            var csSent = self.msgBody.pop() + self.msgBody.pop()*256;
            var cs = 42;
            for (var i = 0; i<self.msgBody.length; i++) {
               cs += self.msgBody[i];
            }
            if (cs == csSent) {
               // remove message from input buffer
               self.buf.splice(0, self.bufIndex);
               self.msgBody.shift();
               if (self.ignoreMsgCount > 0) {
                  self.ignoreMsgCount--;
               } else {
                  routeIncomingMessage(self.msgBody);
               }
            } else {
               console.log("Check sum error.");
               self.buf.shift();
            }
            self.commHandler = self.scanForStx;
            self.scanForStx();
         }

         self.commHandler = self.scanForStx;

         self.tty.on('data', function(data) {
            // network byte order is big endian... let's go with that
            self.buf = self.buf.concat((new stream.Reader(data, stream.BIG_ENDIAN)).readBytes(data.length));
            self.commHandler();
         });

         self.tty.on('error', function(err) {
            console.log('serial error:');
            console.log(err);
         });

         self.tty.on('disconnect', function(err) {
            console.log('error disconnecting?');
            console.log(err);
         });

         self.send = function(data) {
            // parse data into the format that usb devices expect and transmit it
            var dataBytes = parsers.parseJsonToStream(data);
            var writer = new stream.Writer(dataBytes.length+1, stream.BIG_ENDIAN);
            writer.writeUInt8(dataBytes.length);
            writer.writeBytes(dataBytes);
            var buf = writer.toArray();
            var outBuf = [];

            if (buf.length > 255) {
               console.log("ERROR: Cannot send a message longer than 255 bytes!!!");
               return;
            }

            var encodeByteToArray = function(b, a) {
               if (b == ESC || b == STX) {
                  a.push(ESC);
               }
               a.push(b);
            }

            // Take message body, wrap in STX and checksum, add escapes as needed.
            outBuf.push(STX);
            encodeByteToArray(buf.length, outBuf);
            var cs = 42;
            for (var i=0; i<buf.length; i++) {
               encodeByteToArray(buf[i], outBuf);
               cs += buf[i];
            }
            encodeByteToArray(commons.getNibble(cs, 2), outBuf);
            encodeByteToArray(commons.getNibble(cs, 1), outBuf);
            self.tty.write(outBuf, function(err) {
               if (err) {
                  console.log('error writing to serial');
                  console.log(err);
               }
            });
            return;

            writer.writeUInt8(dataBytes.length);
            writer.writeBytes(dataBytes);
            console.log("WRITER in send",writer.toArray());
            self.tty.write(writer.toArray(), function(err) {
               if (err) {
                  console.log('error writing to serial');
                  console.log(err);
               }
            });
         };

         console.log("Calling open callback...");
         if (self.onOpenCallback) {
            self.onOpenCallback();
         }
      });<commit_msg>Changed from checksum to CRC for attachments.<commit_after>var serial = require('serialport');
var fs = require('fs');
var stream = require('binary-stream');
var sets = require('simplesets');

var _ = require("underscore");

var commons = require('./commons');
var parsers = require('./parsing');
var CronJob = require('cron').CronJob;
var CRC = require('crc');
var attachments = {};

function ChillhubDevice(ttyPath, receive, announce) {
   var self = this;

   var ESC = 0xfe;
   var STX = 0xff;
   self.deviceType = '';
   self.UUID = '';
   self.subscriptions = new sets.Set([]);
   self.buf = [];
   self.msgBody = [];
   self.cronJobs = {};
   self.resources = {};
   self.registered = false;
   self.ignoreMsgCount = 10;

   self.uid = ttyPath;
   self.tty = new serial.SerialPort('/dev/'+ttyPath, { 
      baudrate: 115200, 
      disconnectedCallback: function(err) {
         if (err) {
            console.log('error in disconnectedCallback');
            console.log(err);
            console.log('UUID is ' + self.UUID);
         }
         if (self.resources.hasOwnProperty("setStatus")) {
            self.resources.setStatus("disconnected");
         } else {
            console.log("Could not set status, property setStatus not found.");
         }
      }
   });

   self.tty.open(function(err) {
      if (err) {
         console.log('error opening serial port');
         console.log(err);
         return;
      }

      self.tty.flush(function(err) {
         if (err) {
            console.log("Error flushing the serial port input buffer.");
            console.log(err);
            return;
         } 

         console.log("Serial port input buffer flushed.");

         self.scanForStx = function() {
            while (self.buf.length > 0) {
               var c = self.buf.shift();
               if (c == STX) {
                  self.commHandler = self.waitForLength;
                  self.waitForLength();
                  break;
               }
            }
         }

         self.waitForLength = function() {
            if (self.buf.length > 1) {
               if (self.buf[0] == ESC) {
                 self.buf.shift();
               }
               self.msgLen = self.buf.shift();
               self.bufIndex = 0;
               self.msgBody.length = 0;
               self.commHandler = self.waitForMessage;
               self.waitForMessage();
            }
         }

         self.waitForMessage = function() {
            while (self.buf.length > self.bufIndex) {
               if (self.buf[self.bufIndex] == ESC) {
                  if ((self.buf.length - self.bufIndex) > 1) {
                     self.bufIndex++;
                  } else {
                     return;
                  }
               }
               self.msgBody.push(self.buf[self.bufIndex++]);
               // wait for message plus sent checksum
               if (self.msgBody.length >= self.msgLen+2) {
                  self.checkMessage();
               }
            }
         }

         self.checkMessage= function() {
            var crcSent = self.msgBody.pop() + self.msgBody.pop()*256;
            crc =  CRC.crc16ccitt(self.msgBody);
            if (crc == crcSent) {
               // remove message from input buffer
               self.buf.splice(0, self.bufIndex);
               self.msgBody.shift();
               if (self.ignoreMsgCount > 0) {
                  self.ignoreMsgCount--;
               } else {
                  routeIncomingMessage(self.msgBody);
               }
            } else {
               console.log("Check sum error.");
               self.buf.shift();
            }
            self.commHandler = self.scanForStx;
            self.scanForStx();
         }

         self.commHandler = self.scanForStx;

         self.tty.on('data', function(data) {
            // network byte order is big endian... let's go with that
            self.buf = self.buf.concat((new stream.Reader(data, stream.BIG_ENDIAN)).readBytes(data.length));
            self.commHandler();
         });

         self.tty.on('error', function(err) {
            console.log('serial error:');
            console.log(err);
         });

         self.tty.on('disconnect', function(err) {
            console.log('error disconnecting?');
            console.log(err);
         });

         self.send = function(data) {
            // parse data into the format that usb devices expect and transmit it
            var dataBytes = parsers.parseJsonToStream(data);
            var writer = new stream.Writer(dataBytes.length+1, stream.BIG_ENDIAN);
            writer.writeUInt8(dataBytes.length);
            writer.writeBytes(dataBytes);
            var buf = writer.toArray();
            var outBuf = [];

            if (buf.length > 255) {
               console.log("ERROR: Cannot send a message longer than 255 bytes!!!");
               return;
            }

            var encodeByteToArray = function(b, a) {
               if (b == ESC || b == STX) {
                  a.push(ESC);
               }
               a.push(b);
            }

            // Take message body, wrap in STX and checksum, add escapes as needed.
            outBuf.push(STX);
            encodeByteToArray(buf.length, outBuf);
            var crc = CRC.crc16ccitt(buf);
            for (var i=0; i<buf.length; i++) {
               encodeByteToArray(buf[i], outBuf);
            }
            encodeByteToArray(commons.getNibble(crc, 2), outBuf);
            encodeByteToArray(commons.getNibble(crc, 1), outBuf);
            self.tty.write(outBuf, function(err) {
               if (err) {
                  console.log('error writing to serial');
                  console.log(err);
               }
            });
            return;

            writer.writeUInt8(dataBytes.length);
            writer.writeBytes(dataBytes);
            console.log("WRITER in send",writer.toArray());
            self.tty.write(writer.toArray(), function(err) {
               if (err) {
                  console.log('error writing to serial');
                  console.log(err);
               }
            });
         };

         console.log("Calling open callback...");
         if (self.onOpenCallback) {
            self.onOpenCallback();
         }
      });
{'fraction_non_alphanumeric': 0.1266613924050633, 'fraction_numerical': 0.0060917721518987345, 'mean_word_length': 1.4267613745440584, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1158738', 'n_tokens_mistral': 3558, 'n_tokens_neox': 3397, 'n_words': 706}