<commit_before>#!/usr/bin/python

import os
import sys
import pickle
import signal
import argparse
import json
import shutil

sys.path.append(os.environ['ANTELOPE'] + "/data/python")

import antelope.orb as orb
import antelope.Pkt as Pkt
import antelope.brttpkt as brttpkt

"""
python orbstart.py --help
python orbstart.py taexport.ucsd.edu:usarrayTA TA_109C/MGENC/M40
"""


def parse_packet(pkt):
  """
  Parse a generic orb packet into a dict with primitive types.
  @param pkt The orb packet.
  @retval A dictionary representation of the packet.
  """
  pkt_data = {}
  pkt_data['db'] = pkt.db
  pkt_data['dfile'] = pkt.dfile
  if pkt.pf:
    pkt_data['pf'] = pkt.pf.pf2dict()
  else:
    pkt_data['pf'] = None
  pkt_data['srcname'] = str(pkt.srcname)
  pkt_data['string'] = pkt.string
  pkt_data['type'] = {}
  pkt_data['type']['content'] = pkt.type.content
  pkt_data['type']['name'] = pkt.type.name
  pkt_data['type']['suffix'] = pkt.type.suffix
  pkt_data['type']['hdrcode'] = pkt.type.hdrcode
  pkt_data['type']['bodycode'] = pkt.type.bodycode
  pkt_data['type']['desc'] = pkt.type.desc
  pkt_data['version'] = pkt.version
  pkt_data['channels'] = []
  for c in pkt.channels:
    channel = {}
    channel['calib'] = c.calib
    channel['calper'] = c.calper
    channel['chan'] = c.chan
    channel['cuser1'] = c.cuser1
    channel['cuser2'] = c.cuser2
    #channel['data'] = 'array of int or float data'
    channel['data'] = c.data
    channel['duser1'] = c.duser1
    channel['duser2'] = c.duser2
    channel['iuser1'] = c.iuser1
    channel['iuser2'] = c.iuser2
    channel['iuser3'] = c.iuser3
    channel['loc'] = c.loc
    channel['net'] = c.net
    channel['nsamp'] = c.nsamp
    channel['samprate'] = c.samprate
    channel['segtype'] = c.segtype
    channel['sta'] = c.sta
    channel['time'] = c.time
    pkt_data['channels'].append(channel)
  return pkt_data

done = False

def orb_start(orbname, select=None, reject=None, after=-1, timeout=-1, queuesize=100):
  """
  Start and read an orb reap thread until signaled. 
  """
  # Handle break signal.
  def sig_handler(signum, frame):
    print 'Orb reap got signal ', signum
    global done
    done = True
  signal.signal(signal.SIGTERM, sig_handler)

  print 'entering orb reap...'
  with brttpkt.OrbreapThr(orbname, select=select, reject=reject, after=after, timeout=timeout, queuesize=queuesize) as orbth:
    data_dir = None
    while not done:
      try:
        pktid, srcname, time, packet = orbth.get()
      except brttpkt.Timeout:
        print 'Timeout waiting for orb...'
      except brttpkt.NoData:
        print 'No source data in orb...'
      else:
        orbpkt = Pkt.Packet(srcname, time, packet)
        print pktid, srcname, time, len(packet), str(done)
        pkt_data = parse_packet(orbpkt)
        if not data_dir:
          data_dir = '/tmp/scion-data/%s/' % srcname.replace('/','-')
          os.makedirs(data_dir)
        fpath = '/tmp/scion-data/%s/pkt_%i.json' % (srcname.replace('/','-'), pktid)
        print 'writing packet %s' % fpath
        f = open(fpath, 'w')
        json.dump(pkt_data, f)
        f.close()
<commit_msg>Reduced verbosity of orb reap<commit_after>#!/usr/bin/python

import os
import sys
import pickle
import signal
import argparse
import json
import shutil

sys.path.append(os.environ['ANTELOPE'] + "/data/python")

import antelope.orb as orb
import antelope.Pkt as Pkt
import antelope.brttpkt as brttpkt

"""
python orbstart.py --help
python orbstart.py taexport.ucsd.edu:usarrayTA TA_109C/MGENC/M40
"""


def parse_packet(pkt):
  """
  Parse a generic orb packet into a dict with primitive types.
  @param pkt The orb packet.
  @retval A dictionary representation of the packet.
  """
  pkt_data = {}
  pkt_data['db'] = pkt.db
  pkt_data['dfile'] = pkt.dfile
  if pkt.pf:
    pkt_data['pf'] = pkt.pf.pf2dict()
  else:
    pkt_data['pf'] = None
  pkt_data['srcname'] = str(pkt.srcname)
  pkt_data['string'] = pkt.string
  pkt_data['type'] = {}
  pkt_data['type']['content'] = pkt.type.content
  pkt_data['type']['name'] = pkt.type.name
  pkt_data['type']['suffix'] = pkt.type.suffix
  pkt_data['type']['hdrcode'] = pkt.type.hdrcode
  pkt_data['type']['bodycode'] = pkt.type.bodycode
  pkt_data['type']['desc'] = pkt.type.desc
  pkt_data['version'] = pkt.version
  pkt_data['channels'] = []
  for c in pkt.channels:
    channel = {}
    channel['calib'] = c.calib
    channel['calper'] = c.calper
    channel['chan'] = c.chan
    channel['cuser1'] = c.cuser1
    channel['cuser2'] = c.cuser2
    #channel['data'] = 'array of int or float data'
    channel['data'] = c.data
    channel['duser1'] = c.duser1
    channel['duser2'] = c.duser2
    channel['iuser1'] = c.iuser1
    channel['iuser2'] = c.iuser2
    channel['iuser3'] = c.iuser3
    channel['loc'] = c.loc
    channel['net'] = c.net
    channel['nsamp'] = c.nsamp
    channel['samprate'] = c.samprate
    channel['segtype'] = c.segtype
    channel['sta'] = c.sta
    channel['time'] = c.time
    pkt_data['channels'].append(channel)
  return pkt_data

done = False

def orb_start(orbname, select=None, reject=None, after=-1, timeout=-1, queuesize=100):
  """
  Start and read an orb reap thread until signaled. 
  """
  # Handle break signal.
  def sig_handler(signum, frame):
    print 'Orb reap got signal ', signum
    global done
    done = True
  signal.signal(signal.SIGTERM, sig_handler)

  print 'entering orb reap...'
  with brttpkt.OrbreapThr(orbname, select=select, reject=reject, after=after, timeout=timeout, queuesize=queuesize) as orbth:
    data_dir = None
    while not done:
      try:
        pktid, srcname, time, packet = orbth.get()
      except brttpkt.Timeout:
        #print 'Timeout waiting for orb...'
        pass
      except brttpkt.NoData:
        print 'No source data in orb...'
      else:
        orbpkt = Pkt.Packet(srcname, time, packet)
        #print pktid, srcname, time, len(packet), str(done)
        pkt_data = parse_packet(orbpkt)
        if not data_dir:
          data_dir = '/tmp/scion-data/%s/' % srcname.replace('/','-')
          os.makedirs(data_dir)
        fpath = '/tmp/scion-data/%s/pkt_%i.json' % (srcname.replace('/','-'), pktid)
        #print 'writing packet %s' % fpath
        f = open(fpath, 'w')
        json.dump(pkt_data, f)
        f.close()

{'fraction_non_alphanumeric': 0.1526850969176994, 'fraction_numerical': 0.00794407372100413, 'mean_word_length': 3.2078877005347595, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7233411', 'n_tokens_mistral': 2248, 'n_tokens_neox': 2293, 'n_words': 513}