<commit_before>import json
import psutil
import os
import re
import sys
import select
import subprocess
import requests
import time
import threading

from socket import gethostname
from bottle import Bottle, response, request

app = Bottle()
globalWatcherThread = None
globalResourceThread = None
globalProcess = None

# This function will be called from Watcher or Resource class
def send(self, payload):
  success = False
  pause = 1
  while not success and self.working:
    try:
      requests.post(self.host + '/event', data=payload)
      success = True
    except requests.ConnectionError:
      print 'Connection problems. Retrying in', pause, ' sec..'
      self.sleep(pause)
      pause *= 2

class WatcherThread:
  def __init__(self, host, path, prefix):
    self.host = host
    self.path = path
    self.prefix = prefix  

    self.index = {}
    self.skip_patterns = ['DEBUG', 'TRACE']

  def args(self):
    return (self.host, self.path, self.prefix)

  def update(self, path):
    if path not in self.index:
      #print 'created:', path
      payload = {'event': 'created', 'path': path[len(self.prefix):], 'data': ''}
      send(self, payload)
      self.index[path] = 0

    size = os.path.getsize(path)
    if size > self.index[path]:
      with open(path, 'r') as fin:
        fin.seek(self.index[path])
        data = fin.read()

        filteredData = data
        for pattern in self.skip_patterns:
          filteredData = re.sub('[0-9]+:[0-9][^\n=]*' + pattern + '[^\n]*\n', '', filteredData)
        data = filteredData

        #print 'new data:', path, len(data)
        while len(data) and self.working:
          batch = data[:500000]
          data = data[len(batch):]
          payload = {'event': 'new line', 'path': path[len(self.prefix):], 'data': batch}
          send(self, payload)

      self.index[path] = size

  def scan(self):
    for root, dirs, files in os.walk(self.path):
      for file in files:
        self.update(root + '/' + file)

  def run(self):
    self.working = True
    while self.working:
      self.scan()
      self.sleep(30)

  def stop(self):
    print 'Stoping watcher thread...'
    self.working = False

  def sleep(self, seconds):
    cnt = 0
    while self.working and cnt < seconds:
      time.sleep(1)
      cnt += 1

class ResourceThread:
  def __init__(self, host, seep_root):
    self.info = {}
    self.host = host
    self.seep_root = seep_root
    self.procs = set()

  def args(self):
    return (self.host, self.seep_root)

  def scanWorkers(self):
    workers = []
    pids = []
    for proc in psutil.process_iter():
      if proc.name() == 'java' and not proc in self.procs:
        pids.append(proc.pid)

    for pid in pids:
      proc = psutil.Process(pid)
      try:
        pinfo = proc.as_dict(attrs=['cmdline'])
      except psutil.NoSuchProcess:
        pass
      else:
        cmdline = pinfo['cmdline']<commit_msg>Fix bug of changed function in psutil.<commit_after>import json
import psutil
import os
import re
import sys
import select
import subprocess
import requests
import time
import threading

from socket import gethostname
from bottle import Bottle, response, request

app = Bottle()
globalWatcherThread = None
globalResourceThread = None
globalProcess = None

# This function will be called from Watcher or Resource class
def send(self, payload):
  success = False
  pause = 1
  while not success and self.working:
    try:
      requests.post(self.host + '/event', data=payload)
      success = True
    except requests.ConnectionError:
      print 'Connection problems. Retrying in', pause, ' sec..'
      self.sleep(pause)
      pause *= 2

class WatcherThread:
  def __init__(self, host, path, prefix):
    self.host = host
    self.path = path
    self.prefix = prefix  

    self.index = {}
    self.skip_patterns = ['DEBUG', 'TRACE']

  def args(self):
    return (self.host, self.path, self.prefix)

  def update(self, path):
    if path not in self.index:
      #print 'created:', path
      payload = {'event': 'created', 'path': path[len(self.prefix):], 'data': ''}
      send(self, payload)
      self.index[path] = 0

    size = os.path.getsize(path)
    if size > self.index[path]:
      with open(path, 'r') as fin:
        fin.seek(self.index[path])
        data = fin.read()

        filteredData = data
        for pattern in self.skip_patterns:
          filteredData = re.sub('[0-9]+:[0-9][^\n=]*' + pattern + '[^\n]*\n', '', filteredData)
        data = filteredData

        #print 'new data:', path, len(data)
        while len(data) and self.working:
          batch = data[:500000]
          data = data[len(batch):]
          payload = {'event': 'new line', 'path': path[len(self.prefix):], 'data': batch}
          send(self, payload)

      self.index[path] = size

  def scan(self):
    for root, dirs, files in os.walk(self.path):
      for file in files:
        self.update(root + '/' + file)

  def run(self):
    self.working = True
    while self.working:
      self.scan()
      self.sleep(30)

  def stop(self):
    print 'Stoping watcher thread...'
    self.working = False

  def sleep(self, seconds):
    cnt = 0
    while self.working and cnt < seconds:
      time.sleep(1)
      cnt += 1

class ResourceThread:
  def __init__(self, host, seep_root):
    self.info = {}
    self.host = host
    self.seep_root = seep_root
    self.procs = set()

  def args(self):
    return (self.host, self.seep_root)

  def scanWorkers(self):
    workers = []
    pids = []
    for proc in psutil.process_iter():
      if ((isinstance(proc.name, str) and proc.name == 'java') or (hasattr(proc.name, '__call__') and proc.name() == 'java')) and not proc in self.procs:
        pids.append(proc.pid)

    for pid in pids:
      proc = psutil.Process(pid)
      try:
        pinfo = proc.as_dict(attrs=['cmdline'])
      except psutil.NoSuchProcess:
        pass
      else:
        cmdline = pinfo['cmdline']
{'fraction_non_alphanumeric': 0.12870619946091644, 'fraction_numerical': 0.006064690026954178, 'mean_word_length': 2.6179159049360146, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '700647', 'n_tokens_mistral': 1921, 'n_tokens_neox': 1874, 'n_words': 546}