<commit_before>import sys
import os
import re
import json
import subprocess
import traceback
import logging
import base64
import datetime

MYPATH = os.path.abspath(os.path.dirname(__file__))
sys.path.insert(0, os.path.abspath(os.path.join(MYPATH, '..')))
sys.path.insert(0, os.path.abspath(os.path.join(MYPATH, '..', '..')))
sys.path.insert(0, os.path.normpath(os.path.join(MYPATH, "depends")))

os.environ["DJANGO_SETTINGS_MODULE"] = 'settings'

from django.utils.html import escape
from django.conf import settings
from django.utils.importlib import import_module
from django.contrib.auth.models import User
ENGINE = import_module(settings.SESSION_ENGINE)

from tornado.websocket import WebSocketHandler
from tornado import ioloop

import ide.terminal
import ide.settings

SCREEN_COMMAND = False
if os.path.exists(ide.settings.TERMINAL_SCREEN):
  SCREEN_COMMAND = ide.settings.TERMINAL_SCREEN + ' ' + ide.settings.TERMINAL_SHELL
  
class TerminalWebSocket (WebSocketHandler):
  def open (self):
    logging.info('Opening Terminal Web Socket')
    self.last_sent = None
    self.terminals = {}
    self.current_tsid = None
    self.cols = None
    self.lines = None
    
    self.io_loop = ioloop.IOLoop.instance()
    #self.io_loop.set_blocking_signal_threshold(3)
    self.scheduler = ioloop.PeriodicCallback(self.refresh_loop, 100, io_loop=self.io_loop)
    self.scheduler.start()
    
    if SCREEN_COMMAND:
      old = []
      pipe = subprocess.Popen(ide.settings.TERMINAL_SCREEN + " -ls", shell=True, stdout=subprocess.PIPE).stdout
      for line in pipe.readlines():
        regex = re.search("(\d+\.\S+)\s+\(.*\)\s+\(\S+\)", line)
        if regex:
          old.append(regex.group(1))
          
      if old:
        self.write_message(json.dumps({'action': 'oldterms', 'data': old}))
        
  def refresh_loop (self):
    try:
      if self.current_tsid is not None:
        self.term_refresh()
        
    except:
      import traceback
      traceback.print_exc()
      
  def create_terminal (self, tsid, user, width, height, sock=None):
    self.terminals[tsid] = ide.terminal.Terminal()
    
    cmd = ide.settings.TERMINAL_SHELL
    if SCREEN_COMMAND:
      cmd = SCREEN_COMMAND
      if sock:
        cmd = ide.settings.TERMINAL_SCREEN + ' -d -r ' + sock + ' ' + ide.settings.TERMINAL_SHELL
        
    self.terminals[tsid].start(cmd, user.preferences.basedir, width, height, tsid=tsid, onclose=self.remove_terminal)
    
  def process_line (self, num, line):
    html = ''<commit_msg>added resize check<commit_after>import sys
import os
import re
import json
import subprocess
import traceback
import logging
import base64
import datetime

MYPATH = os.path.abspath(os.path.dirname(__file__))
sys.path.insert(0, os.path.abspath(os.path.join(MYPATH, '..')))
sys.path.insert(0, os.path.abspath(os.path.join(MYPATH, '..', '..')))
sys.path.insert(0, os.path.normpath(os.path.join(MYPATH, "depends")))

os.environ["DJANGO_SETTINGS_MODULE"] = 'settings'

from django.utils.html import escape
from django.conf import settings
from django.utils.importlib import import_module
from django.contrib.auth.models import User
ENGINE = import_module(settings.SESSION_ENGINE)

from tornado.websocket import WebSocketHandler
from tornado import ioloop

import ide.terminal
import ide.settings

SCREEN_COMMAND = False
if os.path.exists(ide.settings.TERMINAL_SCREEN):
  SCREEN_COMMAND = ide.settings.TERMINAL_SCREEN + ' ' + ide.settings.TERMINAL_SHELL
  
class TerminalWebSocket (WebSocketHandler):
  def open (self):
    logging.info('Opening Terminal Web Socket')
    self.last_sent = None
    self.terminals = {}
    self.current_tsid = None
    self.cols = None
    self.lines = None
    
    self.io_loop = ioloop.IOLoop.instance()
    #self.io_loop.set_blocking_signal_threshold(3)
    self.scheduler = ioloop.PeriodicCallback(self.refresh_loop, 100, io_loop=self.io_loop)
    self.scheduler.start()
    
    if SCREEN_COMMAND:
      old = []
      pipe = subprocess.Popen(ide.settings.TERMINAL_SCREEN + " -ls", shell=True, stdout=subprocess.PIPE).stdout
      for line in pipe.readlines():
        regex = re.search("(\d+\.\S+)\s+\(.*\)\s+\(\S+\)", line)
        if regex:
          old.insert(0, regex.group(1))
          
      if old:
        self.write_message(json.dumps({'action': 'oldterms', 'data': old}))
        
  def refresh_loop (self):
    try:
      if self.current_tsid is not None:
        self.term_refresh()
        
    except:
      import traceback
      traceback.print_exc()
      
  def create_terminal (self, tsid, user, width, height, sock=None):
    self.terminals[tsid] = ide.terminal.Terminal()
    
    cmd = ide.settings.TERMINAL_SHELL
    if SCREEN_COMMAND:
      cmd = SCREEN_COMMAND
      if sock:
        cmd = ide.settings.TERMINAL_SCREEN + ' -d -r ' + sock + ' ' + ide.settings.TERMINAL_SHELL
        
    self.terminals[tsid].start(cmd, user.preferences.basedir, width, height, tsid=tsid, onclose=self.remove_terminal)
    
  def process_line (self, num, line):
    html = ''
{'fraction_non_alphanumeric': 0.1243533625149224, 'fraction_numerical': 0.004178272980501393, 'mean_word_length': 3.356152512998267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1842431', 'n_tokens_mistral': 1744, 'n_tokens_neox': 1738, 'n_words': 337}