<commit_before>import sublime, sublime_plugin

import lib.util
import sys
# reload lib.util on update/reload of primary module
# so improvements will be loaded without a sublime restart
sys.modules['lib.util'] = reload(lib.util)
from lib.util import communicate, which, popen

import os
import re
import shlex

import subprocess
import thread
import time
import traceback
import Queue

INDENTATION = '  '

class BoundaryError(Exception): pass

if not 'already' in globals():
	already = True
	commands = {}

def spawn(view, edit, indent, cmd, sel):
	q = Queue.Queue()
	def fold(region):
		regions = view.get_regions(region)
		for region in regions:
			lines = view.split_by_newlines(region)
			if len(lines) > 24:

				lines = lines[1:-24]
				area = lines.pop()
				for sub in lines:
					area = area.cover(sub)

				view.unfold(area)
				view.fold(area)

	def merge(region, count):
		if q.empty(): return
		regions = view.get_regions(region)
		if not regions: return

		pos = view.line(regions[0].end() - 1)
		edit = view.begin_edit()
		try:
			start = time.time()
			lines = []
			while start - time.time() < 0.05:
				try:
					lines.append(q.get(False))
					q.task_done()
					count += 1
				except Queue.Empty:
					break

			if not lines: return
			insert(view, edit, pos, '\n'.join(lines), indent + INDENTATION)

			fold(region)
		except:
			print traceback.format_exc()
		finally:
			view.end_edit(edit)

	def stderr(p, region):
		count = 0
		while p.poll() is None:
			line = p.stderr.readline()
			q.put(line.rstrip('\r\n'))
			sublime.set_timeout(make_callback(merge, region, count), 100)

			if count % 3 == 0:
				time.sleep(0.01)

		# if the process wasn't terminated
		if p.returncode >= 0:
			q.put(p.stderr.read().rstrip('\r\n'))
			sublime.set_timeout(make_callback(merge, region, count), 100)

	def stdout(p, region):
		count = 0
		while p.poll() is None:
			line = p.stdout.readline()
			q.put(line.rstrip('\r\n'))
			sublime.set_timeout(make_callback(merge, region, count), 100)

			if count % 3 == 0:
				time.sleep(0.01)

		# if the process wasn't terminated
		if p.returncode >= 0:
			q.put(p.stdout.read().rstrip('\r\n'))
			sublime.set_timeout(make_callback(merge, region, count), 100)

		sublime.set_timeout(make_callback(view.erase_regions, region), 100)
		del commands[region]
		return

	p = popen(cmd, return_error=True)
	if isinstance(p, subprocess.Popen):
		region = 'xiki sub %i' % p.pid
		line = view.full_line(sel.b)
		spread = sublime.Region(line.a, line.b)
		commands[region] = p
		view.add_regions(region, [spread], 'keyword', '', sublime.DRAW_OUTLINED)

		thread.start_new_thread(stdout, (p, region))
		thread.start_new_thread(stderr, (p, region))
	else:
		insert(view, edit, sel, 'Error: ' + p, indent + INDENTATION)

def xiki(view):
	settings = view.settings()

	if settings.get('xiki'):
		for sel in view.sel():
			output = None
			cmd = None
			persist = False
			oldcwd = None

			view.sel().subtract(sel)<commit_msg>improve stderr handling<commit_after>import sublime, sublime_plugin

import lib.util
import sys
# reload lib.util on update/reload of primary module
# so improvements will be loaded without a sublime restart
sys.modules['lib.util'] = reload(lib.util)
from lib.util import communicate, which, popen

import os
import re
import shlex

import subprocess
import thread
import time
import traceback
import Queue

INDENTATION = '  '

class BoundaryError(Exception): pass

if not 'already' in globals():
	already = True
	commands = {}

def spawn(view, edit, indent, cmd, sel):
	q = Queue.Queue()
	def fold(region):
		regions = view.get_regions(region)
		for region in regions:
			lines = view.split_by_newlines(region)
			if len(lines) > 24:

				lines = lines[1:-24]
				area = lines.pop()
				for sub in lines:
					area = area.cover(sub)

				view.unfold(area)
				view.fold(area)

	def merge(region, count):
		if q.empty(): return
		regions = view.get_regions(region)
		if not regions: return

		pos = view.line(regions[0].end() - 1)
		edit = view.begin_edit()
		try:
			start = time.time()
			lines = []
			while start - time.time() < 0.05:
				try:
					lines.append(q.get(False))
					q.task_done()
					count += 1
				except Queue.Empty:
					break

			if not lines: return
			insert(view, edit, pos, '\n'.join(lines), indent + INDENTATION)

			fold(region)
		except:
			print traceback.format_exc()
		finally:
			view.end_edit(edit)

	def poll(p, region, fd):
		count = 0
		while p.poll() is None:
			line = fd.readline()
			if line:
				q.put(line.rstrip('\r\n'))
				sublime.set_timeout(make_callback(merge, region, count), 100)

			if count % 3 == 0:
				time.sleep(0.01)

		# if the process wasn't terminated
		if p.returncode >= 0:
			out = fd.read()
			if out:
				q.put(out.rstrip('\r\n'))
				sublime.set_timeout(make_callback(merge, region, count), 100)

	def stderr(p, region):
		poll(p, region, p.stderr)

	def stdout(p, region):
		poll(p, region, p.stdout)

		sublime.set_timeout(make_callback(view.erase_regions, region), 500)
		del commands[region]
		return

	p = popen(cmd, return_error=True)
	if isinstance(p, subprocess.Popen):
		region = 'xiki sub %i' % p.pid
		line = view.full_line(sel.b)
		spread = sublime.Region(line.a, line.b)
		commands[region] = p
		view.add_regions(region, [spread], 'keyword', '', sublime.DRAW_OUTLINED)

		thread.start_new_thread(stdout, (p, region))
		thread.start_new_thread(stderr, (p, region))
	else:
		insert(view, edit, sel, 'Error: ' + p, indent + INDENTATION)

def xiki(view):
	settings = view.settings()

	if settings.get('xiki'):
		for sel in view.sel():
			output = None
			cmd = None
			persist = False
			oldcwd = None

			view.sel().subtract(sel)
{'fraction_non_alphanumeric': 0.13544973544973546, 'fraction_numerical': 0.01181657848324515, 'mean_word_length': 3.529552715654952, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4677141', 'n_tokens_mistral': 2402, 'n_tokens_neox': 2017, 'n_words': 526}