<commit_before># coding=utf-8

__author__ = "<NAME> <<EMAIL>>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'

import logging
import os
import threading
import time
import fnmatch
import datetime
import sys
import shutil
import Queue
import requests

import octoprint.util as util

from octoprint.settings import settings
from octoprint.events import eventManager, Events

import sarge
import collections

import re

# currently configured timelapse
current = None

# currently active render job, if any
current_render_job = None

# filename formats
_capture_format = "{prefix}-%d.jpg"
_output_format = "{prefix}.mpg"

# old capture format, needed to delete old left-overs from
# versions <1.2.9
_old_capture_format_re = re.compile("^tmp_\d{5}.jpg$")

# valid timelapses
_valid_timelapse_types = ["off", "timed", "zchange"]

# callbacks for timelapse config updates
_update_callbacks = []


def _extract_prefix(filename):
	"""
	>>> _extract_prefix("some_long_filename_without_hyphen.jpg")
	>>> _extract_prefix("-first_char_is_hyphen.jpg")
	>>> _extract_prefix("some_long_filename_with-stuff.jpg")
	'some_long_filename_with'
	"""
	pos = filename.rfind("-")
	if not pos or pos < 0:
		return None
	return filename[:pos]


def get_finished_timelapses():
	files = []
	basedir = settings().getBaseFolder("timelapse")
	for osFile in os.listdir(basedir):
		if not fnmatch.fnmatch(osFile, "*.mp[g4]"):
			continue
		statResult = os.stat(os.path.join(basedir, osFile))
		files.append({
			"name": osFile,
			"size": util.get_formatted_size(statResult.st_size),
			"bytes": statResult.st_size,
			"date": util.get_formatted_datetime(datetime.datetime.fromtimestamp(statResult.st_ctime))
		})
	return files


def get_unrendered_timelapses():
	delete_old_unrendered_timelapses()

	basedir = settings().getBaseFolder("timelapse_tmp")
	jobs = collections.defaultdict(lambda: dict(count=0, size=None, bytes=0, date=None, timestamp=None))
	for osFile in os.listdir(basedir):
		if not fnmatch.fnmatch(osFile, "*.jpg"):
			continue

		prefix = _extract_prefix(osFile)
		if prefix is None:
			continue

		statResult = os.stat(os.path.join(basedir, osFile))
		jobs[prefix]["count"] += 1
		jobs[prefix]["bytes"] += statResult.st_size
		if jobs[prefix]["timestamp"] is None or statResult.st_ctime < jobs[prefix]["timestamp"]:
			jobs[prefix]["timestamp"] = statResult.st_ctime

	def finalize_fields(job):
		job["size"] = util.get_formatted_size(job["bytes"])
		job["date"] = util.get_formatted_datetime(datetime.datetime.fromtimestamp(job["timestamp"]))
		del job["timestamp"]
		return job

	return sorted([util.dict_merge(dict(name=key), finalize_fields(value)) for key, value in jobs.items()], key=lambda x: x["name"])


def delete_unrendered_timelapse(name):
	basedir = settings().getBaseFolder("timelapse_tmp")
	for filename in os.listdir(basedir):
		try:
			if fnmatch.fnmatch(filename, "{}*.jpg".format(name)):
				os.remove(os.path.join(basedir, filename))
		except:
			logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(filename))


def render_unrendered_timelapse(name, gcode=None, postfix=None, fps=25):
	capture_dir = settings().getBaseFolder("timelapse_tmp")
	output_dir = settings().getBaseFolder("timelapse")
	threads = settings().get(["webcam", "ffmpegThreads"])

	job = TimelapseRenderJob(capture_dir, output_dir, name,
	                         postfix=postfix,
	                         capture_format=_capture_format,
	                         output_format=_output_format,
	                         fps=fps,
	                         threads=threads,
	                         on_start=_create_render_start_handler(name, gcode=gcode),
	                         on_success=_create_render_success_handler(name, gcode=gcode),
	                         on_fail=_create_render_fail_handler(name, gcode=gcode),
	                         on_always=_create_render_always_handler(name, gcode=gcode))
	job.process()


def delete_old_unrendered_timelapses():
	basedir = settings().getBaseFolder("timelapse_tmp")
	clean_after_days = settings().getInt(["webcam", "cleanTmpAfterDays"])
	cutoff = time.time() - clean_after_days * 24 * 60 * 60

	prefixes_to_clean = []
	for filename in os.listdir(basedir):
		try:
			path = os.path.join(basedir, filename)

			prefix = _extract_prefix(filename)
			if prefix is None:
				# might be an old tmp_00000.jpg kinda frame. we can't
				# render those easily anymore, so delete that stuff
				if _old_capture_format_re.match(filename):
					os.remove(path)
				continue

			if prefix in prefixes_to_clean:
				continue

			if os.path.getmtime(path) < cutoff:
				prefixes_to_clean.append(prefix)
		except:
			logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(filename))

	for prefix in prefixes_to_clean:
		delete_unrendered_timelapse(prefix)


def _create_render_start_handler(name, gcode=None):
	def f(movie):
		global current_render_job
		event_payload = {"gcode": gcode if gcode is not None else "unknown",
		                 "movie": movie,
		                 "movie_basename": os.path.basename(movie)}
		current_render_job = event_payload
		eventManager().fire(Events.MOVIE_RENDERING, event_payload)
	return f


def _create_render_success_handler(name, gcode=None):
	def f(movie):
		event_payload = {"gcode": gcode if gcode is not None else "unknown",
		                 "movie": movie,
		                 "movie_basename": os.path.basename(movie)}
		eventManager().fire(Events.MOVIE_DONE, event_payload)
		delete_unrendered_timelapse(name)
	return f


def _create_render_fail_handler(name, gcode=None):
	def f(movie, returncode=255, stdout="Unknown error", stderr="Unknown error"):
		event_payload = {"gcode": gcode if gcode is not None else "unknown",
		                 "movie": movie,
		                 "movie_basename": os.path.basename(movie)}
		payload = dict(event_payload)
		payload.update(dict(returncode=returncode, error=stderr))
		eventManager().fire(Events.MOVIE_FAILED, payload)<commit_msg>Don't concurrently delete unrendered timelapses<commit_after># coding=utf-8

__author__ = "<NAME> <<EMAIL>>"
__license__ = 'GNU Affero General Public License http://www.gnu.org/licenses/agpl.html'

import logging
import os
import threading
import time
import fnmatch
import datetime
import sys
import shutil
import Queue
import requests

import octoprint.util as util

from octoprint.settings import settings
from octoprint.events import eventManager, Events

import sarge
import collections

import re

# currently configured timelapse
current = None

# currently active render job, if any
current_render_job = None

# filename formats
_capture_format = "{prefix}-%d.jpg"
_output_format = "{prefix}.mpg"

# old capture format, needed to delete old left-overs from
# versions <1.2.9
_old_capture_format_re = re.compile("^tmp_\d{5}.jpg$")

# valid timelapses
_valid_timelapse_types = ["off", "timed", "zchange"]

# callbacks for timelapse config updates
_update_callbacks = []

# lock for timelapse cleanup, must be re-entrant
_cleanup_lock = threading.RLock()


def _extract_prefix(filename):
	"""
	>>> _extract_prefix("some_long_filename_without_hyphen.jpg")
	>>> _extract_prefix("-first_char_is_hyphen.jpg")
	>>> _extract_prefix("some_long_filename_with-stuff.jpg")
	'some_long_filename_with'
	"""
	pos = filename.rfind("-")
	if not pos or pos < 0:
		return None
	return filename[:pos]


def get_finished_timelapses():
	files = []
	basedir = settings().getBaseFolder("timelapse")
	for osFile in os.listdir(basedir):
		if not fnmatch.fnmatch(osFile, "*.mp[g4]"):
			continue
		statResult = os.stat(os.path.join(basedir, osFile))
		files.append({
			"name": osFile,
			"size": util.get_formatted_size(statResult.st_size),
			"bytes": statResult.st_size,
			"date": util.get_formatted_datetime(datetime.datetime.fromtimestamp(statResult.st_ctime))
		})
	return files


def get_unrendered_timelapses():
	delete_old_unrendered_timelapses()

	basedir = settings().getBaseFolder("timelapse_tmp")
	jobs = collections.defaultdict(lambda: dict(count=0, size=None, bytes=0, date=None, timestamp=None))
	for osFile in os.listdir(basedir):
		if not fnmatch.fnmatch(osFile, "*.jpg"):
			continue

		prefix = _extract_prefix(osFile)
		if prefix is None:
			continue

		statResult = os.stat(os.path.join(basedir, osFile))
		jobs[prefix]["count"] += 1
		jobs[prefix]["bytes"] += statResult.st_size
		if jobs[prefix]["timestamp"] is None or statResult.st_ctime < jobs[prefix]["timestamp"]:
			jobs[prefix]["timestamp"] = statResult.st_ctime

	def finalize_fields(job):
		job["size"] = util.get_formatted_size(job["bytes"])
		job["date"] = util.get_formatted_datetime(datetime.datetime.fromtimestamp(job["timestamp"]))
		del job["timestamp"]
		return job

	return sorted([util.dict_merge(dict(name=key), finalize_fields(value)) for key, value in jobs.items()], key=lambda x: x["name"])


def delete_unrendered_timelapse(name):
	global _cleanup_lock

	basedir = settings().getBaseFolder("timelapse_tmp")
	with _cleanup_lock:
		for filename in os.listdir(basedir):
			try:
				if fnmatch.fnmatch(filename, "{}*.jpg".format(name)):
					os.remove(os.path.join(basedir, filename))
			except:
				if logging.getLogger(__name__).isEnabledFor(logging.DEBUG):
					logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(filename))


def render_unrendered_timelapse(name, gcode=None, postfix=None, fps=25):
	capture_dir = settings().getBaseFolder("timelapse_tmp")
	output_dir = settings().getBaseFolder("timelapse")
	threads = settings().get(["webcam", "ffmpegThreads"])

	job = TimelapseRenderJob(capture_dir, output_dir, name,
	                         postfix=postfix,
	                         capture_format=_capture_format,
	                         output_format=_output_format,
	                         fps=fps,
	                         threads=threads,
	                         on_start=_create_render_start_handler(name, gcode=gcode),
	                         on_success=_create_render_success_handler(name, gcode=gcode),
	                         on_fail=_create_render_fail_handler(name, gcode=gcode),
	                         on_always=_create_render_always_handler(name, gcode=gcode))
	job.process()


def delete_old_unrendered_timelapses():
	global _cleanup_lock

	basedir = settings().getBaseFolder("timelapse_tmp")
	clean_after_days = settings().getInt(["webcam", "cleanTmpAfterDays"])
	cutoff = time.time() - clean_after_days * 24 * 60 * 60

	prefixes_to_clean = []

	with _cleanup_lock:
		for filename in os.listdir(basedir):
			try:
				path = os.path.join(basedir, filename)

				prefix = _extract_prefix(filename)
				if prefix is None:
					# might be an old tmp_00000.jpg kinda frame. we can't
					# render those easily anymore, so delete that stuff
					if _old_capture_format_re.match(filename):
						os.remove(path)
					continue

				if prefix in prefixes_to_clean:
					continue

				if os.path.getmtime(path) < cutoff:
					prefixes_to_clean.append(prefix)
			except:
				if logging.getLogger(__name__).isEnabledFor(logging.DEBUG):
					logging.getLogger(__name__).exception("Error while processing file {} during cleanup".format(filename))

		for prefix in prefixes_to_clean:
			delete_unrendered_timelapse(prefix)
			logging.getLogger(__name__).info("Deleted old unrendered timelapse {}".format(prefix))


def _create_render_start_handler(name, gcode=None):
	def f(movie):
		global current_render_job
		event_payload = {"gcode": gcode if gcode is not None else "unknown",
		                 "movie": movie,
		                 "movie_basename": os.path.basename(movie)}
		current_render_job = event_payload
		eventManager().fire(Events.MOVIE_RENDERING, event_payload)
	return f


def _create_render_success_handler(name, gcode=None):
	def f(movie):
		delete_unrendered_timelapse(name)
		event_payload = {"gcode": gcode if gcode is not None else "unknown",
		                 "movie": movie,
		                 "movie_basename": os.path.basename(movie)}
		eventManager().fire(Events.MOVIE_DONE, event_payload)
	return f


def _create_render_fail_handler(name, gcode=None):
	def f(movie, returncode=255, stdout="Unknown error", stderr="Unknown error"):
		event_payload = {"gcode": gcode if gcode is not None else "unknown",
		                 "movie": movie,
		                 "movie_basename": os.path.basename(movie)}
		payload = dict(event_payload)
		payload.update(dict(returncode=returncode, error=stderr))
		eventManager().fire(Events.MOVIE_FAILED, payload)
{'fraction_non_alphanumeric': 0.1160423128927066, 'fraction_numerical': 0.004135846655531695, 'mean_word_length': 4.276542173730592, 'pattern_counts': {'":': 26, '<': 17, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4793197', 'n_tokens_mistral': 4395, 'n_tokens_neox': 3989, 'n_words': 845}