<commit_before>#!/usr/bin/env python2.7

from __future__ import with_statement
import os, sys
import struct
import pprint; pp = pprint.pprint
import glob
import array
import ctypes
from funcs import *
from filetools import *
import trecformat

# ======================================================================
# MP4/MOV constants
# http://developer.apple.com/library/mac/#documentation/QuickTime/QTFF/QTFFChap1/qtff1.html#//apple_ref/doc/uid/TP40000939-CH203-BBCGDDDF

# some info on mvhd atom
# http://www.koders.com/c/fid7340F6A06EE399155C40FEAD653B61D43AC72C8C.aspx
# http://www.koders.com/c/fidC79D25A829F98C03F813CCB72649A0B62825F6AA.aspx

#toplevels = [
#	'moov',
#	'mdat',
#	'free',
#	'junk',
#	'pnot',
#	'skip',
#	'wide',
#	'pict',
#	'ftyp',
#	'uuid', # Used by Sony's MSNV brand of MP4
#]

# ======================================================================

class AtomIncomplete(Exception):
	def __init__(self, atomtype, atomstart, atomend, realend):
		self.atomtype  = atomtype
		self.atomstart = atomstart
		self.atomend   = atomend
		self.realend   = realend
	
	def __str__(self):
		return 'Atom %s@%d Should end at %d, does end at %d (%d missing)' % (self.atomtype, self.atomstart, self.atomend, self.realend, self.atomend - self.realend)



lineindent = lambda n: n * '    '

def blockindent(n, text):
	lines = text.split('\n')

	return '\n'.join(
		(lineindent(n) + line) if line else (line)
		for line in lines
	)

class Atom(object):
	def __init__(self, start, length, type, content=None):
		self.start   = start
		self.length  = length
		self.type    = type
		self.content = content
	
	def __repr__(self, indent=0, index=0):
		label = self.type
		
		sublabel = ''
		
		if (self.content is not None) and isinstance(self.content, list):
			sublabel = ' // %d children...' % len(self.content)
		
		res = [lineindent(indent) + '[%d] %s  [@%d + %d]%s' % (
			index+1,
			label,
			self.start,
			self.length,
			sublabel
		)]
		
		if self.content is not None:
			multiline = False
			
			if isinstance(self.content, list):
				multiline = True
			elif '\n' in repr(self.content):
				multiline = True
			
			if multiline:
				res.append('\n')
				res.append(lineindent(indent) + '{\n')

				if isinstance(self.content, list):
					for i,item in enumerate(self.content):
						if isinstance(item, Atom):
#							if item.type == 'skip':
#								import pdb
#								pdb.set_trace()
							res.append(item.__repr__(indent+1, index=i))
						else:
							res.append(blockindent(indent+1, ('[%d] ' % (i+1)) + repr(item) + "\n"))
				else:
					res.append(blockindent(indent+1, repr(self.content) + '\n'))

				res.append(lineindent(indent) + '}\n')

			else:
				res.append(' { ')
				res.append(repr(self.content))
				res.append(' }\n')<commit_msg>don't ask me why that was still there<commit_after>#!/usr/bin/env python2.7

from __future__ import with_statement
import os, sys
import struct
import pprint; pp = pprint.pprint
import glob
import array
import ctypes
from funcs import *
from filetools import *
import trecformat

# ======================================================================
# MP4/MOV constants
# http://developer.apple.com/library/mac/#documentation/QuickTime/QTFF/QTFFChap1/qtff1.html#//apple_ref/doc/uid/TP40000939-CH203-BBCGDDDF

# some info on mvhd atom
# http://www.koders.com/c/fid7340F6A06EE399155C40FEAD653B61D43AC72C8C.aspx
# http://www.koders.com/c/fidC79D25A829F98C03F813CCB72649A0B62825F6AA.aspx

#toplevels = [
#	'moov',
#	'mdat',
#	'free',
#	'junk',
#	'pnot',
#	'skip',
#	'wide',
#	'pict',
#	'ftyp',
#	'uuid', # Used by Sony's MSNV brand of MP4
#]

# ======================================================================

class AtomIncomplete(Exception):
	def __init__(self, atomtype, atomstart, atomend, realend):
		self.atomtype  = atomtype
		self.atomstart = atomstart
		self.atomend   = atomend
		self.realend   = realend
	
	def __str__(self):
		return 'Atom %s@%d Should end at %d, does end at %d (%d missing)' % (self.atomtype, self.atomstart, self.atomend, self.realend, self.atomend - self.realend)



lineindent = lambda n: n * '    '

def blockindent(n, text):
	lines = text.split('\n')

	return '\n'.join(
		(lineindent(n) + line) if line else (line)
		for line in lines
	)

class Atom(object):
	def __init__(self, start, length, type, content=None):
		self.start   = start
		self.length  = length
		self.type    = type
		self.content = content
	
	def __repr__(self, indent=0, index=0):
		label = self.type
		
		sublabel = ''
		
		if (self.content is not None) and isinstance(self.content, list):
			sublabel = ' // %d children...' % len(self.content)
		
		res = [lineindent(indent) + '[%d] %s  [@%d + %d]%s' % (
			index+1,
			label,
			self.start,
			self.length,
			sublabel
		)]
		
		if self.content is not None:
			multiline = False
			
			if isinstance(self.content, list):
				multiline = True
			elif '\n' in repr(self.content):
				multiline = True
			
			if multiline:
				res.append('\n')
				res.append(lineindent(indent) + '{\n')

				if isinstance(self.content, list):
					for i,item in enumerate(self.content):
						if isinstance(item, Atom):
							res.append(item.__repr__(indent+1, index=i))
						else:
							res.append(blockindent(indent+1, ('[%d] ' % (i+1)) + repr(item) + "\n"))
				else:
					res.append(blockindent(indent+1, repr(self.content) + '\n'))

				res.append(lineindent(indent) + '}\n')

			else:
				res.append(' { ')
				res.append(repr(self.content))
				res.append(' }\n')
{'fraction_non_alphanumeric': 0.19621060302423027, 'fraction_numerical': 0.027327382036800876, 'mean_word_length': 3.5979899497487438, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2164284', 'n_tokens_mistral': 2375, 'n_tokens_neox': 1982, 'n_words': 438}