<commit_before>#!/usr/bin/python2.7

import argparse, os, re, random, struct
from itertools import chain

print " _                        _        "
print "| |                      | |       "
print "| |_ ___  _ __ ___   __ _| |_ ___  "
print "| __/ _ \| '_ ` _ \ / _` | __/ _ \ "
print "| || (_) | | | | | | (_| | || (_) |"
print " \__\___/|_| |_| |_|\__,_|\__\___/ "
print "tomato.py v1.2 last update 05.10.2017"
print "\\\\ Audio Video Interleave index breaker"
print " "
print "\"je demande a ce qu'on tienne pour un cretin"
print "celui qui se refuserait encore, par exemple,"
print "a voir un cheval galoper sur une tomate.\""
print " - <NAME>"
print " "
print "___________________________________"
print " "

#################
### ARGUMENTS ###
#################

parser = argparse.ArgumentParser(add_help=True)
parser.add_argument("-i", "--input", help="input file")
parser.add_argument('-m', "--mode", action='store', dest='modevalue',help='choose mode, one of:\nshuffle irep ikill bloom pulse reverse invert')
parser.add_argument('-c', action='store', dest='countframes',help='var1', default=1)
parser.add_argument('-n', action='store', dest='positframes',help='var2', default=1)
parser.add_argument('-l','--lim', help='fraction of file to search before giving up, default: 4', default=4)
parser.add_argument("file", help="input file")

args = parser.parse_args()

fileout = args.file
filein = args.input
mode = args.modevalue
countframes = args.countframes
positframes = args.positframes

####################
### OPENING FILE ###
####################

with open(filein,'rb') as rd:
	print "Opening File\n"
	filesize = os.path.getsize(filein)
	chunk = 1024
	lim = args.lim
	idx = ''
	with open(fileout,'wb') as wr:
		print "Streaming File\n"
		for pos in xrange(filesize-chunk, filesize-filesize/lim, -chunk): # move backwards through data
			rd.seek(pos)
			buffer = rd.read(chunk)

			if buffer.find(b'idx1') >= 0:
				split = buffer.split(b'idx1', 1)
				rd.seek(0)
				wr.write(rd.read(pos)) # start the read at the beginning again,
				wr.write(split[0])  # spit out data up to this point plus the stuff before idx
				idx = split[1] + rd.read()
				break

		if len(idx) == 0:
			print('Could not locate index!\n')
			raise SystemExit # quit


	print "Getting list of Tomatoes\n"
	## get the length of the index and store it
	idx, index_length = idx[4:], idx[:4]

	## get the first iframe and store it
	n = 16
	first_frame, idx = idx[:n], idx[n:]
	check = bytearray()
	check.extend(first_frame)
	#print([i for i in check])
	## put all frames in array ignoring sound frames
	regex = re.compile(b'.*wb.*')
	idx = [idx[i:i+n] for i in range(0, len(idx), n) if not re.match(regex,idx[i:i+n])]

	## calculate number of frames
	number_of_frames = len(idx)

	print "Ready for the serious shitz\n"

#########################
### OPERATIONS TO IDX ###
#########################

	if mode == "void":
		print "### MODE - VOID"<commit_msg>fixed positioning of file pointer<commit_after>#!/usr/bin/python2.7

import argparse, os, re, random, struct
from itertools import chain

print " _                        _        "
print "| |                      | |       "
print "| |_ ___  _ __ ___   __ _| |_ ___  "
print "| __/ _ \| '_ ` _ \ / _` | __/ _ \ "
print "| || (_) | | | | | | (_| | || (_) |"
print " \__\___/|_| |_| |_|\__,_|\__\___/ "
print "tomato.py v1.2 last update 05.10.2017"
print "\\\\ Audio Video Interleave index breaker"
print " "
print "\"je demande a ce qu'on tienne pour un cretin"
print "celui qui se refuserait encore, par exemple,"
print "a voir un cheval galoper sur une tomate.\""
print " - <NAME>"
print " "
print "___________________________________"
print " "

#################
### ARGUMENTS ###
#################

parser = argparse.ArgumentParser(add_help=True)
parser.add_argument("-i", "--input", help="input file")
parser.add_argument('-m', "--mode", action='store', dest='modevalue',help='choose mode, one of:\nshuffle irep ikill bloom pulse reverse invert')
parser.add_argument('-c', action='store', dest='countframes',help='var1', default=1)
parser.add_argument('-n', action='store', dest='positframes',help='var2', default=1)
parser.add_argument('-l','--lim', help='fraction of file to search before giving up, default: 4', default=4)
parser.add_argument("file", help="input file")

args = parser.parse_args()

fileout = args.file
filein = args.input
mode = args.modevalue
countframes = args.countframes
positframes = args.positframes

####################
### OPENING FILE ###
####################

with open(filein,'rb') as rd:
	print "Opening File\n"
	filesize = os.path.getsize(filein)
	chunk = 1024
	lim = args.lim
	idx = ''
	with open(fileout,'wb') as wr:
		print "Streaming File\n"
		for pos in xrange(filesize-chunk, filesize-filesize/lim, -chunk): # move backwards through data
			rd.seek(pos)
			buffer = rd.read(chunk)

			if buffer.find(b'idx1') >= 0:
				split = buffer.split(b'idx1', 1)
				rd.seek(0)
				wr.write(rd.read(pos)) # start the read at the beginning again,
				wr.write(split[0])  # spit out data up to this point plus the stuff before idx
                                print(len(split[0]))
                                rd.seek(len(split[0])+8,1)
				idx = split[1] + rd.read()
				break

		if len(idx) == 0:
			print('Could not locate index!\n')
			raise SystemExit # quit


	print "Getting list of Tomatoes\n"
	## get the length of the index and store it
	idx, index_length = idx[4:], idx[:4]

	## get the first iframe and store it
	n = 16
	first_frame, idx = idx[:n], idx[n:]
	check = bytearray()
 	check.extend(first_frame)
	#print([i for i in check])
	## put all frames in array ignoring sound frames
	regex = re.compile(b'.*wb.*')
	idx = [idx[i:i+n] for i in range(0, len(idx), n) if not re.match(regex,idx[i:i+n])]

	## calculate number of frames
	number_of_frames = len(idx)

	print "Ready for the serious shitz\n"

#########################
### OPERATIONS TO IDX ###
#########################

	if mode == "void":
		print "### MODE - VOID"
{'fraction_non_alphanumeric': 0.1931572828433815, 'fraction_numerical': 0.01229031722305265, 'mean_word_length': 3.61455938697318, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '698464', 'n_tokens_mistral': 2177, 'n_tokens_neox': 2031, 'n_words': 557}