<commit_before>from random import random
from functools import partial
import csv
import json

from kivy.app import App
from kivy.uix.widget import Widget
from kivy.uix.button import Button
from kivy.uix.label import Label
from kivy.uix.textinput import TextInput
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.spinner import Spinner
from kivy.graphics import Color, Line, Ellipse


import tensorflow as tf
import numpy as np

maxLen = 60

class Gesture:
	def __init__(self):
		self.classId = 0
		self.points = []
	
	def from_points(self, input):
		for i in range(2, len(input), 2):
			self.points.append((input[i] - input[i-2], input[i+1]-input[i-1]))
			
	def to_dict(self):
		dict = {
			"classId" : self.classId,
			"points" : self.points
		}
		return dict
		
	def from_dict(dict):
		g = Gesture()
		g.classId = dict["classId"]
		g.points = dict["points"]
		return g
		
	def to_tensor(self):
		return self.points[:maxLen] + [self.points[-1]]*(maxLen - len(self.points))
		

class GestureBase:
	def __init__(self):
		self.gestures = []
		self.gestureIds = {'None': 0}
	
	def save(self, path):
		print("Gestures %d" % len(self.gestures))
		with open(path, 'w') as file:
			data = {
				"classes": self.gestureIds,
				"gestures": [g.to_dict() for g in self.gestures], 
			}
			json.dump(data, file, indent=2)
			
	def load(self, path):
		with open(path, 'r') as file:
			data = json.load(file)
			self.gestureIds = data["classes"]
			self.gestures = [Gesture.from_dict(g) for g in data["gestures"]]
	
	def get_classes_in_order(self):
		items = sorted(self.gestureIds.items(), key=lambda p: p[1])
		return [i[0] for i in items]
	
	def add_gesture(self, className, points):
		gesture = Gesture()
		if className not in self.gestureIds:
			self.gestureIds[className] = gesture.classId = len(self.gestureIds)
		else:
			gesture.classId = self.gestureIds[className]
		
		gesture.from_points(points)
		self.gestures.append(gesture)
		
	def to_tensor(self):
		return np.array([g.to_tensor() for g in self.gestures])
		
	def classes_to_tensor(self):
		ret = []
		for g in self.gestures:
			list = [0] * 10
			list[g.classId] = 1
			ret.append(list)
		return np.array(ret)
		
	def lengths_to_tensor(self):
		ret = [len(g.points) for g in self.gestures]
		return np.array(ret)
		#return [100 for g in self.gestures]
		
class GestureRecognizer:
	
	numberOfExamples = None #dynamic
	sampleVectorLen = 2 # x, y coords
	numMemCells = 24
	
	
	def __init__(self):
		self.inputData = tf.placeholder(tf.float32, [None, maxLen, self.sampleVectorLen])
		self.expectedClasses = tf.placeholder(tf.float32, [None, 10])
		self.inputLengths = tf.placeholder(tf.int32, [None])
		print("########################")
		print(tf.shape(self.inputData)[0])
				
		cell = tf.nn.rnn_cell.LSTMCell(self.numMemCells, state_is_tuple=True)
		cellOut, cellState = tf.nn.dynamic_rnn(
			cell, self.inputData, dtype=tf.float32, sequence_length=self.inputLengths)
		
		#last = cellState[1]
		#cellOut = tf.transpose(cellOut, [1, 0, 2])
		batchSize = tf.shape(cellOut)[0]
		index = tf.range(0, batchSize) * maxLen + (self.inputLengths - 1)
		flat = tf.reshape(cellOut, [-1, self.numMemCells])
		last = tf.gather(flat, index)
		print(last.get_shape())
		#last = tf.gather(cellOutTrans, int(cellOutTrans.get_shape()[0]) - 1)
		
		weight = tf.Variable(tf.truncated_normal([self.numMemCells, int(self.expectedClasses.get_shape()[1])], stddev = 0.1))
		bias = tf.Variable(tf.constant(0.1, shape=[self.expectedClasses.get_shape()[1]]))
		#prediction = tf.nn.softmax(tf.matmul(last, weight) + bias)
		prediction = tf.matmul(last, weight) + bias
		
		cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(prediction, self.expectedClasses))
		#cross_entropy = -tf.reduce_sum(self.expectedClasses * tf.log(tf.clip_by_value(prediction,1e-10,1.0)))
		optimizer = tf.train.GradientDescentOptimizer(0.1)
		self.trainer = optimizer.minimize(cross_entropy)
		
		correct_prediction = tf.equal(tf.argmax(prediction,1), tf.argmax(self.expectedClasses,1))
		self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
		
		self.predictionMax = tf.argmax(prediction, 1)
		self.classifier = tf.nn.softmax(prediction)
		
		self.sess = tf.Session()
		init_op = tf.global_variables_initializer()
		self.sess.run(init_op)
		<commit_msg>Updated to Tensorflow 1.0<commit_after>from random import random
from functools import partial
import csv
import json

from kivy.app import App
from kivy.uix.widget import Widget
from kivy.uix.button import Button
from kivy.uix.label import Label
from kivy.uix.textinput import TextInput
from kivy.uix.boxlayout import BoxLayout
from kivy.uix.spinner import Spinner
from kivy.graphics import Color, Line, Ellipse


import tensorflow as tf
import numpy as np

maxLen = 60

class Gesture:
	def __init__(self):
		self.classId = 0
		self.points = []
	
	def from_points(self, input):
		for i in range(2, len(input), 2):
			self.points.append((input[i] - input[i-2], input[i+1]-input[i-1]))
			
	def to_dict(self):
		dict = {
			"classId" : self.classId,
			"points" : self.points
		}
		return dict
		
	def from_dict(dict):
		g = Gesture()
		g.classId = dict["classId"]
		g.points = dict["points"]
		return g
		
	def to_tensor(self):
		return self.points[:maxLen] + [self.points[-1]]*(maxLen - len(self.points))
		

class GestureBase:
	def __init__(self):
		self.gestures = []
		self.gestureIds = {'None': 0}
	
	def save(self, path):
		print("Gestures %d" % len(self.gestures))
		with open(path, 'w') as file:
			data = {
				"classes": self.gestureIds,
				"gestures": [g.to_dict() for g in self.gestures], 
			}
			json.dump(data, file, indent=2)
			
	def load(self, path):
		with open(path, 'r') as file:
			data = json.load(file)
			self.gestureIds = data["classes"]
			self.gestures = [Gesture.from_dict(g) for g in data["gestures"]]
	
	def get_classes_in_order(self):
		items = sorted(self.gestureIds.items(), key=lambda p: p[1])
		return [i[0] for i in items]
	
	def add_gesture(self, className, points):
		gesture = Gesture()
		if className not in self.gestureIds:
			self.gestureIds[className] = gesture.classId = len(self.gestureIds)
		else:
			gesture.classId = self.gestureIds[className]
		
		gesture.from_points(points)
		self.gestures.append(gesture)
		
	def to_tensor(self):
		return np.array([g.to_tensor() for g in self.gestures])
		
	def classes_to_tensor(self):
		ret = []
		for g in self.gestures:
			list = [0] * 10
			list[g.classId] = 1
			ret.append(list)
		return np.array(ret)
		
	def lengths_to_tensor(self):
		ret = [len(g.points) for g in self.gestures]
		return np.array(ret)
		#return [100 for g in self.gestures]
		
class GestureRecognizer:
	
	numberOfExamples = None #dynamic
	sampleVectorLen = 2 # x, y coords
	numMemCells = 24
	
	
	def __init__(self):
		self.inputData = tf.placeholder(tf.float32, [None, maxLen, self.sampleVectorLen])
		self.expectedClasses = tf.placeholder(tf.float32, [None, 10])
		self.inputLengths = tf.placeholder(tf.int32, [None])
				
		cell = tf.contrib.rnn.LSTMCell(self.numMemCells, state_is_tuple=True)
		cellOut, cellState = tf.nn.dynamic_rnn(
			cell, self.inputData, dtype=tf.float32, sequence_length=self.inputLengths)
		
		#last = cellState[1]
		#cellOut = tf.transpose(cellOut, [1, 0, 2])
		batchSize = tf.shape(cellOut)[0]
		index = tf.range(0, batchSize) * maxLen + (self.inputLengths - 1)
		flat = tf.reshape(cellOut, [-1, self.numMemCells])
		last = tf.gather(flat, index)
		print(last.get_shape())
		#last = tf.gather(cellOutTrans, int(cellOutTrans.get_shape()[0]) - 1)
		
		weight = tf.Variable(tf.truncated_normal([self.numMemCells, int(self.expectedClasses.get_shape()[1])], stddev = 0.1))
		bias = tf.Variable(tf.constant(0.1, shape=[self.expectedClasses.get_shape()[1]]))
		#prediction = tf.nn.softmax(tf.matmul(last, weight) + bias)
		prediction = tf.matmul(last, weight) + bias
		
		cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=self.expectedClasses))
		#cross_entropy = -tf.reduce_sum(self.expectedClasses * tf.log(tf.clip_by_value(prediction,1e-10,1.0)))
		optimizer = tf.train.GradientDescentOptimizer(0.1)
		self.trainer = optimizer.minimize(cross_entropy)
		
		correct_prediction = tf.equal(tf.argmax(prediction,1), tf.argmax(self.expectedClasses,1))
		self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
		
		self.predictionMax = tf.argmax(prediction, 1)
		self.classifier = tf.nn.softmax(prediction)
		
		self.sess = tf.Session()
		init_op = tf.global_variables_initializer()
		self.sess.run(init_op)
		
{'fraction_non_alphanumeric': 0.14005373204064947, 'fraction_numerical': 0.014601098002569793, 'mean_word_length': 4.2559852670349905, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6469894', 'n_tokens_mistral': 3539, 'n_tokens_neox': 3154, 'n_words': 655}