<commit_before>from flask import Flask, request, render_template, redirect, url_for, flash, send_from_directory
from flask_sqlalchemy import SQLAlchemy
import logging
from werkzeug import secure_filename
import os
import lzma
import datetime 

COMPRESSION_EXTENSIONS = set(['.xz', '.tar', '.zip'])

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = 'uploads'
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://localhost/visualizer'
db = SQLAlchemy(app)
app.secret_key = 'f43fee5tbt'

# create the model
class Url(db.Model):
	__tablename__ = 'url_counter'
	id = db.Column(db.Integer, primary_key=True, unique=True)
	url = db.Column(db.String(120), unique=True)

	def __init__(self, url):
		self.url = url

# model for the log parser
class LogParser(db.Model):
	__tablename__ = 'log_parser'
	id = db.Column(db.Integer, primary_key=True, unique=True)
	log_name = db.Column(db.String(80), unique=False)
	# think of version representation

	def __init__(self, log_name):
		self.log_name = log_name 

@app.route('/', methods=['POST', 'GET'])
def main():
	# ensure the data is stored
	if request.method == 'POST':
		url = request.form['submit_url']
		result = Url.query.filter_by(url=url).first()
		if result:
			flash("nope")
			return redirect(url_for('main'))
		else:
			entry = Url(url)
			db.session.add(entry)
			db.session.commit()
			return redirect(url_for('main'))
	else:
		return render_template('main.html')

# extract file, ToDo: other decompression algos
def extract_file(filename):
	# currently only for 1 level .xz files
	inF = filename
	print(inF)
	outF =  os.path.splitext(inF)[0] #"{}_{}".format(datetime.datetime.now().strftime("%y%m%d%H%M"), os.path.splitext(inF)[0])
	print(str(outF))
	with lzma.open(inF, 'rb') as i:
		with open(outF, 'wb') as o:
			o.write(i.read())

# function for the logs upload		
@app.route('/logs', methods=['GET', 'POST'])
def upload_log():
	if request.method == 'POST':
		file = request.files['upload_log']
		# we trust the user for now
		if file:
			log = LogParser(os.path.splitext(file.filename)[0])
			db.session.add(log)
			db.session.commit()
			# ToDo improve unique names
			if os.path.splitext(file.filename)[-1] in COMPRESSION_EXTENSIONS:
				# save the compressed file. decompress it and delete compressed version
				path_name = os.path.join(app.config['UPLOAD_FOLDER'], file.filename)
				file.save(path_name)
				extract_file(path_name) 
				os.remove(path_name)
				return redirect(url_for('upload_log'))
	return render_template('logs_form.html')
<commit_msg>both archived and non-archived files allowed for the upload<commit_after>from flask import Flask, request, render_template, redirect, url_for, flash, send_from_directory
from flask_sqlalchemy import SQLAlchemy
import logging
from werkzeug import secure_filename
import os
import lzma
import datetime 

COMPRESSION_EXTENSIONS = set(['.xz', '.tar', '.zip'])

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = 'uploads'
app.config['SQLALCHEMY_DATABASE_URI'] = 'postgresql://localhost/visualizer'
db = SQLAlchemy(app)
app.secret_key = 'f43fee5tbt'

# create the model
class Url(db.Model):
	__tablename__ = 'url_counter'
	id = db.Column(db.Integer, primary_key=True, unique=True)
	url = db.Column(db.String(120), unique=True)

	def __init__(self, url):
		self.url = url

# model for the log parser
class LogParser(db.Model):
	__tablename__ = 'log_parser'
	id = db.Column(db.Integer, primary_key=True, unique=True)
	log_name = db.Column(db.String(80), unique=False)
	# think of version representation

	def __init__(self, log_name):
		self.log_name = log_name 

@app.route('/', methods=['POST', 'GET'])
def main():
	# ensure the data is stored
	if request.method == 'POST':
		url = request.form['submit_url']
		result = Url.query.filter_by(url=url).first()
		if result:
			flash("nope")
			return redirect(url_for('main'))
		else:
			entry = Url(url)
			db.session.add(entry)
			db.session.commit()
			return redirect(url_for('main'))
	else:
		return render_template('main.html')

# extract file, ToDo: other decompression algos
def extract_file(filename):
	# currently only for 1 level .xz files
	inF = filename
	print(inF)
	outF =  os.path.splitext(inF)[0] #"{}_{}".format(datetime.datetime.now().strftime("%y%m%d%H%M"), os.path.splitext(inF)[0])
	print(str(outF))
	with lzma.open(inF, 'rb') as i:
		with open(outF, 'wb') as o:
			o.write(i.read())

# function for the logs upload		
@app.route('/logs', methods=['GET', 'POST'])
def upload_log():
	if request.method == 'POST':
		file = request.files['upload_log']
		# we trust the user for now
		if file:
			log = LogParser(os.path.splitext(file.filename)[0])
			db.session.add(log)
			db.session.commit()
			# ToDo improve unique names
			if os.path.splitext(file.filename)[-1] in COMPRESSION_EXTENSIONS:
				# save the compressed file. decompress it and delete compressed version
				path_name = os.path.join(app.config['UPLOAD_FOLDER'], file.filename)
				file.save(path_name)
				extract_file(path_name) 
				os.remove(path_name)
				return redirect(url_for('upload_log'))
			else:
				path_name = os.path.join(app.config['UPLOAD_FOLDER'], file.filename)
				file.save(path_name)
				return redirect(url_for('upload_log'))
	return render_template('logs_form.html')

{'fraction_non_alphanumeric': 0.14304812834224598, 'fraction_numerical': 0.004965622612681436, 'mean_word_length': 4.910835214446952, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3443803', 'n_tokens_mistral': 2028, 'n_tokens_neox': 1870, 'n_words': 425}