<commit_before>from flask import Flask, Response, request, jsonify
from sqlalchemy import create_engine, select, text, MetaData, Table, String, Integer, BigInteger, Boolean, Column, DateTime, String, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from geoalchemy2 import Geometry
from geoalchemy2.functions import ST_X, ST_Y
import ipaddress
import datetime

app = Flask(__name__)

if not app.debug:
    import logging
    import logging.handlers
    handler = logging.handlers.RotatingFileHandler("/var/log/piecewise/collector.log", maxBytes = 10 * 1000 * 1000, backupCount = 5)
    handler.setLevel(logging.WARNING)
    app.logger.addHandler(handler)

db_engine = create_engine("postgresql+psycopg2://postgres:@/piecewise")
Base = declarative_base()
Session = sessionmaker(bind=db_engine)
db_session = Session()

metadata = MetaData()
metadata.bind = db_engine
extra_data = Table('extra_data', metadata, 
    Column('id', Integer, primary_key = True),
    Column('timestamp', DateTime),
    Column('verified', Boolean),
    Column('bigquery_key', String),
    Column('location', Geometry("Point", srid=4326)),
    Column('connection_type', String),
    Column('advertised_download', Integer),
    Column('advertised_upload', Integer),
    Column('location_type', String),
    Column('cost_of_service', Integer))
metadata.create_all()

class ExtraData(Base):
    __tablename__ = 'extra_data'
    id = Column('id', Integer, primary_key = True)
    timestamp = Column('timestamp', DateTime)
    verified = Column('verified', Boolean)
    bigquery_key = Column('bigquery_key', String)
    location = Column('location', Geometry("Point", srid=4326))
    connection_type = Column('connection_type', String)
    advertised_download = Column('advertised_download', Integer)
    advertised_upload = Column('advertised_upload', Integer)
    location_type = Column('location_type', String)
    cost_of_service = Column('cost_of_service', Integer)

class results(Base):
    __tablename__ = 'results'
    id = Column('id', BigInteger, primary_key = True)
    datetime = Column('time', DateTime)
    location = Column('location', Geometry("POINT", srid=4326))
    client_ip = Column('client_ip', BigInteger)
    server_ip = Column('server_ip', BigInteger)
    countrtt = Column('countrtt', BigInteger)
    sumrtt = Column('sumrtt', BigInteger)
    download_flag = Column('download_flag', Boolean)
    download_time = Column('download_time', BigInteger)
    download_octets = Column('download_octets', BigInteger)
    upload_time =  Column('upload_time', BigInteger)
    upload_octets = Column('upload_octets', BigInteger)

@app.route("/datadump", methods=['GET'])
def retrieve_datadump():
    r_count = int(db_session.query(results).count())
    results = db_session.query(ExtraData, ST_X(ExtraData.location).label('lon'),
            ST_Y(ExtraData.location).label('lat')).order_by(
            order_by).limit(limit).offset(offset).all()

    result = db_session.query(results).all()
    
    records = []
    for row in result:
        record = {}
        record['id'] = row[0].id
        record['datetime'] = row[0].datetime
        record['client_ip'] = row[0].client_ip
        record['server_ip'] = row[0].server_ip
        record['countrtt'] = row[0].countrtt
        record['sumrtt'] = row[0].sumrtt
        record['download_flag'] = row[0].download_flag
        record['download_time'] = row[0].download_time
        record['download_octets'] = row[0].download_octets
        records.append(record)

    if len(records):
        return (jsonify(record_count=record_count, records=records), 200, {})
    else:
        return ('', 500, {})



@app.route("/unverify", methods=['GET'])
def unverify_extra_data():
    if not request.args.get('id'):
        return ('', 400, {})

    try:
        result = db_session.query(ExtraData).filter_by(
            id=request.args.get('id')).update({"verified": "f"})
        db_session.commit()
    except:
        db_session.rollback()

    if result:
        return ('', 200, {})
    else:
        return ('', 500, {})


@app.route("/verify", methods=['GET'])
def verify_extra_data():
    if not request.args.get('id'):
        return ('', 400, {})

    try:<commit_msg>modified datadump endpoint for testing<commit_after>from flask import Flask, Response, request, jsonify
from sqlalchemy import create_engine, select, text, MetaData, Table, String, Integer, BigInteger, Boolean, Column, DateTime, String, Integer
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
from geoalchemy2 import Geometry
from geoalchemy2.functions import ST_X, ST_Y
import ipaddress
import datetime

app = Flask(__name__)

if not app.debug:
    import logging
    import logging.handlers
    handler = logging.handlers.RotatingFileHandler("/var/log/piecewise/collector.log", maxBytes = 10 * 1000 * 1000, backupCount = 5)
    handler.setLevel(logging.WARNING)
    app.logger.addHandler(handler)

db_engine = create_engine("postgresql+psycopg2://postgres:@/piecewise")
Base = declarative_base()
Session = sessionmaker(bind=db_engine)
db_session = Session()

metadata = MetaData()
metadata.bind = db_engine
extra_data = Table('extra_data', metadata, 
    Column('id', Integer, primary_key = True),
    Column('timestamp', DateTime),
    Column('verified', Boolean),
    Column('bigquery_key', String),
    Column('location', Geometry("Point", srid=4326)),
    Column('connection_type', String),
    Column('advertised_download', Integer),
    Column('advertised_upload', Integer),
    Column('location_type', String),
    Column('cost_of_service', Integer))
metadata.create_all()

class ExtraData(Base):
    __tablename__ = 'extra_data'
    id = Column('id', Integer, primary_key = True)
    timestamp = Column('timestamp', DateTime)
    verified = Column('verified', Boolean)
    bigquery_key = Column('bigquery_key', String)
    location = Column('location', Geometry("Point", srid=4326))
    connection_type = Column('connection_type', String)
    advertised_download = Column('advertised_download', Integer)
    advertised_upload = Column('advertised_upload', Integer)
    location_type = Column('location_type', String)
    cost_of_service = Column('cost_of_service', Integer)

class results(Base):
    __tablename__ = 'results'
    id = Column('id', BigInteger, primary_key = True)
    datetime = Column('time', DateTime)
    location = Column('location', Geometry("POINT", srid=4326))
    client_ip = Column('client_ip', BigInteger)
    server_ip = Column('server_ip', BigInteger)
    countrtt = Column('countrtt', BigInteger)
    sumrtt = Column('sumrtt', BigInteger)
    download_flag = Column('download_flag', Boolean)
    download_time = Column('download_time', BigInteger)
    download_octets = Column('download_octets', BigInteger)
    upload_time =  Column('upload_time', BigInteger)
    upload_octets = Column('upload_octets', BigInteger)

@app.route("/datadump", methods=['GET'])
def retrieve_datadump():
    r_count = int(db_session.query(results).count())
    result = db_session.query(results).all()
    
    records = []
    for row in result:
        record = {}
        record['id'] = row[0].id
        record['datetime'] = row[0].datetime
        record['client_ip'] = row[0].client_ip
        record['server_ip'] = row[0].server_ip
        record['countrtt'] = row[0].countrtt
        record['sumrtt'] = row[0].sumrtt
        record['download_flag'] = row[0].download_flag
        record['download_time'] = row[0].download_time
        record['download_octets'] = row[0].download_octets
        records.append(record)

    if len(records):
        return (jsonify(record_count=record_count, records=records), 200, {})
    else:
        return ('', 500, {})

@app.route("/unverify", methods=['GET'])
def unverify_extra_data():
    if not request.args.get('id'):
        return ('', 400, {})

    try:
        result = db_session.query(ExtraData).filter_by(
            id=request.args.get('id')).update({"verified": "f"})
        db_session.commit()
    except:
        db_session.rollback()

    if result:
        return ('', 200, {})
    else:
        return ('', 500, {})


@app.route("/verify", methods=['GET'])
def verify_extra_data():
    if not request.args.get('id'):
        return ('', 400, {})

    try:
{'fraction_non_alphanumeric': 0.14032567049808428, 'fraction_numerical': 0.012691570881226053, 'mean_word_length': 4.1978842563783445, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5257047', 'n_tokens_mistral': 2621, 'n_tokens_neox': 2526, 'n_words': 524}