<commit_before>#!/usr/bin/env python
"""Creates a DB table with the extent information of the files in a Potree OctTree.
The files are not opened, instead we get the information from the HRC files and the 
known extent of the OctTree nodes"""

import argparse, traceback, time, os, psycopg2, json
import utils

USERNAME = utils.getUserName()
COMMIT_INTERVAL = 1000

counter = 0

def argument_parser():
    """ Define the arguments and return the parser object"""
    parser = argparse.ArgumentParser(
    description="""Creates a DB table with the extent information of the files in a Potree OctTree.
The files are not opened, instead we get the information from the HRC files and the 
known extent of the OctTree nodes""")
    parser.add_argument('-i','--input',default='',help='Input folder with the Potree OctTree',type=str, required=True)
    parser.add_argument('-s','--srid',default='',help='SRID',type=int, required=True)
    parser.add_argument('-d','--dbname',default=utils.DB_NAME,help='Postgres DB name [default ' + utils.DB_NAME + ']',type=str)
    parser.add_argument('-u','--dbuser',default=USERNAME,help='DB user [default ' + USERNAME + ']',type=str)
    parser.add_argument('-p','--dbpass',default='',help='DB pass',type=str)
    parser.add_argument('-b','--dbhost',default='',help='DB host',type=str)
    parser.add_argument('-r','--dbport',default='',help='DB port',type=str)
    return parser

def getChildBC(minX,minY,minZ,maxX,maxY,maxZ,childIndex):
    rX = (maxX - minX) / 2.
    rY = (maxY - minY) / 2.
    rZ = (maxZ - minZ) / 2.
    if childIndex == 0:
        return (minX,      minY,      minZ,      minX + rX, minY + rY, minZ + rZ)
    elif childIndex == 1:
        return (minX,      minY,      minZ + rZ, minX + rX, minY + rY, maxZ)
    elif childIndex == 2:
        return (minX,      minY + rY, minZ,      minX + rX, maxY,      minZ + rZ)
    elif childIndex == 3:
        return (minX,      minY + rY, minZ + rZ, minX + rX, maxY,      maxZ)
    elif childIndex == 4:
        return (minX + rX, minY,      minZ,      maxX,      minY + rY, minZ + rZ)
    elif childIndex == 5:
        return (minX + rX, minY,      minZ + rZ, maxX,      minY + rY, maxZ)
    elif childIndex == 6:
        return (minX + rX, minY + rY, minZ,      maxX,      maxY,      minZ + rZ)
    elif childIndex == 7:
        return (minX + rX, minY + rY, minZ + rZ, maxX,      maxY,      maxZ)
    else:
        raise Exception('Child index must be [0,7]!')
    
def addNodeFolder(cursor, node, nodeAbsPath, hierarchyStepSize, extension, minX, minY, minZ, maxX, maxY, maxZ, srid):
    hrcFile = node + '.hrc'
    hrc = None
    if os.path.isfile(nodeAbsPath + '/' + hrcFile):
        # Check if there is data in this node in Octtree A (we check if the HRC file for this node exist)
        hrc = utils.readHRC(nodeAbsPath + '/' + hrcFile, hierarchyStepSize)
        for level in range(hierarchyStepSize+1):
            for i in range(len(hrc[level])):
                if hrc[level][i]:
                    (childNode, isFile) = utils.getNodeName(level, i, node, hierarchyStepSize, extension)
                    relativeNode = childNode.replace(node,'').replace('.' + extension, '')
                    (lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ) = (minX, minY, minZ, maxX, maxY, maxZ)
                    for pNode in relativeNode:
                        (lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ) = getChildBC(lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ, int(pNode))
                    if isFile: 
                        addNodeFile(cursor, [nodeAbsPath + '/' + childNode, len(childNode) - 5, int(hrc[level][i]), lminZ, lmaxZ, lminX, lminY, lmaxX, lmaxY, int(srid)])
                    else:
                        addNodeFolder(cursor, node + childNode, nodeAbsPath + '/' + childNode, hierarchyStepSize, extension, lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ, srid)       

def addNodeFile(cursor, insertArgs):
    insertStatement = """INSERT INTO """ + utils.DB_TABLE_POTREE + """(filepath,level,numberpoints,minz,maxz,geom) VALUES (%s, %s, %s, %s, %s, ST_MakeEnvelope(%s, %s, %s, %s, %s))"""
    cursor.execute(insertStatement, insertArgs)
    counter += 1
    if counter == COMMIT_INTERVAL:
        cursor.connection.commit()
        counter = 0


def run(inputFolder, srid, dbName, dbTable, dbPass, dbUser, dbHost, dbPort):
    # Make connection
    connectionString = utils.getConnectString(dbName, dbUser, dbPass, dbHost, dbPort)
    connection = psycopg2.connect(connectionString)
    cursor = connection.cursor()
    
    # Make it absolute path
    inputFolder = os.path.abspath(inputFolder)
    
    # Create table
    cursor.execute('CREATE TABLE ' + utils.DB_TABLE_POTREE + ' (filepath text, level integer, numberpoints integer, minz double precision, maxz double precision, geom public.geometry(Geometry, %s))', [srid, ])
    connection.commit()
    connection.close()
    

    cloudJSAbsPath = inputFolder + '/cloud.js'
    if not os.path.isfile(cloudJSAbsPath):
        raise Exception('Error: ' + cloudJSAbsPath + ' is not found!')        
    
    cloudJSData = json.loads(open(cloudJSAbsPath, 'r').read())
    hierarchyStepSize = cloudJSData['hierarchyStepSize']
    cloudJSBBox = cloudJSData['boundingBox']
    (minX,minY,minZ,maxX,maxY,maxZ) = (cloudJSBBox['lx'],cloudJSBBox['ly'],cloudJSBBox['lz'],cloudJSBBox['ux'],cloudJSBBox['uy'],cloudJSBBox['uz'])
    
    connection = psycopg2.connect(connectionString)<commit_msg>Declare global counter<commit_after>#!/usr/bin/env python
"""Creates a DB table with the extent information of the files in a Potree OctTree.
The files are not opened, instead we get the information from the HRC files and the 
known extent of the OctTree nodes"""

import argparse, traceback, time, os, psycopg2, json
import utils

USERNAME = utils.getUserName()
COMMIT_INTERVAL = 1000

counter = 0

def argument_parser():
    """ Define the arguments and return the parser object"""
    parser = argparse.ArgumentParser(
    description="""Creates a DB table with the extent information of the files in a Potree OctTree.
The files are not opened, instead we get the information from the HRC files and the 
known extent of the OctTree nodes""")
    parser.add_argument('-i','--input',default='',help='Input folder with the Potree OctTree',type=str, required=True)
    parser.add_argument('-s','--srid',default='',help='SRID',type=int, required=True)
    parser.add_argument('-d','--dbname',default=utils.DB_NAME,help='Postgres DB name [default ' + utils.DB_NAME + ']',type=str)
    parser.add_argument('-u','--dbuser',default=USERNAME,help='DB user [default ' + USERNAME + ']',type=str)
    parser.add_argument('-p','--dbpass',default='',help='DB pass',type=str)
    parser.add_argument('-b','--dbhost',default='',help='DB host',type=str)
    parser.add_argument('-r','--dbport',default='',help='DB port',type=str)
    return parser

def getChildBC(minX,minY,minZ,maxX,maxY,maxZ,childIndex):
    rX = (maxX - minX) / 2.
    rY = (maxY - minY) / 2.
    rZ = (maxZ - minZ) / 2.
    if childIndex == 0:
        return (minX,      minY,      minZ,      minX + rX, minY + rY, minZ + rZ)
    elif childIndex == 1:
        return (minX,      minY,      minZ + rZ, minX + rX, minY + rY, maxZ)
    elif childIndex == 2:
        return (minX,      minY + rY, minZ,      minX + rX, maxY,      minZ + rZ)
    elif childIndex == 3:
        return (minX,      minY + rY, minZ + rZ, minX + rX, maxY,      maxZ)
    elif childIndex == 4:
        return (minX + rX, minY,      minZ,      maxX,      minY + rY, minZ + rZ)
    elif childIndex == 5:
        return (minX + rX, minY,      minZ + rZ, maxX,      minY + rY, maxZ)
    elif childIndex == 6:
        return (minX + rX, minY + rY, minZ,      maxX,      maxY,      minZ + rZ)
    elif childIndex == 7:
        return (minX + rX, minY + rY, minZ + rZ, maxX,      maxY,      maxZ)
    else:
        raise Exception('Child index must be [0,7]!')
    
def addNodeFolder(cursor, node, nodeAbsPath, hierarchyStepSize, extension, minX, minY, minZ, maxX, maxY, maxZ, srid):
    hrcFile = node + '.hrc'
    hrc = None
    if os.path.isfile(nodeAbsPath + '/' + hrcFile):
        # Check if there is data in this node in Octtree A (we check if the HRC file for this node exist)
        hrc = utils.readHRC(nodeAbsPath + '/' + hrcFile, hierarchyStepSize)
        for level in range(hierarchyStepSize+1):
            for i in range(len(hrc[level])):
                if hrc[level][i]:
                    (childNode, isFile) = utils.getNodeName(level, i, node, hierarchyStepSize, extension)
                    relativeNode = childNode.replace(node,'').replace('.' + extension, '')
                    (lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ) = (minX, minY, minZ, maxX, maxY, maxZ)
                    for pNode in relativeNode:
                        (lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ) = getChildBC(lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ, int(pNode))
                    if isFile: 
                        addNodeFile(cursor, [nodeAbsPath + '/' + childNode, len(childNode) - 5, int(hrc[level][i]), lminZ, lmaxZ, lminX, lminY, lmaxX, lmaxY, int(srid)])
                    else:
                        addNodeFolder(cursor, node + childNode, nodeAbsPath + '/' + childNode, hierarchyStepSize, extension, lminX, lminY, lminZ, lmaxX, lmaxY, lmaxZ, srid)       

def addNodeFile(cursor, insertArgs):
    global counter
    insertStatement = """INSERT INTO """ + utils.DB_TABLE_POTREE + """(filepath,level,numberpoints,minz,maxz,geom) VALUES (%s, %s, %s, %s, %s, ST_MakeEnvelope(%s, %s, %s, %s, %s))"""
    cursor.execute(insertStatement, insertArgs)
    counter += 1
    if counter == COMMIT_INTERVAL:
        cursor.connection.commit()
        counter = 0


def run(inputFolder, srid, dbName, dbPass, dbUser, dbHost, dbPort):
    # Make connection
    connectionString = utils.getConnectString(dbName, dbUser, dbPass, dbHost, dbPort)
    connection = psycopg2.connect(connectionString)
    cursor = connection.cursor()
    
    # Make it absolute path
    inputFolder = os.path.abspath(inputFolder)
    
    # Create table
    cursor.execute('CREATE TABLE ' + utils.DB_TABLE_POTREE + ' (filepath text, level integer, numberpoints integer, minz double precision, maxz double precision, geom public.geometry(Geometry, %s))', [srid, ])
    connection.commit()
    connection.close()
    

    cloudJSAbsPath = inputFolder + '/cloud.js'
    if not os.path.isfile(cloudJSAbsPath):
        raise Exception('Error: ' + cloudJSAbsPath + ' is not found!')        
    
    cloudJSData = json.loads(open(cloudJSAbsPath, 'r').read())
    hierarchyStepSize = cloudJSData['hierarchyStepSize']
    cloudJSBBox = cloudJSData['boundingBox']
    (minX,minY,minZ,maxX,maxY,maxZ) = (cloudJSBBox['lx'],cloudJSBBox['ly'],cloudJSBBox['lz'],cloudJSBBox['ux'],cloudJSBBox['uy'],cloudJSBBox['uz'])
    
    connection = psycopg2.connect(connectionString)
{'fraction_non_alphanumeric': 0.1351599010354623, 'fraction_numerical': 0.004581691560524146, 'mean_word_length': 3.0184094256259204, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3916133', 'n_tokens_mistral': 3759, 'n_tokens_neox': 3707, 'n_words': 968}