<commit_before>#! /usr/bin/env python
# -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Collaboration
# Licensed under the 2-clause BSD license.

"""
This is meant to hold utility scripts for parts and connections

"""
from __future__ import absolute_import, division, print_function
from tabulate import tabulate

from hera_mc import part_connect, mc, geo_location, correlator_levels

# Pass part using a dictionary with a superset of part data:
#   [hpn]{hptype, manufacturer_number, install_date, short_description, repr,
#         portA[name(s)],  portB[name(s)],  geo[E, N, z, station, subarray]}
# Pass connections by using a dictionary of lists:
#   {up[name(s)],   b_on_up[name(s)],   start_on_up[time(s)],   stop_on_up[time(s)],
#    down[name(s)], a_on_down[name(s)], start_on_down[time(s)], stop_on_down[time(s)]}


class PartsAndConnections:
    no_connection_designator = '--'
    def __init__(self):
        pass

    def show_part(self, args, part_dict):
        """
        Print out part information.  Uses tabulate package.

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        part_dict:  input dictionary of parts, generated by self.get_part
        """

        table_data = []
        if args.verbosity == 'm':
            headers = ['HERA P/N','Part Type','Mfg #','Date']
        elif args.verbosity == 'h':
            headers = ['HERA P/N','Part Type','Mfg #','Date','A ports','B ports','Info','Geo']
        for hpn in sorted(part_dict.keys()):
            if args.verbosity == 'h':
                td = [hpn,part_dict[hpn]['hptype'],
                      part_dict[hpn]['manufacturer_number'],part_dict[hpn]['install_date']]
                pts = ''
                for a in part_dict[hpn]['a_ports']:
                    pts+=(a+', ')
                td.append(pts.strip().strip(','))
                pts = ''
                for b in part_dict[hpn]['b_ports']:
                    pts+=(b+', ')
                td.append(pts.strip().strip(','))
                td.append(part_dict[hpn]['short_description'])
                if part_dict[hpn]['geo'] is not None:
                    s = "{:.1f}E, {:.1f}N, {:.1f}m".format(part_dict[hpn]['geo']['easting'],
                         part_dict[hpn]['geo']['northing'],part_dict[hpn]['geo']['elevation'])
                    td.append(s)
                table_data.append(td)
            elif args.verbosity == 'm':
                table_data.append([hpn,part_dict[hpn]['hptype'],
                    part_dict[hpn]['manufacturer_number'],part_dict[hpn]['install_date']])
            else:
                print(hpn, part_dict[hpn]['repr'])
        if args.verbosity=='m' or args.verbosity=='h':
            print(tabulate(table_data,headers=headers,tablefmt='orgtbl'))

    def get_part(self, args, hpn_query=None, exact_match=False, show_part=False):
        """
        Return information on a part.  It will return all matching first characters unless exact_match==True.

        Returns part_dict, a dictionary keyed on hera part number with part data as found in hera_mc tables

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        hpn_query:  the input hera part number (whole or first part thereof)
        exact_match:  boolean to enforce full part number match
        show_part:  boolean to call show_part or not
        """
        if hpn_query is None:
            hpn_query = args.hpn
            exact_match = args.exact_match
        if not exact_match and hpn_query[-1]!='%':
            hpn_query = hpn_query+'%'
        connection_dict = self.get_connection(args, hpn_query=hpn_query, port_query='all', exact_match=exact_match, show_connection=False)
        part_dict = {}
        db = mc.connect_to_mc_db(args)
        with db.sessionmaker() as session:
            for part in session.query(part_connect.Parts).filter(part_connect.Parts.hpn.like(hpn_query)):
                part_dict[part.hpn] = {'hptype': part.hptype,
                                       'manufacturer_number': part.manufacturer_number,
                                       'install_date': part.install_date,
                                       'a_ports': [], 'b_ports': [], 'short_description':'', 'geo':None}
                part_dict[part.hpn]['repr'] = part.__repr__()  # Keep for now
                for part_info in session.query(part_connect.PartInfo).filter(part_connect.PartInfo.hpn == part.hpn):
                    part_dict[part.hpn]['short_description'] = part_info.short_description
                part_dict[part.hpn]['a_ports'] = connection_dict[part.hpn]['a_ports']
                part_dict[part.hpn]['b_ports'] = connection_dict[part.hpn]['b_ports']
                if part.hptype == 'station':
                    args.locate = part.hpn
                    part_dict[part.hpn]['geo'] = geo_location.locate_station(args, show_geo=False)
        if show_part:
            self.show_part(args, part_dict)
        return part_dict

    def __pull_out_component(self,cmpt_list,i):
        try:
            c = cmpt_list[i]
        except IndexError:
            try:
                c = cmpt_list[-1]
            except IndexError:
                c = self.no_connection_designator
        return c
    def show_connection(self, args, connection_dict):
        """
        Print out connection information.  Uses tabulate package.

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        connection_dict:  input dictionary of parts, generated by self.get_connection
        """
        table_data = []
        if args.verbosity == 'm':
            headers = ['Upstream', '<Port b:', ':Port a>', 'Part', '<Port b:', ':Port a>', 'Downstream']
        elif args.verbosity == 'h':
            headers = ['Upstream', '<Port b:', ':Port a>', 'Part', '<Port b:', ':Port a>', 'Downstream']
        for pkey in sorted(connection_dict.keys()): #Note that get_connection makes sure len(a_ports)=len(b_ports)
            for i in range(len(connection_dict[pkey]['a_ports'])):
                up  = self.__pull_out_component(connection_dict[pkey]['up_parts'],i)
                dn  = self.__pull_out_component(connection_dict[pkey]['down_parts'],i)
                pta = self.__pull_out_component(connection_dict[pkey]['a_ports'],i)
                ptb = self.__pull_out_component(connection_dict[pkey]['b_ports'],i)
                bup = self.__pull_out_component(connection_dict[pkey]['b_on_up'],i)
                adn = self.__pull_out_component(connection_dict[pkey]['a_on_down'],i)
                rup = self.__pull_out_component(connection_dict[pkey]['repr_up'],i)
                rdown = self.__pull_out_component(connection_dict[pkey]['repr_down'],i)
                if args.verbosity == 'h':
                    table_data.append([up,bup,pta,pkey,ptb,adn,dn])
                elif args.verbosity == 'm':
                    table_data.append([up,bup,pta,pkey,ptb,adn,dn])
                else:
                    print(rup,rdown)
        if args.verbosity=='m' or args.verbosity=='h':
            print(tabulate(table_data,headers=headers,tablefmt='orgtbl'))


    def get_connection(self, args, hpn_query=None, port_query=None, exact_match=False, show_connection=False):
        """
        Return information on parts connected to args.connection -- NEED TO INCLUDE USING START/STOP_TIME!!!
        It should get connections immediately adjacent to one part (upstream and downstream).

        Returns connection_dict, a dictionary keyed on part number of adjacent connections

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        hpn_query:  the input hera part number (whole or first part thereof)
        port_query:  a specifiable port name,  default is 'all'
        exact_match:  boolean to enforce full part number match
        show_connection:  boolean to call show_part or not
        """

        if hpn_query is None:
            hpn_query = args.connection
            exact_match = args.exact_match
        if not exact_match and hpn_query[-1]!='%':
            hpn_query = hpn_query+'%'
        if port_query is None:
            port_query = args.specify_port
        connection_dict = {}
        db = mc.connect_to_mc_db(args)
        with db.sessionmaker() as session:
            for connection in session.query(part_connect.Connections).filter(part_connect.Connections.up.like(hpn_query)):
                if port_query=='all' or connection.b_on_up == port_query:
                    if connection.up not in connection_dict.keys():
                        connection_dict[connection.up] = {'a_ports':[], 'up_parts':[],   'b_on_up':[], 
                                                          'b_ports':[], 'down_parts':[], 'a_on_down':[],
                                                          'start_on_down':[], 'stop_on_down':[], 'repr_down':[],
                                                          'start_on_up':[],   'stop_on_up':[],   'repr_up':[]}
                    connection_dict[connection.up]['b_ports'].append(connection.b_on_up)
                    connection_dict[connection.up]['down_parts'].append(connection.down)
                    connection_dict[connection.up]['a_on_down'].append(connection.a_on_down)
                    connection_dict[connection.up]['start_on_down'].append(connection.start_time)
                    connection_dict[connection.up]['stop_on_down'].append(connection.stop_time)
                    connection_dict[connection.up]['repr_down'].append(connection.__repr__())
            for connection in session.query(part_connect.Connections).filter(part_connect.Connections.down.like(hpn_query)):
                if port_query=='all' or connection.a_on_down == port_query:
                    if connection.down not in connection_dict.keys():  #Should only find at most 1 for last component
                        connection_dict[connection.down] = {'a_ports':[], 'up_parts':[],  'b_on_up':[], 
                                                            'b_ports':[], 'down_parts':[],'a_on_down':[],
                                                            'start_on_down':[], 'stop_on_down':[], 'repr_down':[],
                                                            'start_on_up':[],   'stop_on_up':[],   'repr_up':[]}
                    connection_dict[connection.down]['a_ports'].append(connection.a_on_down)
                    connection_dict[connection.down]['up_parts'].append(connection.up)
                    connection_dict[connection.down]['b_on_up'].append(connection.b_on_up)
                    connection_dict[connection.down]['start_on_up'].append(connection.start_time)
                    connection_dict[connection.down]['stop_on_up'].append(connection.stop_time)
                    connection_dict[connection.down]['repr_up'].append(connection.__repr__())
        for pkey in connection_dict.keys(): #Check port consistency on every part, fix if needed NOT FULLY IMPLEMENTED!
            ##Check and balance ports
            number_of_ports = {'A':len(connection_dict[pkey]['a_ports']),'B':len(connection_dict[pkey]['b_ports'])}
            if number_of_ports['A']==0:
                connection_dict[pkey]['a_ports'] = [self.no_connection_designator]
            if number_of_ports['B']==0:
                connection_dict[pkey]['b_ports'] = [self.no_connection_designator]
            if number_of_ports['A'] > number_of_ports['B']:
                for i in range(number_of_ports['A']-1):
                    connection_dict[pkey]['b_ports'].append(self.no_connection_designator)
            elif number_of_ports['B'] > number_of_ports['A']:
                for i in range(number_of_ports['B']-1):
                    connection_dict[pkey]['a_ports'].append(self.no_connection_designator)
            elif number_of_ports['A']>1:
                part_to_check = {}
                part_to_check[pkey] = {'a_ports':connection_dict[pkey]['a_ports'], 'b_ports':connection_dict[pkey]['a_ports']}
                for i,p in enumerate(part_to_check[pkey]['a_ports']):
                    check_port = self.__get_next_port(args,pkey,p,direction='down',check_part=part_to_check)
                    if check_port != part_to_check[pkey]['b_ports'][i]:
                        print('Ports differ ',p,check_port,part_to_check[pkey]['b_ports'][i])
                        print('This is undoubtedly just an ordering issue within the database')
                        print('   so this check needs to fix it here for this purpose. NOT YET IMPLEMENTED')
        if show_connection:
            self.show_connection(args, connection_dict)
        return connection_dict

    def __get_next_port(self,args,hpn,port,direction,check_part):
        """
        Get port on correct side of a given part to move up/down stream.
        """
        if check_part:
            part_dict = check_part
        else:
            part_dict = self.get_part(args,hpn_query=hpn,exact_match=True,show_part=False)
        if direction=='up':
            ports = [part_dict[hpn]['a_ports'],part_dict[hpn]['b_ports']]
            replacing = ['b','a']
        elif direction=='down':
            ports = [part_dict[hpn]['b_ports'],part_dict[hpn]['a_ports']]
            replacing = ['a','b']
        number_of_ports = [len(ports[0]),len(ports[1])]
        for i,p in enumerate(ports):
            if self.no_connection_designator in p: #This will only take out one.
                number_of_ports[i]-=1
        if port in ports[0]:
            return_port = port
        elif number_of_ports[0] == 1:
            return_port = ports[0][0]
        elif number_of_ports[0] == 0:
            return_port = None
        elif port in ports[1]:
            if number_of_ports[1] == number_of_ports[0]:
                return_port = port.replace(replacing[0],replacing[1])
            elif number_of_ports[1] > number_of_ports[0]: # SHOULDN'T GET HERE
                return_port = ports[0][0]                 #   BUT JUST IN CASE PICK ONE
            else:                                         # HERE IS THE ILL-DEFINED BRANCHING CASE
                return_port = ports[0][0]                 #   PICK ONE FOR NOW
        else:
            print('Error:  port not found',port)
            return_port = None 
        return return_port

    def __go_upstream(self, args, hpn, port, connection_dict=None):
        """
        Find the next connection up the signal chain.
        """
        up_port = self.__get_next_port(args,hpn,port,direction='up',check_part=False)
        if up_port == self.no_connection_designator or up_port is None:
            return
        stays_none = False
        if connection_dict is None:
            stays_none = True
            connection_dict = self.get_connection(args, hpn_query=hpn, port_query=up_port, 
                                                  exact_match=True, show_connection=False)
        if hpn not in connection_dict.keys():
            print('Get it and build up a self.connection_dict within the run.')
        for i,hpn_up in enumerate(connection_dict[hpn]['up_parts']):
            port = connection_dict[hpn]['b_on_up'][i]
            if hpn_up not in self.upstream:
                self.upstream.append([hpn_up,port])
            if stays_none:
                connection_dict = None
            self.__go_upstream(args, hpn_up, port, connection_dict)

    def __go_downstream(self, args, hpn, port, connection_dict=None):
        """
        Find the next connection down the signal chain
        """
        down_port = self.__get_next_port(args,hpn,port,direction='down',check_part=False)
        if down_port == self.no_connection_designator or down_port is None:
            return
        stays_none = False
        if connection_dict is None:
            stays_none = True
            connection_dict = self.get_connection(args, hpn_query=hpn, port_query=down_port, 
                                                  exact_match=True, show_connection=False)
        if hpn not in connection_dict.keys():
            print('Get it and build up a self.connection_dict within the run.')
        for i,hpn_down in enumerate(connection_dict[hpn]['down_parts']):
            port = connection_dict[hpn]['a_on_down'][i]
            if hpn_down not in self.downstream:
                self.downstream.append([hpn_down,port])
            if stays_none:
                connection_dict=None
            self.__go_downstream(args, hpn_down, port,connection_dict)


    def get_hookup(self, args, hpn_query=None, port_query=None, show_hookup=False):
        """
        Return the full hookup.  Note that if a part is selected up or down stream of a branching part, 
        it picks one and doesn't give all options -- something to work on.
        Returns hookup_dict, a dictionary keyed on derived key of hpn:port

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        hpn_query:  the input hera part number (whole or first part thereof)
        port_query:  a specifiable port name,  default is 'all'
        show_hookup:  boolean to call show_hookup or not

        """
        exact_match = False
        if hpn_query is None:
            hpn_query = args.mapr
            exact_match = args.exact_match
        if port_query is None:
            port_query = args.specify_port
        parts = self.get_part(args, hpn_query=hpn_query, exact_match=exact_match, show_part=False)
        #-#connections = get_connection(args, hpn_query=hpn_query, port_query=port_query, 
        #-#                             exact_match=exact_match, show_connection=False)
        #-#print('RESET TO NONE UNTIL IMPLEMENT BUILD SELF.CONNECTION_DICT')
        connections = None
        hookup_dict = {}
        #-#print(parts)
        for hpn in parts.keys():
            number_a_ports = len(parts[hpn]['a_ports'])
            number_b_ports = len(parts[hpn]['b_ports'])
            if port_query == 'all':
                if number_b_ports>number_a_ports:
                    port_query = parts[hpn]['b_ports']
                else:
                    port_query = parts[hpn]['a_ports']
            elif type(port_query) is not list:
                port_query = [port_query]
            for p in port_query:
                if p == self.no_connection_designator:
                    continue
                self.upstream = [[hpn,p]]
                self.downstream = []
                self.__go_upstream(args, hpn, p, connections)
                self.__go_downstream(args, hpn, p, connections)
                furthest_up = self.upstream[-1][0]
                try_station = self.get_part(args,hpn_query=furthest_up,exact_match=True,show_part=False)
                hukey = hpn+':'+p
                if try_station[furthest_up]['hptype'] == 'station':
                    hookup_dict[hukey] = [[try_station[furthest_up]['geo']['station_number'],'S']]<commit_msg>some additional minor cleanup<commit_after>#! /usr/bin/env python
# -*- mode: python; coding: utf-8 -*-
# Copyright 2016 the HERA Collaboration
# Licensed under the 2-clause BSD license.

"""
This is meant to hold utility scripts for parts and connections

"""
from __future__ import absolute_import, division, print_function
from tabulate import tabulate

from hera_mc import part_connect, mc, geo_location, correlator_levels


class PartsAndConnections:
    """
    Class to allow various manipulations of parts and their properties etc.  Things are 
    manipulated/passed as dictionaries currently.
    """
    no_connection_designator = '--'

    def __init__(self):
        pass

    def show_part(self, args, part_dict):
        """
        Print out part information.  Uses tabulate package.

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        part_dict:  input dictionary of parts, generated by self.get_part
        """

        table_data = []
        if args.verbosity == 'm':
            headers = ['HERA P/N','Part Type','Mfg #','Date']
        elif args.verbosity == 'h':
            headers = ['HERA P/N','Part Type','Mfg #','Date','A ports','B ports','Info','Geo']
        for hpn in sorted(part_dict.keys()):
            if args.verbosity == 'h':
                td = [hpn,part_dict[hpn]['hptype'],
                      part_dict[hpn]['manufacturer_number'],part_dict[hpn]['install_date']]
                pts = ''
                for a in part_dict[hpn]['a_ports']:
                    pts+=(a+', ')
                td.append(pts.strip().strip(','))
                pts = ''
                for b in part_dict[hpn]['b_ports']:
                    pts+=(b+', ')
                td.append(pts.strip().strip(','))
                td.append(part_dict[hpn]['short_description'])
                if part_dict[hpn]['geo'] is not None:
                    s = "{:.1f}E, {:.1f}N, {:.1f}m".format(part_dict[hpn]['geo']['easting'],
                         part_dict[hpn]['geo']['northing'],part_dict[hpn]['geo']['elevation'])
                    td.append(s)
                table_data.append(td)
            elif args.verbosity == 'm':
                table_data.append([hpn,part_dict[hpn]['hptype'],
                    part_dict[hpn]['manufacturer_number'],part_dict[hpn]['install_date']])
            else:
                print(hpn, part_dict[hpn]['repr'])
        if args.verbosity=='m' or args.verbosity=='h':
            print(tabulate(table_data,headers=headers,tablefmt='orgtbl'))

    def get_part(self, args, hpn_query=None, exact_match=False, show_part=False):
        """
        Return information on a part.  It will return all matching first characters unless exact_match==True.

        Returns part_dict, a dictionary keyed on hera part number with part data as found in hera_mc tables

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        hpn_query:  the input hera part number (whole or first part thereof)
        exact_match:  boolean to enforce full part number match
        show_part:  boolean to call show_part or not
        """
        if hpn_query is None:
            hpn_query = args.hpn
            exact_match = args.exact_match
        if not exact_match and hpn_query[-1]!='%':
            hpn_query = hpn_query+'%'
        connection_dict = self.get_connection(args, hpn_query=hpn_query, port_query='all', exact_match=exact_match, show_connection=False)
        part_dict = {}
        db = mc.connect_to_mc_db(args)
        with db.sessionmaker() as session:
            for part in session.query(part_connect.Parts).filter(part_connect.Parts.hpn.like(hpn_query)):
                part_dict[part.hpn] = {'hptype': part.hptype,
                                       'manufacturer_number': part.manufacturer_number,
                                       'install_date': part.install_date,
                                       'a_ports': [], 'b_ports': [], 'short_description':'', 'geo':None}
                part_dict[part.hpn]['repr'] = part.__repr__()  # Keep for now
                for part_info in session.query(part_connect.PartInfo).filter(part_connect.PartInfo.hpn == part.hpn):
                    part_dict[part.hpn]['short_description'] = part_info.short_description
                part_dict[part.hpn]['a_ports'] = connection_dict[part.hpn]['a_ports']
                part_dict[part.hpn]['b_ports'] = connection_dict[part.hpn]['b_ports']
                if part.hptype == 'station':
                    args.locate = part.hpn
                    part_dict[part.hpn]['geo'] = geo_location.locate_station(args, show_geo=False)
        if show_part:
            self.show_part(args, part_dict)
        return part_dict

    def __pull_out_component(self,cmpt_list,i):
        try:
            c = cmpt_list[i]
        except IndexError:
            try:
                c = cmpt_list[-1]
            except IndexError:
                c = self.no_connection_designator
        return c
    def show_connection(self, args, connection_dict):
        """
        Print out connection information.  Uses tabulate package.

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        connection_dict:  input dictionary of parts, generated by self.get_connection
        """

        table_data = []
        if args.verbosity == 'm':
            headers = ['Upstream', '<Port b:', ':Port a>', 'Part', '<Port b:', ':Port a>', 'Downstream']
        elif args.verbosity == 'h':
            headers = ['Upstream', '<Port b:', ':Port a>', 'Part', '<Port b:', ':Port a>', 'Downstream']
        for pkey in sorted(connection_dict.keys()): #Note that get_connection makes sure len(a_ports)=len(b_ports)
            for i in range(len(connection_dict[pkey]['a_ports'])):
                up  = self.__pull_out_component(connection_dict[pkey]['up_parts'],i)
                dn  = self.__pull_out_component(connection_dict[pkey]['down_parts'],i)
                pta = self.__pull_out_component(connection_dict[pkey]['a_ports'],i)
                ptb = self.__pull_out_component(connection_dict[pkey]['b_ports'],i)
                bup = self.__pull_out_component(connection_dict[pkey]['b_on_up'],i)
                adn = self.__pull_out_component(connection_dict[pkey]['a_on_down'],i)
                rup = self.__pull_out_component(connection_dict[pkey]['repr_up'],i)
                rdown = self.__pull_out_component(connection_dict[pkey]['repr_down'],i)
                if args.verbosity == 'h':
                    table_data.append([up,bup,pta,pkey,ptb,adn,dn])
                elif args.verbosity == 'm':
                    table_data.append([up,bup,pta,pkey,ptb,adn,dn])
                else:
                    print(rup,rdown)
        if args.verbosity=='m' or args.verbosity=='h':
            print(tabulate(table_data,headers=headers,tablefmt='orgtbl'))


    def get_connection(self, args, hpn_query=None, port_query=None, exact_match=False, show_connection=False):
        """
        Return information on parts connected to args.connection -- NEED TO INCLUDE USING START/STOP_TIME!!!
        It should get connections immediately adjacent to one part (upstream and downstream).

        Returns connection_dict, a dictionary keyed on part number of adjacent connections

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        hpn_query:  the input hera part number (whole or first part thereof)
        port_query:  a specifiable port name,  default is 'all'
        exact_match:  boolean to enforce full part number match
        show_connection:  boolean to call show_part or not
        """

        if hpn_query is None:
            hpn_query = args.connection
            exact_match = args.exact_match
        if not exact_match and hpn_query[-1]!='%':
            hpn_query = hpn_query+'%'
        hpn_query = hpn_query.strip()
        if port_query is None:
            port_query = args.specify_port
        connection_dict = {}
        db = mc.connect_to_mc_db(args)
        with db.sessionmaker() as session:
            for connection in session.query(part_connect.Connections).filter(part_connect.Connections.up.like(hpn_query)):
                if port_query=='all' or connection.b_on_up == port_query:
                    if connection.up not in connection_dict.keys():
                        connection_dict[connection.up] = {'a_ports':[], 'up_parts':[],   'b_on_up':[], 
                                                          'b_ports':[], 'down_parts':[], 'a_on_down':[],
                                                          'start_on_down':[], 'stop_on_down':[], 'repr_down':[],
                                                          'start_on_up':[],   'stop_on_up':[],   'repr_up':[]}
                    connection_dict[connection.up]['b_ports'].append(connection.b_on_up)
                    connection_dict[connection.up]['down_parts'].append(connection.down)
                    connection_dict[connection.up]['a_on_down'].append(connection.a_on_down)
                    connection_dict[connection.up]['start_on_down'].append(connection.start_time)
                    connection_dict[connection.up]['stop_on_down'].append(connection.stop_time)
                    connection_dict[connection.up]['repr_down'].append(connection.__repr__())
            for connection in session.query(part_connect.Connections).filter(part_connect.Connections.down.like(hpn_query)):
                if port_query=='all' or connection.a_on_down == port_query:
                    if connection.down not in connection_dict.keys():  #Should only find at most 1 for last component
                        connection_dict[connection.down] = {'a_ports':[], 'up_parts':[],  'b_on_up':[], 
                                                            'b_ports':[], 'down_parts':[],'a_on_down':[],
                                                            'start_on_down':[], 'stop_on_down':[], 'repr_down':[],
                                                            'start_on_up':[],   'stop_on_up':[],   'repr_up':[]}
                    connection_dict[connection.down]['a_ports'].append(connection.a_on_down)
                    connection_dict[connection.down]['up_parts'].append(connection.up)
                    connection_dict[connection.down]['b_on_up'].append(connection.b_on_up)
                    connection_dict[connection.down]['start_on_up'].append(connection.start_time)
                    connection_dict[connection.down]['stop_on_up'].append(connection.stop_time)
                    connection_dict[connection.down]['repr_up'].append(connection.__repr__())
        for pkey in connection_dict.keys(): #Check port consistency on every part, fix if needed NOT FULLY IMPLEMENTED!
            ##Check and balance ports
            number_of_ports = {'A':len(connection_dict[pkey]['a_ports']),'B':len(connection_dict[pkey]['b_ports'])}
            if number_of_ports['A']==0:
                connection_dict[pkey]['a_ports'] = [self.no_connection_designator]
            if number_of_ports['B']==0:
                connection_dict[pkey]['b_ports'] = [self.no_connection_designator]
            if number_of_ports['A'] > number_of_ports['B']:
                for i in range(number_of_ports['A']-1):
                    connection_dict[pkey]['b_ports'].append(self.no_connection_designator)
            elif number_of_ports['B'] > number_of_ports['A']:
                for i in range(number_of_ports['B']-1):
                    connection_dict[pkey]['a_ports'].append(self.no_connection_designator)
            elif number_of_ports['A']>1:
                part_to_check = {}
                part_to_check[pkey] = {'a_ports':connection_dict[pkey]['a_ports'], 'b_ports':connection_dict[pkey]['a_ports']}
                for i,p in enumerate(part_to_check[pkey]['a_ports']):
                    check_port = self.__get_next_port(args,pkey,p,direction='down',check_part=part_to_check)
                    if check_port != part_to_check[pkey]['b_ports'][i]:
                        print('Ports differ ',p,check_port,part_to_check[pkey]['b_ports'][i])
                        print('This is undoubtedly just an ordering issue within the database')
                        print('   so this check needs to fix it here for this purpose. NOT YET IMPLEMENTED')
        if show_connection:
            self.show_connection(args, connection_dict)
        return connection_dict

    def __get_next_port(self,args,hpn,port,direction,check_part):
        """
        Get port on correct side of a given part to move up/down stream.
        """
        if check_part:
            part_dict = check_part
        else:
            part_dict = self.get_part(args,hpn_query=hpn,exact_match=True,show_part=False)
        if direction=='up':
            ports = [part_dict[hpn]['a_ports'],part_dict[hpn]['b_ports']]
            replacing = ['b','a']
        elif direction=='down':
            ports = [part_dict[hpn]['b_ports'],part_dict[hpn]['a_ports']]
            replacing = ['a','b']
        number_of_ports = [len(ports[0]),len(ports[1])]
        for i,p in enumerate(ports):
            if self.no_connection_designator in p: #This will only take out one.
                number_of_ports[i]-=1
        if port in ports[0]:
            return_port = port
        elif number_of_ports[0] == 1:
            return_port = ports[0][0]
        elif number_of_ports[0] == 0:
            return_port = None
        elif port in ports[1]:
            if number_of_ports[1] == number_of_ports[0]:
                return_port = port.replace(replacing[0],replacing[1])
            elif number_of_ports[1] > number_of_ports[0]: # SHOULDN'T GET HERE
                return_port = ports[0][0]                 #   BUT JUST IN CASE PICK ONE
            else:                                         # HERE IS THE ILL-DEFINED BRANCHING CASE
                return_port = ports[0][0]                 #   PICK ONE FOR NOW
        else:
            print('Error:  port not found',port)
            return_port = None 
        return return_port

    def __go_upstream(self, args, hpn, port):
        """
        Find the next connection up the signal chain.
        """
        up_port = self.__get_next_port(args,hpn,port,direction='up',check_part=False)
        if up_port == self.no_connection_designator or up_port is None:
            return
        connection_dict = self.get_connection(args, hpn_query=hpn, port_query=up_port, 
                                              exact_match=True, show_connection=False)
        if hpn not in connection_dict.keys():
            print('Get it and build up a self.connection_dict within the run.')
        for i,hpn_up in enumerate(connection_dict[hpn]['up_parts']):
            port = connection_dict[hpn]['b_on_up'][i]
            if hpn_up not in self.upstream:
                self.upstream.append([hpn_up,port])
            self.__go_upstream(args, hpn_up, port)

    def __go_downstream(self, args, hpn, port):
        """
        Find the next connection down the signal chain
        """
        down_port = self.__get_next_port(args,hpn,port,direction='down',check_part=False)
        if down_port == self.no_connection_designator or down_port is None:
            return
        connection_dict = self.get_connection(args, hpn_query=hpn, port_query=down_port, 
                                              exact_match=True, show_connection=False)
        if hpn not in connection_dict.keys():
            print('Get it and build up a self.connection_dict within the run.')
        for i,hpn_down in enumerate(connection_dict[hpn]['down_parts']):
            port = connection_dict[hpn]['a_on_down'][i]
            if hpn_down not in self.downstream:
                self.downstream.append([hpn_down,port])
            self.__go_downstream(args, hpn_down, port)


    def get_hookup(self, args, hpn_query=None, port_query=None, show_hookup=False):
        """
        Return the full hookup.  Note that if a part is selected up or down stream of a branching part, 
        it picks one and doesn't give all options -- something to work on.
        Returns hookup_dict, a dictionary keyed on derived key of hpn:port

        Parameters
        -----------
        args:  arguments as per mc and parts argument parser
        hpn_query:  the input hera part number (whole or first part thereof)
        port_query:  a specifiable port name,  default is 'all'
        show_hookup:  boolean to call show_hookup or not

        """
        exact_match = False
        if hpn_query is None:
            hpn_query = args.mapr
            exact_match = args.exact_match
        if port_query is None:
            port_query = args.specify_port
        parts = self.get_part(args, hpn_query=hpn_query, exact_match=exact_match, show_part=False)
        connections = None
        hookup_dict = {}
        for hpn in parts.keys():
            number_a_ports = len(parts[hpn]['a_ports'])
            number_b_ports = len(parts[hpn]['b_ports'])
            if port_query == 'all':
                if number_b_ports>number_a_ports:
                    port_query = parts[hpn]['b_ports']
                else:
                    port_query = parts[hpn]['a_ports']
            elif type(port_query) is not list:
                port_query = [port_query]
            for p in port_query:
                if p == self.no_connection_designator:
                    continue
                self.upstream = [[hpn,p]]
                self.downstream = []
                self.__go_upstream(args, hpn, p, connections)
                self.__go_downstream(args, hpn, p, connections)
                furthest_up = self.upstream[-1][0]
                try_station = self.get_part(args,hpn_query=furthest_up,exact_match=True,show_part=False)
                hukey = hpn+':'+p
                if try_station[furthest_up]['hptype'] == 'station':
                    hookup_dict[hukey] = [[try_station[furthest_up]['geo']['station_number'],'S']]
{'fraction_non_alphanumeric': 0.11522335107240768, 'fraction_numerical': 0.002203945600172015, 'mean_word_length': 2.1366548642724665, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 43, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7357248', 'n_tokens_mistral': 10971, 'n_tokens_neox': 10606, 'n_words': 2284}