<commit_before><?php
#
#  Copyright 2003-2015 Opmantek Limited (www.opmantek.com)
#
#  ALL CODE MODIFICATIONS MUST BE SENT TO <EMAIL>
#
#  This file is part of Open-AudIT.
#
#  Open-AudIT is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Open-AudIT is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with Open-AudIT (most likely in a file named LICENSE).
#  If not, see <http://www.gnu.org/licenses/>
#
#  For further information on Open-AudIT or for a license other than AGPL please see
#  www.opmantek.com or email <EMAIL>
#
# *****************************************************************************

/**
* @category  Helper
* @package   Open-AudIT
* @author    <NAME> <<EMAIL>>
* @copyright 2014 Opmantek
* @license   http://www.gnu.org/licenses/agpl-3.0.html aGPL v3
* @version   3.0.0
* @link      http://www.open-audit.org
 */
if (!defined('BASEPATH')) {
    exit('No direct script access allowed');
}



if (!function_exists('process_scan')) {
    function process_scan($input = '')
    {
        if (empty($input) or empty($input->ip)) {
            return false;
        }

        if (empty($input->nmap_ports)) {
            $input->nmap_ports = '';
        }

        $log = new stdClass();
        $log->discovery_id = null;
        $log->system_id = null;
        $log->timestamp = null;
        $log->severity = 7;
        $log->pid = getmypid();
        $log->file = 'discovery_helper';
        $log->function = 'process_scan';
        $log->message = 'Data accepted';
        $log->ip = $input->ip;
        if (empty($input->system_id)) {
            $log->system_id = null;
        } else {
            $log->system_id = intval($input->system_id);
        }
        stdlog($log);
        $individual_ip_start = microtime(true);

        $CI = get_instance();

        // our required models
        $CI->load->model('m_audit_log');
        $CI->load->model('m_credentials');
        $CI->load->model('m_devices');
        $CI->load->model('m_device');
        $CI->load->model('m_devices_components');
        $CI->load->model('m_orgs');
        $CI->load->model('m_scripts');

        // our required helpers
        $CI->load->helper('audit');
        $CI->load->helper('mac');
        $CI->load->helper('network');
        $CI->load->helper('snmp');
        $CI->load->helper('snmp_oid');
        $CI->load->helper('ssh');
        $CI->load->helper('wmi');

        if (php_uname('s') != 'Windows NT') {
            $filepath = $CI->config->config['base_path'] . '/other';
        } else {
            $filepath = $CI->config->config['base_path'] . '\\other';
        }

        // Get the discovery details
        if (!empty($input->discovery_id)) {
            $input->discovery_id = intval($input->discovery_id);
            $log->severity = 7;
            $log->summary = 'Discovery id ' . $input->discovery_id . ' provided';
            $log->message = 'When processing discover_subnet, discovery_id ' . $input->discovery_id . ' was provided in the input.';
            $log->discovery_id = intval($input->discovery_id);
            $log->command_status = 'notice';
            $GLOBALS['discovery_id'] = $input->discovery_id;
            discovery_log($log);

            $sql = '/* input::discoveries */ ' . 'SELECT * FROM `discoveries` WHERE id = ?';
            $data = array($input->discovery_id);
            $query = $CI->db->query($sql, $data);
            $result = $query->result();
            $discovery = $result[0];
            # The below MUST be set if not already
            if (empty($discovery->org_id)) {
                $discovery->org_id = 1;
            }
            if (empty($discovery->device_count)) {
                $discovery->device_count = 0;
            }
            if (empty($discovery->limit)) {
                $discovery->limit = 0;
            }
        }

        // Create a blank discovery object if not already created above
        if (empty($discovery)) {
            $discovery = new stdClass();
            $discovery->id = '';
            $discovery->discard = '';
            $discovery->org_id = 1;
            $discovery->device_count = 0;
            $discovery->limit = 0;
            $proto = 'http';
            if ($CI->config->config['is_ssl'] === true) {
                $proto = 'https';
            }
            if (stripos($CI->config->config['default_network_address'], 'https:') !== false) {
                $proto = 'https';
            }
            $discovery->network_address = $proto . '://127.0.0.1/open-audit/';
        }



        // The end submit from the script that indicates there are no more items to be submitted
        if (!empty($input->complete) and $input->complete === 'y' and !empty($input->discovery_id)) {
            $sql = "/* input::discoveries */ " . "UPDATE `discoveries` SET `complete` = 'y' WHERE id = ?";
            $data = array(intval($input->discovery_id));
            $query = $CI->db->query($sql, $data);
            $log->severity = 7;
            $log->summary = 'Set discovery entry status to complete';
            $log->detail = $CI->db->last_query();
            discovery_log($log);
            if ($discovery->discard == 'y') {
                $sql = "/* input::discoveries */ " . "DELETE FROM `discoveries` WHERE id = ?";
                $data = array($log->discovery_id);
                $query = $CI->db->query($sql, $data);
                $log->severity = 7;
                $log->summary = 'Deleteing discovery entry';
                $log->message = $CI->db->last_query();
                stdlog($log);
            }
            if (!empty($discovery->other->single) and $discovery->other->single == 'y' and $discovery->discard == 'y') {
                sleep(10);
                $sql = "/* input::discoveries */ " . "DELETE FROM `credentials` WHERE description = 'Discovery " . $discovery->other->subnet . "'";
                $data = array();
                $query = $CI->db->query($sql, $data);
                $log->severity = 7;
                $log->summary = 'Deleteing credentials entries';
                $log->message = $CI->db->last_query();
                stdlog($log);
            }
            return true;
        }

        // check the IP isn't in the excluded list
        if (stripos(' ' . $CI->config->config['discovery_ip_exclude'] . ' ', ' ' . $input->ip . ' ') !== false) {
            # Our ip address matched an ip in the discovery_ip_exclude list - skip it
            $log->message = $input->ip . ' is in the list of excluded ip addresses - skipping.';
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            return false;
        }

        // check we have an actual valid device IP
        if (filter_var($input->ip, FILTER_VALIDATE_IP) === false) {
            $log->message = "Invalid IP address submitted (" . $input->ip . "), skipping.";
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            return false;
        }

        # Check we're obeying the limit
        if ($discovery->device_count >= $discovery->limit and $discovery->limit != 0) {
            $log->message = "License count exceeded. Not processing device " . $input->ip;
            discovery_log($log);
            return false;
        }

        # Update the count
        if (!empty($discovery->id)) {
            $sql = "/* input::discoveries */ " . "UPDATE discoveries SET device_count = device_count + 1 WHERE id = ?";
            $data = array($discovery->id);
            $query = $CI->db->query($sql, $data);
        }

        # Let's go!
        $log->message = 'Received data for ' . $input->ip . ', now starting to process';
        $individual_log_id_start = discovery_log($log);
        unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);


        $device = new stdClass();
        if (!empty($log->system_id)) {
            $device->id = $log->system_id;
        } else {
            $device->id = '';
        }

        $device->audits_ip = (string)$input->ip;
        $device->discovery_id = $discovery->id;
        $device->os_family = '';
        $device->os_group = '';
        $device->ip = (string)$input->ip;
        $device->last_seen = $CI->config->config['timestamp'];
        $device->last_seen_by = 'nmap';
        $device->mac_address = @(string)$input->mac_address;
        $device->manufacturer = '';
        $device->name = '';
        $device->sysDescr = '';
        $device->timestamp = $CI->config->config['timestamp'];
        $device->type = '';
        $device->uuid = '';
        $device->vm_uuid = '';

        // Use local DNS if requested
        if ($CI->config->item('discovery_use_dns') == 'y') {
            $device = dns_validate($device);
            if (!empty($device->dns_hostname)) {
                $log->command_status = 'notice';
                $log->message = 'IP ' . $device->ip . ' resolved to DNS hostname ' . $device->dns_hostname;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
            if (!empty($device->domain)) {
                $log->command_status = 'notice';
                $log->message = 'IP ' . $device->ip . ' resolved to DNS domain ' . $device->dns_domain;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
        }

        // See if what little we have matches an existing device
        if (empty($device->id)) {
            $parameters = new stdCLass();
            $parameters->details = $device;
            $parameters->log = $log;
            $device->id = $CI->m_device->match($parameters);

            if (!empty($device->id)) {
                $log->system_id = $device->id;
                $log->message = "Device with ID " . $device->id . " found on initial Nmap result.";
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
        }

        // Remove old logs
        if (!empty($device->id)) {
            $sql = "/* input::discoveries */ " . "DELETE FROM discovery_log WHERE system_id = " . $device->id . " AND (`discovery_id` != " . $discovery->id . " OR pid != " . $log->pid . ")";
            $log->message = 'Delete the previous log entries for this device';
            $log->command = $sql;
            $command_log_id = discovery_log($log);
            $command_start = microtime(true);
            $query = $CI->db->query($sql);
            $command_end = microtime(true);
            $log->command = $CI->db->last_query();
            $log->command_time_to_execute = $command_end - $command_start;
            $log->id = $command_log_id;
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            unset($log->id, $command_log_id);
        }

        // Update the logs for THIS run if we didn't get a system_id in the input
        if (!empty($device->id) and empty($input->system_id)) {
            $sql = "/* input::discoveries */ " . "UPDATE discovery_log SET system_id = " . intval($log->system_id) . " WHERE discovery_id = " . $discovery->id . " and ip = '" . $device->ip . "'";
            $log->message = 'Update the current log entries with our new device';
            $log->command = $sql;
            $command_log_id = discovery_log($log);
            $command_start = microtime(true);
            $query = $CI->db->query($sql);
            $command_end = microtime(true);
            $log->command = $CI->db->last_query();
            $log->command_time_to_execute = $command_end - $command_start;
            $log->id = $command_log_id;
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            unset($log->id, $command_log_id);
        }

        // We need a user object when retrieving various collections and
        // we should set the 'user' orgs to the org of this particular discovery run (and its children)
        $CI->user = new stdClass();
        if (!empty($discovery->org_id)) {
            $CI->user->orgs = $CI->m_orgs->get_children($discovery->org_id);
            if (count($CI->user->orgs) > 0) {
                $CI->user->org_list = $discovery->org_id . ',' . implode(',', $CI->user->orgs);
            } else {
                $CI->user->org_list = $discovery->org_id;
            }
        } else {
            $CI->user->org_list = '';
        }


        // Get any credential sets
        if (empty($credentials)) {
            $credentials = array();
        }
        $temp = $CI->m_credentials->collection();
        if (count($temp) > 0) {
            $credentials = array_merge($credentials, $temp);
        } else {
            $log->severity = 5;
            $log->message = "No credentials returned from database";
            discovery_log($log);
            $log->severity = 7;
        }
        unset($temp);
        $creds = array();
        foreach ($credentials as $credential) {
            $creds[] = $credential->attributes;
        }
        unset($credentials);
        $credentials = $creds;
        unset($creds);

        $log->command_status = 'notice';
        $log->message = 'WMI Status is '.$input->wmi_status.' on '.$device->ip;
        if (!empty($device->id)) {
            $log->message .= ' (System ID ' . $device->id . ')';
        }
        discovery_log($log);

        $input->ssh_port = '22';
        if ($discovery->other->nmap->ssh_ports != '22') {
            $nmap_ports = explode(',', $discovery->other->nmap->ssh_ports);
            foreach (explode(',', $input->nmap_ports) as $port) {
                $temp = explode('/', $port);
                $port = $temp[0];
                foreach ($nmap_ports as $nmap_port) {
                    if ($port == $nmap_port) {
                        $input->ssh_port = $port;
                        $input->ssh_status = 'true';
                    }
                }
                unset($nmap_item);
                unset($temp);
            }
        }

        $log->message = 'SSH Status is '.$input->ssh_status.' on '.$device->ip;
        if (!empty($device->id)) {
            $log->message .= ' (System ID ' . $device->id . ')';
        }
        discovery_log($log);

        $log->message = 'SNMP Status is '.$input->snmp_status.' on '.$device->ip;
        if (!empty($device->id)) {
            $log->message .= ' (System ID ' . $device->id . ')';
        }
        discovery_log($log);

        # SNMP
        $credentials_snmp = false;
        if (!extension_loaded('snmp') and $input->snmp_status == 'true') {
            $log->message = 'PHP extension not loaded, skipping SNMP data retrieval for ' . $device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            discovery_log($log);
        }
        if (extension_loaded('snmp') and $input->snmp_status == 'true') {
            $run_snmp = false;
            foreach ($credentials as $item) {
                if ($item->type === 'snmp' or $item->type === 'snmp_v2' or $item->type === 'snmp_v3') {
                    $run_snmp = true;
                    break;
                }
            }
            if ($run_snmp) {
                $log->message = 'Testing SNMP credentials for '.$device->ip;
                if (!empty($device->id)) {
                    $log->message .= ' (System ID ' . $device->id . ')';
                }
                discovery_log($log);
                $credentials_snmp = snmp_credentials($device->ip, $credentials, $log);
            } else {
                $log->message = 'No SNMP credentials, skipping SNMP data retrieval for ' . $device->ip;
                discovery_log($log);
            }
        }
        if ($credentials_snmp) {
            $temp_array = snmp_audit($device->ip, $credentials_snmp,$log);
            if (!empty($temp_array['details'])) {
                foreach ($temp_array['details'] as $key => $value) {
                    if (!empty($value)) {
                        $device->$key = $value;
                    }
                }
                $device->last_seen_by = 'snmp';
                $device->audits_ip = '127.0.0.1';
            }
            if (!empty($temp_array['interfaces'])) {
                $network_interfaces = $temp_array['interfaces'];
            }
            if (!empty($temp_array['modules'])) {
                $modules = $temp_array['modules'];
            }
            if (!empty($temp_array['ip'])) {
                $ip = $temp_array['ip'];
            }
            if (!empty($temp_array['guests'])) {
                $guests = $temp_array['guests'];
            }
        }
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';
        if ($device->type != 'computer' and $device->type != '' and $device->type != 'unknown' and $device->os_family != 'DD-WRT' and stripos($device->sysDescr, 'dd-wrt') === false and stripos($device->manufacturer, 'Ubiquiti') === false ) {
            $log->severity = 7;
            $log->message = 'Not a computer and not a DD-WRT device, setting SSH status to false for ' . $device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            $log->severity = 5;
            discovery_log($log);
            $log->severity = 7;
            $input->ssh_status = 'false';
        }
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';
        $log->command_status = 'notice';


        # SSH
        if ($input->ssh_status == 'true') {
            $log->message = 'Testing SSH credentials for '.$device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            discovery_log($log);
            $parameters = new stdClass();
            $parameters->ip = $device->ip;
            $parameters->system_id = '';
            if (!empty($device->id)) {
                $parameters->system_id = $device->id;
            }
            $parameters->log = $log;
            $parameters->credentials = $credentials;
            $parameters->ssh_port = $input->ssh_port;
            $ssh_details = ssh_audit($parameters);
            if (!empty($ssh_details)) {

                if (!empty($ssh_details->credentials)) {
                    $credentials_ssh = $ssh_details->credentials;
                }
                unset($ssh_details->credentials);
                $device->last_seen_by = 'ssh';
                $device->audits_ip = '127.0.0.1';
                foreach ($ssh_details as $key => $value) {
                    if (!empty($value) and empty($device->$key)) {
                        $device->$key = $value;
                    }
                }
            }
        }

        $log->file = 'discovery_helper';
        $log->function = 'discoveries';
        $log->command_status = 'notice';


        # We do not want to attempt to audit using WMI anything that's not a Windows machine
        if (!empty($device->os_group) and $device->os_group != 'Windows') {
            $input->wmi_status = 'false';
            $log->message = 'Setting WMI to false because we have an os_group that is not Windows, it is: ' . $device->os_group;
            discovery_log($log);
        }

        # WMI
        if ($input->wmi_status == 'true') {
            $log->message = 'Testing Windows credentials for ' . $device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            discovery_log($log);
            $credentials_windows = windows_credentials($device->ip, $credentials, $log);
        } else {
            $credentials_windows = false;
        }
        if ($input->wmi_status == 'true' and $credentials_windows) {
            $windows_details = wmi_audit($device->ip, $credentials_windows, $log);
            if (!empty($windows_details)) {
                $device->last_seen_by = 'windows';
                $device->audits_ip = '127.0.0.1';
                foreach ($windows_details as $key => $value) {
                    if (!empty($value)) {
                        $device->$key = $value;
                    }
                }
            }
        }
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';

        # Intelligent guesses at various attributes

        # Set manufacturer based on MAC address (if not already set)
        if (empty($device->manufacturer) and !empty($input->mac_address)) {
            $device->manufacturer = get_manufacturer_from_mac($input->mac_address);
                $log->severity = 7;
                $log->command_status = 'notice';
                $log->message = 'MAC ' . $input->mac_address . ' (input) matched to manufacturer ' . $device->manufacturer;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
        }
        if (empty($device->manufacturer) and !empty($device->mac_address)) {
            $device->manufacturer = get_manufacturer_from_mac($device->mac_address);
                $log->severity = 7;
                $log->command_status = 'notice';
                $log->message = 'MAC ' . $device->mac_address . ' (device) matched to manufacturer ' . $device->manufacturer;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
        }
        # in the case where port 5060 is detected and we have no other information, assign type 'voip phone'
        if (empty($device->type) and empty($device->snmp_oid) and empty($device->uuid) and stripos($input->nmap_ports, '5060/') !== false) {
            $device->type = 'voip phone';
        }
        # Port 62078 is used by IOS for iTunes wifi sync
        if (stripos($input->nmap_ports, '62078/tcp/iphone-sync') !== false) {
            # this could be an iPad (tablet), iPod (media device) or iPhone (smart phone).
            $log->message = 'Detected port TCP 62078 open. Assuming an Apple IOS device';
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            $device->type = 'iphone';
            $device->model = 'Apple iPhone';
            $device->manufacturer = 'Apple';
            $device->os_group = 'Apple IOS';
            $device->os_family = 'Apple IOS';
            $device->os_name = 'Apple IOS';
            if (stripos($device->hostname, 'ipad') !== false or stripos($device->dns_hostname, 'ipad') !== false) {
                $device->type = 'ipad';
                $device->model = 'Apple iPad';
            }
            if (stripos($device->hostname, 'ipod') !== false or stripos($device->dns_hostname, 'ipod') !== false) {
                $device->type = 'ipod';
                $device->model = 'Apple iPod';
            }
        }
        # Android devices typically have a hostname of android-***
        if (stripos($device->hostname, 'android') !== false or stripos($device->dns_hostname, 'android') !== false) {
            # Could be a table or smart phone or anything else.
            # We have no way of knowing so simply setting it to android.
            $device->type = 'android';
            $device->os_group = 'Android';
            $device->os_family = 'Android';
            $device->os_name = 'Android';
        }
        # Ubiquiti guessing
        if (empty($device->model) and stripos($device->manufacturer, 'Ubiquiti') !== false) {
            $device = $CI->m_devices->model_guess($device);
            if (!empty($device->model)) {
                $log->message = 'Best guess at Ubiquiti model to be ' . $device->model . ' for sysDesc: ' . $device->sysDescr;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
            if (empty($device->type)) {
                $device->type = 'router';
                $log->message = 'No device type assigned to Ubiquiti device, assigning router.';
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
        }
        # Playstation guess
        if (stripos($device->manufacturer, 'Sony') !== false and ($device->hostname == 'playstation' or $device->dns_hostname == 'playstation')) {
            $device->type = 'game console';
            $log->message = 'Assigning type = game console to Sony Playstation.';
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
        }

        # If we don't have a device.id, check with our updated device attributes (if any)
        if (empty($device->id)) {
            $parameters = new stdCLass();
            $parameters->details = $device;
            $parameters->log = $log;
            $device->id = $CI->m_device->match($parameters);
            if (!empty($device->id)) {
                $log->system_id = $device->id;
                // remove any old logs for this device
                $sql = "/* input::discoveries */ " . "DELETE FROM discovery_log WHERE system_id = " . $device->id . " AND (discovery_id != " . $discovery->id . " OR pid != " . $log->pid . ")";
                $log->message = 'Delete the previous log entries for this system_id';
                $log->command_status = 'notice';
                $log->command = $sql;
                $command_log_id = discovery_log($log);
                $command_start = microtime(true);
                $query = $CI->db->query($sql);
                $command_end = microtime(true);
                $log->command = $CI->db->last_query();
                $log->command_time_to_execute = $command_end - $command_start;
                $log->id = $command_log_id;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
                unset($log->id, $command_log_id);
                // update the previous log entries with our new system_id
                $sql = "/* input::discoveries */ " . "UPDATE discovery_log SET system_id = " . intval($log->system_id) . " WHERE discovery_id = " . $discovery->id . " and ip = '" . $device->ip . "'";
                $log->message = 'Update the previous log entries with the system_id';
                $log->command_status = 'notice';
                $log->command = $sql;
                $command_log_id = discovery_log($log);
                $command_start = microtime(true);
                $query = $CI->db->query($sql);
                $command_end = microtime(true);
                $log->command = $CI->db->last_query();
                $log->command_time_to_execute = $command_end - $command_start;
                $log->id = $command_log_id;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
                unset($log->id, $command_log_id);
            }
        }

        # Set the device org_id based on this discovery
        $device->org_id = $discovery->org_id;

        # If we have specifically assigned another org_id, set it
        if (!empty($discovery->devices_assigned_to_org)) {
            $device->org_id = $discovery->devices_assigned_to_org;
        }

        # If we have specifically assigned a location_id, set it
        if (!empty($discovery->devices_assigned_to_location)) {
            $device->location_id = $discovery->devices_assigned_to_location;
        }

        # See if we have a Mac Address for the device's IP
        if (!empty($network_interfaces) and empty($device->mac_address)) {
            foreach ($network_interfaces as $interface) {
                if ($interface->ip == $device->ip) {
                    $device->mac_address = $interface->mac;
                    $device->subnet = $interface->subnet;
                }
            }
        }

        // insert or update the device
        if (!empty($device->id)) {
            // we have a system id - UPDATE
            $action = 'update';
            $log->severity = 7;
            $log->command_status = 'notice';
            $log->message = 'Start of ' . strtoupper($device->last_seen_by) . ' update for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
            $CI->m_device->update($device);
            $device->ip = ip_address_from_db($device->ip);
            $log->file = 'discovery_helper';
            $log->function = 'discoveries';
            $log->ip = $device->ip;
            $log->message = 'End of ' . strtoupper($device->last_seen_by) . ' update for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
        } else {
            // we have a new system - INSERT
            $action = 'insert';
            $log->severity = 7;
            $log->ip = $device->ip;
            $log->command_status = 'notice';
            $log->message = 'Start of ' . strtoupper($device->last_seen_by) . ' insert for ' . $device->ip;
            discovery_log($log);
            $device->id = $CI->m_device->insert($device);
            $device->ip = ip_address_from_db($device->ip);
            $log->system_id = $device->id;
            $log->file = 'discovery_helper';
            $log->function = 'discoveries';
            $log->message = 'End of ' . strtoupper($device->last_seen_by) . ' insert for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
            // update the previous log entries with our new system_id
            $sql = "/* input::discoveries */ " . "UPDATE discovery_log SET system_id = " . intval($log->system_id) . " WHERE pid = " . intval($log->pid) . " and ip = '" . $device->ip . "'";
            $log->message = 'Update the previous log entries with our new system_id';
            $log->command = $sql;
            $command_log_id = discovery_log($log);
            $command_start = microtime(true);
            $query = $CI->db->query($sql);
            $command_end = microtime(true);
            $log->command = $CI->db->last_query();
            $log->command_time_to_execute = $command_end - $command_start;
            $log->id = $command_log_id;
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            unset($log->id, $command_log_id);
        }

        // grab some timestamps
        $device->last_seen = $CI->m_devices_components->read($device->id, 'y', 'system', '', 'last_seen');
        $device->first_seen = $CI->m_devices_components->read($device->id, 'y', 'system', '', 'first_seen');

        // Insert an audit log
        $user = '';
        if (!empty($CI->user->full_name)) {
            $user = $CI->user->full_name;
        }
        $CI->m_audit_log->create($device->id, $user, $device->last_seen_by, $device->audits_ip, '', '', $device->last_seen);
        unset($user);

        if (!empty($device->ip)) {
            $log->ip = $device->ip;
        }
        $log->discovery_id = $device->discovery_id;
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';

        // update any network interfaces retrieved by SNMP
        if (isset($network_interfaces) and is_array($network_interfaces) and count($network_interfaces) > 0) {
            $log->message = 'Processing found network interfaces for ' . $device->ip . ' (System ID ' . $device->id . ')';
            #$log->command = json_encode($network_interfaces);
            $log->command_output = '';
            discovery_log($log);
            $parameters = new stdClass();
            $parameters->table = 'network';
            $parameters->details = $device;
            $parameters->input = $network_interfaces;
            $parameters->log = $log;
            $CI->m_devices_components->process_component($parameters);
        }

        // update any ip addresses retrieved by SNMP
        if (!empty($ip) and is_array($ip) and count($ip) > 0) {
            $log->ip = $device->ip;
            $log->discovery_id = $device->discovery_id;
            $log->file = 'discovery_helper';
            $log->function = 'discoveries';
            $log->command_status = 'notice';
            $log->message = 'Processing found ip addresses for ' . $device->ip . ' (System ID ' . $device->id . ')';
            #$log->command = json_encode($ip);
            $log->command_output = '';
            discovery_log($log);
            $parameters = new stdClass();
            $parameters->table = 'ip';
            $parameters->details = $device;
            $parameters->input = $ip;
            $parameters->log = $log;
            $CI->m_devices_components->process_component($parameters);
        }

        // create or update the entry in the ip table from non-SNMP data
        //     so our 'networks' endpoint and functions can find the device
        if (empty($ip)) {
            $log->command_status = 'notice';
            $log->message = 'Processing found ip addresses (non-snmp) for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
            $item = new stdClass();
            $item->system_id = $device->id;
            $item->ip = $device->ip;
            $item->version = 4;<commit_msg>Fix IP from SNMP processing in discoveries.<commit_after><?php
#
#  Copyright 2003-2015 Opmantek Limited (www.opmantek.com)
#
#  ALL CODE MODIFICATIONS MUST BE SENT TO <EMAIL>
#
#  This file is part of Open-AudIT.
#
#  Open-AudIT is free software: you can redistribute it and/or modify
#  it under the terms of the GNU Affero General Public License as published
#  by the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  Open-AudIT is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU Affero General Public License for more details.
#
#  You should have received a copy of the GNU Affero General Public License
#  along with Open-AudIT (most likely in a file named LICENSE).
#  If not, see <http://www.gnu.org/licenses/>
#
#  For further information on Open-AudIT or for a license other than AGPL please see
#  www.opmantek.com or email <EMAIL>
#
# *****************************************************************************

/**
* @category  Helper
* @package   Open-AudIT
* @author    <NAME> <<EMAIL>>
* @copyright 2014 Opmantek
* @license   http://www.gnu.org/licenses/agpl-3.0.html aGPL v3
* @version   3.0.0
* @link      http://www.open-audit.org
 */
if (!defined('BASEPATH')) {
    exit('No direct script access allowed');
}



if (!function_exists('process_scan')) {
    function process_scan($input = '')
    {
        if (empty($input) or empty($input->ip)) {
            return false;
        }

        if (empty($input->nmap_ports)) {
            $input->nmap_ports = '';
        }

        $log = new stdClass();
        $log->discovery_id = null;
        $log->system_id = null;
        $log->timestamp = null;
        $log->severity = 7;
        $log->pid = getmypid();
        $log->file = 'discovery_helper';
        $log->function = 'process_scan';
        $log->message = 'Data accepted';
        $log->ip = $input->ip;
        if (empty($input->system_id)) {
            $log->system_id = null;
        } else {
            $log->system_id = intval($input->system_id);
        }
        stdlog($log);
        $individual_ip_start = microtime(true);

        $CI = get_instance();

        // our required models
        $CI->load->model('m_audit_log');
        $CI->load->model('m_credentials');
        $CI->load->model('m_devices');
        $CI->load->model('m_device');
        $CI->load->model('m_devices_components');
        $CI->load->model('m_orgs');
        $CI->load->model('m_scripts');

        // our required helpers
        $CI->load->helper('audit');
        $CI->load->helper('mac');
        $CI->load->helper('network');
        $CI->load->helper('snmp');
        $CI->load->helper('snmp_oid');
        $CI->load->helper('ssh');
        $CI->load->helper('wmi');

        if (php_uname('s') != 'Windows NT') {
            $filepath = $CI->config->config['base_path'] . '/other';
        } else {
            $filepath = $CI->config->config['base_path'] . '\\other';
        }

        // Get the discovery details
        if (!empty($input->discovery_id)) {
            $input->discovery_id = intval($input->discovery_id);
            $log->severity = 7;
            $log->summary = 'Discovery id ' . $input->discovery_id . ' provided';
            $log->message = 'When processing discover_subnet, discovery_id ' . $input->discovery_id . ' was provided in the input.';
            $log->discovery_id = intval($input->discovery_id);
            $log->command_status = 'notice';
            $GLOBALS['discovery_id'] = $input->discovery_id;
            discovery_log($log);

            $sql = '/* input::discoveries */ ' . 'SELECT * FROM `discoveries` WHERE id = ?';
            $data = array($input->discovery_id);
            $query = $CI->db->query($sql, $data);
            $result = $query->result();
            $discovery = $result[0];
            # The below MUST be set if not already
            if (empty($discovery->org_id)) {
                $discovery->org_id = 1;
            }
            if (empty($discovery->device_count)) {
                $discovery->device_count = 0;
            }
            if (empty($discovery->limit)) {
                $discovery->limit = 0;
            }
        }

        // Create a blank discovery object if not already created above
        if (empty($discovery)) {
            $discovery = new stdClass();
            $discovery->id = '';
            $discovery->discard = '';
            $discovery->org_id = 1;
            $discovery->device_count = 0;
            $discovery->limit = 0;
            $proto = 'http';
            if ($CI->config->config['is_ssl'] === true) {
                $proto = 'https';
            }
            if (stripos($CI->config->config['default_network_address'], 'https:') !== false) {
                $proto = 'https';
            }
            $discovery->network_address = $proto . '://127.0.0.1/open-audit/';
        }



        // The end submit from the script that indicates there are no more items to be submitted
        if (!empty($input->complete) and $input->complete === 'y' and !empty($input->discovery_id)) {
            $sql = "/* input::discoveries */ " . "UPDATE `discoveries` SET `complete` = 'y' WHERE id = ?";
            $data = array(intval($input->discovery_id));
            $query = $CI->db->query($sql, $data);
            $log->severity = 7;
            $log->summary = 'Set discovery entry status to complete';
            $log->detail = $CI->db->last_query();
            discovery_log($log);
            if ($discovery->discard == 'y') {
                $sql = "/* input::discoveries */ " . "DELETE FROM `discoveries` WHERE id = ?";
                $data = array($log->discovery_id);
                $query = $CI->db->query($sql, $data);
                $log->severity = 7;
                $log->summary = 'Deleteing discovery entry';
                $log->message = $CI->db->last_query();
                stdlog($log);
            }
            if (!empty($discovery->other->single) and $discovery->other->single == 'y' and $discovery->discard == 'y') {
                sleep(10);
                $sql = "/* input::discoveries */ " . "DELETE FROM `credentials` WHERE description = 'Discovery " . $discovery->other->subnet . "'";
                $data = array();
                $query = $CI->db->query($sql, $data);
                $log->severity = 7;
                $log->summary = 'Deleteing credentials entries';
                $log->message = $CI->db->last_query();
                stdlog($log);
            }
            return true;
        }

        // check the IP isn't in the excluded list
        if (stripos(' ' . $CI->config->config['discovery_ip_exclude'] . ' ', ' ' . $input->ip . ' ') !== false) {
            # Our ip address matched an ip in the discovery_ip_exclude list - skip it
            $log->message = $input->ip . ' is in the list of excluded ip addresses - skipping.';
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            return false;
        }

        // check we have an actual valid device IP
        if (filter_var($input->ip, FILTER_VALIDATE_IP) === false) {
            $log->message = "Invalid IP address submitted (" . $input->ip . "), skipping.";
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            return false;
        }

        # Check we're obeying the limit
        if ($discovery->device_count >= $discovery->limit and $discovery->limit != 0) {
            $log->message = "License count exceeded. Not processing device " . $input->ip;
            discovery_log($log);
            return false;
        }

        # Update the count
        if (!empty($discovery->id)) {
            $sql = "/* input::discoveries */ " . "UPDATE discoveries SET device_count = device_count + 1 WHERE id = ?";
            $data = array($discovery->id);
            $query = $CI->db->query($sql, $data);
        }

        # Let's go!
        $log->message = 'Received data for ' . $input->ip . ', now starting to process';
        $individual_log_id_start = discovery_log($log);
        unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);


        $device = new stdClass();
        if (!empty($log->system_id)) {
            $device->id = $log->system_id;
        } else {
            $device->id = '';
        }

        $device->audits_ip = (string)$input->ip;
        $device->discovery_id = $discovery->id;
        $device->os_family = '';
        $device->os_group = '';
        $device->ip = (string)$input->ip;
        $device->last_seen = $CI->config->config['timestamp'];
        $device->last_seen_by = 'nmap';
        $device->mac_address = @(string)$input->mac_address;
        $device->manufacturer = '';
        $device->name = '';
        $device->sysDescr = '';
        $device->timestamp = $CI->config->config['timestamp'];
        $device->type = '';
        $device->uuid = '';
        $device->vm_uuid = '';

        // Use local DNS if requested
        if ($CI->config->item('discovery_use_dns') == 'y') {
            $device = dns_validate($device);
            if (!empty($device->dns_hostname)) {
                $log->command_status = 'notice';
                $log->message = 'IP ' . $device->ip . ' resolved to DNS hostname ' . $device->dns_hostname;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
            if (!empty($device->domain)) {
                $log->command_status = 'notice';
                $log->message = 'IP ' . $device->ip . ' resolved to DNS domain ' . $device->dns_domain;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
        }

        // See if what little we have matches an existing device
        if (empty($device->id)) {
            $parameters = new stdCLass();
            $parameters->details = $device;
            $parameters->log = $log;
            $device->id = $CI->m_device->match($parameters);

            if (!empty($device->id)) {
                $log->system_id = $device->id;
                $log->message = "Device with ID " . $device->id . " found on initial Nmap result.";
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
        }

        // Remove old logs
        if (!empty($device->id)) {
            $sql = "/* input::discoveries */ " . "DELETE FROM discovery_log WHERE system_id = " . $device->id . " AND (`discovery_id` != " . $discovery->id . " OR pid != " . $log->pid . ")";
            $log->message = 'Delete the previous log entries for this device';
            $log->command = $sql;
            $command_log_id = discovery_log($log);
            $command_start = microtime(true);
            $query = $CI->db->query($sql);
            $command_end = microtime(true);
            $log->command = $CI->db->last_query();
            $log->command_time_to_execute = $command_end - $command_start;
            $log->id = $command_log_id;
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            unset($log->id, $command_log_id);
        }

        // Update the logs for THIS run if we didn't get a system_id in the input
        if (!empty($device->id) and empty($input->system_id)) {
            $sql = "/* input::discoveries */ " . "UPDATE discovery_log SET system_id = " . intval($log->system_id) . " WHERE discovery_id = " . $discovery->id . " and ip = '" . $device->ip . "'";
            $log->message = 'Update the current log entries with our new device';
            $log->command = $sql;
            $command_log_id = discovery_log($log);
            $command_start = microtime(true);
            $query = $CI->db->query($sql);
            $command_end = microtime(true);
            $log->command = $CI->db->last_query();
            $log->command_time_to_execute = $command_end - $command_start;
            $log->id = $command_log_id;
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            unset($log->id, $command_log_id);
        }

        // We need a user object when retrieving various collections and
        // we should set the 'user' orgs to the org of this particular discovery run (and its children)
        $CI->user = new stdClass();
        if (!empty($discovery->org_id)) {
            $CI->user->orgs = $CI->m_orgs->get_children($discovery->org_id);
            if (count($CI->user->orgs) > 0) {
                $CI->user->org_list = $discovery->org_id . ',' . implode(',', $CI->user->orgs);
            } else {
                $CI->user->org_list = $discovery->org_id;
            }
        } else {
            $CI->user->org_list = '';
        }


        // Get any credential sets
        if (empty($credentials)) {
            $credentials = array();
        }
        $temp = $CI->m_credentials->collection();
        if (count($temp) > 0) {
            $credentials = array_merge($credentials, $temp);
        } else {
            $log->severity = 5;
            $log->message = "No credentials returned from database";
            discovery_log($log);
            $log->severity = 7;
        }
        unset($temp);
        $creds = array();
        foreach ($credentials as $credential) {
            $creds[] = $credential->attributes;
        }
        unset($credentials);
        $credentials = $creds;
        unset($creds);

        $log->command_status = 'notice';
        $log->message = 'WMI Status is '.$input->wmi_status.' on '.$device->ip;
        if (!empty($device->id)) {
            $log->message .= ' (System ID ' . $device->id . ')';
        }
        discovery_log($log);

        $input->ssh_port = '22';
        if ($discovery->other->nmap->ssh_ports != '22') {
            $nmap_ports = explode(',', $discovery->other->nmap->ssh_ports);
            foreach (explode(',', $input->nmap_ports) as $port) {
                $temp = explode('/', $port);
                $port = $temp[0];
                foreach ($nmap_ports as $nmap_port) {
                    if ($port == $nmap_port) {
                        $input->ssh_port = $port;
                        $input->ssh_status = 'true';
                    }
                }
                unset($nmap_item);
                unset($temp);
            }
        }

        $log->message = 'SSH Status is '.$input->ssh_status.' on '.$device->ip;
        if (!empty($device->id)) {
            $log->message .= ' (System ID ' . $device->id . ')';
        }
        discovery_log($log);

        $log->message = 'SNMP Status is '.$input->snmp_status.' on '.$device->ip;
        if (!empty($device->id)) {
            $log->message .= ' (System ID ' . $device->id . ')';
        }
        discovery_log($log);

        # SNMP
        $credentials_snmp = false;
        if (!extension_loaded('snmp') and $input->snmp_status == 'true') {
            $log->message = 'PHP extension not loaded, skipping SNMP data retrieval for ' . $device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            discovery_log($log);
        }
        if (extension_loaded('snmp') and $input->snmp_status == 'true') {
            $run_snmp = false;
            foreach ($credentials as $item) {
                if ($item->type === 'snmp' or $item->type === 'snmp_v2' or $item->type === 'snmp_v3') {
                    $run_snmp = true;
                    break;
                }
            }
            if ($run_snmp) {
                $log->message = 'Testing SNMP credentials for '.$device->ip;
                if (!empty($device->id)) {
                    $log->message .= ' (System ID ' . $device->id . ')';
                }
                discovery_log($log);
                $credentials_snmp = snmp_credentials($device->ip, $credentials, $log);
            } else {
                $log->message = 'No SNMP credentials, skipping SNMP data retrieval for ' . $device->ip;
                discovery_log($log);
            }
        }
        if ($credentials_snmp) {
            $temp_array = snmp_audit($device->ip, $credentials_snmp,$log);
            if (!empty($temp_array['details'])) {
                foreach ($temp_array['details'] as $key => $value) {
                    if (!empty($value)) {
                        $device->$key = $value;
                    }
                }
                $device->last_seen_by = 'snmp';
                $device->audits_ip = '127.0.0.1';
            }
            if (!empty($temp_array['interfaces'])) {
                $network_interfaces = $temp_array['interfaces'];
            }
            if (!empty($temp_array['modules'])) {
                $modules = $temp_array['modules'];
            }
            if (!empty($temp_array['ip'])) {
                $ip = $temp_array['ip'];
            }
            if (!empty($temp_array['guests'])) {
                $guests = $temp_array['guests'];
            }
        }
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';
        if ($device->type != 'computer' and $device->type != '' and $device->type != 'unknown' and $device->os_family != 'DD-WRT' and stripos($device->sysDescr, 'dd-wrt') === false and stripos($device->manufacturer, 'Ubiquiti') === false ) {
            $log->severity = 7;
            $log->message = 'Not a computer and not a DD-WRT device, setting SSH status to false for ' . $device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            $log->severity = 5;
            discovery_log($log);
            $log->severity = 7;
            $input->ssh_status = 'false';
        }
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';
        $log->command_status = 'notice';


        # SSH
        if ($input->ssh_status == 'true') {
            $log->message = 'Testing SSH credentials for '.$device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            discovery_log($log);
            $parameters = new stdClass();
            $parameters->ip = $device->ip;
            $parameters->system_id = '';
            if (!empty($device->id)) {
                $parameters->system_id = $device->id;
            }
            $parameters->log = $log;
            $parameters->credentials = $credentials;
            $parameters->ssh_port = $input->ssh_port;
            $ssh_details = ssh_audit($parameters);
            if (!empty($ssh_details)) {

                if (!empty($ssh_details->credentials)) {
                    $credentials_ssh = $ssh_details->credentials;
                }
                unset($ssh_details->credentials);
                $device->last_seen_by = 'ssh';
                $device->audits_ip = '127.0.0.1';
                foreach ($ssh_details as $key => $value) {
                    if (!empty($value) and empty($device->$key)) {
                        $device->$key = $value;
                    }
                }
            }
        }

        $log->file = 'discovery_helper';
        $log->function = 'discoveries';
        $log->command_status = 'notice';


        # We do not want to attempt to audit using WMI anything that's not a Windows machine
        if (!empty($device->os_group) and $device->os_group != 'Windows') {
            $input->wmi_status = 'false';
            $log->message = 'Setting WMI to false because we have an os_group that is not Windows, it is: ' . $device->os_group;
            discovery_log($log);
        }

        # WMI
        if ($input->wmi_status == 'true') {
            $log->message = 'Testing Windows credentials for ' . $device->ip;
            if (!empty($device->id)) {
                $log->message .= ' (System ID ' . $device->id . ')';
            }
            discovery_log($log);
            $credentials_windows = windows_credentials($device->ip, $credentials, $log);
        } else {
            $credentials_windows = false;
        }
        if ($input->wmi_status == 'true' and $credentials_windows) {
            $windows_details = wmi_audit($device->ip, $credentials_windows, $log);
            if (!empty($windows_details)) {
                $device->last_seen_by = 'windows';
                $device->audits_ip = '127.0.0.1';
                foreach ($windows_details as $key => $value) {
                    if (!empty($value)) {
                        $device->$key = $value;
                    }
                }
            }
        }
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';

        # Intelligent guesses at various attributes

        # Set manufacturer based on MAC address (if not already set)
        if (empty($device->manufacturer) and !empty($input->mac_address)) {
            $device->manufacturer = get_manufacturer_from_mac($input->mac_address);
                $log->severity = 7;
                $log->command_status = 'notice';
                $log->message = 'MAC ' . $input->mac_address . ' (input) matched to manufacturer ' . $device->manufacturer;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
        }
        if (empty($device->manufacturer) and !empty($device->mac_address)) {
            $device->manufacturer = get_manufacturer_from_mac($device->mac_address);
                $log->severity = 7;
                $log->command_status = 'notice';
                $log->message = 'MAC ' . $device->mac_address . ' (device) matched to manufacturer ' . $device->manufacturer;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
        }
        # in the case where port 5060 is detected and we have no other information, assign type 'voip phone'
        if (empty($device->type) and empty($device->snmp_oid) and empty($device->uuid) and stripos($input->nmap_ports, '5060/') !== false) {
            $device->type = 'voip phone';
        }
        # Port 62078 is used by IOS for iTunes wifi sync
        if (stripos($input->nmap_ports, '62078/tcp/iphone-sync') !== false) {
            # this could be an iPad (tablet), iPod (media device) or iPhone (smart phone).
            $log->message = 'Detected port TCP 62078 open. Assuming an Apple IOS device';
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            $device->type = 'iphone';
            $device->model = 'Apple iPhone';
            $device->manufacturer = 'Apple';
            $device->os_group = 'Apple IOS';
            $device->os_family = 'Apple IOS';
            $device->os_name = 'Apple IOS';
            if (stripos($device->hostname, 'ipad') !== false or stripos($device->dns_hostname, 'ipad') !== false) {
                $device->type = 'ipad';
                $device->model = 'Apple iPad';
            }
            if (stripos($device->hostname, 'ipod') !== false or stripos($device->dns_hostname, 'ipod') !== false) {
                $device->type = 'ipod';
                $device->model = 'Apple iPod';
            }
        }
        # Android devices typically have a hostname of android-***
        if (stripos($device->hostname, 'android') !== false or stripos($device->dns_hostname, 'android') !== false) {
            # Could be a table or smart phone or anything else.
            # We have no way of knowing so simply setting it to android.
            $device->type = 'android';
            $device->os_group = 'Android';
            $device->os_family = 'Android';
            $device->os_name = 'Android';
        }
        # Ubiquiti guessing
        if (empty($device->model) and stripos($device->manufacturer, 'Ubiquiti') !== false) {
            $device = $CI->m_devices->model_guess($device);
            if (!empty($device->model)) {
                $log->message = 'Best guess at Ubiquiti model to be ' . $device->model . ' for sysDesc: ' . $device->sysDescr;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
            if (empty($device->type)) {
                $device->type = 'router';
                $log->message = 'No device type assigned to Ubiquiti device, assigning router.';
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            }
        }
        # Playstation guess
        if (stripos($device->manufacturer, 'Sony') !== false and ($device->hostname == 'playstation' or $device->dns_hostname == 'playstation')) {
            $device->type = 'game console';
            $log->message = 'Assigning type = game console to Sony Playstation.';
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
        }

        # If we don't have a device.id, check with our updated device attributes (if any)
        if (empty($device->id)) {
            $parameters = new stdCLass();
            $parameters->details = $device;
            $parameters->log = $log;
            $device->id = $CI->m_device->match($parameters);
            if (!empty($device->id)) {
                $log->system_id = $device->id;
                // remove any old logs for this device
                $sql = "/* input::discoveries */ " . "DELETE FROM discovery_log WHERE system_id = " . $device->id . " AND (discovery_id != " . $discovery->id . " OR pid != " . $log->pid . ")";
                $log->message = 'Delete the previous log entries for this system_id';
                $log->command_status = 'notice';
                $log->command = $sql;
                $command_log_id = discovery_log($log);
                $command_start = microtime(true);
                $query = $CI->db->query($sql);
                $command_end = microtime(true);
                $log->command = $CI->db->last_query();
                $log->command_time_to_execute = $command_end - $command_start;
                $log->id = $command_log_id;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
                unset($log->id, $command_log_id);
                // update the previous log entries with our new system_id
                $sql = "/* input::discoveries */ " . "UPDATE discovery_log SET system_id = " . intval($log->system_id) . " WHERE discovery_id = " . $discovery->id . " and ip = '" . $device->ip . "'";
                $log->message = 'Update the previous log entries with the system_id';
                $log->command_status = 'notice';
                $log->command = $sql;
                $command_log_id = discovery_log($log);
                $command_start = microtime(true);
                $query = $CI->db->query($sql);
                $command_end = microtime(true);
                $log->command = $CI->db->last_query();
                $log->command_time_to_execute = $command_end - $command_start;
                $log->id = $command_log_id;
                discovery_log($log);
                unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
                unset($log->id, $command_log_id);
            }
        }

        # Set the device org_id based on this discovery
        $device->org_id = $discovery->org_id;

        # If we have specifically assigned another org_id, set it
        if (!empty($discovery->devices_assigned_to_org)) {
            $device->org_id = $discovery->devices_assigned_to_org;
        }

        # If we have specifically assigned a location_id, set it
        if (!empty($discovery->devices_assigned_to_location)) {
            $device->location_id = $discovery->devices_assigned_to_location;
        }

        # See if we have a Mac Address for the device's IP
        if (!empty($network_interfaces) and empty($device->mac_address)) {
            foreach ($network_interfaces as $interface) {
                if ($interface->ip == $device->ip) {
                    $device->mac_address = $interface->mac;
                    $device->subnet = $interface->subnet;
                }
            }
        }

        // insert or update the device
        if (!empty($device->id)) {
            // we have a system id - UPDATE
            $action = 'update';
            $log->severity = 7;
            $log->command_status = 'notice';
            $log->message = 'Start of ' . strtoupper($device->last_seen_by) . ' update for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
            $CI->m_device->update($device);
            $device->ip = ip_address_from_db($device->ip);
            $log->file = 'discovery_helper';
            $log->function = 'discoveries';
            $log->ip = $device->ip;
            $log->message = 'End of ' . strtoupper($device->last_seen_by) . ' update for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
        } else {
            // we have a new system - INSERT
            $action = 'insert';
            $log->severity = 7;
            $log->ip = $device->ip;
            $log->command_status = 'notice';
            $log->message = 'Start of ' . strtoupper($device->last_seen_by) . ' insert for ' . $device->ip;
            discovery_log($log);
            $device->id = $CI->m_device->insert($device);
            $device->ip = ip_address_from_db($device->ip);
            $log->system_id = $device->id;
            $log->file = 'discovery_helper';
            $log->function = 'discoveries';
            $log->message = 'End of ' . strtoupper($device->last_seen_by) . ' insert for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
            // update the previous log entries with our new system_id
            $sql = "/* input::discoveries */ " . "UPDATE discovery_log SET system_id = " . intval($log->system_id) . " WHERE pid = " . intval($log->pid) . " and ip = '" . $device->ip . "'";
            $log->message = 'Update the previous log entries with our new system_id';
            $log->command = $sql;
            $command_log_id = discovery_log($log);
            $command_start = microtime(true);
            $query = $CI->db->query($sql);
            $command_end = microtime(true);
            $log->command = $CI->db->last_query();
            $log->command_time_to_execute = $command_end - $command_start;
            $log->id = $command_log_id;
            discovery_log($log);
            unset($log->title, $log->message, $log->command, $log->command_time_to_execute, $log->command_error_message);
            unset($log->id, $command_log_id);
        }

        // grab some timestamps
        $device->last_seen = $CI->m_devices_components->read($device->id, 'y', 'system', '', 'last_seen');
        $device->first_seen = $CI->m_devices_components->read($device->id, 'y', 'system', '', 'first_seen');

        // Insert an audit log
        $user = '';
        if (!empty($CI->user->full_name)) {
            $user = $CI->user->full_name;
        }
        $CI->m_audit_log->create($device->id, $user, $device->last_seen_by, $device->audits_ip, '', '', $device->last_seen);
        unset($user);

        if (!empty($device->ip)) {
            $log->ip = $device->ip;
        }
        $log->discovery_id = $device->discovery_id;
        $log->file = 'discovery_helper';
        $log->function = 'discoveries';

        // update any network interfaces retrieved by SNMP
        if (isset($network_interfaces) and is_array($network_interfaces) and count($network_interfaces) > 0) {
            $log->message = 'Processing found network interfaces for ' . $device->ip . ' (System ID ' . $device->id . ')';
            #$log->command = json_encode($network_interfaces);
            $log->command_output = '';
            discovery_log($log);
            $parameters = new stdClass();
            $parameters->table = 'network';
            $parameters->details = $device;
            $parameters->input = $network_interfaces;
            $parameters->log = $log;
            $CI->m_devices_components->process_component($parameters);
        }

        // update any ip addresses retrieved by SNMP
        if (!empty($ip) and is_array($ip->item) and count($ip->item) > 0) {
            $log->ip = $device->ip;
            $log->discovery_id = $device->discovery_id;
            $log->file = 'discovery_helper';
            $log->function = 'discoveries';
            $log->command_status = 'notice';
            $log->message = 'Processing found ip addresses for ' . $device->ip . ' (System ID ' . $device->id . ')';
            #$log->command = json_encode($ip);
            $log->command_output = '';
            discovery_log($log);
            $parameters = new stdClass();
            $parameters->table = 'ip';
            $parameters->details = $device;
            $parameters->input = $ip->item;
            $parameters->log = $log;
            $CI->m_devices_components->process_component($parameters);
        }

        // create or update the entry in the ip table from non-SNMP data
        //     so our 'networks' endpoint and functions can find the device
        if (empty($ip)) {
            $log->command_status = 'notice';
            $log->message = 'Processing found ip addresses (non-snmp) for ' . $device->ip . ' (System ID ' . $device->id . ')';
            discovery_log($log);
            $item = new stdClass();
            $item->system_id = $device->id;
            $item->ip = $device->ip;
            $item->version = 4;
{'fraction_non_alphanumeric': 0.15916542045323823, 'fraction_numerical': 0.002962153456804325, 'mean_word_length': 1.885325478156246, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 1368, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4377564', 'n_tokens_mistral': 20489, 'n_tokens_neox': 19652, 'n_words': 4329}