<commit_before>#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The IP Module
======================

This module is consuming the global channel.

It first performs a regex to find IP addresses and then matches those IPs to
some configured ip ranges.

The list of IP ranges are expected to be in CIDR format (e.g. 192.168.0.0/16)
and should be defined in the config.cfg file, under the [IP] section

"""

import time
import re
import sys
from pubsublogger import publisher
from packages import Paste
from Helper import Process
from ipaddress import IPv4Network, IPv4Address


def search_ip(message):
    paste = Paste.Paste(message)
    content = paste.get_p_content()
    # regex to find IPs
    reg_ip = re.compile(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', flags=re.MULTILINE)
    # list of the regex results in the Paste, may be null
    results = reg_ip.findall(content)
    matching_ips = []

    for res in results:
        address = IPv4Address(res)
        for network in ip_networks:
            if address in network:
                matching_ips.append(address)

    if len(matching_ips) > 0:
        print('{} contains {} IPs'.format(paste.p_name, len(matching_ips)))
        publisher.warning('{} contains {} IPs'.format(paste.p_name, len(matching_ips)))

        #Tag message with IP
        msg = 'infoleak:automatic-detection="ip";{}'.format(message)
        p.populate_set_out(msg, 'Tags')<commit_msg>fix: [IPAdress] remove leading zeros #465<commit_after>#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The IP Module
======================

This module is consuming the global channel.

It first performs a regex to find IP addresses and then matches those IPs to
some configured ip ranges.

The list of IP ranges are expected to be in CIDR format (e.g. 192.168.0.0/16)
and should be defined in the config.cfg file, under the [IP] section

"""

import time
import re
import sys
from pubsublogger import publisher
from packages import Paste
from Helper import Process
from ipaddress import IPv4Network, IPv4Address


def search_ip(message):
    paste = Paste.Paste(message)
    content = paste.get_p_content()
    # regex to find IPs
    reg_ip = re.compile(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)', flags=re.MULTILINE)
    # list of the regex results in the Paste, may be null
    results = reg_ip.findall(content)
    matching_ips = []

    for ip in results:
        ip = '.'.join([str(int(x)) for x in ip.split('.')])
        address = IPv4Address(ip)
        for network in ip_networks:
            if address in network:
                matching_ips.append(address)

    if len(matching_ips) > 0:
        print('{} contains {} IPs'.format(paste.p_name, len(matching_ips)))
        publisher.warning('{} contains {} IPs'.format(paste.p_name, len(matching_ips)))

        #Tag message with IP
        msg = 'infoleak:automatic-detection="ip";{}'.format(message)
        p.populate_set_out(msg, 'Tags')
{'fraction_non_alphanumeric': 0.14828269484808454, 'fraction_numerical': 0.03203434610303831, 'mean_word_length': 3.409024745269287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '605797', 'n_tokens_mistral': 1069, 'n_tokens_neox': 1010, 'n_words': 294}