<commit_before>import socket
import io
import struct
import time
from twisted.internet import reactor
from twisted.internet.endpoints import TCP4ServerEndpoint

from twisted.python import log

from config import bindIp as interface_ip

import blocks


blockShipList = {
    12100: "172.16.31.10",
    12200: "172.16.17.32",
    12300: "192.168.3.11",
    12400: "172.16.17.32",
    12500: "172.16.17.32",
    12600: "192.168.127.12",
    12700: "192.168.3.11",
    12800: "172.16.17.32",
    12900: "172.16.31.10",
    12000: "172.16.31.10",
}

queryShipArr = [(12199, "172.16.31.10"),
                (12299, "172.16.17.32"),
                (12399, "192.168.3.11"),
                (12499, "172.16.17.32"),
                (12599, "172.16.17.32"),
                (12699, "192.168.127.12"),
                (12799, "192.168.3.11"),
                (12899, "172.16.17.32"),
                (12999, "172.16.31.10"),
                (12099, "172.16.31.10")]
curShipIndex = 0


def get_ship_from_port(port):
    port_string = str(port)
    ship_num = int(port_string[2])
    if ship_num == 0:
        ship_num = 10
    return ship_num

cachedBlocks = {}


def get_first_block(ship_port, destination_ip):
    if ship_port not in blockShipList:
        return None

    if ship_port not in cachedBlocks:
        cachedBlocks[ship_port] = {'time_scraped': time.time(), 'data': scrape_block_packet(blockShipList[ship_port], ship_port, destination_ip)}
        print("[BlockCache] Cached new block for ship %i, Holding onto it for a minute..." % ship_port)
        return cachedBlocks[ship_port]['data']

    last_time = cachedBlocks[ship_port]['time_scraped']
    current_time = time.time()
    if current_time > last_time + 60:
        cachedBlocks[ship_port] = {'time_scraped': time.time(), 'data': scrape_block_packet(blockShipList[ship_port], ship_port, destination_ip)}
        print("[BlockCache] Cached new block for ship %i, Holding onto it for a minute..." % ship_port)
    return cachedBlocks[ship_port]['data']


def get_ship_query(my_ip_address):
    global curShipIndex, queryShipArr
    ship_port, ship_address = queryShipArr[curShipIndex]
    data = scrape_ship_packet(ship_address, ship_port, my_ip_address)
    curShipIndex += 1
    if curShipIndex > len(queryShipArr):
        curShipIndex = 0
    return data


def scrape_block_packet(ship_ip, ship_port, destination_ip):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    log.msg("[BlockQuery] Scraping %s:%i for a initial block..." % (ship_ip, ship_port))
    s.connect((ship_ip, ship_port))
    data = io.BytesIO()
    data.write(s.recv(4))
    actual_size = struct.unpack_from('i', data.getvalue(), 0x0)[0]
    data.write(s.recv(actual_size - 4))
    s.close()
    data.flush()
    data = bytearray(data.getvalue())
    name = data[0x24:0x64].decode('utf-16le')
    o1, o2, o3, o4, port = struct.unpack_from('BBBBH', buffer(data), 0x64)
    ip_string = '%i.%i.%i.%i' % (o1, o2, o3, o4)
    if port not in blocks.blockList:
        log.msg("[BlockList] Discovered new block %s at addr %s:%i! Recording..." % (name.encode('ascii', errors='xmlcharrefreplace'), ip_string, port))
        blocks.blockList[port] = (ip_string, name)<commit_msg>Fix off by one error.<commit_after>import socket
import io
import struct
import time
from twisted.internet import reactor
from twisted.internet.endpoints import TCP4ServerEndpoint

from twisted.python import log

from config import bindIp as interface_ip

import blocks


blockShipList = {
    12100: "172.16.31.10",
    12200: "172.16.17.32",
    12300: "192.168.3.11",
    12400: "172.16.17.32",
    12500: "172.16.17.32",
    12600: "192.168.127.12",
    12700: "192.168.3.11",
    12800: "172.16.17.32",
    12900: "172.16.31.10",
    12000: "172.16.31.10",
}

queryShipArr = [(12199, "172.16.31.10"),
                (12299, "172.16.17.32"),
                (12399, "192.168.3.11"),
                (12499, "172.16.17.32"),
                (12599, "172.16.17.32"),
                (12699, "192.168.127.12"),
                (12799, "192.168.3.11"),
                (12899, "172.16.17.32"),
                (12999, "172.16.31.10"),
                (12099, "172.16.31.10")]
curShipIndex = 0


def get_ship_from_port(port):
    port_string = str(port)
    ship_num = int(port_string[2])
    if ship_num == 0:
        ship_num = 10
    return ship_num

cachedBlocks = {}


def get_first_block(ship_port, destination_ip):
    if ship_port not in blockShipList:
        return None

    if ship_port not in cachedBlocks:
        cachedBlocks[ship_port] = {'time_scraped': time.time(), 'data': scrape_block_packet(blockShipList[ship_port], ship_port, destination_ip)}
        print("[BlockCache] Cached new block for ship %i, Holding onto it for a minute..." % ship_port)
        return cachedBlocks[ship_port]['data']

    last_time = cachedBlocks[ship_port]['time_scraped']
    current_time = time.time()
    if current_time > last_time + 60:
        cachedBlocks[ship_port] = {'time_scraped': time.time(), 'data': scrape_block_packet(blockShipList[ship_port], ship_port, destination_ip)}
        print("[BlockCache] Cached new block for ship %i, Holding onto it for a minute..." % ship_port)
    return cachedBlocks[ship_port]['data']


def get_ship_query(my_ip_address):
    global curShipIndex, queryShipArr
    ship_port, ship_address = queryShipArr[curShipIndex]
    data = scrape_ship_packet(ship_address, ship_port, my_ip_address)
    curShipIndex += 1
    if curShipIndex >= len(queryShipArr):
        curShipIndex = 0
    return data


def scrape_block_packet(ship_ip, ship_port, destination_ip):
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    log.msg("[BlockQuery] Scraping %s:%i for a initial block..." % (ship_ip, ship_port))
    s.connect((ship_ip, ship_port))
    data = io.BytesIO()
    data.write(s.recv(4))
    actual_size = struct.unpack_from('i', data.getvalue(), 0x0)[0]
    data.write(s.recv(actual_size - 4))
    s.close()
    data.flush()
    data = bytearray(data.getvalue())
    name = data[0x24:0x64].decode('utf-16le')
    o1, o2, o3, o4, port = struct.unpack_from('BBBBH', buffer(data), 0x64)
    ip_string = '%i.%i.%i.%i' % (o1, o2, o3, o4)
    if port not in blocks.blockList:
        log.msg("[BlockList] Discovered new block %s at addr %s:%i! Recording..." % (name.encode('ascii', errors='xmlcharrefreplace'), ip_string, port))
        blocks.blockList[port] = (ip_string, name)
{'fraction_non_alphanumeric': 0.14438751746623196, 'fraction_numerical': 0.09874243129948766, 'mean_word_length': 3.0875634517766497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 40, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3994758', 'n_tokens_mistral': 2675, 'n_tokens_neox': 2289, 'n_words': 491}