<commit_before># relay.py: PyLink Relay plugin
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import pickle
import sched
import threading
import time
import string
from collections import defaultdict

import utils
from log import log

dbname = "pylinkrelay.db"
relayusers = defaultdict(dict)

def normalizeNick(irc, netname, nick, separator="/"):
    orig_nick = nick
    protoname = irc.proto.__name__
    maxnicklen = irc.maxnicklen
    if protoname == 'charybdis':
        # Charybdis doesn't allow / in usernames, and will quit with
        # a protocol violation if there is one.
        separator = separator.replace('/', '|')
        nick = nick.replace('/', '|')
    if nick.startswith(tuple(string.digits)):
        # On TS6 IRCd-s, nicks that start with 0-9 are only allowed if
        # they match the UID of the originating server. Otherwise, you'll
        # get nasty protocol violations!
        nick = '_' + nick
    tagnicks = True

    suffix = separator + netname
    nick = nick[:maxnicklen]
    # Maximum allowed length of a nickname.
    allowedlength = maxnicklen - len(suffix)
    # If a nick is too long, the real nick portion must be cut off, but the
    # /network suffix must remain the same.

    nick = nick[:allowedlength]
    nick += suffix
    while utils.nickToUid(irc, nick):
        # The nick we want exists? Darn, create another one then.
        # Increase the separator length by 1 if the user was already tagged,
        # but couldn't be created due to a nick conflict.
        # This can happen when someone steals a relay user's nick.
        new_sep = separator + separator[-1]
        nick = normalizeNick(irc, netname, orig_nick, separator=new_sep)
    finalLength = len(nick)
    assert finalLength <= maxnicklen, "Normalized nick %r went over max " \
        "nick length (got: %s, allowed: %s!" % (nick, finalLength, maxnicklen)

    return nick

def loadDB():
    global db
    try:
        with open(dbname, "rb") as f:
            db = pickle.load(f)
    except (ValueError, IOError):
        log.exception("Relay: failed to load links database %s"
            ", creating a new one in memory...", dbname)
        db = {}

def exportDB(scheduler):
    scheduler.enter(30, 1, exportDB, argument=(scheduler,))
    log.debug("Relay: exporting links database to %s", dbname)
    with open(dbname, 'wb') as f:
        pickle.dump(db, f, protocol=4)

def getPrefixModes(irc, remoteirc, channel, user):
    modes = ''
    for pmode in ('owner', 'admin', 'op', 'halfop', 'voice'):
        if pmode not in remoteirc.cmodes:  # Mode not supported by IRCd
            continue
        if user in irc.channels[channel].prefixmodes[pmode+'s']:
            modes += remoteirc.cmodes[pmode]
    return modes

def getRemoteUser(irc, remoteirc, user):
    # If the user (stored here as {(netname, UID):
    # {network1: UID1, network2: UID2}}) exists, don't spawn it
    # again!
    try:
        u = relayusers[(irc.name, user)][remoteirc.name]
    except KeyError:
        userobj = irc.users.get(user)
        if userobj is None:
            # The query wasn't actually a valid user... Oh well!
            return
        nick = normalizeNick(remoteirc, irc.name, userobj.nick)
        ident = userobj.ident
        host = userobj.host
        realname = userobj.realname
        u = remoteirc.proto.spawnClient(remoteirc, nick, ident=ident,
                                        host=host, realname=realname).uid
        remoteirc.users[u].remote = irc.name
    relayusers[(irc.name, user)][remoteirc.name] = u
    remoteirc.users[u].remote = irc.name
    return u

def getLocalUser(irc, user):
    # Our target is an internal client, which means someone
    # is kicking a remote user over the relay.
    # We have to find the real target for the KICK. This is like
    # findRemoteUser, but in reverse.
    # First, iterate over everyone!
    for k, v in relayusers.items():
        log.debug('(%s) getLocalUser: processing %s, %s in relayusers', irc.name, k, v)
        if k[0] == irc.name:
            # We don't need to do anything if the target users is on
            # the same network as us.
            log.debug('(%s) getLocalUser: skipping %s since the target network matches the source network.', irc.name, k)
            continue
        if v.get(irc.name) == user:
            # If the stored pseudoclient UID for the kicked user on
            # this network matches the target we have, set that user
            # as the one we're kicking! It's a handful, but remember
            # we're mapping (home network, UID) pairs to their
            # respective relay pseudoclients on other networks.
            remoteuser = k
            log.debug('(%s) getLocalUser: found %s to correspond to %s.', irc.name, v, k)
            return remoteuser

def findRelay(chanpair):
    if chanpair in db:  # This chanpair is a shared channel; others link to it
        return chanpair
    # This chanpair is linked *to* a remote channel
    for name, dbentry in db.items():
        if chanpair in dbentry['links']:
            return name

def findRemoteChan(irc, remoteirc, channel):
    query = (irc.name, channel)
    remotenetname = remoteirc.name
    chanpair = findRelay(query)
    if chanpair is None:
        return
    if chanpair[0] == remotenetname:
        return chanpair[1]
    else:
        for link in db[chanpair]['links']:
            if link[0] == remotenetname:
                return link[1]

def initializeChannel(irc, channel):
    # We're initializing a relay that already exists. This can be done at
    # ENDBURST, or on the LINK command.
    irc.proto.joinClient(irc, irc.pseudoclient.uid, channel)
    c = irc.channels[channel]
    relay = findRelay((irc.name, channel))
    log.debug('(%s) initializeChannel being called on %s', irc.name, channel)
    log.debug('(%s) initializeChannel: relay pair found to be %s', irc.name, relay)
    queued_users = []
    if relay:
        all_links = db[relay]['links'].copy()
        all_links.update((relay,))
        log.debug('(%s) initializeChannel: all_links: %s', irc.name, all_links)
        for link in all_links:
            modes = []
            remotenet, remotechan = link
            if remotenet == irc.name:
                continue
            remoteirc = utils.networkobjects[remotenet]
            rc = remoteirc.channels[remotechan]
            for user in remoteirc.channels[remotechan].users:
                # Don't spawn our pseudoclients again.
                if not utils.isInternalClient(remoteirc, user):
                    log.debug('(%s) initializeChannel: should be joining %s/%s to %s', irc.name, user, remotenet, channel)
                    remoteuser = getRemoteUser(remoteirc, irc, user)
                    userpair = (getPrefixModes(remoteirc, irc, remotechan, user), remoteuser)
                    log.debug('(%s) initializeChannel: adding %s to queued_users for %s', irc.name, userpair, channel)
                    queued_users.append(userpair)
            if queued_users:
                irc.proto.sjoinServer(irc, irc.sid, channel, queued_users, ts=rc.ts)
            relayModes(remoteirc, irc, remoteirc.sid, remotechan)
            relayModes(irc, remoteirc, irc.sid, channel)

    log.debug('(%s) initializeChannel: joining our users: %s', irc.name, c.users)
    relayJoins(irc, channel, c.users, c.ts, c.modes)
    topic = remoteirc.channels[relay[1]].topic
    # XXX: find a more elegant way to do this
    # Only update the topic if it's different from what we already have.
    if topic != irc.channels[channel].topic:
        irc.proto.topicServer(irc, irc.sid, channel, topic)

def handle_join(irc, numeric, command, args):
    channel = args['channel']
    if not findRelay((irc.name, channel)):
        # No relay here, return.
        return
    modes = args['modes']
    ts = args['ts']
    users = set(args['users'])
    # users.update(irc.channels[channel].users)
    relayJoins(irc, channel, users, ts, modes)
utils.add_hook(handle_join, 'JOIN')
<commit_msg>relay: missing line in initializeChannel<commit_after># relay.py: PyLink Relay plugin
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import pickle
import sched
import threading
import time
import string
from collections import defaultdict

import utils
from log import log

dbname = "pylinkrelay.db"
relayusers = defaultdict(dict)

def normalizeNick(irc, netname, nick, separator="/"):
    orig_nick = nick
    protoname = irc.proto.__name__
    maxnicklen = irc.maxnicklen
    if protoname == 'charybdis':
        # Charybdis doesn't allow / in usernames, and will quit with
        # a protocol violation if there is one.
        separator = separator.replace('/', '|')
        nick = nick.replace('/', '|')
    if nick.startswith(tuple(string.digits)):
        # On TS6 IRCd-s, nicks that start with 0-9 are only allowed if
        # they match the UID of the originating server. Otherwise, you'll
        # get nasty protocol violations!
        nick = '_' + nick
    tagnicks = True

    suffix = separator + netname
    nick = nick[:maxnicklen]
    # Maximum allowed length of a nickname.
    allowedlength = maxnicklen - len(suffix)
    # If a nick is too long, the real nick portion must be cut off, but the
    # /network suffix must remain the same.

    nick = nick[:allowedlength]
    nick += suffix
    while utils.nickToUid(irc, nick):
        # The nick we want exists? Darn, create another one then.
        # Increase the separator length by 1 if the user was already tagged,
        # but couldn't be created due to a nick conflict.
        # This can happen when someone steals a relay user's nick.
        new_sep = separator + separator[-1]
        nick = normalizeNick(irc, netname, orig_nick, separator=new_sep)
    finalLength = len(nick)
    assert finalLength <= maxnicklen, "Normalized nick %r went over max " \
        "nick length (got: %s, allowed: %s!" % (nick, finalLength, maxnicklen)

    return nick

def loadDB():
    global db
    try:
        with open(dbname, "rb") as f:
            db = pickle.load(f)
    except (ValueError, IOError):
        log.exception("Relay: failed to load links database %s"
            ", creating a new one in memory...", dbname)
        db = {}

def exportDB(scheduler):
    scheduler.enter(30, 1, exportDB, argument=(scheduler,))
    log.debug("Relay: exporting links database to %s", dbname)
    with open(dbname, 'wb') as f:
        pickle.dump(db, f, protocol=4)

def getPrefixModes(irc, remoteirc, channel, user):
    modes = ''
    for pmode in ('owner', 'admin', 'op', 'halfop', 'voice'):
        if pmode not in remoteirc.cmodes:  # Mode not supported by IRCd
            continue
        if user in irc.channels[channel].prefixmodes[pmode+'s']:
            modes += remoteirc.cmodes[pmode]
    return modes

def getRemoteUser(irc, remoteirc, user):
    # If the user (stored here as {(netname, UID):
    # {network1: UID1, network2: UID2}}) exists, don't spawn it
    # again!
    try:
        u = relayusers[(irc.name, user)][remoteirc.name]
    except KeyError:
        userobj = irc.users.get(user)
        if userobj is None:
            # The query wasn't actually a valid user... Oh well!
            return
        nick = normalizeNick(remoteirc, irc.name, userobj.nick)
        ident = userobj.ident
        host = userobj.host
        realname = userobj.realname
        u = remoteirc.proto.spawnClient(remoteirc, nick, ident=ident,
                                        host=host, realname=realname).uid
        remoteirc.users[u].remote = irc.name
    relayusers[(irc.name, user)][remoteirc.name] = u
    remoteirc.users[u].remote = irc.name
    return u

def getLocalUser(irc, user):
    # Our target is an internal client, which means someone
    # is kicking a remote user over the relay.
    # We have to find the real target for the KICK. This is like
    # findRemoteUser, but in reverse.
    # First, iterate over everyone!
    for k, v in relayusers.items():
        log.debug('(%s) getLocalUser: processing %s, %s in relayusers', irc.name, k, v)
        if k[0] == irc.name:
            # We don't need to do anything if the target users is on
            # the same network as us.
            log.debug('(%s) getLocalUser: skipping %s since the target network matches the source network.', irc.name, k)
            continue
        if v.get(irc.name) == user:
            # If the stored pseudoclient UID for the kicked user on
            # this network matches the target we have, set that user
            # as the one we're kicking! It's a handful, but remember
            # we're mapping (home network, UID) pairs to their
            # respective relay pseudoclients on other networks.
            remoteuser = k
            log.debug('(%s) getLocalUser: found %s to correspond to %s.', irc.name, v, k)
            return remoteuser

def findRelay(chanpair):
    if chanpair in db:  # This chanpair is a shared channel; others link to it
        return chanpair
    # This chanpair is linked *to* a remote channel
    for name, dbentry in db.items():
        if chanpair in dbentry['links']:
            return name

def findRemoteChan(irc, remoteirc, channel):
    query = (irc.name, channel)
    remotenetname = remoteirc.name
    chanpair = findRelay(query)
    if chanpair is None:
        return
    if chanpair[0] == remotenetname:
        return chanpair[1]
    else:
        for link in db[chanpair]['links']:
            if link[0] == remotenetname:
                return link[1]

def initializeChannel(irc, channel):
    # We're initializing a relay that already exists. This can be done at
    # ENDBURST, or on the LINK command.
    irc.proto.joinClient(irc, irc.pseudoclient.uid, channel)
    c = irc.channels[channel]
    relay = findRelay((irc.name, channel))
    log.debug('(%s) initializeChannel being called on %s', irc.name, channel)
    log.debug('(%s) initializeChannel: relay pair found to be %s', irc.name, relay)
    queued_users = []
    if relay:
        all_links = db[relay]['links'].copy()
        all_links.update((relay,))
        log.debug('(%s) initializeChannel: all_links: %s', irc.name, all_links)
        for link in all_links:
            modes = []
            remotenet, remotechan = link
            if remotenet == irc.name:
                continue
            remoteirc = utils.networkobjects[remotenet]
            rc = remoteirc.channels[remotechan]
            for user in remoteirc.channels[remotechan].users:
                # Don't spawn our pseudoclients again.
                if not utils.isInternalClient(remoteirc, user):
                    log.debug('(%s) initializeChannel: should be joining %s/%s to %s', irc.name, user, remotenet, channel)
                    remoteuser = getRemoteUser(remoteirc, irc, user)
                    userpair = (getPrefixModes(remoteirc, irc, remotechan, user), remoteuser)
                    log.debug('(%s) initializeChannel: adding %s to queued_users for %s', irc.name, userpair, channel)
                    queued_users.append(userpair)
            if queued_users:
                irc.proto.sjoinServer(irc, irc.sid, channel, queued_users, ts=rc.ts)
            relayModes(remoteirc, irc, remoteirc.sid, remotechan)
            relayModes(irc, remoteirc, irc.sid, channel)

    log.debug('(%s) initializeChannel: joining our users: %s', irc.name, c.users)
    relayJoins(irc, channel, c.users, c.ts, c.modes)
    remoteirc = utils.networkobjects[relay[0]]
    topic = remoteirc.channels[relay[1]].topic
    # XXX: find a more elegant way to do this
    # Only update the topic if it's different from what we already have.
    if topic != irc.channels[channel].topic:
        irc.proto.topicServer(irc, irc.sid, channel, topic)

def handle_join(irc, numeric, command, args):
    channel = args['channel']
    if not findRelay((irc.name, channel)):
        # No relay here, return.
        return
    modes = args['modes']
    ts = args['ts']
    users = set(args['users'])
    # users.update(irc.channels[channel].users)
    relayJoins(irc, channel, users, ts, modes)
utils.add_hook(handle_join, 'JOIN')

{'fraction_non_alphanumeric': 0.10434622014282197, 'fraction_numerical': 0.0024008864811622755, 'mean_word_length': 2.539986925256047, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '182123', 'n_tokens_mistral': 5040, 'n_tokens_neox': 4817, 'n_words': 1600}