<commit_before>import Message, { MessageConstructor, MessageParam, MessagePrefix } from './Message';
import { all as coreMessageTypes } from './MessageTypes';
import { ServerProperties, defaultServerProperties } from '../ServerProperties';
import { splitWithLimit } from '../Toolkit/StringTools';

export default function parseMessage(
  line: string,
  serverProperties: ServerProperties = defaultServerProperties,
  knownCommands: Map<string, MessageConstructor> = coreMessageTypes,
  isServer: boolean = false,
  nonConformingCommands: string[] = [],
): Message {
  const splitLine: string[] = line.split(' ');
  let token: string;

  let command: string | undefined;
  const params: MessageParam[] = [];
  let tags: Map<string, string> = new Map();
  let prefix: MessagePrefix | undefined;

  while (splitLine.length) {
    token = splitLine[0];
    if (token[0] === '@' && !tags && !command) {
      tags = parseTags(token.substr(1), tags);
    } else if (token[0] === ':') {
      if (!prefix && !command) {
        if (token.length > 1) {
          // Not an empty prefix
          prefix = parsePrefix(token.substr(1));
        }
      } else {
        params.push({
          value: splitLine.join(' ').substr(1),
          trailing: true,
        });
        break;
      }
    } else if (!command) {
      command = token.toUpperCase();
    } else {
      params.push({
        value: token,
        trailing: false,
      });
    }
    splitLine.shift();
  }

  if (!command) {
    throw new Error(`line without command received: ${line}`);
  }

  let message: Message;

  let messageClass: MessageConstructor = Message;
  if (knownCommands.has(command)) {
    messageClass = knownCommands.get(command)!;
  }

  // tslint:disable-next-line:no-inferred-empty-object-type
  message = new messageClass(
    command,
    params,
    tags,
    prefix,
    serverProperties,
    line,
    isServer,
    !nonConformingCommands.includes(command),
  );

  return message;
}

export function parsePrefix(raw: string): MessagePrefix {
  const [nick, hostName] = splitWithLimit(raw, '!', 2);
  if (hostName) {
    const [user, host] = splitWithLimit(hostName, '@', 2);
    if (host) {
      return { nick, user, host };
    } else {
      return { nick, host: user };
    }
  } else {
    return { nick };
  }
}

const tagUnescapeMap: { [char: string]: string } = {
  '\\': '\\',
  ':': ';',
  n: '\n',
  r: '\r',
  s: ' ',
};

export function parseTags(raw: string, map?: Map<string, string>): Map<string, string> {
  const tags: Map<string, string> = map || new Map();
  const tagStrings = raw.split(';');
  for (const tagString of tagStrings) {
    const [tagName, tagValue] = splitWithLimit(tagString, '=', 2);
    if (tagName === '') {
      continue; // Ignore empty tags: @ @; @x; etc.
    }
    // unescape according to http://ircv3.net/specs/core/message-tags-3.2.html#escaping-values
    tags.set(tagName, tagValue.replace(/\\([\\:nrs])/g, (_, match) => tagUnescapeMap[match]));
  }

  return tags;
}
<commit_msg>Revert "Ensure tags: Map<string, string> is always defined"<commit_after>import Message, { MessageConstructor, MessageParam, MessagePrefix } from './Message';
import { all as coreMessageTypes } from './MessageTypes';
import { ServerProperties, defaultServerProperties } from '../ServerProperties';
import { splitWithLimit } from '../Toolkit/StringTools';

export default function parseMessage(
	line: string,
	serverProperties: ServerProperties = defaultServerProperties,
	knownCommands: Map<string, MessageConstructor> = coreMessageTypes,
	isServer: boolean = false,
	nonConformingCommands: string[] = []
): Message {
	const splitLine: string[] = line.split(' ');
	let token: string;

	let command: string | undefined;
	const params: MessageParam[] = [];
	let tags: Map<string, string> | undefined;
	let prefix: MessagePrefix | undefined;

	while (splitLine.length) {
		token = splitLine[0];
		if (token[0] === '@' && !tags && !command) {
			tags = parseTags(token.substr(1));
		} else if (token[0] === ':') {
			if (!prefix && !command) {
				if (token.length > 1) { // Not an empty prefix
					prefix = parsePrefix(token.substr(1));
				}
			} else {
				params.push({
					value: splitLine.join(' ').substr(1),
					trailing: true
				});
				break;
			}
		} else if (!command) {
			command = token.toUpperCase();
		} else {
			params.push({
				value: token,
				trailing: false
			});
		}
		splitLine.shift();
	}

	if (!command) {
		throw new Error(`line without command received: ${line}`);
	}

	let message: Message;

	let messageClass: MessageConstructor = Message;
	if (knownCommands.has(command)) {
		messageClass = knownCommands.get(command)!;
	}

	// tslint:disable-next-line:no-inferred-empty-object-type
	message = new messageClass(command, params, tags, prefix, serverProperties, line, isServer, !nonConformingCommands.includes(command));

	return message;
}

export function parsePrefix(raw: string): MessagePrefix {
	const [nick, hostName] = splitWithLimit(raw, '!', 2);
	if (hostName) {
		const [user, host] = splitWithLimit(hostName, '@', 2);
		if (host) {
			return { nick, user, host };
		} else {
			return { nick, host: user };
		}
	} else {
		return { nick };
	}
}

const tagUnescapeMap: { [char: string]: string } = {
	'\\': '\\',
	':': ';',
	n: '\n',
	r: '\r',
	s: ' '
};

export function parseTags(raw: string): Map<string, string> | undefined {
	const tags: Map<string, string> = new Map();
	const tagStrings = raw.split(';');
	for (const tagString of tagStrings) {
		const [tagName, tagValue] = splitWithLimit(tagString, '=', 2);
		if (tagName === '') {
			continue; // Ignore empty tags: @ @; @x; etc.
		}
		// unescape according to http://ircv3.net/specs/core/message-tags-3.2.html#escaping-values
		tags.set(tagName, tagValue.replace(/\\([\\:nrs])/g, (_, match) => tagUnescapeMap[match]));
	}

	return tags.size > 0 ? tags : undefined;
}

{'fraction_non_alphanumeric': 0.16270783847980996, 'fraction_numerical': 0.004580929759077028, 'mean_word_length': 3.3959731543624163, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3075085', 'n_tokens_mistral': 1973, 'n_tokens_neox': 1883, 'n_words': 470}