<commit_before>'use strict';

const word = '[a-fA-F\\d:]';
const b = options => options && options.includeBoundaries ?
	`(?:(?<=\\s|^)(?=${word})|(?<=${word})(?=\\s|$))` :
	'';

const v4 = '(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3}';

const v6seg = '[a-fA-F\\d]{1,4}';
const v6 = `
(
(?:${v6seg}:){7}(?:${v6seg}|:)|                                // 1:2:3:4:5:6:7::  1:2:3:4:5:6:7:8
(?:${v6seg}:){6}(?:${v4}|:${v6seg}|:)|                         // 1:2:3:4:5:6::    fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b   fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b  fc00:e968:6179::de52:7100.2.3.4
(?:${v6seg}:){5}(?::${v4}|(:${v6seg}){1,2}|:)|                 // 1:2:3:4:5::      fc00:db20:35b:7399::5:8   fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b    fc00:db20:35b:7399::5:1.2.3.4
(?:${v6seg}:){4}(?:(:${v6seg}){0,1}:${v4}|(:${v6seg}){1,3}|:)| // 1:2:3:4::        fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:7:8   fc00:db20:35b:7399::5      fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:7:1.2.3.4
(?:${v6seg}:){3}(?:(:${v6seg}){0,2}:${v4}|(:${v6seg}){1,4}|:)| // 1:2:3::          fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:6:7:8   fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b        fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:6:7:1.2.3.4
(?:${v6seg}:){2}(?:(:${v6seg}){0,3}:${v4}|(:${v6seg}){1,5}|:)| // 1:2::            fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:5:6:7:8   fc00:db20:35b:7399::5          fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:5:6:7:1.2.3.4
(?:${v6seg}:){1}(?:(:${v6seg}){0,4}:${v4}|(:${v6seg}){1,6}|:)| // 1::              fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:4:5:6:7:8   fc00:db20:35b:7399::5            fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:4:5:6:7:1.2.3.4
(?::((?::${v6seg}){0,5}:${v4}|(?::${v6seg}){1,7}|:))           // fdf8:f53e:61e4::18:4:5:6:7:8  fdf8:f53e:61e4::18:4:5:6:7:8  fc00:e968:6179::de52:7100             ::1.2.3.4
)(%[0-9a-zA-Z]{1,})?                                           // %eth0            %1
`.replace(/\s*\/\/.*$/gm, '').replace(/\n/g, '').trim();

// Pre-compile only the exact regexes because adding a global flag make regexes stateful
const v46Exact = new RegExp(`(?:^${v4}$)|(?:^${v6}$)`);
const v4exact = new RegExp(`^${v4}$`);
const v6exact = new RegExp(`^${v6}$`);

const ip = options => options && options.exact ?
	v46Exact :
	new RegExp(`(?:${b(options)}${v4}${b(options)})|(?:${b(options)}${v6}${b(options)})`, 'g');

ip.v4 = options => options && options.exact ? v4exact : new RegExp(`${b(options)}${v4}${b(options)}`, 'g');
ip.v6 = options => options && options.exact ? v6exact : new RegExp(`${b(options)}${v6}${b(options)}`, 'g');

module.exports = ip;
<commit_msg>Remove empty captured groups from match (#31)<commit_after>'use strict';

const word = '[a-fA-F\\d:]';
const b = options => options && options.includeBoundaries ?
	`(?:(?<=\\s|^)(?=${word})|(?<=${word})(?=\\s|$))` :
	'';

const v4 = '(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)(?:\\.(?:25[0-5]|2[0-4]\\d|1\\d\\d|[1-9]\\d|\\d)){3}';

const v6seg = '[a-fA-F\\d]{1,4}';
const v6 = `
(?:
(?:${v6seg}:){7}(?:${v6seg}|:)|                                    // 1:2:3:4:5:6:7::  1:2:3:4:5:6:7:8
(?:${v6seg}:){6}(?:${v4}|:${v6seg}|:)|                             // 1:2:3:4:5:6::    fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b   fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b  1:2:3:4:5:6::1.2.3.4
(?:${v6seg}:){5}(?::${v4}|(?::${v6seg}){1,2}|:)|                   // 1:2:3:4:5::      fc00:db20:35b:7399::5:8   fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b    fc00:db20:35b:7399::5:1.2.3.4
(?:${v6seg}:){4}(?:(?::${v6seg}){0,1}:${v4}|(?::${v6seg}){1,3}|:)| // 1:2:3:4::        fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:7:8   fc00:db20:35b:7399::5      fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:7:1.2.3.4
(?:${v6seg}:){3}(?:(?::${v6seg}){0,2}:${v4}|(?::${v6seg}){1,4}|:)| // 1:2:3::          fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:6:7:8   fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b        fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:6:7:1.2.3.4
(?:${v6seg}:){2}(?:(?::${v6seg}){0,3}:${v4}|(?::${v6seg}){1,5}|:)| // 1:2::            fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:5:6:7:8   fc00:db20:35b:7399::5          fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:5:6:7:1.2.3.4
(?:${v6seg}:){1}(?:(?::${v6seg}){0,4}:${v4}|(?::${v6seg}){1,6}|:)| // 1::              fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:4:5:6:7:8   fc00:db20:35b:7399::5            fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b:4:5:6:7:1.2.3.4
(?::(?:(?::${v6seg}){0,5}:${v4}|(?::${v6seg}){1,7}|:))             // fdf8:f53e:61e4::18:4:5:6:7:8  fdf8:f53e:61e4::18:4:5:6:7:8  fc00:e968:6179::de52:7100             ::1.2.3.4
)(?:%[0-9a-zA-Z]{1,})?                                             // %eth0            %1
`.replace(/\s*\/\/.*$/gm, '').replace(/\n/g, '').trim();

// Pre-compile only the exact regexes because adding a global flag make regexes stateful
const v46Exact = new RegExp(`(?:^${v4}$)|(?:^${v6}$)`);
const v4exact = new RegExp(`^${v4}$`);
const v6exact = new RegExp(`^${v6}$`);

const ip = options => options && options.exact ?
	v46Exact :
	new RegExp(`(?:${b(options)}${v4}${b(options)})|(?:${b(options)}${v6}${b(options)})`, 'g');

ip.v4 = options => options && options.exact ? v4exact : new RegExp(`${b(options)}${v4}${b(options)}`, 'g');
ip.v6 = options => options && options.exact ? v6exact : new RegExp(`${b(options)}${v6}${b(options)}`, 'g');

module.exports = ip;

{'fraction_non_alphanumeric': 0.33570492417150344, 'fraction_numerical': 0.1980902452724209, 'mean_word_length': 4.04438149197356, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 14, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '689506', 'n_tokens_mistral': 3208, 'n_tokens_neox': 2877, 'n_words': 236}