<commit_before># configure simulated AU for v1
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.root=V1
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.branch=3
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.depth=2
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.numFiles=4

# AU suitable for V3 testing
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.root=V3
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.branch=1
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.depth=0
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.fileTypes=17
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.numFiles=20
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.binFileSize=2048

# LOCKSS & LCAP tuning parameters
org.lockss.log.default.level=debug

#lockss config stuff
org.lockss.platform.diskSpacePaths=./

org.lockss.config.reloadInterval = 1m

#hasher settings
#org.lockss.hasher.numSteps
#org.lockss.hasher.priority
#org.lockss.hasher.stepBytes

#comm settings
org.lockss.ui.start=yes
org.lockss.proxy.start=no
org.lockss.comm.multicast.group=172.16.31.10
#org.lockss.comm.multicast.port=3456
org.lockss.comm.multicast.port=localIp
org.lockss.comm.unicast.port=1025
org.lockss.comm.multicast.verify=no

# lcap protocol settings
org.lockss.protocol.ttl=2
org.lockss.protocol.hashAlgorithm=SHA-1

# poll settings
org.lockss.poll.maxpolls=20
org.lockss.poll.quorum=3
org.lockss.poll.agreeVerify=10
org.lockss.poll.disagreeVerify=50
org.lockss.poll.voteMargin=51
org.lockss.poll.trustedWeight=350
org.lockss.poll.namepoll.deadline=5m

org.lockss.poll.contentpoll.min=4m
org.lockss.poll.contentpoll.max=12h

org.lockss.treewalk.initial.estimate=15m
org.lockss.treewalk.interval.min=1h
org.lockss.treewalk.interval.max=3h
org.lockss.treewalk.start.delay=10s
org.lockss.comm.router.beacon.interval=1m
org.lockss.baseau.toplevel.poll.interval.min=5m
org.lockss.baseau.toplevel.poll.interval.max=15m

# V3
org.lockss.poll.defaultPollProtocol=3
org.lockss.scomm.enabled=true
org.lockss.scomm.maxMessageSize=33554430
org.lockss.poll.v3.quorum=2
org.lockss.poll.v3.minPollSize=3
org.lockss.poll.v3.maxPollSize=3
org.lockss.poll.v3.minNominationSize=1
org.lockss.poll.v3.maxNominationSize=1
org.lockss.id.initialV3PeerList=\
    TCP:[127.0.0.1]:9720;\
    TCP:[127.0.0.1]:9721;\
    TCP:[1192.168.127.12]:9722;\
    TCP:[127.0.0.1]:9723

org.lockss.metrics.slowest.hashrate = 250
org.lockss.state.recall.delay=5m

org.lockss.crawler.startCrawlsInterval=5m
org.lockss.crawler.threadPool.max=3

# prod group modes
org.lockss.baseau.defaultFetchRateLimiterSource = plugin
org.lockss.crawler.maxRepairRate = 1000/1d
org.lockss.poll.v3.deleteExtraFiles = true
org.lockss.poll.v3.voteBlockThreshold = 0
org.lockss.repository.globalNodeCache.enabled = true
org.lockss.repository.globalNodeCache.size = 500
org.lockss.state.globalNodeCache.enabled = true
org.lockss.state.globalNodeCache.size = 500
org.lockss.scheduler.maxBackgroundLoad = 50
org.lockss.state.pollHistory.trimRewrite = true
org.lockss.treewalk.v1.mode = off

# Standard debugging settings
org.lockss.monitor.inputStreams=true
org.lockss.thread.hungThreadDump = true<commit_msg>set to keep all the URL lists in crawl status after the crawl finishes<commit_after># configure simulated AU for v1
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.root=V1
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.branch=3
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.depth=2
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V1.numFiles=4

# AU suitable for V3 testing
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.root=V3
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.branch=1
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.depth=0
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.fileTypes=17
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.numFiles=20
#org.lockss.au.org|lockss|plugin|simulated|SimulatedPlugin.root~V3.binFileSize=2048

# LOCKSS & LCAP tuning parameters
org.lockss.log.default.level=debug

#lockss config stuff
org.lockss.platform.diskSpacePaths=./

org.lockss.config.reloadInterval = 1m

#hasher settings
#org.lockss.hasher.numSteps
#org.lockss.hasher.priority
#org.lockss.hasher.stepBytes

#comm settings
org.lockss.ui.start=yes
org.lockss.proxy.start=no
org.lockss.comm.multicast.group=172.16.31.10
#org.lockss.comm.multicast.port=3456
org.lockss.comm.multicast.port=localIp
org.lockss.comm.unicast.port=1025
org.lockss.comm.multicast.verify=no

# lcap protocol settings
org.lockss.protocol.ttl=2
org.lockss.protocol.hashAlgorithm=SHA-1

# poll settings
org.lockss.poll.maxpolls=20
org.lockss.poll.quorum=3
org.lockss.poll.agreeVerify=10
org.lockss.poll.disagreeVerify=50
org.lockss.poll.voteMargin=51
org.lockss.poll.trustedWeight=350
org.lockss.poll.namepoll.deadline=5m

org.lockss.poll.contentpoll.min=4m
org.lockss.poll.contentpoll.max=12h

org.lockss.treewalk.initial.estimate=15m
org.lockss.treewalk.interval.min=1h
org.lockss.treewalk.interval.max=3h
org.lockss.treewalk.start.delay=10s
org.lockss.comm.router.beacon.interval=1m
org.lockss.baseau.toplevel.poll.interval.min=5m
org.lockss.baseau.toplevel.poll.interval.max=15m

# V3
org.lockss.poll.defaultPollProtocol=3
org.lockss.scomm.enabled=true
org.lockss.scomm.maxMessageSize=33554430
org.lockss.poll.v3.quorum=2
org.lockss.poll.v3.minPollSize=3
org.lockss.poll.v3.maxPollSize=3
org.lockss.poll.v3.minNominationSize=1
org.lockss.poll.v3.maxNominationSize=1
org.lockss.id.initialV3PeerList=\
    TCP:[127.0.0.1]:9720;\
    TCP:[127.0.0.1]:9721;\
    TCP:[127.0.0.1]:9722;\
    TCP:[127.0.0.1]:9723

org.lockss.metrics.slowest.hashrate = 250
org.lockss.state.recall.delay=5m

org.lockss.crawler.startCrawlsInterval=5m
org.lockss.crawler.threadPool.max=3

# prod group modes
org.lockss.baseau.defaultFetchRateLimiterSource = plugin
org.lockss.crawler.maxRepairRate = 1000/1d
org.lockss.poll.v3.deleteExtraFiles = true
org.lockss.poll.v3.voteBlockThreshold = 0
org.lockss.repository.globalNodeCache.enabled = true
org.lockss.repository.globalNodeCache.size = 500
org.lockss.state.globalNodeCache.enabled = true
org.lockss.state.globalNodeCache.size = 500
org.lockss.scheduler.maxBackgroundLoad = 50
org.lockss.state.pollHistory.trimRewrite = true
org.lockss.treewalk.v1.mode = off

# Standard debugging settings
org.lockss.monitor.inputStreams=true
org.lockss.thread.hungThreadDump = true

org.lockss.crawlStatus.keepUrls=all
{'fraction_non_alphanumeric': 0.13307416267942584, 'fraction_numerical': 0.04784688995215311, 'mean_word_length': 6.247020585048754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 10, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '363656', 'n_tokens_mistral': 2818, 'n_tokens_neox': 2668, 'n_words': 251}