<commit_before>#! /usr/bin/env python
"""
Partition a graph.

% python scripts/partition-graph.py <base>

This will output many <base>.subset.N.pmap files.

Use '-h' for parameter help.
"""

import sys
import threading
import Queue
import gc
import os.path
import argparse

import khmer

DEFAULT_SUBSET_SIZE = int(1e5)
DEFAULT_N_THREADS = 4

###

def worker(q, basename, stop_big_traversals):
    while 1:
        try:
            (ht, n, start, stop) = q.get(False)
        except Queue.Empty:
            print 'exiting'
            return

        outfile = basename + '.subset.%d.pmap' % (n,)
        if os.path.exists(outfile):
            print 'SKIPPING', outfile, ' -- already exists'
            continue
        
        print 'starting:', basename, n

        # pay attention to stoptags when partitioning; take command line<commit_msg>Add virtual memory debugging support on Linux for partitioning code.<commit_after>#! /usr/bin/env python
"""
Partition a graph.

% python scripts/partition-graph.py <base>

This will output many <base>.subset.N.pmap files.

Use '-h' for parameter help.
"""

import sys
import threading
import Queue
import gc
import os.path
import argparse

# Debugging Support
import re
import platform
if "Linux" == platform.system( ):
    def __debug_vm_usage( msg ):
	print "===> DEBUG: " + msg
	for vmstat in re.findall( r".*Vm.*", file( "/proc/self/status" ).read( ) ):
	    print vmstat
else:
    def __debug_vm_usage( msg ): pass

import khmer

DEFAULT_SUBSET_SIZE = int(1e5)
DEFAULT_N_THREADS = 4

###

def worker(q, basename, stop_big_traversals):
    while 1:
        try:
            (ht, n, start, stop) = q.get(False)
        except Queue.Empty:
            print 'exiting'
            return

        outfile = basename + '.subset.%d.pmap' % (n,)
        if os.path.exists(outfile):
            print 'SKIPPING', outfile, ' -- already exists'
            continue
        
        print 'starting:', basename, n

        # pay attention to stoptags when partitioning; take command line
{'fraction_non_alphanumeric': 0.1123152709359606, 'fraction_numerical': 0.003940886699507389, 'mean_word_length': 2.582010582010582, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3596385', 'n_tokens_mistral': 665, 'n_tokens_neox': 642, 'n_words': 198}