<commit_before>import os, sys, math, random, os
from subprocess import *
 
files = os.listdir(sys.argv[1])
if sys.argv[1][-1] == "/":
	name = sys.argv[1][:-1].split("/")[-1]
else:
	name = sys.argv[1].split("/")[-1]

nodes = file(sys.argv[2]).readlines()
fno = int(math.ceil(max(1, float(len(files)) / len(nodes))))

print >> sys.stderr, "Dataset name: %s" % name
print >> sys.stderr, "%d files and %d nodes: %d files per node" %\
		(len(files), len(nodes), fno)

random.shuffle(nodes)
nodes = iter(nodes)
procs = []

while True:
	nset = [sys.argv[1] + "/" + f  for f  in files[:fno]]
	if not nset:
		break
	files = files[fno:]
	node = nodes.next().strip()
	
	if "REMOVE_FIRST" in os.environ:
		call(["ssh", node, "rm -Rf /var/disco/" + name])

	if call(["ssh", node, "mkdir /var/disco/%s" % name]):
		print >> sys.stderr, "Couldn't mkdir %s:/var/disco/%s" % (node, name)

	print >> sys.stderr, "Copying %d files to %s" % (len(nset), node)
	p = Popen(["scp"] + nset + ["%s:/var/disco/%s/" % (node, name)])
	procs.append((node, p, nset))

print >> sys.stderr, "Waiting for copyers to finish.."

for node, p, nset in procs:
	if p.wait():
		print >> sys.stderr, "Copying to %s failed" % node
	else:
		print >> sys.stderr, "%s ok" % node
		print "\n".join(("disco://%s/%s" % (node, f) for f in nset))

print >> sys.stderr, "Done"


<commit_msg>bugfix: Distrfiles.py didn't handle long paths correctly. Thanks to Lexiang for reporting the bug.<commit_after>import os, sys, math, random, os
from subprocess import *
 
files = os.listdir(sys.argv[1])
if sys.argv[1][-1] == "/":
	name = os.path.basename(sys.argv[1][:-1])
else:
	name = os.path.basename(sys.argv[1])

nodes = file(sys.argv[2]).readlines()
fno = int(math.ceil(max(1, float(len(files)) / len(nodes))))

print >> sys.stderr, "Dataset name: %s" % name
print >> sys.stderr, "%d files and %d nodes: %d files per node" %\
		(len(files), len(nodes), fno)

random.shuffle(nodes)
nodes = iter(nodes)
procs = []

while True:
	nset = [sys.argv[1] + "/" + f  for f  in files[:fno]]
	if not nset:
		break
	files = files[fno:]
	node = nodes.next().strip()
	
	if "REMOVE_FIRST" in os.environ:
		call(["ssh", node, "rm -Rf /var/disco/" + name])

	if call(["ssh", node, "mkdir /var/disco/%s" % name]):
		print >> sys.stderr, "Couldn't mkdir %s:/var/disco/%s" % (node, name)

	print >> sys.stderr, "Copying %d files to %s" % (len(nset), node)
	p = Popen(["scp"] + nset + ["%s:/var/disco/%s/" % (node, name)])
	procs.append((node, p, nset))

print >> sys.stderr, "Waiting for copyers to finish.."

for node, p, nset in procs:
	if p.wait():
		print >> sys.stderr, "Copying to %s failed" % node
	else:
		print >> sys.stderr, "%s ok" % node
		print "\n".join(("disco://%s/%s/%s" %
                        (node, name, os.path.basename(f)) for f in nset))

print >> sys.stderr, "Done"



{'fraction_non_alphanumeric': 0.21560283687943263, 'fraction_numerical': 0.0070921985815602835, 'mean_word_length': 3.617021276595745, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2313380', 'n_tokens_mistral': 1179, 'n_tokens_neox': 1132, 'n_words': 304}