<commit_before>import os
import sys
###
from collections import defaultdict
###
from prettyplotlib import plt
###
import boto
conn = boto.connect_s3(anon=True)
pds = conn.get_bucket('aws-publicdatasets')

# Get all segments
target = 'CC-MAIN-2014-35'
segments = list(pds.list('common-crawl/crawl-data/{}/segments/'.format(target), delimiter='/'))
# Record the total size and all file paths for the segments
files = dict(warc=[], wet=[], wat=[], segment=[x.name for x in segments])
size = dict(warc=[], wet=[], wat=[])

# Traverse each segment and all the files they contain
for i, segment in enumerate(segments):
  sys.stderr.write('\rProcessing segment {} of {}'.format(i, len(segments)))
  for ftype in ['warc', 'wat', 'wet']:
    for f in pds.list(segment.name + ftype + '/'):
      files[ftype].append(f.name)
      size[ftype].append(f.size)
sys.stderr.write('\n')

# Write total size and file paths to files
prefix = 'crawl_stats/{}/'.format(target)
if not os.path.exists(prefix):
  os.makedirs(prefix)
###
f = open(prefix + 'crawl.size', 'w')
for ftype, val in size.items():
  f.write('{}\t{}\n'.format(ftype, sum(val)))
f.close()
###
for ftype in files:
  f = open(prefix + '{}.path'.format(ftype), 'w')
  for fn in files[ftype]:
    f.write(fn + '\n')
  f.close()
###
# Kid friendly stats (i.e. console)
for ftype, fsize in size.items():
  sys.stderr.write('{} files contain {} bytes over {} files\n'.format(ftype.upper(), sum(fsize), len(files[ftype])))
###
# Note: you might want to run `gzip *.zip` before upload
# s3cmd put --acl-public *.path.gz s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-YYYY-WW/

###
# Plot
for ftype, fsize in size.items():
  if not fsize:
    continue
  plt.hist(fsize, bins=50)
  plt.xlabel('Size (bytes)')
  plt.ylabel('Count')<commit_msg>Fixing filename suffixes and adding additional info to crawl size files<commit_after>import os
import sys
###
from collections import defaultdict
###
from prettyplotlib import plt
###
import boto
conn = boto.connect_s3(anon=True)
pds = conn.get_bucket('aws-publicdatasets')

# Get all segments
target = 'CC-MAIN-2014-41'
segments = list(pds.list('common-crawl/crawl-data/{}/segments/'.format(target), delimiter='/'))
# Record the total size and all file paths for the segments
files = dict(warc=[], wet=[], wat=[], segment=[x.name for x in segments])
size = dict(warc=[], wet=[], wat=[])

# Traverse each segment and all the files they contain
for i, segment in enumerate(segments):
  sys.stderr.write('\rProcessing segment {} of {}'.format(i, len(segments)))
  for ftype in ['warc', 'wat', 'wet']:
    for f in pds.list(segment.name + ftype + '/'):
      files[ftype].append(f.name)
      size[ftype].append(f.size)
sys.stderr.write('\n')

# Write total size and file paths to files
prefix = 'crawl_stats/{}/'.format(target)
if not os.path.exists(prefix):
  os.makedirs(prefix)
###
f = open(prefix + 'crawl.size', 'w')
for ftype, val in size.items():
  f.write('{}\t{}\t{}\n'.format(ftype, sum(val), len(val)))
f.close()
###
for ftype in files:
  f = open(prefix + '{}.paths'.format(ftype), 'w')
  for fn in files[ftype]:
    f.write(fn + '\n')
  f.close()
###
# Kid friendly stats (i.e. console)
for ftype, fsize in size.items():
  sys.stderr.write('{} files contain {} bytes over {} files\n'.format(ftype.upper(), sum(fsize), len(files[ftype])))
###
# To upload to the correct spot on S3
# gzip *.paths
# s3cmd put --acl-public *.paths.gz s3://aws-publicdatasets/common-crawl/crawl-data/CC-MAIN-YYYY-WW/

###
# Plot
for ftype, fsize in size.items():
  if not fsize:
    continue
  plt.hist(fsize, bins=50)
  plt.xlabel('Size (bytes)')
  plt.ylabel('Count')
{'fraction_non_alphanumeric': 0.18530701754385964, 'fraction_numerical': 0.006304824561403508, 'mean_word_length': 4.334795321637427, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2232182', 'n_tokens_mistral': 1333, 'n_tokens_neox': 1300, 'n_words': 354}