<commit_before>#!/usr/bin/python
# This fixes some of the image URLs in records.pickle.
import record
import re
import cPickle
rs = record.AllRecords()

for idx, r in enumerate(rs):
  url = r.photo_url

  # Some images have thumbnails but are missing the full photo URL.
  # Convert
  #    http://webbie1.sfpl.org/multimedia/thumbnails/aaa9774_x.jpg
  # -> http://webbie1.sfpl.org/multimedia/sfphotos/aaa-9774.jpg
  if not url:
    url = r.thumbnail_url
    if 'aaf' in url:
      # There are only two of these:
      #   http://sflib1.sfpl.org:82/record=b1026391~S0
      #   http://sflib1.sfpl.org:82/record=b1036043~S0
      # They're both links to larger collections of images and don't really fit
      # the mold of the other pages. So we remove them.
      del rs[idx]
      continue
    else:
      # Two of these... I think they're just omissions.
      url = re.sub(r'(.*)/thumbnails/(...)(\d+)_x\.jpg', r'\1/sfphotos/\2-\3.jpg', url)

  # Remove trailing spaces from image URLs.
  # Maybe ~4 of these.
  if url[-1] == ' ':
    url = url[0:-1]

  # Change 'foojpg' -> 'foo.jpg'
  # Just a typo. There are ~4 of these, too.
  if url[-3:] == 'jpg' and url[-4:] != '.jpg':
    url = url[:-3] + '.jpg'

print "Re-pickling"
output_file = "records.pickle"
f = file(output_file, "w")
p = cPickle.Pickler(f, 2)
for r in rs:
  count += 1
  p.dump(r)
  if count % 100 == 0:
    print "Pickled %d records" % count<commit_msg>fix bug<commit_after>#!/usr/bin/python
# This fixes some of the image URLs in records.pickle.
import record
import re
import cPickle
rs = record.AllRecords()

for idx, r in enumerate(rs):
  url = r.photo_url

  # Some images have thumbnails but are missing the full photo URL.
  # Convert
  #    http://webbie1.sfpl.org/multimedia/thumbnails/aaa9774_x.jpg
  # -> http://webbie1.sfpl.org/multimedia/sfphotos/aaa-9774.jpg
  if not url:
    url = r.thumbnail_url
    if 'aaf' in url:
      # There are only two of these:
      #   http://sflib1.sfpl.org:82/record=b1026391~S0
      #   http://sflib1.sfpl.org:82/record=b1036043~S0
      # They're both links to larger collections of images and don't really fit
      # the mold of the other pages. So we remove them.
      del rs[idx]
      continue
    else:
      # Two of these... I think they're just omissions.
      url = re.sub(r'(.*)/thumbnails/(...)(\d+)_x\.jpg', r'\1/sfphotos/\2-\3.jpg', url)

  # Remove trailing spaces from image URLs.
  # Maybe ~4 of these.
  if url[-1] == ' ':
    url = url[0:-1]

  # Change 'foojpg' -> 'foo.jpg'
  # Just a typo. There are ~4 of these, too.
  if url[-3:] == 'jpg' and url[-4:] != '.jpg':
    url = url[:-3] + '.jpg'

print "Re-pickling"
output_file = "records.pickle"
f = file(output_file, "w")
p = cPickle.Pickler(f, 2)
for i, r in enumerate(rs):
  p.dump(r)
  if i % 100 == 0:
    print "Pickled %d records" % i
{'fraction_non_alphanumeric': 0.15132275132275133, 'fraction_numerical': 0.03421516754850088, 'mean_word_length': 2.673575129533679, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7412439', 'n_tokens_mistral': 1147, 'n_tokens_neox': 1050, 'n_words': 309}