<commit_before>#!/usr/bin/python

import argparse
import os
import cPickle as pickle
import sys

import cache


def _shift_file_left(shift_file, start_offset, length):
  """ Shifts the contents of an open file toward the beginning of that file.
  Args:
    start_offset: The offset in the file to start shifting at.
    length: How many bytes to shift everything after start_offset. """
  # Get size.
  shift_file.seek(0, 2)
  file_size = shift_file.tell()

  # Shift everything in chunks.
  chunk_size = 1000000000
  while start_offset < file_size:
    shift_file.seek(start_offset)
    chunk = shift_file.read(chunk_size)

    # Move it backwards.
    shift_file.seek(start_offset - length)
    shift_file.write(chunk)

    start_offset += len(chunk)

def repair_cache(cache_map, cache_data):
  """ Actually performs the repair of a particular cache.
  Args:
    cache_map: The location of the cache_map file.
    cache_data: The location of the cache_data file. """
  print "Loading cache map..."
  map_file = file(cache_map, "rb")
  map_data, offsets, free_start, free_end = pickle.load(map_file)

  # Go through each entry in the cache one by one. No data should be unnacounted
  # for.
  print "Analyzing cache..."
  data_file = file(cache_data, "r+b")

  # Start with the first element.
  sorted_offsets = sorted(offsets.keys())
  first_img_id = offsets[sorted_offsets[0]]
  label, name = first_img_id.split("_")
  offset, size = map_data[label][name]

  # Go through the entire thing, hopping from one element to the next.
  total_size = os.path.getsize(cache_data)
  while offset <= total_size:
    offset += size

    if offset == free_start:
      # We're at the free section. Skip to the end.
      offset = free_end
    if (offset > free_start and offset < free_end):
      # We somehow ended up in the middle of the free section.
      print "ERROR: Cache data extends into free section."
      # Fix it.
      free_start = offset
      offset = free_end
      print "(Repaired.)"

    # There should be another image here.
    if offset not in offsets:
      print "ERROR: Unexpected empty space in cache. (Repaired.)"
      space_start = offset
      # Scan forward until we find the end of the empty region.
      while offset not in offsets:
        offset += 1
      space_end = offset

      # Move everything to cover the empty region.
      _shift_file_left(data_file, space_end, space_end - space_start)

    # Go to the next image.<commit_msg>Fix issue with cache repairing.<commit_after>#!/usr/bin/python

import argparse
import os
import cPickle as pickle
import sys

import cache


def _shift_file_left(shift_file, start_offset, length):
  """ Shifts the contents of an open file toward the beginning of that file.
  Args:
    start_offset: The offset in the file to start shifting at.
    length: How many bytes to shift everything after start_offset. """
  # Get size.
  shift_file.seek(0, 2)
  file_size = shift_file.tell()

  # Shift everything in chunks.
  chunk_size = 1000000000
  while start_offset < file_size:
    shift_file.seek(start_offset)
    chunk = shift_file.read(chunk_size)

    # Move it backwards.
    shift_file.seek(start_offset - length)
    shift_file.write(chunk)

    start_offset += len(chunk)

def repair_cache(cache_map, cache_data):
  """ Actually performs the repair of a particular cache.
  Args:
    cache_map: The location of the cache_map file.
    cache_data: The location of the cache_data file. """
  print "Loading cache map..."
  map_file = file(cache_map, "rb")
  map_data, offsets, free_start, free_end = pickle.load(map_file)

  # Go through each entry in the cache one by one. No data should be unnacounted
  # for.
  print "Analyzing cache..."
  data_file = file(cache_data, "r+b")

  # Start with the first element.
  sorted_offsets = sorted(offsets.keys())
  first_img_id = offsets[sorted_offsets[0]]
  label, name = first_img_id.split("_")
  offset, size = map_data[label][name]

  # Go through the entire thing, hopping from one element to the next.
  total_size = os.path.getsize(cache_data)
  while offset <= total_size:
    offset += size

    if offset == free_start:
      # We're at the free section. Skip to the end.
      offset = free_end
    if (offset > free_start and offset < free_end):
      # We somehow ended up in the middle of the free section.
      print "ERROR: Cache data extends into free section."
      # Fix it.
      free_start = offset
      offset = free_end
      print "(Repaired.)"

    # There should be another image here.
    if offset not in offsets:
      print "ERROR: Unexpected empty space in cache."
      space_start = offset
      # Scan forward until we find the end of the empty region.
      while offset not in offsets:
        if offset >= total_size:
          # We read off the end of the cache, in which case we can just cut the
          # file here.
          data_file.seek(space_start)
          data_file.truncate()
          break

        offset += 1
      space_end = offset

      # Move everything to cover the empty region.
      _shift_file_left(data_file, space_end, space_end - space_start)
      print "(Repaired.)"

    # Go to the next image.
{'fraction_non_alphanumeric': 0.08134155744024672, 'fraction_numerical': 0.005397070161912105, 'mean_word_length': 3.041277258566978, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1352037', 'n_tokens_mistral': 1623, 'n_tokens_neox': 1555, 'n_words': 576}