<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import io
import math
import optparse
import os.path
import signal
import struct
import zlib

import starbound
import starbound.btreedb4


try:
    # Don't break on pipe signal.
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
except:
    # Probably a Windows machine.
    pass

# Override range with xrange when running Python 2.x.
try:
    range = xrange
except:
    pass



def main():
    p = optparse.OptionParser('Usage: %prog [options] <input file>')

    p.add_option('-f', '--force', dest='force',
                 action='store_true', default=False,
                 help='ignore some errors')

    p.add_option('-o', '--output', dest='output',
                 help='where to output repaired world (defaults to input file '
                      'path with .repaired added to the end)')

    # TODO
    #p.add_option('-r', '--replace', action='append',
    #             dest='replace', metavar='FROM,TO',
    #             help='replace one tile material with another')

    p.add_option('-w', '--blank-world', dest='world',
                 help='the blank .world file that was created in place of the '
                      '.fail one (for metadata recovery)')

    options, arguments = p.parse_args()

    if len(arguments) != 1:
        p.error('Incorrect number of arguments')

    try:
        world = starbound.open_file(arguments[0])
        if not isinstance(world, starbound.World):
            raise Exception('Not a world')
    except Exception as e:
        p.error('Could not open fail file (%s)' % e)

    if options.output:
        out_name = options.output
    else:
        out_name = arguments[0] + '.repaired'

    if os.path.isfile(out_name):
        if options.force:
            print 'warning: overwriting existing file'
        else:
            p.error('"%s" already exists' % out_name)

    if options.world:
        fail_name = os.path.basename(arguments[0])
        world_name = os.path.basename(options.world)

        if fail_name[:len(world_name)] != world_name:
            if options.force:
                print 'warning: .fail and .world filenames do not match'
            else:
                p.error('.fail and .world filenames do not match')

        try:
            blank_world = starbound.open_file(options.world)
        except Exception as e:
            p.error('Could not open blank world (%s)' % e)

    try:
        metadata, version = world.get_metadata()
    except Exception as e:
        if options.world:
            try:
                print 'warning: restoring metadata using blank world'
                metadata, version = blank_world.get_metadata()
            except Exception as e:
                p.error('Failed to restore metadata (%s)' % e)
        else:
            p.error('Metadata section is corrupt (%s)' % e)

    if version == 1:
        size = metadata['planet']['size']
    elif version in (2, 3):
        size = metadata['worldTemplate']['size']
    else:
        p.error('Unsupported metadata version %d' % version)

    regions_x = int(math.ceil(size[0] / 32))
    regions_y = int(math.ceil(size[1] / 32))
    print 'Attempting to recover %d×%d regions...' % (regions_x, regions_y)

    blocks_per_percent = world.num_blocks // 100
    entries_recovered = 0
    percent = 0

    # This dict will contain all the keys and their data.
    data = dict()

    # Find all leaves and try to read them individually.
    for index in range(world.num_blocks):
        if index % blocks_per_percent == 0:
            print '%d%% (%d entries recovered)' % (percent, entries_recovered)
            percent += 1

        block = world.get_block(index)
        if not isinstance(block, starbound.btreedb4.BTreeLeaf):
            continue
<commit_msg>repair: Actually restore metadata from the other world<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import io
import math
import optparse
import os.path
import signal
import struct
import zlib

import starbound
import starbound.btreedb4


try:
    # Don't break on pipe signal.
    signal.signal(signal.SIGPIPE, signal.SIG_DFL)
except:
    # Probably a Windows machine.
    pass

# Override range with xrange when running Python 2.x.
try:
    range = xrange
except:
    pass



def main():
    p = optparse.OptionParser('Usage: %prog [options] <input file>')

    p.add_option('-f', '--force', dest='force',
                 action='store_true', default=False,
                 help='ignore some errors')

    p.add_option('-o', '--output', dest='output',
                 help='where to output repaired world (defaults to input file '
                      'path with .repaired added to the end)')

    # TODO
    #p.add_option('-r', '--replace', action='append',
    #             dest='replace', metavar='FROM,TO',
    #             help='replace one tile material with another')

    p.add_option('-w', '--blank-world', dest='world',
                 help='the blank .world file that was created in place of the '
                      '.fail one (for metadata recovery)')

    options, arguments = p.parse_args()

    if len(arguments) != 1:
        p.error('Incorrect number of arguments')

    try:
        world = starbound.open_file(arguments[0])
        if not isinstance(world, starbound.World):
            raise Exception('Not a world')
    except Exception as e:
        p.error('Could not open fail file (%s)' % e)

    if options.output:
        out_name = options.output
    else:
        out_name = arguments[0] + '.repaired'

    if os.path.isfile(out_name):
        if options.force:
            print 'warning: overwriting existing file'
        else:
            p.error('"%s" already exists' % out_name)

    if options.world:
        fail_name = os.path.basename(arguments[0])
        world_name = os.path.basename(options.world)

        if fail_name[:len(world_name)] != world_name:
            if options.force:
                print 'warning: .fail and .world filenames do not match'
            else:
                p.error('.fail and .world filenames do not match')

        try:
            blank_world = starbound.open_file(options.world)
        except Exception as e:
            p.error('Could not open blank world (%s)' % e)

    # This dict will contain all the keys and their data.
    data = dict()

    try:
        metadata, version = world.get_metadata()
    except Exception as e:
        if options.world:
            try:
                print 'warning: restoring metadata using blank world'
                metadata, version = blank_world.get_metadata()
                data['\x00\x00\x00\x00\x00'] = blank_world.get_raw((0, 0, 0))
            except Exception as e:
                p.error('Failed to restore metadata (%s)' % e)
        else:
            p.error('Metadata section is corrupt (%s)' % e)

    if version == 1:
        size = metadata['planet']['size']
    elif version in (2, 3):
        size = metadata['worldTemplate']['size']
    else:
        p.error('Unsupported metadata version %d' % version)

    regions_x = int(math.ceil(size[0] / 32))
    regions_y = int(math.ceil(size[1] / 32))
    print 'Attempting to recover %d×%d regions...' % (regions_x, regions_y)

    blocks_per_percent = world.num_blocks // 100
    entries_recovered = 0
    percent = 0

    # Find all leaves and try to read them individually.
    for index in range(world.num_blocks):
        if index % blocks_per_percent == 0:
            print '%d%% (%d entries recovered)' % (percent, entries_recovered)
            percent += 1

        block = world.get_block(index)
        if not isinstance(block, starbound.btreedb4.BTreeLeaf):
            continue

{'fraction_non_alphanumeric': 0.11479028697571744, 'fraction_numerical': 0.007921049214387741, 'mean_word_length': 2.178704085843995, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3887068', 'n_tokens_mistral': 2299, 'n_tokens_neox': 2188, 'n_words': 677}