<commit_before>#!/usr/bin/env python

import optparse
import os
import sys

import starbound

def main():
    p = optparse.OptionParser()
    p.add_option('-d', '--destination', dest='path',
                 help='Destination directory')
    options, arguments = p.parse_args()

    if len(arguments) != 1:
        raise ValueError('Only one argument is supported (package path)')
    package_path = arguments[0]

    base = options.path if options.path else '.'

    with starbound.open_file(package_path) as package:
        if not isinstance(package, starbound.Package):
            raise ValueError('Provided path is not a package')

        print 'Loading index...'

        # Get the paths from the index in the database.
        paths = list(package.get_index())

        print 'Index loaded. Extracting %d files...' % len(paths)

        num_files = 0
        percentage_count = len(paths) // 100

        for path in paths:
            dest_path = base + path

            dir_path = os.path.dirname(dest_path)
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)

            try:
                data = package.get(path)
            except:
                print
                print 'Failed to read', path
                continue

            with open(dest_path, 'w') as file:
                file.write(data)

            num_files += 1
            if not num_files % percentage_count:
                sys.stdout.write('.')
                sys.stdout.flush()
<commit_msg>export: Don't crash when <100 files in .pak<commit_after>#!/usr/bin/env python

import optparse
import os
import sys

import starbound

def main():
    p = optparse.OptionParser()
    p.add_option('-d', '--destination', dest='path',
                 help='Destination directory')
    options, arguments = p.parse_args()

    if len(arguments) != 1:
        raise ValueError('Only one argument is supported (package path)')
    package_path = arguments[0]

    base = options.path if options.path else '.'

    with starbound.open_file(package_path) as package:
        if not isinstance(package, starbound.Package):
            raise ValueError('Provided path is not a package')

        print 'Loading index...'

        # Get the paths from the index in the database.
        paths = list(package.get_index())

        print 'Index loaded. Extracting %d files...' % len(paths)

        num_files = 0
        percentage_count = max(len(paths) // 100, 1)

        for path in paths:
            dest_path = base + path

            dir_path = os.path.dirname(dest_path)
            if not os.path.exists(dir_path):
                os.makedirs(dir_path)

            try:
                data = package.get(path)
            except:
                print
                print 'Failed to read', path
                continue

            with open(dest_path, 'w') as file:
                file.write(data)

            num_files += 1
            if not num_files % percentage_count:
                sys.stdout.write('.')
                sys.stdout.flush()

{'fraction_non_alphanumeric': 0.09993489583333333, 'fraction_numerical': 0.005859375, 'mean_word_length': 1.8089579524680073, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2576935', 'n_tokens_mistral': 872, 'n_tokens_neox': 821, 'n_words': 246}