<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""A module to compress and sort legends exports from DF 0.40.09 and later.

.bmp converted to .png where possible.
Archive for Legends Viewer created if possible, or just
    compress the (huge) xml.
Sort files into region folder, with maps subfolders,
    and move to user content folder if found.
"""

from __future__ import print_function, unicode_literals, absolute_import

import os, zipfile, glob, subprocess
from . import paths
from .lnp import lnp

def get_region_info():
    """Returns a tuple of strings for an available region and date.
    Eg: ('region1', '00250-01-01')
    """
    files = [f for f in glob.glob(paths.get('df', 'region*-*')) if
             os.path.isfile(f)]
    if files:
        fname = files[0]
        idx = fname.index('-')
        date = fname[idx+1:idx+12]
        if date[6] == '-':
            date = fname[idx+1:idx+13]
        elif date[7] == '-':
            date = fname[idx+1:idx+14]
        return (fname[:idx], date)

def compress_bitmaps():
    """Compresses all bitmap maps."""
    try:
        from PIL import Image
    except ImportError:
        try:
            import Image
        except ImportError:
            print('Please install PIL or Pillow to compress bitmaps.')
            call_optipng()
    else:
        for fname in glob.glob(paths.get(
                'df', '-'.join(get_region_info()) + '-*.bmp')):
            f = Image.open(fname)
            f.save(fname[:-3] + 'png', format='PNG', optimize=True)
            os.remove(fname)

def call_optipng():
    """Calling optipng can work well, but isn't very portable."""
    if os.name == 'nt' and os.path.isfile(paths.get('df', 'optipng.exe')):
        print('Falling back to optipng for image compression.')
        for fname in glob.glob(paths.get(
                'df', '-'.join(get_region_info()) + '-*.bmp')):
            ret = subprocess.call([paths.get('df', 'optipng'), '-zc9', '-zm9',
                                   '-zs0', '-f0', fname],
                                  creationflags=0x00000008)
            if ret == 0:
                os.remove(fname)

def choose_region_map():
    """Returns the most-prefered region map available, or fallback."""
    pattern = paths.get('df', '-'.join(get_region_info()) + '-')
    for name in ('detailed', 'world_map'):
        for ext in ('.png', '.bmp'):
            if os.path.isfile(pattern + name + ext):
                return pattern + name + ext
    return pattern + 'world_map.bmp'

def create_archive():
    """Creates a legends archive, or zips the xml if files are missing."""
    pattern = paths.get('df', '-'.join(get_region_info()) + '-')
    l = [pattern + 'legends.xml', pattern + 'world_history.txt',
         choose_region_map(), pattern + 'world_sites_and_pops.txt']
    if all([os.path.isfile(f) for f in l]):
        with zipfile.ZipFile(pattern[:-1] + '_legends_archive.zip',
                             'w', zipfile.ZIP_DEFLATED) as zipped:
            for f in l:
                zipped.write(f, os.path.basename(f))
                os.remove(f)
    elif os.path.isfile(pattern + 'legends.xml'):
        with zipfile.ZipFile(pattern[:-1] + '_legends_xml.zip',
                             'w', zipfile.ZIP_DEFLATED) as zipped:
            zipped.write(pattern + 'legends.xml')
            os.remove(pattern + 'legends.xml')

def move_files():
    """Moves files to a subdir, and subdir to ../User Generated Content if
    that dir exists."""
    pattern = paths.get('df', '-'.join(get_region_info()))
    dirname = get_region_info()[0] + '_legends_exports'
    if os.path.isdir(os.path.join(lnp.BASEDIR, 'User Generated Content')):
        dirname = os.path.join(lnp.BASEDIR, 'User Generated Content', dirname)
    else:
        dirname = paths.get('df', dirname)
    for site_map in glob.glob(pattern + '-site_map-*'):
        os.renames(site_map, os.path.join(dirname, 'site_maps',
                                          os.path.basename(site_map)))
    maps = ('world_map', 'bm', 'detailed', 'dip', 'drn', 'el', 'elw',
            'evil', 'hyd', 'nob', 'rain', 'sal', 'sav', 'str', 'tmp',
            'trd', 'veg', 'vol')
    for m in maps:
        m = glob.glob(pattern + '-' + m + '.???')
        if m:
            os.renames(m[0], os.path.join(dirname, 'region_maps',
                                          os.path.basename(m[0])))
    for file in glob.glob(paths.get('df', get_region_info()[0] + '*')):
        if os.path.isfile(file):
            os.renames(file, os.path.join(dirname, os.path.basename(file)))
    for f in glob.glob(paths.get('df', '*color_key.txt')):
        os.remove(f)

def process_legends():
    """Process all legends exports in sets."""
    if lnp.df_info.version >= '0.40.09':
        i = 0
        while get_region_info():
            compress_bitmaps()
            create_archive()
            move_files()
            i += 1
        return i
    else:
        # TODO:  support legends exports from before 40.09 (best-effort)
        pass<commit_msg>Fix *another* path issue causing an infinite loop...<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""A module to compress and sort legends exports from DF 0.40.09 and later.

.bmp converted to .png where possible.
Archive for Legends Viewer created if possible, or just
    compress the (huge) xml.
Sort files into region folder, with maps subfolders,
    and move to user content folder if found.
"""

from __future__ import print_function, unicode_literals, absolute_import

import os, zipfile, glob, subprocess
from . import paths
from .lnp import lnp

def get_region_info():
    """Returns a tuple of strings for an available region and date.
    Eg: ('region1', '00250-01-01')
    """
    files = [f for f in glob.glob(paths.get('df', 'region*-*-??-??-*')) if
             os.path.isfile(f)]
    if files:
        fname = os.path.basename(files[0])
        idx = fname.index('-')
        date = fname[idx+1:idx+12]
        if date[6] == '-':
            date = fname[idx+1:idx+13]
        elif date[7] == '-':
            date = fname[idx+1:idx+14]
        return (fname[:idx], date)

def compress_bitmaps():
    """Compresses all bitmap maps."""
    try:
        from PIL import Image
    except ImportError:
        try:
            import Image
        except ImportError:
            print('Please install PIL or Pillow to compress bitmaps.')
            call_optipng()
    else:
        for fname in glob.glob(paths.get(
                'df', '-'.join(get_region_info()) + '-*.bmp')):
            f = Image.open(fname)
            f.save(fname[:-3] + 'png', format='PNG', optimize=True)
            os.remove(fname)

def call_optipng():
    """Calling optipng can work well, but isn't very portable."""
    if os.name == 'nt' and os.path.isfile(paths.get('df', 'optipng.exe')):
        print('Falling back to optipng for image compression.')
        for fname in glob.glob(paths.get(
                'df', '-'.join(get_region_info()) + '-*.bmp')):
            ret = subprocess.call([paths.get('df', 'optipng'), '-zc9', '-zm9',
                                   '-zs0', '-f0', fname],
                                  creationflags=0x00000008)
            if ret == 0:
                os.remove(fname)

def choose_region_map():
    """Returns the most-prefered region map available, or fallback."""
    pattern = paths.get('df', '-'.join(get_region_info()) + '-')
    for name in ('detailed', 'world_map'):
        for ext in ('.png', '.bmp'):
            if os.path.isfile(pattern + name + ext):
                return pattern + name + ext
    return pattern + 'world_map.bmp'

def create_archive():
    """Creates a legends archive, or zips the xml if files are missing."""
    pattern = paths.get('df', '-'.join(get_region_info()) + '-')
    l = [pattern + 'legends.xml', pattern + 'world_history.txt',
         choose_region_map(), pattern + 'world_sites_and_pops.txt']
    if all([os.path.isfile(f) for f in l]):
        with zipfile.ZipFile(pattern[:-1] + '_legends_archive.zip',
                             'w', zipfile.ZIP_DEFLATED) as zipped:
            for f in l:
                zipped.write(f, os.path.basename(f))
                os.remove(f)
    elif os.path.isfile(pattern + 'legends.xml'):
        with zipfile.ZipFile(pattern[:-1] + '_legends_xml.zip',
                             'w', zipfile.ZIP_DEFLATED) as zipped:
            zipped.write(pattern + 'legends.xml')
            os.remove(pattern + 'legends.xml')

def move_files():
    """Moves files to a subdir, and subdir to ../User Generated Content if
    that dir exists."""
    pattern = paths.get('df', '-'.join(get_region_info()))
    dirname = get_region_info()[0] + '_legends_exports'
    if os.path.isdir(os.path.join(lnp.BASEDIR, 'User Generated Content')):
        dirname = os.path.join(lnp.BASEDIR, 'User Generated Content', dirname)
    else:
        dirname = paths.get('df', dirname)
    for site_map in glob.glob(pattern + '-site_map-*'):
        os.renames(site_map, os.path.join(dirname, 'site_maps',
                                          os.path.basename(site_map)))
    maps = ('world_map', 'bm', 'detailed', 'dip', 'drn', 'el', 'elw',
            'evil', 'hyd', 'nob', 'rain', 'sal', 'sav', 'str', 'tmp',
            'trd', 'veg', 'vol')
    for m in maps:
        m = glob.glob(pattern + '-' + m + '.???')
        if m:
            os.renames(m[0], os.path.join(dirname, 'region_maps',
                                          os.path.basename(m[0])))
    for file in glob.glob(paths.get('df', get_region_info()[0] + '*')):
        if os.path.isfile(file):
            os.renames(file, os.path.join(dirname, os.path.basename(file)))
    for f in glob.glob(paths.get('df', '*color_key.txt')):
        os.remove(f)

def process_legends():
    """Process all legends exports in sets."""
    if lnp.df_info.version >= '0.40.09':
        i = 0
        while get_region_info():
            compress_bitmaps()
            create_archive()
            move_files()
            i += 1
        return i
{'fraction_non_alphanumeric': 0.15424317617866004, 'fraction_numerical': 0.0115136476426799, 'mean_word_length': 2.168553459119497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2327537', 'n_tokens_mistral': 3315, 'n_tokens_neox': 3185, 'n_words': 836}