<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import

import argparse
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os
import multiprocessing as mp
import tempfile
from contextlib import contextmanager
from functools import partial
import fitsio
import subprocess as sp
import shutil
import sys
import numpy as np
import logging

logger = mp.log_to_stderr()
logger.setLevel(logging.INFO)

class NullPool(object):
    def __init__(self, *args, **kwargs): pass

    def map(self, fn, args):
        return map(fn, args)

def pack_images(dirname, output_name, kind='png'):
    full_out_path = os.path.realpath(output_name)
    with change_directory(dirname):
        cmd = ['tar', 'czf', full_out_path]
        cmd.extend([fname for fname in os.listdir('.')
                    if '.png' in fname])
        str_cmd = map(str, cmd)
        logger.debug("Running command [{}]".format(' '.join(str_cmd)))
        sp.check_call(str_cmd)


def build_image((i, input_fname), outdir, frame_min=0.8, frame_max=1.2):
    output_fname = os.path.join(outdir,
                                '{:05d}_{}.png'.format(
                                    i,
                                    os.path.basename(input_fname)))
    logger.debug('Building image {} => {}'.format(input_fname,
                                                  output_fname))

    if os.path.isfile(output_fname):
        logger.debug("Image {} exists, skipping".format(output_fname))
        return

    fig, axis = plt.subplots(figsize=(8, 8))
    with fitsio.FITS(input_fname) as infile:
        image_data = infile[0].read()
        header = infile[0].read_header()

    if image_data.shape == (2048, 2088):
        overscan = image_data[4:, -15:].mean()
        logger.warning('Image is raw, subtracting overscan {}'.format(
            overscan)
        )
        image_data = image_data[:, 20:-20] - overscan

    med_image = np.median(image_data)
    z1, z2 = (med_image * frame_min, med_image * frame_max)
    axis.imshow(image_data, interpolation='None', origin='lower',
                cmap=plt.cm.afmhot, vmin=z1, vmax=z2)
    for dimension in ['xaxis', 'yaxis']:
        getattr(axis, dimension).set_visible(False)
    axis.set_title(header.get('image_id', None))
    fig.tight_layout()
    fig.savefig(output_fname, bbox_inches='tight')
    plt.close(fig)

def sort_images(images):
    logger.info('Sorting images by mjd')
    return sorted(images, key=lambda fname: fitsio.read_header(fname)['mjd'])

def generate_movie(image_directory, output_filename, fps=15):
    logger.info('Building movie file {}, fps {}'.format(
        output_filename, fps)
    )
    n_cpu = mp.cpu_count()
    with change_directory(image_directory):
        cmd = ['mencoder', 'mf://*.png', '-mf',
               'w=800:h=600:fps={}:type=png'.format(fps),
                '-ovc', 'x264', '-x264encopts',
                'crf=18:nofast_pskip:nodct_decimate:nocabac:global_header:threads={}'.format(n_cpu),
                '-of', 'lavf', '-lavfopts', 'format=mp4',
                '-o', output_filename]
        sp.check_call(cmd)<commit_msg>Extract function to compute image data<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import

import argparse
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import os
import multiprocessing as mp
import tempfile
from contextlib import contextmanager
from functools import partial
import fitsio
import subprocess as sp
import shutil
import sys
import numpy as np
import logging

logger = mp.log_to_stderr()
logger.setLevel(logging.INFO)

class NullPool(object):
    def __init__(self, *args, **kwargs): pass

    def map(self, fn, args):
        return map(fn, args)

def pack_images(dirname, output_name, kind='png'):
    full_out_path = os.path.realpath(output_name)
    with change_directory(dirname):
        cmd = ['tar', 'czf', full_out_path]
        cmd.extend([fname for fname in os.listdir('.')
                    if '.png' in fname])
        str_cmd = map(str, cmd)
        logger.debug("Running command [{}]".format(' '.join(str_cmd)))
        sp.check_call(str_cmd)


def extract_image_data(input_fname):
    with fitsio.FITS(input_fname) as infile:
        image_data = infile[0].read()
        header = infile[0].read_header()

    if image_data.shape == (2048, 2088):
        overscan = image_data[4:, -15:].mean()
        logger.warning('Image is raw, subtracting overscan {}'.format(
            overscan)
        )
        image_data = image_data[:, 20:-20] - overscan

    return header, image_data

def build_image((i, input_fname), outdir, frame_min=0.8, frame_max=1.2):
    output_fname = os.path.join(outdir,
                                '{:05d}_{}.png'.format(
                                    i,
                                    os.path.basename(input_fname)))
    logger.debug('Building image {} => {}'.format(input_fname,
                                                  output_fname))

    if os.path.isfile(output_fname):
        logger.debug("Image {} exists, skipping".format(output_fname))
        return

    fig, axis = plt.subplots(figsize=(8, 8))
    header, image_data = extract_image_data(input_fname)
    med_image = np.median(image_data)
    z1, z2 = (med_image * frame_min, med_image * frame_max)
    axis.imshow(image_data, interpolation='None', origin='lower',
                cmap=plt.cm.afmhot, vmin=z1, vmax=z2)
    for dimension in ['xaxis', 'yaxis']:
        getattr(axis, dimension).set_visible(False)
    axis.set_title(header.get('image_id', None))
    fig.tight_layout()
    fig.savefig(output_fname, bbox_inches='tight')
    plt.close(fig)

def sort_images(images):
    logger.info('Sorting images by mjd')
    return sorted(images, key=lambda fname: fitsio.read_header(fname)['mjd'])

def generate_movie(image_directory, output_filename, fps=15):
    logger.info('Building movie file {}, fps {}'.format(
        output_filename, fps)
    )
    n_cpu = mp.cpu_count()
    with change_directory(image_directory):
        cmd = ['mencoder', 'mf://*.png', '-mf',
               'w=800:h=600:fps={}:type=png'.format(fps),
                '-ovc', 'x264', '-x264encopts',
                'crf=18:nofast_pskip:nodct_decimate:nocabac:global_header:threads={}'.format(n_cpu),
                '-of', 'lavf', '-lavfopts', 'format=mp4',
                '-o', output_filename]
        sp.check_call(cmd)
{'fraction_non_alphanumeric': 0.13439218081857054, 'fraction_numerical': 0.014355528405620037, 'mean_word_length': 2.7855491329479767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2023516', 'n_tokens_mistral': 2203, 'n_tokens_neox': 2110, 'n_words': 446}