<commit_before>#!/usr/bin/env python3

import os
import sys
import traceback
import functools
from shutil import copyfile

import numpy as np
from PyQt4.QtCore import *
from PyQt4.QtGui import *

sys.path.append('..')
import qtutil
import tifffile as tiff

from .util import fileloader, fileconverter
from .util import mse_ui_elements as mue


class NotConvertedError(Exception):
  pass

class FileAlreadyInProjectError(Exception):
  def __init__(self, filename):
    self.filename = filename

class Widget(QWidget):
  def __init__(self, project, plugin_position, parent=None):
    super(Widget, self).__init__(parent)

    if not project:
      return

    self.plugin_position = plugin_position
    self.project = project
    self.setup_ui()
    if isinstance(plugin_position, int):
        self.params = project.pipeline[self.plugin_position]
        assert(self.params['name'] == 'fileimporter')
        self.setup_param_signals()
        self.setup_params()

    self.listview.setModel(QStandardItemModel())
    for f in self.project.files:
      if f['type'] == 'video':
        self.listview.model().appendRow(QStandardItem(f['path']))

  def setup_params(self):
    self.scale_factor.setValue(self.params['Scale Factor'])


  def setup_ui(self):
    vbox = QVBoxLayout()

    ## Related to importing Raws
    self.setWindowTitle('Import Raw File')

    #vbox.addWidget(QLabel('Set the size all data are to be rescaled to'))

    grid = QGridLayout()
    grid.addWidget(QLabel('Set channel and scale factor used for all imported data'), 0, 0)

    # grid.addWidget(QLabel('Width:'), 0, 0)
    # self.rescale_width = QSpinBox()
    # self.rescale_width.setMinimum(1)
    # self.rescale_width.setMaximum(1024)
    # self.rescale_width.setValue(256)
    # grid.addWidget(self.rescale_width, 0, 1)
    #
    # grid.addWidget(QLabel('Height:'), 1, 0)
    # self.rescale_height = QSpinBox()
    # self.rescale_height.setMinimum(1)
    # self.rescale_height.setMaximum(1024)
    # self.rescale_height.setValue(256)
    # grid.addWidget(self.rescale_height, 1, 1)

    grid.addWidget(QLabel('Scale Factor:'), 1, 0)
    self.scale_factor = QDoubleSpinBox()
    self.scale_factor.setSingleStep(0.25)
    self.scale_factor.setMinimum(0.25)
    self.scale_factor.setMaximum(1.00)
    self.scale_factor.setValue(1.0)
    grid.addWidget(self.scale_factor, 1, 1)

    grid.addWidget(QLabel('Channel:'), 2, 0)
    self.channel = QSpinBox()
    self.channel.setMinimum(1)
    self.channel.setMaximum(3)
    self.channel.setValue(2)
    grid.addWidget(self.channel, 2, 1)

    grid.addWidget(qtutil.separator(), 3, 0)
    grid.addWidget(qtutil.separator(), 3, 1)
    grid.addWidget(QLabel('Set paramaters for all imported raws'), 4, 0)
    grid.addWidget(QLabel('All raws will be rescaled and specified channel imported upon conversion'), 5, 0)
    grid.addWidget(QLabel('Width:'), 6, 0)
    self.sb_width = QSpinBox()
    self.sb_width.setMinimum(1)
    self.sb_width.setMaximum(1024)
    self.sb_width.setValue(256)
    grid.addWidget(self.sb_width, 6, 1)
    grid.addWidget(QLabel('Height:'), 7, 0)
    self.sb_height = QSpinBox()
    self.sb_height.setMinimum(1)
    self.sb_height.setMaximum(1024)
    self.sb_height.setValue(256)
    grid.addWidget(self.sb_height, 7, 1)
    grid.addWidget(QLabel('Number of channels:'), 8, 0)
    self.sb_channel = QSpinBox()
    self.sb_channel.setMinimum(1)
    self.sb_channel.setMaximum(3)
    self.sb_channel.setValue(3)
    grid.addWidget(self.sb_channel, 8, 1)
    grid.addWidget(QLabel('dtype:'), 9, 0)
    self.cb_dtype = QComboBox()
    for t in 'uint8', 'float32', 'float64':
      self.cb_dtype.addItem(t)
    grid.addWidget(self.cb_dtype, 9, 1)
    grid.addWidget(qtutil.separator(), 10, 0)
    grid.addWidget(qtutil.separator(), 10, 1)
    vbox.addLayout(grid)
    vbox.addStretch()

    self.setLayout(vbox)
    self.resize(400, 220)

    vbox.addWidget(mue.WarningWidget('Warning. This application has not yet been memory optimized for conversion.'
                                     ' We advise you only import files no larger than 1/4 of your memory'))
    pb = QPushButton('New Video')
    pb.clicked.connect(self.new_video)
    vbox.addWidget(pb)

    self.listview = QListView()
    self.listview.setStyleSheet('QListView::item { height: 26px; }')
    self.listview.setSelectionMode(QAbstractItemView.NoSelection)
    vbox.addWidget(self.listview)

    vbox.addStretch()
    self.setLayout(vbox)

  def setup_param_signals(self):
      self.scale_factor.valueChanged[float].connect(functools.partial(self.update_plugin_params, 'Scale Factor'))


  def update_plugin_params(self, key, val):
      self.params[key] = val
      self.project.pipeline[self.plugin_position] = self.params



  def convert_raw(self, filename):
    rescale_value = float(self.scale_factor.value())
    dtype = str(self.cb_dtype.currentText())
    width = int(self.sb_width.value())
    height = int(self.sb_height.value())
    rescale_width = int(width * rescale_value)
    rescale_height = int(height * rescale_value)
    channels = int(self.sb_channel.value())
    channel = int(self.channel.value())
    path = os.path.splitext(os.path.basename(filename))[0] + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting raw to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.raw2npy(filename, path, dtype, width, height,
        channels, channel, callback)
    except:
      qtutil.critical('Converting raw to npy failed.')
      progress.close()
    else:
      ret_filename = path
      if rescale_value != 1.00:
        unscaled = np.load(path)
        no_frames = len(unscaled)
        try:
          scaled = self.bin_ndarray(unscaled, (no_frames, rescale_height,
                                      rescale_width), callback, operation='mean')
          np.save(path, scaled)
        except:
          qtutil.critical('Rebinning raw failed. We can only scale down from larger sized files to preserve data')
          progress.close()
    return ret_filename

  def convert_tif(self, filename):
    rescale_value = float(self.scale_factor.value())
    channel = int(self.channel.value())

    path = os.path.splitext(os.path.basename(filename))[0] + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting tif to npy...', 'Abort', 0, 100, self)<commit_msg>can import different channels of the same file (channel used included in file name)<commit_after>#!/usr/bin/env python3

import os
import sys
import traceback
import functools
from shutil import copyfile

import numpy as np
from PyQt4.QtCore import *
from PyQt4.QtGui import *

sys.path.append('..')
import qtutil
import tifffile as tiff

from .util import fileloader, fileconverter
from .util import mse_ui_elements as mue


class NotConvertedError(Exception):
  pass

class FileAlreadyInProjectError(Exception):
  def __init__(self, filename):
    self.filename = filename

class Widget(QWidget):
  def __init__(self, project, plugin_position, parent=None):
    super(Widget, self).__init__(parent)

    if not project:
      return

    self.plugin_position = plugin_position
    self.project = project
    self.setup_ui()
    if isinstance(plugin_position, int):
        self.params = project.pipeline[self.plugin_position]
        assert(self.params['name'] == 'fileimporter')
        self.setup_param_signals()
        self.setup_params()

    self.listview.setModel(QStandardItemModel())
    for f in self.project.files:
      if f['type'] == 'video':
        self.listview.model().appendRow(QStandardItem(f['path']))

  def setup_params(self):
    self.scale_factor.setValue(self.params['Scale Factor'])


  def setup_ui(self):
    vbox = QVBoxLayout()

    ## Related to importing Raws
    self.setWindowTitle('Import Raw File')

    #vbox.addWidget(QLabel('Set the size all data are to be rescaled to'))

    grid = QGridLayout()
    grid.addWidget(QLabel('Set channel and scale factor used for all imported data'), 0, 0)

    # grid.addWidget(QLabel('Width:'), 0, 0)
    # self.rescale_width = QSpinBox()
    # self.rescale_width.setMinimum(1)
    # self.rescale_width.setMaximum(1024)
    # self.rescale_width.setValue(256)
    # grid.addWidget(self.rescale_width, 0, 1)
    #
    # grid.addWidget(QLabel('Height:'), 1, 0)
    # self.rescale_height = QSpinBox()
    # self.rescale_height.setMinimum(1)
    # self.rescale_height.setMaximum(1024)
    # self.rescale_height.setValue(256)
    # grid.addWidget(self.rescale_height, 1, 1)

    grid.addWidget(QLabel('Scale Factor:'), 1, 0)
    self.scale_factor = QDoubleSpinBox()
    self.scale_factor.setSingleStep(0.25)
    self.scale_factor.setMinimum(0.25)
    self.scale_factor.setMaximum(1.00)
    self.scale_factor.setValue(1.0)
    grid.addWidget(self.scale_factor, 1, 1)

    grid.addWidget(QLabel('Channel:'), 2, 0)
    self.channel = QSpinBox()
    self.channel.setMinimum(1)
    self.channel.setMaximum(3)
    self.channel.setValue(2)
    grid.addWidget(self.channel, 2, 1)

    grid.addWidget(qtutil.separator(), 3, 0)
    grid.addWidget(qtutil.separator(), 3, 1)
    grid.addWidget(QLabel('Set paramaters for all imported raws'), 4, 0)
    grid.addWidget(QLabel('All raws will be rescaled and specified channel imported upon conversion'), 5, 0)
    grid.addWidget(QLabel('Width:'), 6, 0)
    self.sb_width = QSpinBox()
    self.sb_width.setMinimum(1)
    self.sb_width.setMaximum(1024)
    self.sb_width.setValue(256)
    grid.addWidget(self.sb_width, 6, 1)
    grid.addWidget(QLabel('Height:'), 7, 0)
    self.sb_height = QSpinBox()
    self.sb_height.setMinimum(1)
    self.sb_height.setMaximum(1024)
    self.sb_height.setValue(256)
    grid.addWidget(self.sb_height, 7, 1)
    grid.addWidget(QLabel('Number of channels:'), 8, 0)
    self.sb_channel = QSpinBox()
    self.sb_channel.setMinimum(1)
    self.sb_channel.setMaximum(3)
    self.sb_channel.setValue(3)
    grid.addWidget(self.sb_channel, 8, 1)
    grid.addWidget(QLabel('dtype:'), 9, 0)
    self.cb_dtype = QComboBox()
    for t in 'uint8', 'float32', 'float64':
      self.cb_dtype.addItem(t)
    grid.addWidget(self.cb_dtype, 9, 1)
    grid.addWidget(qtutil.separator(), 10, 0)
    grid.addWidget(qtutil.separator(), 10, 1)
    vbox.addLayout(grid)
    vbox.addStretch()

    self.setLayout(vbox)
    self.resize(400, 220)

    vbox.addWidget(mue.WarningWidget('Warning. This application has not yet been memory optimized for conversion.'
                                     ' We advise you only import files no larger than 1/4 of your memory'))
    pb = QPushButton('New Video')
    pb.clicked.connect(self.new_video)
    vbox.addWidget(pb)

    self.listview = QListView()
    self.listview.setStyleSheet('QListView::item { height: 26px; }')
    self.listview.setSelectionMode(QAbstractItemView.NoSelection)
    vbox.addWidget(self.listview)

    vbox.addStretch()
    self.setLayout(vbox)

  def setup_param_signals(self):
      self.scale_factor.valueChanged[float].connect(functools.partial(self.update_plugin_params, 'Scale Factor'))


  def update_plugin_params(self, key, val):
      self.params[key] = val
      self.project.pipeline[self.plugin_position] = self.params



  def convert_raw(self, filename):
    rescale_value = float(self.scale_factor.value())
    dtype = str(self.cb_dtype.currentText())
    width = int(self.sb_width.value())
    height = int(self.sb_height.value())
    rescale_width = int(width * rescale_value)
    rescale_height = int(height * rescale_value)
    channels = int(self.sb_channel.value())
    channel = int(self.channel.value())
    path = os.path.splitext(os.path.basename(filename))[0] + '_channel_' + str(channel) + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting raw to npy...', 'Abort', 0, 100, self)
    progress.setAutoClose(True)
    progress.setMinimumDuration(0)
    progress.setValue(0)

    def callback(value):
      progress.setValue(int(value * 100))
      QApplication.processEvents()

    try:
      fileconverter.raw2npy(filename, path, dtype, width, height,
        channels, channel, callback)
    except:
      qtutil.critical('Converting raw to npy failed.')
      progress.close()
    else:
      ret_filename = path
      if rescale_value != 1.00:
        unscaled = np.load(path)
        no_frames = len(unscaled)
        try:
          scaled = self.bin_ndarray(unscaled, (no_frames, rescale_height,
                                      rescale_width), callback, operation='mean')
          np.save(path, scaled)
        except:
          qtutil.critical('Rebinning raw failed. We can only scale down from larger sized files to preserve data')
          progress.close()
    return ret_filename

  def convert_tif(self, filename):
    rescale_value = float(self.scale_factor.value())
    #channel = int(self.channel.value())

    path = os.path.splitext(os.path.basename(filename))[0] + '.npy'
    path = os.path.join(self.project.path, path)

    progress = QProgressDialog('Converting tif to npy...', 'Abort', 0, 100, self)
{'fraction_non_alphanumeric': 0.12466104248267551, 'fraction_numerical': 0.020186803253992167, 'mean_word_length': 3.479419703103914, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1655088', 'n_tokens_mistral': 4695, 'n_tokens_neox': 4380, 'n_words': 880}