<jupyter_start><jupyter_text>**Mouse brain research**   The main goals are measure cloride anione concentration in mouse brain cells and build functional coonectivity maps. The functionality is presented by 3 parts:1.   import modules and defining constans, regions of interest (ROI)2.   processing data via reading, filtering and etc3.   display results.  List of connected with this project literature and other things:  1.   [github](https://github.com/kuratko-hub/brain_Cl_measure)2.   [literature](https://drive.google.com/drive/folders/1e_wlOevjWSAggrUYb5L28uWNNflXI-Q3?usp=sharing) the files are located by next order: `'condition exp stim/w/exp_num/mouse'  `1.   condition : anesthetized or awake2.   exp: number of days after surgery (00d, 3d, 10d, 15d)3.   stim: stimulated or not (stim, no_st)4.   w: detecton wavelength (430, 505..)5.   exp_num: experiment number (1,2..)6.   mouse: mouse number (1,2..)for example: `'anesthetized 00d no_st/430/1/Flourescence 1_430_M1T1.tif'  `to start working please:1.   set path to data_directory and cortex_map_directory below2.   run beginning, data preparation functions and analysis functions sections3.   sort files by prepare functions and use analysis functions<jupyter_code>data_directory = "/content/drive/MyDrive/data"
cortex_map_directory = "/content/drive/MyDrive/Colab Notebooks/Brain/Mouse cortex map.xlsx"<jupyter_output><empty_output><jupyter_text>**beginning**    modules and constants<jupyter_code>from google.colab import drive
from IPython.display import clear_output

import numpy as np
import pandas as pd
from scipy import signal, stats, ndimage, fft
import tifffile as tf
import numba
import copy

import os, ipywidgets, warnings, sys

from tqdm import tqdm
from glob import glob
import matplotlib.ticker as ticker
import matplotlib.pyplot as plt
import matplotlib.colors as mcolors
import PIL.Image as Image
import concurrent.futures

if "BaselineRemoval" not in sys.modules:
  !pip install BaselineRemoval
# if "ffmpeg" not in sys.modules:
#   !pip install ffmpeg-python
# if "vidstab" not in sys.modules:
#   !pip install vidstab

from BaselineRemoval import BaselineRemoval
# import ffmpeg, vidstab, cv2

drive.mount('/content/drive/')
os.chdir(data_directory)
clear_output()

warnings.filterwarnings("ignore")
CORTEX_MAP = pd.read_excel(cortex_map_directory)

ZONE_NAMES = np.array(list(map(lambda c: c.replace('cent ', '').replace('\t',''), 
                               CORTEX_MAP.columns[1::2].to_list()
                               )))

FREQS = {'{}-{}'.format(f[0], f[1]) : '{}-{}'.format(f[0], f[1]) for f in [[0.001, 0.1],
                                                                           [0.1,   0.25],
                                                                           [0.25,  1.0],
                                                                           [1.0,   3.0],
                                                                           [3.0,   5.0],
                                                                           [5.0,   8.0]]}

BRAIN_IMAGE = tf.imread(glob('anesthetized 00d no_st/430/1/*M1*.tif')[0], key = [0])

CORES = os.cpu_count()
MIN = 50

ZONES = ['M1', 'M2', 'S1HL', 'S1FL', 'RS']<jupyter_output><empty_output><jupyter_text>get_roi<jupyter_code>def get_roi(k1 = 0.01, angle = 4 * np.pi / 180, k2 = 0.0196, center = (177, 145), dpi = 114):
  
  results = {'left' : {}, 'right' :{}}
  
  for n, name in enumerate(ZONE_NAMES):

    n = 2 * n + 1

    x = np.append(CORTEX_MAP[CORTEX_MAP.iloc[:,n].notna()].iloc[:,n].to_numpy() + 3.08 * k1, 
                  CORTEX_MAP[CORTEX_MAP.iloc[:,n+1].notna()].iloc[-1::-1,n+1].to_numpy() - 3.08 * k1)
    x = np.append(x, x[0])
    
    y = np.append(CORTEX_MAP[CORTEX_MAP.iloc[:,n].notna()].iloc[:,0].to_numpy(), 
                  CORTEX_MAP[CORTEX_MAP.iloc[:,n].notna()].iloc[-1::-1,0].to_numpy()) * (1 - k1)
    y = np.append(y, y[0])

    results['left'].update({name: [(-x * np.cos(angle) - y * np.sin(angle)) / k2 + center[0], 
                                   (x * np.sin(angle) - y * np.cos(angle))  / k2 + center[1]
                                   ]
                            })
    
    results['right'].update({name: [(x * np.cos(angle) - y * np.sin(angle))  / k2 + center[0], 
                                    (-x * np.sin(angle) - y * np.cos(angle)) / k2 + center[1]
                                    ]
                             })

  def in_func(x, y):

    fig = plt.figure()
    ax = plt.subplot(111)
    ax.imshow(np.zeros((344,344)), cmap = 'gray')

    plt.axis('off')
    ax.plot(x, y, c = 'w')
    ax.fill(x, y, color = 'w')
    ax.set_xlim(0, 344)
    ax.set_ylim(344, 0)

    plt.savefig('test.png', bbox_inches = 'tight', pad_inches = 0, cmap = 'gray', dpi = dpi)
    plt.close()

    result = np.asarray(Image.open('test.png'))[:,:,2] > 0
    result[-1] = False
    return result.flatten()

  return {k1: {k2: in_func(v2[0], v2[1]) for k2, v2 in v1.items()} for k1, v1 in results.items()}
ROI = get_roi()<jupyter_output><empty_output><jupyter_text>**data preparation functions**    prepare<jupyter_code>def prepare(indicies = None, **kwargs):

  _params = [{x : '*' for x in ['cond', 'exp', 'stim', w, 'exp_num', 'mouse']} for w in ['w1', 'w2']]
  for p in _params:
    for key, value in kwargs.items():
      p.update({key : value})

  paths = [glob("{} {} {}/{}/{}/*M{}T*.tif".format(*list(d.values()))) for d in _params]

  # replace wavelengths on 'w' char in paths
  paths_compare = [[p.replace(str(w), 'w') for p in pp] for w, pp in zip([_params[0]['w1'], _params[1]['w2']], paths)]

  # looking for a common paths
  paths_common = list(set(paths_compare[0]).intersection(paths_compare[1]))

  if len(paths_common) == 0:
    return None

  else:
    # looking for a indexes of common paths in paths_compare and updating paths by them
    paths = [[p[pc.index(com)] for pc, p in zip(paths_compare, paths)] for com in paths_common]
    names = []

    for i, s in enumerate(paths_common):

      name = {}
      s = s.replace('/', ' ')
      spl = s.split(' ')

      for j, d in zip([0, 1, 2, 4], ['cond', 'exp', 'stim', 'exp_num']):
        name.update({d : spl[j]})

      name.update({'mouse' : s[s.find('M') + 1]})
          
      common = list(set(name).intersection(kwargs))
      [name.pop(x) for x in common]

      names.append(' '.join([key + '=' + str(value) for key, value in name.items()]))

    if indicies is not None:
      paths = [paths[i] for i in indicies]
      names = [names[i] for i in indicies]

    return paths, names, kwargs
# test
# pack = prepare(cond = 'anesthetized', w1 = 505, w2 = 430, exp_num = 1, mouse = 1)
# pack = prepare(cond = 'anesthetized', w1 = 505, w2 = 430, exp_num = 1, exp = '00d', stim = 'stim')
# pack<jupyter_output><empty_output><jupyter_text>processing<jupyter_code>def processing(paths, filt, key):

  _data = [tf.imread(paths[i], key = key) for i in range(2)]

  if _data[0].shape[0] > 1000:

    m = _data[0].mean(axis = 0)
    m = m > np.percentile(m, 50)

    _data = np.divide(_data[0], _data[1], dtype = np.float32)
    _data = np.nan_to_num(_data, posinf = 0)
    _data = _data / _data.max()

    return _data, m

  else:

    return 0
# test
# paths = ['anesthetized 10d no_st/505/1/Fluorescence_Cube_505_M0T3.tif', 
#          'anesthetized 10d no_st/430/1/Fluorescence_Cube_430_M0T3.tif']
         
# data, m = processing(paths, ndimage.uniform_filter)<jupyter_output><empty_output><jupyter_text>butter_bandpass_filter<jupyter_code>def butter_bandpass_filter(data, N = 5, freq = np.array([0, 1]), fs = 16.66):

  freq = 2 * freq / fs
  if freq[0] == 0:
    b, a = signal.butter(N = N, Wn = freq[1], btype = 'lowpass')
  else:
    b, a = signal.butter(N = N, Wn = freq, btype = 'bandpass')
  
  return signal.filtfilt(b, a, data).astype(np.uint8)<jupyter_output><empty_output><jupyter_text>get_signals<jupyter_code>def get_signals(pack, filt = lambda x: x, key = None):

  paths, file_names, args = [p.copy() for p in pack]
  _signals = {}

  for path, name in zip(paths, file_names):

    _temp_1 = processing(path, filt, key)

    if _temp_1 != 0:

      _data, mask = _temp_1
      _data = np.reshape(_data, (_data.shape[0], -1))
      mask = mask.flatten()

      # structure of _signals dictionary
      # file name ->
      _signals.update({name : {}})
        
      for k1, v1 in ROI.items():
        # -> left/right side -> 
        _signals[name].update({k1 : {}})

        for k2, v2 in v1.items():
          # checking mask for True values
          _temp = v2 * mask

          if ~(~_temp).all() and _temp.sum() > 40:
            
            # -> zone name -> signal
            _signals[name][k1].update({k2: np.mean(np.compress(_temp, _data, axis = 1), axis = 1)})

  return _signals, args
# test
# ss = get_signals(pack)
# ss<jupyter_output><empty_output><jupyter_text>get_data_fft<jupyter_code>def get_data_fft(pack, filt = lambda x: x, key = None, fs = 16.66):

  paths, file_names, args = [p.copy() for p in pack]
  _maps, _file_frequencies, _file_masks = {}, {}, {}

  for path, file_name in zip(paths, file_names):

    _temp_1 = processing(path, filt, key)

    if _temp_1 != 0:

      _temp, mask = _temp_1

      N = _temp.shape[0]
      freq = fft.fftfreq(N, 1 / fs)[1: N // 2]
      
      _file_masks.update({file_name: mask})
      _file_frequencies.update({file_name : freq})

      _temp = fft.fft(_temp, axis = 0)[1: N//2]
      _temp = {'amplitude' : np.abs(_temp),  'phase': np.arctan2(np.imag(_temp), np.real(_temp) )}
      _temp_2 = {}
      for k1, v1 in _temp.items():
        _temp_2.update({k1 : {}})
        for k2 in FREQS:        # frequencies
          f = k2.split('-')
          f = (float(f[0]), float(f[1]))
          t = np.mean(v1[(freq > f[0]) & (freq <= f[1])], axis = 0)
          _temp_2[k1].update({k2 : t})

      _maps.update({file_name : _temp_2})

  return _maps, _file_frequencies, _file_masks, args
# pack = prepare(cond = 'anesthetized', w1 = 505, w2 = 430, exp_num = 1, mouse = 1)
# pack = prepare(cond = 'awake', w1 = 505, w2 = 430, exp_num = 1, exp = '00d', stim = 'no_st')
# pack
# pack = prepare('cond' : 'awake',        'w1' : 505, 'w2' : 430, 'exp_num' : 1, 'exp' : exp, 'stim' : 'stim')
# dd = get_data_fft(pack)
# dd<jupyter_output><empty_output><jupyter_text>**analysis functions**    interface<jupyter_code>def interface_1(file_names, zone_names):

  items = [ipywidgets.SelectMultiple(
           options = {x : x for x in file_names},
           value = [x for x in file_names],
           description = 'file_names'
           ), # 0
           ipywidgets.SelectMultiple(
           options = {x : x for x in zone_names},
           value = zone_names,
           description = 'zone_names'
           ), # 1
           
           ipywidgets.Select(
           options = {x : x for x in ['no', 'ZhangFit', 'trend']},
           value = 'trend',
           description = 'method'
           ), # 2
           ipywidgets.BoundedIntText(
           value = 2,
           min = 0,
           max = 10,
           step = 1,
           description = 'degree'
           ), # 3
           ipywidgets.FloatText(
           value = 16.66,
           min = 0,
           max = 50.0,
           description = 'fs'
           ), # 4
           ipywidgets.Select(
           options = {x : x for x in ['original', 'fourier']},
           value = 'fourier',
           description = 'dtype'
           ), # 5
           ipywidgets.BoundedIntText(
           value = 3,
           min = 0,
           max = 20,
           step = 1,
           description = 'start'
           ), # 6

           ipywidgets.IntText(
           value = 25,
           description = 'length'
           ), # 7
           ipywidgets.IntText(
           value = 4,
           description = 'width'
           ), # 8
           ipywidgets.FloatText(
           value = -0.5,
           description = 'xlim_0'
           ), # 9
           ipywidgets.FloatText(
           value = 9,
           description = 'xlim_1'
           ), # 10
           ipywidgets.BoundedFloatText(
           value = 1,
           min = 0,
           max = 1,
           step = 0.1,
           description = 'major'
           ), # 11
           ipywidgets.BoundedFloatText(
           value = 0.5,
           min = 0,
           max = 1,
           step = 0.1,
           description = 'minor'
           ), # 12
           ipywidgets.Text(
           value = '',
           description = 'print'
           ), # 13
           ipywidgets.BoundedFloatText(
           value = 1,
           min = 0,
           max = 1,
           step = 0.1,
           description = 'linewidth'
           ), # 14
           ipywidgets.BoundedIntText(
           value = 14,
           min = 0,
           max = 20,
           step = 1,
           description = 'fontsize'
           )] # 15

  titles_items  = [item.description for item in items]
  titles_acc    = ['figsize', 'xlim', 'xticks', 'other']
  titles_ui     = ['input', 'procedure', 'output']

  acc = ipywidgets.Accordion([ipywidgets.HBox([items[7],  items[8]]),
                              ipywidgets.HBox([items[9],  items[10]]),
                              ipywidgets.HBox([items[11], items[12]]),
                              ipywidgets.HBox([items[13], items[14], items[15]])
                              ])

  ui = ipywidgets.Tab([ipywidgets.HBox([items[0], items[1]]),
                       ipywidgets.HBox([items[2], items[3], items[4], items[5], items[6]]),
                       acc
                       ])

  for i in range(len(ui.children)):
    ui.set_title(i, titles_ui[i])

  for i in range(len(acc.children)):
    acc.set_title(i, titles_acc[i])

  colors = ['b', 'g', 'r', 'c', 'm', 'y', 'k', 'b', 'g', 'r']

  return {key: value for key, value in zip(titles_items, items)}, ui, colors

def interface_2(file_names):

  items = [ipywidgets.SelectMultiple(
           options = {x : x for x in file_names},
           value = file_names,
           description = 'file_names'
           ), # 0
           ipywidgets.SelectMultiple(
           options = FREQS,
           value = [x for x in FREQS.values()],
           description = 'zone_names'
           ), # 1

           ipywidgets.BoundedIntText(
           value = 50,
           min = 0,
           max = 100,
           step = 1,
           description = 'percentile'
           ), # 2

           ipywidgets.IntText(
           value = 25,
           description = 'length'
           ), # 3
           ipywidgets.IntText(
           value = 10,
           description = 'width'
           ), # 4
           ipywidgets.BoundedFloatText(
           value = 0.7,
           min = 0,
           max = 1,
           step = 0.1,
           description = 'alpha'
           ), # 5
           ipywidgets.BoundedIntText(
           value = 14,
           min = 0,
           max = 20,
           step = 1,
           description = 'fontsize'
           ), # 6
           ipywidgets.Textarea(
           value = '',
           description = 'print'
           ) # 7
           ]

  titles_items = [item.description for item in items]
  titles_acc = ['figsize', 'other']
  titles_ui  = ['input', 'procedure', 'output']

  acc = ipywidgets.Accordion([ipywidgets.HBox([items[3], items[4]]),
                              ipywidgets.HBox([items[5], items[6], items[7]])
                              ])

  ui = ipywidgets.Tab([ipywidgets.HBox([items[0], items[1]]),
                       items[2],
                       acc
                       ])

  for i in range(len(ui.children)):
    ui.set_title(i, titles_ui[i])

  for i in range(len(acc.children)):
    acc.set_title(i, titles_acc[i])
  
  colors = ['Reds', 'Greens', 'Oranges', 'Blues', 'Purples', 'Greys', 'Blues', 'Purples', 'Greys']

  return {key: value for key, value in zip(titles_items, items)}, ui, colors

def interface_3(file_names):

  items = [ipywidgets.SelectMultiple(
           options = {x : x for x in file_names},
           value = [x for x in file_names],
           description = 'file_names',
           ), # 0
           
           ipywidgets.Select(
           options = {x : x for x in ['no', 'ZhangFit', 'trend']},
           value = 'trend',
           description = 'method',
           ), # 1
           ipywidgets.BoundedIntText(
           value = 2,
           min = 0,
           max = 10,
           step = 1,
           description = 'degree',
           ), # 2

           ipywidgets.IntText(
           value = 7,
           description = 'length',
           ), # 3
           ipywidgets.IntText(
           value = 7,
           description = 'width',
           ), # 4 
           ipywidgets.BoundedIntText(
           value = 14,
           description = 'fontsize',
           min = 0,
           max = 20,
           step = 1,
           ), # 5
           ipywidgets.Textarea(
           value = '',
           description = 'print',
           ) # 6
           ]

  titles_items =  [item.description for item in items]
  titles_acc =    ['figsize', 'other']
  titles_ui =     ['input', 'procedure', 'ouput']

  acc = ipywidgets.Accordion([ipywidgets.HBox([items[3], items[4]]),
                              ipywidgets.HBox([items[5], items[6]])
                              ])

  ui = ipywidgets.Tab(children = [items[0],
                                  ipywidgets.HBox([items[1], items[2]]),
                                  acc
                                  ])

  for i in range(len(ui.children)):
    ui.set_title(i, titles_ui[i])

  for i in range(len(acc.children)):
    acc.set_title(i, titles_acc[i])

  return {key: value for key, value in zip(titles_items, items)}, ui

def interface_4(file_names, zone_names):

  items = [ipywidgets.Select(
           options = {x : x for x in file_names},
           value = file_names[0],
           description = 'file_name',
           ), # 0
           ipywidgets.SelectMultiple(
           options = {x : x for x in zone_names},
           value = [x for x in zone_names],
           description = 'zone_names',
           ), # 1
           
           
           ipywidgets.Select(
           options = {x : x for x in ['no', 'ZhangFit', 'trend']},
           value = 'trend',
           description = 'method',
           ), # 2
           ipywidgets.BoundedIntText(
           value = 2,
           min = 0,
           max = 10,
           step = 1,
           description = 'degree',
           ), # 3
           ipywidgets.Select(
           options = {x : x for x in ['psd', 'complex', 'magnitude', 'angle', 'phase']},
           value = 'magnitude',
           description = 'mode',
           ), # 4
           ipywidgets.Select(
           options = {x : x for x in ['density', 'spectrum']},
           value = 'spectrum',
           description = 'scaling',
           ), # 5
           ipywidgets.FloatText(
           value = 16.66,
           min = 0,
           max = 50.0,
           description = 'fs',
           ), # 6
           ipywidgets.IntText(
           value = 0,
           description = 'noverlap',
           ), # 7


           ipywidgets.IntText(
           value = 10,
           description = 'length',
           ), # 8
           ipywidgets.IntText(
           value = 4,
           description = 'width',
           ), # 9 
           ipywidgets.BoundedIntText(
           value = 14,
           min = 0,
           max = 20,
           step = 1,
           description = 'fontsize',
           ), # 10
           ipywidgets.Text(
           value = '',
           description = 'print',
           ) # 11
           ]

  acc1 = ipywidgets.Accordion([ipywidgets.HBox([items[2], items[3]]),
                               ipywidgets.HBox([items[4], items[5]]),
                               ipywidgets.HBox([items[6], items[7]])
                               ])
  
  acc2 = ipywidgets.Accordion([ipywidgets.HBox([items[8],  items[9]]),
                               ipywidgets.HBox([items[10], items[11]])
                               ])

  ui = ipywidgets.Tab([ipywidgets.HBox([items[0], items[1]]),
                       acc1,
                       acc2
                       ])
  
  titles_items  = [item.description for item in items]
  titles_acc1   = ['signal', 'out_types', 'other']
  titles_acc2   = ['figsize', 'other']
  titles_ui     = ['input', 'procedure', 'output']

  for i in range(len(ui.children)):
    ui.set_title(i, titles_ui[i])

  for i in range(len(acc1.children)):
    acc1.set_title(i, titles_acc1[i])

  for i in range(len(acc2.children)):
    acc2.set_title(i, titles_acc2[i])

  titles_items = [item.description for item in items]

  return {key: value for key, value in zip(titles_items, items)}, ui<jupyter_output><empty_output><jupyter_text>plot zones<jupyter_code>def plot_zones():

  colors = list(mcolors.XKCD_COLORS.values())
  k1, angle, k2, center = 0.01, 4 * np.pi / 180, 0.0196, (177, 145)
  image = BRAIN_IMAGE

  fig, ax = plt.subplots(figsize = (13,13))
  ax.imshow(image, cmap = 'gnuplot')
  result = []

  for n in range(len(ZONE_NAMES)):

    n = 2 * n + 1

    x = np.append(CORTEX_MAP[CORTEX_MAP.iloc[:,n].notna()].iloc[:,n].to_numpy() + 3.08 * k1, 
                  CORTEX_MAP[CORTEX_MAP.iloc[:,n+1].notna()].iloc[-1::-1,n+1].to_numpy() - 3.08 * k1)
    
    y = np.append(CORTEX_MAP[CORTEX_MAP.iloc[:,n].notna()].iloc[:,0].to_numpy(), 
                  CORTEX_MAP[CORTEX_MAP.iloc[:,n].notna()].iloc[-1::-1,0].to_numpy()) * (1 - k1)
                  
    x = np.append(x, x[0])
    y = np.append(y, y[0])

    result.append([[(-x * np.cos(angle) - y * np.sin(angle)) / k2 + center[0], 
                    (x * np.sin(angle) - y * np.cos(angle)) / k2 + center[1]], 
                   [(x * np.cos(angle) - y * np.sin(angle)) / k2 + center[0], 
                    (-x * np.sin(angle) - y * np.cos(angle)) / k2 + center[1]]])

  for i, cc in enumerate(result):
    for j, (x, y) in enumerate(cc):
      if j == 0:
        ax.plot(x, y, c = colors[i], label = ZONE_NAMES[i])
      else:
        ax.plot(x, y, c = colors[i])
      ax.fill(x, y, color = colors[i], alpha = 0.5)

  ax.set_xlim(0, 344)
  ax.set_ylim(344, 0)
  ax.set_axis_off()
  box = ax.get_position()
  ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])

  ax.legend(loc = 'center left', bbox_to_anchor = (1, 0.5))

  plt.show()<jupyter_output><empty_output><jupyter_text>fourier analysis - 1<jupyter_code>def func_1(data):

  _signals, args = [copy.deepcopy(d) for d in data]

  file_names_1, zone_names_1 = list(_signals), set()
  for i in _signals.values():
    for j in i.values():
      for k in j.keys():
        zone_names_1.add(k)
  zone_names_1 = list(zone_names_1)

  file_names_1.sort()
  zone_names_1 = [n for n in ZONE_NAMES if n in zone_names_1]

  items, ui, colors = interface_1(file_names_1, zone_names_1)

  functions = {'no' : lambda y: y,
               'ZhangFit' : lambda y: BaselineRemoval(y).ZhangFit(), 
               'trend' : lambda y: y - stats.linregress(np.arange(y.shape[0]), y).intercept
               }

  flag = True

  def process(fs):

    for k1, v1 in _signals.items(): # name
      for k2, v2 in v1.items(): # left/right
        for k3, s in v2.items(): #zone

          N = s.shape[0]

          x = np.linspace(0, N / fs, N)
          xf = fft.fftfreq(N, 1 / fs)[: N // 2]

          _signals[k1][k2][k3] = {}
          for n, f in functions.items():

            ys = f(s)
            yfs = np.abs(fft.fft(ys)[: N // 2])

            _signals[k1][k2][k3].update({n: {'original': {'x' : x,  'y' : ys}, 
                                             'fourier' : {'x' : xf, 'y' : yfs}
                                             }})
            
    return False

  def f(file_names, zone_names,
        method, degree, fs, dtype, start, 
        length, width, xlim_0, xlim_1, major, minor, print, linewidth, fontsize):
    
    nonlocal flag
    if flag:
      flag = process(fs)
    
    fig, axs = plt.subplots(len(zone_names), 2, figsize = (length, width * len(zone_names)))
    handles = []

    if len(zone_names) == 1: 
      axs = axs[np.newaxis, ]

    for n, k1 in enumerate(file_names): # files
      v1 = _signals[k1]
      for i, (k2, v2) in enumerate(v1.items()): # left/right
          for j, k3 in enumerate(zone_names): # zones
            v3 = v2.get(k3)
            if v3 is not None:
              c = colors[file_names_1.index(k1)]
              if dtype == 'original':
                start = 0
              x = v3[method][dtype]['x'][start:]
              y = v3[method][dtype]['y'][start:] + n * 2

              im, = axs[j, i].plot(x, y, label = k1, linewidth = linewidth, c = c)
              if k3 == 'M1' and k2 == 'left':
                handles.append(im)

    for i, k1 in enumerate(zone_names):
      axs[i, 0].set_ylabel(k1 + '\n' + 'intensity', fontsize = fontsize)
      axs[i, 1].set_ylabel('intensity', fontsize = fontsize)
      if len(axs[i, 0].lines) != 0 and len(axs[i, 1].lines) != 0:
        limy = [axs[i, 0].get_ylim(), axs[i, 1].get_ylim()]
        limy = (np.min([limy[0][0], limy[1][0]]), np.max([limy[0][1], limy[1][1]]))
        for m in range(2):
          axs[i, m].set_ylim(limy)

    for a in axs.flatten():
      if len(a.lines) == 0:
        a.axis('off')
      else:
        if dtype is 'original':
          a.set_xlabel('seconds', fontsize = fontsize)
        else:
          if (xlim_0 != -0.5) or (xlim_1 != 9):
            a.set_xlim(xlim_0, xlim_1)
          a.xaxis.set_major_locator(ticker.MultipleLocator(major))
          a.xaxis.set_minor_locator(ticker.MultipleLocator(minor))
          a.set_xlabel('Hz', fontsize = fontsize)

        a.tick_params(axis = 'both', which = 'both', labelsize = fontsize)
        a.grid(which = 'major', color = 'k', linewidth = linewidth * 0.75)
        a.minorticks_on()
        a.grid(which = 'minor', color = 'gray', linewidth = linewidth * 0.5)

    axs[0, 0].set_title('Левое полушарие', fontsize = fontsize)
    axs[0, 1].set_title('Правое полушарие', fontsize = fontsize)

    axs[0, 1].legend(handles = handles, fontsize = fontsize, loc = 'upper right')
    plt.suptitle(print + '//' + 
                 'method=' + method + '//' + 'degree=' + str(degree) + '//' + 'fs=' + str(fs) + 
                 '//' + 'dtype=' + dtype + '//' + 'start=' +  str(start) + '\n' +
                 '//' + '//'.join([key + '=' + str(value) for key, value in args.items()]) + '//', 
                 y = 1.005, fontsize = fontsize)
    plt.tight_layout()
    plt.show()

  inter = ipywidgets.interactive_output(f, items)
  display(ui, inter)
# pack = prepare(cond = 'anesthetized', w1 = 505, w2 = 430, exp_num = 1, mouse = 1)
# pack
# test
# ss = get_signals(pack)
# func_1(ss)<jupyter_output><empty_output><jupyter_text>2d fourier analysis - 2<jupyter_code>def func_2(data):

  _maps, _file_frequencies, _file_masks, args = [copy.deepcopy(d) for d in data]

  _file_names_1 = list(_maps)
  _file_names_1.sort()

  image = BRAIN_IMAGE / BRAIN_IMAGE.max()

  _file_names_1.insert(0, 'original')
  _maps.update({'original': {x : {k3 : image for k3 in list(FREQS)} for x in ['amplitude', 'phase'] }})
  _file_masks.update({'original': image > np.percentile(image, 50)})

  items, ui, colors = interface_2(_file_names_1)
  
  def f(file_names, zone_names,
        percentile, 
        length, width, alpha, fontsize, print):
    
    fig, axs = plt.subplots(len(zone_names), 3, figsize = (length, width * len(zone_names)))

    if len(zone_names) == 1: 
      axs = axs[np.newaxis, ]

    for k1 in file_names:
      v1 = _maps[k1]
      mask = _file_masks[k1]
      for i, (k2, v2) in enumerate(v1.items()):
        for j, k3 in enumerate(zone_names):
          v3 = v2[k3]
          v3 = ndimage.uniform_filter(v3)
          v3 = v3 / v3.max()
          map = v3 * mask / mask
          if k1 == 'original':
            map[v3 < np.percentile(v3, percentile)] = np.nan
          if k1 != 'original' and k2 == 'amplitude':
            map[v3 < np.percentile(v3, 40)] = np.nan
            map[v3 > np.percentile(v3, 99)] = np.nan
          if k1 != 'original' and k2 == 'phase':
            map[v3 < np.percentile(v3, 60)] = np.nan
            map[v3 > np.percentile(v3, 99)] = np.nan

          im = axs[j, i].imshow(map, cmap = colors[_file_names_1.index(k1)], alpha = alpha)
          if k2 == 'phase':
            cbar = fig.colorbar(mappable = im, ax = axs[j, i + 1], label = k1, shrink = 0.75)
            if k1 != 'original':
              cbar.set_ticks([])

    for a in axs.flatten():
      a.axis('off')

    for i, k1 in enumerate(zone_names):
      axs[i, 0].text(-0.05, 0.5, k1 + ' Hz',
                     horizontalalignment = 'center',
                     verticalalignment = 'center',
                     rotation = 'vertical',
                     transform = axs[i, 0].transAxes, fontsize = fontsize)
      
    axs[0, 0].text(0.5, 1.05, 'magnitude',
                   horizontalalignment = 'center',
                   verticalalignment = 'center',
                   rotation = 'horizontal',
                   transform = axs[0, 0].transAxes, fontsize = fontsize)
    
    axs[0, 1].text(0.5, 1.05, 'phase',
                   horizontalalignment = 'center',
                   verticalalignment = 'center',
                   rotation = 'horizontal',
                   transform = axs[0, 1].transAxes, fontsize = fontsize)
    
    plt.suptitle(print + '//' + 
                 'percentile=' + str(percentile) + '//' + '\n' +
                 '//' + '//'.join([key + '=' + str(value) for key, value in args.items()]) + '//', 
                 y = 1.005, fontsize = fontsize)
    plt.tight_layout()
    plt.show()

  inter = ipywidgets.interactive_output(f, items)
  display(ui, inter)
# pack = prepare(cond = 'anesthetized', w1 = 505, w2 = 430, exp_num = 1, mouse = 1)
# pack
# dd = get_data_fft(pack)
# func_2(dd)<jupyter_output><empty_output><jupyter_text>correlation matrix - 3<jupyter_code>def func_3(data):

  _signals, args = [copy.deepcopy(d) for d in data]

  file_names_1 = list(_signals)
  file_names_1.sort()

  items, ui = interface_3(file_names_1)

  functions = {'no' :lambda y: y,
               'ZhangFit': lambda y: BaselineRemoval(y).ZhangFit(), 
               'trend' :lambda y: y - stats.linregress(np.arange(y.shape[0]), y).intercept}

  flag, _signals_m = True, {}

  def process():

    fs = 16.66

    for k1, v1 in _signals.items(): # name

      _signals_m.update({k1: {}})
      for n, f in functions.items():
        _signals_m[k1].update({n: {}})
        for k2, v2 in v1.items(): # left/right
          _signals_m[k1][n].update({k2: {}})
          for k3, s in v2.items(): #zone
            _signals_m[k1][n][k2].update({k3: f(s)})
            
    return False

  def f(file_names,
        method, degree, 
        length, width, fontsize, print):
    
    nonlocal flag
    if flag:
      flag = process()
      
    fig, axs = plt.subplots(2, len(file_names), figsize = (length * len(file_names), width * 2) )

    if len(file_names) == 1: 
      axs = axs[:, np.newaxis]

    for i, k1 in enumerate(file_names):

      v1 = _signals_m[k1][method]

      a, b = list(v1['left']), list(v1['right'])
      inters = list(set(a).intersection(set(b)))

      df = [pd.DataFrame.from_dict(v1['left'])[inters], pd.DataFrame.from_dict(v1['right'])[inters]]

      df1, df2 = df[0].corr(), df[1].corr()

      for j, name_1 in enumerate(list(df1.columns)):
        for k, name_2 in enumerate(list(df2.columns)):
          if k > j:
            df1.at[name_2, name_1] = df2.at[name_2, name_1]

      dfc = [df1, pd.DataFrame(columns = df[0].columns, index = df[1].columns, dtype = float)]

      for col in list(df[0].columns):
        for idx in list(df[1].columns):
          dfc[1].loc[idx, col] = stats.pearsonr(df[0][col], df[1][idx])[0]

      for k in range(2):
        im = axs[k, i].matshow(dfc[k], cmap = 'jet')
        axs[k, i].tick_params(axis = 'both', which = 'major', labelsize = fontsize)
        axs[k, i].set_xticks(range(df[k].shape[1]))
        axs[k, i].set_yticks(range(df[k].shape[1]))
        axs[k, i].set_xticklabels(list(df[k].columns), rotation = 90)
        axs[k, i].set_yticklabels(list(df[k].columns))
        for (m, j), z in np.ndenumerate(dfc[k]):
          axs[k, i].text(j, m, '{:0.2f}'.format(z), ha = 'center', va = 'center', fontsize = fontsize * 0.5)
        axs[k, i].xaxis.set_label_position('top') 
        axs[k, i].set_ylabel('Левое полушарие', fontsize = fontsize)

      axs[0, i].set_xlabel('Правое полушарие' + '\n' + k1, fontsize = fontsize)
      axs[1, i].set_xlabel('Правое полушарие', fontsize = fontsize)
      
    plt.suptitle(print + '//' + 
                 'method=' + method + '//' + 'degree=' + str(degree) + '\n' +
                 '//' + '//'.join([key + '=' + str(value) for key, value in args.items()]) + '//', 
                 y = 1.02, fontsize = fontsize)
    plt.tight_layout()
    plt.show()

  inter = ipywidgets.interactive_output(f, items)
  display(ui, inter)
# pack = prepare(cond = 'anesthetized', w1 = 505, w2 = 430, exp_num = 1, mouse = 1)
# pack
# func_3(ss)<jupyter_output><empty_output><jupyter_text>spectrograms - 4<jupyter_code>def func_4(data):

  _signals, args = [copy.deepcopy(d) for d in data]

  file_names_1, zone_names_1 = list(_signals), set()
  for i in _signals.values():
    for j in i.values():
      for k in j.keys():
        zone_names_1.add(k)
  zone_names_1 = list(zone_names_1)

  file_names_1.sort()
  zone_names_1 = [n for n in ZONE_NAMES if n in zone_names_1]

  items, ui = interface_4(file_names_1, zone_names_1)
  
  functions = {'no' : lambda y: y,
               'ZhangFit' : lambda y: BaselineRemoval(y).ZhangFit(), 
               'trend' : lambda y: y - stats.linregress(np.arange(y.shape[0]), y).intercept
               }

  flag = True
  _signals_m = {}

  def process():

    fs = 16.66

    for k1, v1 in _signals.items():       # name

      _signals_m.update({k1: {}})
      for n, f in functions.items():
        _signals_m[k1].update({n: {}})
        for k2, v2 in v1.items():         # left/right
          _signals_m[k1][n].update({k2: {}})
          for k3, s in v2.items():        # zone
            _signals_m[k1][n][k2].update({k3: f(s)})
            
    return False

  def f(file_name, zone_names,
        method, degree, mode, scaling, fs, noverlap,
        length, width, fontsize, print):

    nonlocal flag
    if flag:
      flag = process()

    v1 = _signals_m[file_name][method]
    a, b = list(v1['left']), list(v1['right'])

    inters = list(set(a).intersection(set(b)))
    
    fig, axs = plt.subplots(len(inters), 2, figsize = (length * 2, width * len(inters)) )

    for i, (k2, v2) in enumerate(v1.items()): # side
      for j, k3 in enumerate(inters):         # zones
        v3 = v2.get(k3)
        if v3 is not None:
          f, t, Sxx = signal.spectrogram(v3, fs, mode = mode, scaling = scaling)
          axs[j, i].pcolormesh(t, f, Sxx, shading = 'gouraud')
        else:
          axs[j, i].axis('off')


    for i, k1 in enumerate(inters):
      axs[i, 0].set_ylabel(k1, fontsize = fontsize)

    axs[0, 0].set_title('Левое полушарие', fontsize = fontsize)
    axs[0, 1].set_title('Правое полушарие', fontsize = fontsize)
    plt.suptitle(print + '//' + 
                 'method=' + method + '//' + 'degree=' + str(degree) + '\n' +
                 '//' + '//'.join([key + '=' + str(value) for key, value in args.items()]) + '//', 
                 y = 1.02, fontsize = fontsize)
    plt.tight_layout()
    plt.show()

  inter = ipywidgets.interactive_output(f, items)
  display(ui, inter)
# pack = prepare(cond = 'anesthetized', w1 = 505, w2 = 430, exp_num = 1, mouse = 1)
# pack
# ss = get_signals(pack)
# func_4(ss)<jupyter_output><empty_output><jupyter_text>**prepare data**<jupyter_code>parameters = [[{'cond' : 'anesthetized', 'w1' : 505, 'w2' : 430, 'exp_num' : 1, 'mouse' : mouse} for mouse in range(10)],
              [{'cond' : 'anesthetized', 'w1' : 505, 'w2' : 430, 'exp_num' : 1, 'exp' : exp, 'stim' : 'no_st'} for exp in ['00d', '03d', '10d', '15d']],
              [{'cond' : 'anesthetized', 'w1' : 505, 'w2' : 430, 'exp_num' : 1, 'exp' : exp, 'stim' : 'stim'}  for exp in ['00d', '03d', '10d', '15d']],
              [{'cond' : 'awake',        'w1' : 505, 'w2' : 430, 'exp_num' : 1, 'mouse' : mouse} for mouse in range(10)],
              [{'cond' : 'awake',        'w1' : 505, 'w2' : 430, 'exp_num' : 1, 'exp' : exp, 'stim' : 'no_st'} for exp in ['00d', '03d', '10d', '15d']],
              [{'cond' : 'awake',        'w1' : 505, 'w2' : 430, 'exp_num' : 1, 'exp' : exp, 'stim' : 'stim'}  for exp in ['00d', '03d', '10d', '15d']],
              ]
packs = [[i for i in [prepare(**p) for p in pp] if i is not None] for pp in parametrs]
temp_numb, temp_data = [], []
for i in packs:
  temp_numb.append(len(i))
  for j in i:
    temp_data.append(j)
with concurrent.futures.ProcessPoolExecutor(2) as executor:
  temp_1 = list(executor.map(get_signals, temp_data))
with concurrent.futures.ProcessPoolExecutor(1) as executor:
  temp_2 = list(executor.map(get_data_fft, temp_data))
c, signals, data_fft = 0, [], []
for i in range(len(temp_numb)):
  signals.append([])
  data_fft.append([])
  for j in range(temp_numb[i]):
    signals[i].append(temp_1[c])
    data_fft[i].append(temp_2[c])
    c += 1<jupyter_output><empty_output><jupyter_text>**results**    plot_zones<jupyter_code>plot_zones()<jupyter_output><empty_output><jupyter_text>fourier analysis<jupyter_code>for ss in signals:
  for s in ss:
    func_1(s)<jupyter_output><empty_output><jupyter_text>2d fourier analysis<jupyter_code>for dd in data_fft[2:]:
  for d in dd:
    func_2(d)
for dd in data_fft[:2]:
  for d in dd:
    func_2(d)<jupyter_output><empty_output><jupyter_text>correlation matrix<jupyter_code>for ss in signals:
  for s in ss:
    func_3(s)<jupyter_output><empty_output><jupyter_text>spectrograms<jupyter_code>for ss in signals:
  for s in ss:
    func_4(s)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15502756629036493, 'fraction_numerical': 0.029482803885534262, 'mean_word_length': 2.0546110665597435, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 95, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '52008', 'license': 'MIT', 'n_tokens_mistral': 14377, 'n_tokens_neox': 13407, 'n_words': 3053, 'path': 'Mouse_brain_research.ipynb', 'repo_name': 'kuratko-hub/brain_cl_measure', 'size': '38090'}