<commit_before># TODO:
# [ ] add description (old one is invalidated)
# [ ] add extra sensors to 10-10 standard (F9, Ns etc)
# [ ] integrate with scalp-plot functions
# [ ] add (small) sensor dots by default in scalp plot, names optional.
import numpy as np

def per_to_cart(theta, phi):
  theta = theta/100. * np.pi
  phi = phi/100. * np.pi - np.pi/2
  return [np.sin(theta) * np.sin(phi), np.cos(theta), 
    np.sin(theta) * np.cos(phi)]

def project_scalp(xyz, v=2.):
  '''
  Project the sensor locations towards a point *below* the cap with distance v
  on the xy plane.
  '''
  # v/(v+z) = x'/x => x' = xv/(v+z)
  x, y, z = xyz
  return (x*v/(v+z), y*v/(v+z))

def gen_10_10():
  # add irregularly placed sensors
  result = {
    'Fp1': (10, 0), 'Fpz': (0, 50), 'Fp2': (10, 100), 'Nz': (-10, 50),
    'O1' : (110, 0), 'Oz' : (100, 50), 'O2' : (110, 100),
    'I1' : (90, -10), 'Iz' : (100, 50), 'I2' : (90, 110)}

  # generate polar sensor positions with names
  lr_pos = [str(i) for i in [9, 7, 5, 3, 1, 'z', 2, 4, 6, 8, 10]]
  fb_slab = ['AF', 'F', 'FC', 'C', 'CP', 'P', 'PO']
  for theta_per, slab in zip(np.linspace(20, 80, 7), fb_slab):
    for phi_per, pos in zip(np.linspace(-10, 110, 11), lr_pos):
      result[slab + pos] = (theta_per, phi_per)
 
  # convert to cartesian coords
  result = dict([(k, per_to_cart(*v)) for (k, v) in result.items()])
 
  # substitute with sepcialized labels and remove illegal sensors
  SUBST = {'C9':'T9', 'C7':'T7', 'C8':'T8', 'C10':'T10',
    'CP9':'TP9', 'CP7':'TP7', 'CP8':'TP8', 'CP10':'TP10',
    'FC9':'FT9', 'FC7':'FT7', 'FC8':'FT8', 'FC10':'FT10'}
  BLACKLIST = ['AF9', 'AF10']
  result = dict([(SUBST.get(k, k), v) for k, v in result.items()
    if k not in BLACKLIST])

  return result

import unittest, matplotlib.pyplot as plt

class Test10_10(unittest.TestCase):
  def test_plot_locs(self):
    locs = []
    print LAYOUT['Nz']
    print LAYOUT['I1']
    print LAYOUT['Iz']
    print LAYOUT['I2']
    for (label, coord) in LAYOUT.items():
      x, y = project_scalp(coord)
      plt.text(x, y + .03, label, fontsize=8, ha='center')
      locs.append((x, y))
    locs = np.asarray(locs)

    plt.plot(locs[:, 0], locs[:, 1], 'ko')
    plt.xlim(-1.5, 1.5)
    plt.ylim(-1.5, 1.5)
    plt.show()
    
LAYOUT = gen_10_10()

class TestPolar(unittest.TestCase):
  def test_extremes(self):
    np.testing.assert_almost_equal(per_to_cart(0, 50), np.array([0, 1, 0]))
    np.testing.assert_almost_equal(per_to_cart(50, 50), np.array([0, 0, 1]))
    np.testing.assert_almost_equal(per_to_cart(100, 50), np.array([0, -1, 0]))
    np.testing.assert_almost_equal(per_to_cart(50, 0), np.array([-1, 0, 0]))
    np.testing.assert_almost_equal(per_to_cart(50, 100), np.array([1, 0, 0]))

if __name__ == '__main__':
  unittest.main()
<commit_msg>Surrendered. Using 3D locations of Oostenveld.<commit_after># TODO:
# [ ] add description (old one is invalidated)
# [ ] add extra sensors to 10-10 standard (F9, Ns etc)
# [ ] integrate with scalp-plot functions
# [ ] add (small) sensor dots by default in scalp plot, names optional.
import numpy as np
from pprint import pprint

OOSTENVELD_10_5 = \
'''
LPA	0.0000	0.9237	-0.3826	
RPA	0.0000	-0.9237	-0.3826
Nz	0.9230	0.0000	-0.3824
Fp1	0.9511	0.3090	0.0001
Fpz	1.0000	-0.0000	0.0001
Fp2	0.9511	-0.3091	0.0000
AF9	0.7467	0.5425	-0.3825
AF7	0.8090	0.5878	0.0000
AF5	0.8553	0.4926	0.1552
AF3	0.8920	0.3554	0.2782
AF1	0.9150	0.1857	0.3558
AFz	0.9230	0.0000	0.3824
AF2	0.9150	-0.1857	0.3558
AF4	0.8919	-0.3553	0.2783
AF6	0.8553	-0.4926	0.1552
AF8	0.8090	-0.5878	0.0000
AF10	0.7467	-0.5425	-0.3825
F9	0.5430	0.7472	-0.3826
F7	0.5878	0.8090	0.0000
F5	0.6343	0.7210	0.2764
F3	0.6726	0.5399	0.5043
F1	0.6979	0.2888	0.6542
Fz	0.7067	0.0000	0.7067
F2	0.6979	-0.2888	0.6542
F4	0.6726	-0.5399	0.5043
F6	0.6343	-0.7210	0.2764
F8	0.5878	-0.8090	0.0000
F10	0.5429	-0.7472	-0.3826
FT9	0.2852	0.8777	-0.3826
FT7	0.3090	0.9511	0.0000
FC5	0.3373	0.8709	0.3549
FC3	0.3612	0.6638	0.6545
FC1	0.3770	0.3581	0.8532
FCz	0.3826	0.0000	0.9233
FC2	0.3770	-0.3581	0.8532
FC4	0.3612	-0.6638	0.6545
FC6	0.3373	-0.8709	0.3549
FT8	0.3090	-0.9511	0.0000
FT10	0.2852	-0.8777	-0.3826
T9	-0.0001	0.9237	-0.3826
T7	0.0000	1.0000	0.0000
C5	0.0001	0.9237	0.3826
C3	0.0001	0.7066	0.7066
C1	0.0002	0.3824	0.9231
Cz	0.0002	0.0000	1.0000
C2	0.0001	-0.3824	0.9231
C4	0.0001	-0.7066	0.7066
C6	0.0001	-0.9237	0.3826
T8	0.0000	-1.0000	0.0000
T10	0.0000	-0.9237	-0.3826
TP9	-0.2852	0.8777	-0.3826
TP7	-0.3090	0.9511	-0.0001
CP5	-0.3372	0.8712	0.3552
CP3	-0.3609	0.6635	0.6543
CP1	-0.3767	0.3580	0.8534
CPz	-0.3822	0.0000	0.9231
CP2	-0.3767	-0.3580	0.8534
CP4	-0.3608	-0.6635	0.6543
CP6	-0.3372	-0.8712	0.3552
TP8	-0.3090	-0.9511	-0.0001
TP10	-0.2853	-0.8777	-0.3826
P9	-0.5429	0.7472	-0.3826
P7	-0.5878	0.8090	-0.0001
P5	-0.6342	0.7211	0.2764
P3	-0.6724	0.5401	0.5045
P1	-0.6975	0.2889	0.6545
Pz	-0.7063	0.0000	0.7065
P2	-0.6975	-0.2889	0.6545
P4	-0.6724	-0.5401	0.5045
P6	-0.6342	-0.7211	0.2764
P8	-0.5878	-0.8090	-0.0001
P10	-0.5429	-0.7472	-0.3826
PO9	-0.7467	0.5425	-0.3825
PO7	-0.8090	0.5878	0.0000
PO5	-0.8553	0.4929	0.1555
PO3	-0.8918	0.3549	0.2776
PO1	-0.9151	0.1858	0.3559
POz	-0.9230	-0.0000	0.3824
PO2	-0.9151	-0.1859	0.3559
PO4	-0.8918	-0.3549	0.2776
PO6	-0.8553	-0.4929	0.1555
PO8	-0.8090	-0.5878	0.0000
PO10	-0.7467	-0.5425	-0.3825
O1	-0.9511	0.3090	0.0000
Oz	-1.0000	0.0000	0.0000
O2	-0.9511	-0.3090	0.0000
I1	-0.8785	0.2854	-0.3824
Iz	-0.9230	0.0000	-0.3823
I2	-0.8785	-0.2854	-0.3824
AFp9h	0.8732	0.4449	-0.1949
AFp7h	0.9105	0.4093	0.0428
AFp5h	0.9438	0.3079	0.1159
AFp3h	0.9669	0.1910	0.1666
AFp1h	0.9785	0.0647	0.1919
AFp2h	0.9785	-0.0647	0.1919
AFp4h	0.9669	-0.1910	0.1666
AFp6h	0.9438	-0.3079	0.1159
AFp8h	0.9105	-0.4093	0.0428
AFp10h	0.8732	-0.4449	-0.1949
AFF9h	0.6929	0.6929	-0.1949
AFF7h	0.7325	0.6697	0.1137
AFF5h	0.7777	0.5417	0.3163
AFF3h	0.8111	0.3520	0.4658
AFF1h	0.8289	0.1220	0.5452
AFF2h	0.8289	-0.1220	0.5452
AFF4h	0.8111	-0.3520	0.4658
AFF6h	0.7777	-0.5417	0.3163
AFF8h	0.7325	-0.6697	0.1138
AFF10h	0.6929	-0.6929	-0.1949
FFT9h	0.4448	0.8730	-0.1950
FFT7h	0.4741	0.8642	0.1647
FFC5h	0.5107	0.7218	0.4651
FFC3h	0.5384	0.4782	0.6925
FFC1h	0.5533	0.1672	0.8148
FFC2h	0.5533	-0.1672	0.8148
FFC4h	0.5384	-0.4782	0.6925
FFC6h	0.5107	-0.7218	0.4651
FFT8h	0.4741	-0.8642	0.1647
FFT10h	0.4448	-0.8730	-0.1950
FTT9h	0.1533	0.9678	-0.1950
FTT7h	0.1640	0.9669	0.1915
FCC5h	0.1779	0.8184	0.5448
FCC3h	0.1887	0.5466	0.8154
FCC1h	0.1944	0.1919	0.9615
FCC2h	0.1944	-0.1919	0.9615
FCC4h	0.1887	-0.5466	0.8154
FCC6h	0.1779	-0.8184	0.5448
FTT8h	0.1640	-0.9669	0.1915
FTT10h	0.1533	-0.9678	-0.1950
TTP9h	-0.1532	0.9678	-0.1950
TTP7h	-0.1639	0.9669	0.1915
CCP5h	-0.1778	0.8185	0.5449
CCP3h	-0.1883	0.5465	0.8153
CCP1h	-0.1940	0.1918	0.9611
CCP2h	-0.1940	-0.1918	0.9611
CCP4h	-0.1884	-0.5465	0.8153
CCP6h	-0.1778	-0.8185	0.5449
TTP8h	-0.1639	-0.9669	0.1915
TTP10h	-0.1533	-0.9678	-0.1950
TPP9h	-0.4448	0.8731	-0.1950
TPP7h	-0.4740	0.8639	0.1646
CPP5h	-0.5106	0.7220	0.4653
CPP3h	-0.5384	0.4786	0.6933
CPP1h	-0.5532	0.1673	0.8155
CPP2h	-0.5532	-0.1673	0.8155
CPP4h	-0.5384	-0.4786	0.6933
CPP6h	-0.5106	-0.7220	0.4653
TPP8h	-0.4740	-0.8638	0.1646
TPP10h	-0.4449	-0.8731	-0.1950
PPO9h	-0.6928	0.6928	-0.1950
PPO7h	-0.7324	0.6700	0.1139
PPO5h	-0.7776	0.5420	0.3167
PPO3h	-0.8108	0.3520	0.4659
PPO1h	-0.8284	0.1220	0.5453
PPO2h	-0.8284	-0.1220	0.5453
PPO4h	-0.8108	-0.3519	0.4659
PPO6h	-0.7775	-0.5421	0.3167
PPO8h	-0.7324	-0.6700	0.1139
PPO10h	-0.6928	-0.6928	-0.1950
POO9h	-0.8730	0.4448	-0.1950
POO7h	-0.9106	0.4097	0.0430
POO5h	-0.9438	0.3080	0.1160
POO3h	-0.9665	0.1908	0.1657
POO1h	-0.9783	0.0647	0.1918
POO2h	-0.9783	-0.0647	0.1918
POO4h	-0.9665	-0.1908	0.1657
POO6h	-0.9438	-0.3080	0.1160
POO8h	-0.9106	-0.4097	0.0430
POO10h	-0.8730	-0.4448	-0.1950
OI1h	-0.9679	0.1533	-0.1950
OI2h	-0.9679	-0.1533	-0.1950
Fp1h	0.9877	0.1564	0.0001
Fp2h	0.9877	-0.1564	0.0001
AF9h	0.7928	0.5759	-0.1949
AF7h	0.8332	0.5463	0.0810
AF5h	0.8750	0.4284	0.2213
AF3h	0.9053	0.2735	0.3231
AF1h	0.9211	0.0939	0.3758
AF2h	0.9210	-0.0939	0.3758
AF4h	0.9053	-0.2735	0.3231
AF6h	0.8750	-0.4284	0.2212
AF8h	0.8332	-0.5463	0.0810
AF10h	0.7927	-0.5759	-0.1949
F9h	0.5761	0.7929	-0.1949
F7h	0.6117	0.7772	0.1420
F5h	0.6549	0.6412	0.3987
F3h	0.6872	0.4214	0.5906
F1h	0.7045	0.1468	0.6933
F2h	0.7045	-0.1468	0.6933
F4h	0.6872	-0.4214	0.5906
F6h	0.6549	-0.6412	0.3987
F8h	0.6117	-0.7772	0.1420
F10h	0.5761	-0.7929	-0.1949
FT9h	0.3027	0.9317	-0.1950
FT7h	0.3235	0.9280	0.1813
FC5h	0.3500	0.7817	0.5146
FC3h	0.3703	0.5207	0.7687
FC1h	0.3811	0.1824	0.9054
FC2h	0.3811	-0.1824	0.9054
FC4h	0.3703	-0.5207	0.7687
FC6h	0.3500	-0.7817	0.5146
FT8h	0.3235	-0.9280	0.1813
FT10h	0.3028	-0.9317	-0.1950
T9h	0.0000	0.9801	-0.1950
T7h	0.0000	0.9801	0.1949
C5h	0.0001	0.8311	0.5552
C3h	0.0002	0.5550	0.8306
C1h	0.0001	0.1950	0.9801
C2h	0.0002	-0.1950	0.9801
C4h	0.0002	-0.5550	0.8306
C6h	0.0001	-0.8311	0.5552
T8h	0.0000	-0.9801	0.1949
T10h	0.0000	-0.9801	-0.1950
TP9h	-0.3028	0.9319	-0.1949
TP7h	-0.3234	0.9278	0.1813
CP5h	-0.3498	0.7818	0.5148
CP3h	-0.3699	0.5206	0.7688
CP1h	-0.3808	0.1825	0.9059
CP2h	-0.3808	-0.1825	0.9059
CP4h	-0.3699	-0.5206	0.7688
CP6h	-0.3498	-0.7818	0.5148
TP8h	-0.3234	-0.9278	0.1813
TP10h	-0.3028	-0.9319	-0.1949
P9h	-0.5761	0.7929	-0.1950
P7h	-0.6116	0.7771	0.1420
P5h	-0.6546	0.6411	0.3985
P3h	-0.6869	0.4217	0.5912
P1h	-0.7041	0.1469	0.6934
P2h	-0.7041	-0.1469	0.6934
P4h	-0.6870	-0.4216	0.5912
P6h	-0.6546	-0.6411	0.3985
P8h	-0.6116	-0.7771	0.1420
P10h	-0.5761	-0.7929	-0.1950
PO9h	-0.7926	0.5759	-0.1950
PO7h	-0.8331	0.5459	0.0809
PO5h	-0.8752	0.4292	0.2219
PO3h	-0.9054	0.2737	0.3233
PO1h	-0.9210	0.0939	0.3757
PO2h	-0.9210	-0.0940	0.3757
PO4h	-0.9054	-0.2737	0.3233
PO6h	-0.8752	-0.4292	0.2219
PO8h	-0.8331	-0.5459	0.0809
PO10h	-0.7926	-0.5758	-0.1950
O1h	-0.9877	0.1564	0.0000
O2h	-0.9877	-0.1564	0.0000
I1h	-0.9118	0.1444	-0.3824
I2h	-0.9118	-0.1444	-0.3824
AFp9	0.8225	0.4190	-0.3825
AFp7	0.8910	0.4540	0.0000
AFp5	0.9282	0.3606	0.0817
AFp3	0.9565	0.2508	0.1438
AFp1	0.9743	0.1287	0.1828
AFpz	0.9799	-0.0000	0.1949
AFp2	0.9743	-0.1287	0.1828
AFp4	0.9565	-0.2508	0.1437
AFp6	0.9282	-0.3606	0.0817
AFp8	0.8910	-0.4540	0.0000
AFp10	0.8225	-0.4191	-0.3825
AFF9	0.6527	0.6527	-0.3825
AFF7	0.7071	0.7071	0.0000
AFF5	0.7564	0.6149	0.2206
AFF3	0.7962	0.4535	0.3990
AFF1	0.8221	0.2404	0.5148
AFFz	0.8312	0.0000	0.5554
AFF2	0.8221	-0.2404	0.5148
AFF4	0.7962	-0.4535	0.3990
AFF6	0.7564	-0.6149	0.2206
AFF8	0.7071	-0.7071	0.0000
AFF10	0.6527	-0.6527	-0.3825
FFT9	0.4192	0.8226	-0.3826
FFT7	0.4540	0.8910	0.0000
FFC5	0.4932	0.8072	0.3215
FFC3	0.5260	0.6110	0.5905
FFC1	0.5477	0.3286	0.7685
FFCz	0.5553	0.0000	0.8310
FFC2	0.5477	-0.3286	0.7685
FFC4	0.5260	-0.6110	0.5905
FFC6	0.4932	-0.8072	0.3216
FFT8	0.4540	-0.8910	0.0000
FFT10	0.4192	-0.8226	-0.3826
FTT9	0.1444	0.9119	-0.3826
FTT7	0.1565	0.9877	0.0000
FCC5	0.1713	0.9099	0.3754
FCC3	0.1838	0.6957	0.6933
FCC1	0.1922	0.3764	0.9059
FCCz	0.1951	0.0000	0.9804
FCC2	0.1922	-0.3764	0.9059
FCC4	0.1838	-0.6957	0.6933
FCC6	0.1713	-0.9099	0.3754
FTT8	0.1564	-0.9877	0.0000
FTT10	0.1444	-0.9119	-0.3826
TTP9	-0.1444	0.9117	-0.3826
TTP7	-0.1564	0.9877	-0.0001
CCP5	-0.1711	0.9100	0.3754
CCP3	-0.1836	0.6959	0.6936
CCP1	-0.1918	0.3763	0.9056
CCPz	-0.1948	0.0000	0.9800
CCP2	-0.1919	-0.3763	0.9056
CCP4	-0.1836	-0.6959	0.6936
CCP6	-0.1711	-0.9100	0.3754
TTP8	-0.1564	-0.9877	-0.0001
TTP10	-0.1444	-0.9117	-0.3826
TPP9	-0.4191	0.8225	-0.3826
TPP7	-0.4540	0.8910	-0.0001
CPP5	-0.4931	0.8073	0.3216
CPP3	-0.5259	0.6109	0.5904
CPP1	-0.5476	0.3285	0.7685
CPPz	-0.5551	0.0000	0.8311
CPP2	-0.5475	-0.3286	0.7685
CPP4	-0.5258	-0.6109	0.5904
CPP6	-0.4931	-0.8073	0.3216
TPP8	-0.4540	-0.8910	-0.0001
TPP10	-0.4191	-0.8225	-0.3826
PPO9	-0.6529	0.6529	-0.3825
PPO7	-0.7071	0.7071	0.0000
PPO5	-0.7561	0.6147	0.2205
PPO3	-0.7960	0.4537	0.3995
PPO1	-0.8218	0.2405	0.5152
PPOz	-0.8306	0.0000	0.5551
PPO2	-0.8218	-0.2405	0.5152
PPO4	-0.7960	-0.4537	0.3995
PPO6	-0.7562	-0.6147	0.2205
PPO8	-0.7071	-0.7071	0.0000
PPO10	-0.6529	-0.6529	-0.3825
POO9	-0.8228	0.4191	-0.3824
POO7	-0.8910	0.4540	0.0000
POO5	-0.9283	0.3608	0.0818
POO3	-0.9567	0.2511	0.1442
POO1	-0.9739	0.1285	0.1822
POOz	-0.9797	-0.0000	0.1949
POO2	-0.9739	-0.1286	0.1822
POO4	-0.9567	-0.2511	0.1442
POO6	-0.9283	-0.3608	0.0818
POO8	-0.8910	-0.4540	0.0000
POO10	-0.8228	-0.4191	-0.3824
OI1	-0.9322	0.3029	-0.1949
OIz	-0.9799	0.0000	-0.1949
OI2	-0.9322	-0.3029	-0.1949
T3	0.0000	1.0000	0.0000
T5	-0.5878	0.8090	-0.0001
T4	0.0000	-1.0000	0.0000
T6	-0.5878	-0.8090	-0.0001
'''

def parse_10_5(line):
  s, y, x, z = line.split()
  return (s, tuple(map(float, [x, y, z])))

OOSTENVELD_10_5 = dict([parse_10_5(l) for l in OOSTENVELD_10_5.splitlines()])

def project_scalp(xyz, v=2.):
  '''
  Project the sensor locations towards a point *below* the cap with distance v
  on the xy plane.
  '''
  # v/(v+z) = x'/x => x' = xv/(v+z)
  x, y, z = xyz
  return (x*v/(v+z), y*v/(v+z))

import unittest, matplotlib.pyplot as plt

class Test10_10(unittest.TestCase):
  def test_dists(self):
    def dist(a, b):
      return np.linalg.norm((np.atleast_1d(a) - np.atleast_1d(b)))

    def test_eq_dists(labs):
      dists = [dist(LAYOUT[a], LAYOUT[b]) for a, b in zip(labs, labs[1:])]
      self.assert_(np.all(np.abs(np.diff(dists)) < 1e-3))

    test_eq_dists('Nz Fpz AFz Fz FCz Cz CPz Pz POz Oz Iz'.split())
    test_eq_dists('Fpz Fp1 AF7 F7 FT7 T7 TP7 P7 PO7 O1 Oz'.split())
    test_eq_dists('Fpz Fp2 AF8 F8 FT8 T8 TP8 P8 PO8 O2 Oz'.split())
    test_eq_dists('T9 T7 C5 C3 C1 Cz C2 C4 C6 T8 T10'.split())

  def test_plot_locs(self):
    locs = []
    for (label, coord) in LAYOUT.items():
      x, y = project_scalp(coord)
      plt.text(x, y + .03, label, fontsize=8, ha='center')
      locs.append((x, y))
    locs = np.asarray(locs)

    plt.plot(locs[:, 0], locs[:, 1], 'ko')
    plt.xlim(-1.5, 1.5)
    plt.ylim(-1.5, 1.5)
    plt.show()
    
LAYOUT = OOSTENVELD_10_5


if __name__ == '__main__':
  unittest.main()

{'fraction_non_alphanumeric': 0.16588294870867396, 'fraction_numerical': 0.4150329161542357, 'mean_word_length': 2.599062744077063, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '185240', 'n_tokens_mistral': 11092, 'n_tokens_neox': 8679, 'n_words': 1876}