# disable warnings bc we good on this one
import warnings
warnings.filterwarnings('ignore')
import os
from pprint import pprint
import numpy as np
import pandas as pd
import pickle
from tqdm import tqdm_notebook as tqdm
from scipy.io import loadmat
from datetime import datetime
from pynwb import NWBFile, TimeSeries, NWBHDF5IO
from pynwb.file import Subject
# install manually with python ./setup.py install
# https://github.com/NeurodataWithoutBorders/nwb-jupyter-widgets
#from nwbwidgets import nwb2widget
# local version of oe's analysis-tools
import OpenEphys as oe
data_dir = '../data/nwb/'
Load our own idiosyncratic metadata format. It is stored as a matlab struct, and contains sorted spikes as well as metadata.
out = loadmat(os.path.join(data_dir, 'OutPSTH_ch2c1.mat'))
out = out['out']
Scipy loads matlab structs as record arrays, to see the fields we can refer to its dtype
out.dtype.names
('IL',
'Nclusters',
'tetrode',
'channel',
'cluster',
'cell',
'M1ON',
'mM1ONspikecount',
'sM1ONspikecount',
'semM1ONspikecount',
'mM1ON',
'mM1spontON',
'sM1spontON',
'semM1spontON',
'LaserStart',
'LaserWidth',
'Lasernumpulses',
'M_LaserStart',
'M_LaserWidth',
'M_LaserNumPulses',
'M_LaserISI',
'M1OFF',
'mM1OFF',
'mM1OFFspikecount',
'sM1OFFspikecount',
'semM1OFFspikecount',
'mM1spontOFF',
'sM1spontOFF',
'semM1spontOFF',
'numpulseamps',
'numgapdurs',
'pulseamps',
'gapdurs',
'gapdelay',
'soa',
'nrepsON',
'nrepsOFF',
'xlimits',
'samprate',
'datadir',
'spiketimes',
'LaserRecorded',
'StimRecorded',
'M1ONLaser',
'mM1ONLaser',
'M1OFFLaser',
'mM1OFFLaser',
'M1ONStim',
'mM1ONStim',
'M1OFFStim',
'mM1OFFStim',
'nb',
'stimlog',
'user')
Wow that's pretty big list of opaque variable names. Thankfully that's the point of converting to NWB.
We are only going to work with a small subset of it anyway so don't get your peppers pickled.
Extract recording metadata. First we get the stim log, we convert it to a pandas DataFrame so it's easier to work with, and do a bit of cleaning.
stim_log = pd.DataFrame.from_records(out['stimlog'][0][0][0])
# remove columns we don't need
stim_log = stim_log[['type', 'stimulus_description', 'timestamp', 'LaserOnOff']]
stim_log = stim_log[stim_log.columns].astype(str)
# remove leading and lagging brackets
for name, col in stim_log.iteritems():
stim_log.loc[:,name] = col.str.lstrip("[u'").str.rstrip("]'")
# clean up timestamps
stim_log.loc[:,'timestamp'] = stim_log['timestamp'] + "000"
stim_log.loc[:,'timestamp'] = pd.to_datetime(stim_log['timestamp'],
format="%B-%d-%Y %H:%M:%S.%f")
Parse this PITA... i mean plaintext stim description... it looks like this:
print(stim_log.loc[0:5, 'stimulus_description'])
0 whitenoise amplitude:80 ramp:0 loop_flg:0 seam... 1 whitenoise amplitude:80 ramp:0 loop_flg:0 seam... 2 whitenoise amplitude:80 ramp:0 loop_flg:0 seam... 3 whitenoise amplitude:80 ramp:0 loop_flg:0 seam... 4 whitenoise amplitude:80 ramp:0 loop_flg:0 seam... 5 GPIAS amplitude:80 ramp:0 soa:50 soaflag:isi l... Name: stimulus_description, dtype: object
so we need to split that first by spaces and then by colons to break it up into new columns.
# Expand it to columns
stim_desc = stim_log.loc[:,'stimulus_description'].str.split(' ',expand=True)
# two types of stim have two sets of params, do white noise first
stim_noise = stim_desc.loc[stim_desc[0]=='whitenoise']
col_names = ['type']
# iterate through columns, splitting on colons and
# assigning the left to the name and the right as the value
for i in range(1,stim_noise.shape[1]):
try:
split_col = stim_noise.loc[:,i].str.split(':', expand=True)
# get name of column
col_name = split_col[0][0]
if col_name is None:
continue
col_names.append(col_name)
# replace column w value
stim_noise.loc[:,i] = split_col[1]
except:
pass
# remove None columns and set names
stim_noise = stim_noise.loc[:,0:len(col_names)-1]
stim_noise.columns = col_names
#######
# YEAH REPEATING OURSELVES
########
stim_gpias = stim_desc.loc[stim_desc[0]=='GPIAS']
col_names = ['type']
for i in range(1,stim_gpias.shape[1]):
try:
split_col = stim_gpias.loc[:,i].str.split(':', expand=True)
# get name of column
col_name = split_col.iloc[0,0]
if col_name is None:
continue
col_names.append(col_name)
# replace column w value
stim_gpias.loc[:,i] = split_col[1]
except:
pass
# remove None columns and set names
stim_gpias = stim_gpias.loc[:,0:len(col_names)-1]
stim_gpias.columns = col_names
# join them by making a fake index column to preserve position later
stim_noise['idx'] = stim_noise.index
stim_gpias['idx'] = stim_gpias.index
stim_combined = stim_noise.merge(stim_gpias, how="outer")
stim_combined.loc[:,'idx'] = stim_combined['idx'].astype('int64')
Now we have some sensible looking stimulus parameters, but they aren't sorted yet.
stim_combined.head(10)
| type | amplitude | ramp | loop_flg | seamless | duration | next | idx | soa | soaflag | gapdelay | gapdur | pulsedur | pulseamp | laser | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 0 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 1 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 2 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 2 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 3 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 3 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 4 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 4 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 5 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 6 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 6 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 7 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 7 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 8 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 8 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 9 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 9 | whitenoise | 80 | 0 | 0 | 1 | 2331 | -500 | 11 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
We now recombine the stimulus details with the rest of the stimulus log, and...
# and recombine with main stim log
stim_log['idx'] = stim_log.index
stim_df = stim_log.merge(stim_combined, how="outer")
stim_df.set_index('idx', inplace=True)
stim_df.sort_index(inplace=True)
# drop old column
stim_df.drop('stimulus_description',axis=1, inplace=True)
The final stimulus dataframe:
stim_df.head(10)
| type | timestamp | LaserOnOff | amplitude | ramp | loop_flg | seamless | duration | next | soa | soaflag | gapdelay | gapdur | pulsedur | pulseamp | laser | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| idx | ||||||||||||||||
| 0 | whitenoise | 2019-03-04 04:46:10.329 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 1 | whitenoise | 2019-03-04 04:46:12.304 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 2 | whitenoise | 2019-03-04 04:46:14.209 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 3 | whitenoise | 2019-03-04 04:46:16.123 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 4 | whitenoise | 2019-03-04 04:46:18.024 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 5 | GPIAS | 2019-03-04 04:46:19.940 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | 50 | isi | 1000 | 256 | 25 | 100 | 0 |
| 6 | whitenoise | 2019-03-04 04:46:21.842 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 7 | whitenoise | 2019-03-04 04:46:23.746 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 8 | whitenoise | 2019-03-04 04:46:25.661 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
| 9 | whitenoise | 2019-03-04 04:46:27.568 | 0 | 80 | 0 | 0 | 1 | 2331 | -500 | NaN | NaN | NaN | NaN | NaN | NaN | NaN |
Next we'll get metadata that's particular to this recording...
# fieldnames are in the dtype attribute
# https://docs.scipy.org/doc/scipy/reference/generated/scipy.io.loadmat.html
field_names = out.dtype.names
get_fields = ['tetrode', 'channel', 'cluster', 'cell',
'spiketimes']
out_dict = {}
for field in get_fields:
# for some reason everything is nested 3 arrays deep...
# I'm going to hardcode this, but you could just keep digging until you get an array that isn't size 1
field_data = out[field][0][0][0]
# if this is an experiment-wide piece of metadata,
# get it out of the array
if field_data.shape == (1,):
field_data = field_data[0]
out_dict[field] = field_data
pprint(out_dict)
{'cell': 1,
'channel': 2,
'cluster': 1,
'spiketimes': array([ 20.9655, 44.8939, 51.1769, 59.727 , 61.1946, 67.7939,
82.6213, 97.8639, 98.7214, 107.901 , 108.0083, 108.318 ,
110.9303, 124.7441, 124.7567, 124.8284, 125.357 , 125.3916,
125.5729, 134.9357, 134.9821, 139.8962, 146.2673, 147.5681,
161.2608, 161.2692, 174.3744, 176.3128, 176.3674, 177.988 ,
178.0086, 178.2294, 178.5374, 179.2267, 179.832 , 198.2626,
199.1443, 199.2332, 199.2403, 199.8766, 201.6041, 202.1028,
207.1975, 208.2051, 237.3535, 243.4475, 252.2219, 289.0311,
290.4128, 307.6082, 307.6908, 307.7069, 328.7144, 328.766 ,
329.1264, 329.2594, 329.2658, 347.2154, 348.9669, 363.6879,
368.3297, 398.0272, 408.5268, 408.5354, 444.4572, 461.1884,
482.9897, 497.9009, 497.9217, 506.5821, 509.7768, 520.3779,
520.4915, 523.6779, 536.2337, 549.3541, 560.9855, 566.6371,
572.6109, 577.803 , 605.4382, 615.1602, 615.1676, 615.173 ,
628.6723, 630.8017, 631.2642, 677.1608, 715.4772, 716.3417,
730.5307, 759.4155, 760.8868, 766.1939, 767.6284, 767.7524,
774.2875, 786.0831, 802.4592, 802.4814, 810.7174, 814.8066,
815.2042, 815.2107, 815.6967, 815.7411, 819.5135, 821.9447,
822.5254, 822.5315, 823.9816, 829.2945, 834.124 , 838.7919,
842.0624, 845.5414, 847.4408, 848.0423, 851.669 , 859.8857,
866.0063, 872.3663, 901.1164, 917.3665, 917.8764, 917.9911,
918.318 , 925.5094, 927.235 , 928.715 , 935.5001, 936.0137,
953.0953, 953.1027, 954.0326, 954.1241, 954.1361, 961.1602,
963.666 , 963.6908, 977.2483, 982.1321, 991.6571, 1024.7936,
1036.1612, 1036.1723, 1053.9281, 1071.2796, 1099.2917, 1099.8015,
1103.0699, 1139.8205, 1149.5919, 1176.7023, 1177.3322, 1233.0499,
1233.0609, 1237.3543, 1269.315 , 1269.4004, 1270.3455, 1271.0548,
1366.8456, 1368.9129, 1444.7188, 1444.7858, 1446.0314, 1446.0349,
1464.9205, 1477.9342, 1482.7394, 1482.7582, 1483.0128, 1524.2914,
1530.9965, 1535.0686, 1536.2521, 1564.7592, 1571.2494, 1572.5929,
1573.6616, 1573.725 , 1573.7327, 1573.7473, 1589.544 , 1610.083 ,
1610.2402, 1613.3386, 1613.3478, 1638.8267, 1642.5698, 1642.5926,
1657.9415, 1665.8156, 1665.8453, 1666.4992, 1666.7568, 1666.8695,
1682.1595, 1682.1712, 1682.1799, 1686.5183, 1686.5256, 1687.9811,
1689.2362, 1692.8254, 1692.8371, 1692.8619, 1697.9051, 1698.2533,
1717.3203, 1717.4 , 1723.9691, 1724.0302, 1724.0649, 1734.7228,
1734.754 , 1735.5312, 1739.3499, 1740.48 , 1758.961 , 1773.398 ,
1773.4064, 1789.3938, 1790.7284, 1795.0465, 1805.2129, 1806.0022,
1806.5657, 1812.5308, 1812.6101, 1816.703 , 1816.8976, 1852.6706,
1861.7364, 1881.472 , 1934.4126, 1935.2656, 1945.6135, 1953.7404,
1981.1332, 1981.3779, 1981.385 , 1981.4898, 1992.6459, 1992.7138,
1995.551 , 1995.8362, 1995.8608, 2004.8935, 2004.9546, 2010.5749,
2049.4204, 2053.3136, 2054.1525, 2065.8965, 2067.2763, 2094.8565,
2127.4138, 2138.0672, 2138.0788, 2187.8077, 2187.8542, 2189.7364,
2198.5361, 2198.5783, 2217.6129, 2222.9587, 2235.1983, 2241.1844,
2244.8964, 2245.0015, 2246.7684, 2249.7938, 2250.888 , 2259.0239,
2287.5913, 2294.9063, 2294.9157, 2301.578 , 2310.124 , 2310.2207,
2310.2265, 2310.3712, 2310.4895, 2310.4953, 2310.7765, 2310.8347,
2310.8613, 2311.0113]),
'tetrode': 2}
and finally the metadata that's particular to the whole experiment. This is stored in a separate notebook file. So yes, we have to parse more matlab structs, but thankfull this time it's less painful (although nothing in MATLAB is painless).
# We also have a separate notebook file...
notebook = loadmat(os.path.join(data_dir, 'notebook.mat'))
notebook_fields = ['user', 'mouseID', 'mouseDOB', 'mouseSex', 'mouseGenotype', 'Drugs']
expt_dict = {}
for field in notebook_fields:
expt_dict[field] = notebook['nb'][field][0][0][0]
pprint(expt_dict)
{'Drugs': u'none',
'mouseDOB': u'11/28/18',
'mouseGenotype': u'100012',
'mouseID': u'9083',
'mouseSex': u'f',
'user': u'Kat'}
We have saved our raw data as OpenEphys .continuous data, use the analysis-tools to load it. I have made some fixes to this code, it being buggy as all hell, and so am using a local version.
I am not including the raw .continuous files in this demo because they are ~560MB, but the following was what was used to load, truncate, and add them to the NWB file.
# oe_files = [os.path.join(data_dir, fn)
# for fn in os.listdir(data_dir)
# if fn.endswith('continuous')]
#
# continuous = []
#
# #oe_evts = oe.load(os.path.join(data_dir,'all_channels.events'))
#
#
# for fn in tqdm(oe_files):
# oe_data = oe.load(fn)
# # construct dict to store data
# oe_dict = {
# 'name': oe_data['header']['channel'].decode().strip("'"),
# 'fs' : oe_data['header']['sampleRate'],
# 'scale': oe_data['header']['bitVolts'],
# 'data': oe_data['data'][0:1000]
# }
#
# continuous.append(oe_dict)
cont_fn = os.path.join(data_dir, 'continuous_dict.pck')
# with open(cont_fn, 'wb+') as cont_file:
# pickle.dump(continuous, cont_file)
with open(cont_fn, 'rb') as cont_file:
continuous = pickle.load(cont_file)
pprint(continuous[0])
{'data': array([ -42.705, -39.78 , -37.05 , -34.515, -31.98 , -29.835,
-28.08 , -26.52 , -25.35 , -24.57 , -24.18 , -23.985,
-24.375, -24.96 , -26.13 , -27.495, -29.25 , -31.395,
-33.93 , -36.66 , -39.585, -42.705, -45.825, -48.75 ,
-51.87 , -54.99 , -58.11 , -60.84 , -63.57 , -66.3 ,
-68.64 , -70.98 , -73.125, -75.27 , -77.415, -79.56 ,
-81.51 , -83.07 , -84.825, -86.19 , -87.555, -88.725,
-89.7 , -90.48 , -91.065, -91.455, -91.65 , -91.65 ,
-91.845, -92.04 , -92.235, -92.625, -93.015, -93.6 ,
-93.99 , -94.575, -95.355, -96.135, -96.915, -97.89 ,
-98.67 , -99.645, -100.425, -101.205, -101.985, -102.96 ,
-103.74 , -104.52 , -105.495, -106.275, -107.055, -107.64 ,
-108.42 , -108.81 , -109.2 , -109.395, -109.2 , -109.005,
-108.81 , -108.42 , -108.03 , -107.445, -106.86 , -106.275,
-105.495, -104.52 , -103.74 , -102.96 , -101.985, -101.205,
-100.23 , -99.255, -98.28 , -97.695, -97.11 , -96.915,
-96.72 , -96.915, -97.11 , -97.695, -98.085, -98.865,
-99.645, -100.425, -101.595, -102.96 , -104.325, -105.69 ,
-107.445, -109.005, -110.76 , -112.515, -114.27 , -115.83 ,
-117.39 , -118.95 , -120.315, -121.68 , -123.24 , -124.8 ,
-126.36 , -127.92 , -129.48 , -131.235, -132.795, -134.355,
-135.915, -137.085, -137.865, -138.645, -138.84 , -138.84 ,
-138.45 , -137.865, -136.89 , -135.525, -134.16 , -132.795,
-131.235, -129.48 , -127.725, -125.97 , -124.215, -122.46 ,
-120.9 , -119.34 , -117.975, -116.61 , -115.245, -113.685,
-112.125, -110.565, -108.81 , -107.055, -105.495, -103.74 ,
-102.18 , -100.815, -99.84 , -98.865, -98.475, -97.89 ,
-97.695, -97.305, -97.11 , -96.915, -96.72 , -96.525,
-96.135, -95.94 , -95.55 , -95.16 , -94.77 , -94.185,
-93.6 , -93.015, -92.43 , -91.845, -91.455, -91.065,
-90.675, -90.48 , -90.285, -90.09 , -89.895, -89.895,
-89.7 , -89.505, -89.31 , -88.92 , -88.335, -87.555,
-86.775, -85.8 , -84.63 , -83.265, -82.095, -80.73 ,
-79.56 , -78.39 , -77.415, -76.635, -76.05 , -75.66 ,
-75.66 , -75.855, -76.245, -77.025, -78. , -79.17 ,
-80.535, -81.9 , -83.265, -84.825, -86.19 , -87.75 ,
-89.505, -91.455, -93.405, -95.355, -97.5 , -99.45 ,
-101.205, -102.96 , -104.715, -106.275, -107.64 , -109.005,
-110.37 , -111.735, -112.905, -113.88 , -114.66 , -115.44 ,
-115.83 , -116.22 , -116.415, -116.805, -117. , -117.39 ,
-117.39 , -117.39 , -117.195, -117. , -116.805, -116.415,
-115.83 , -115.245, -114.855, -114.465, -113.88 , -113.49 ,
-112.905, -112.71 , -112.515, -112.515, -112.515, -112.905,
-113.49 , -114.27 , -115.245, -116.415, -117.78 , -119.34 ,
-121.095, -122.85 , -124.41 , -125.97 , -127.335, -128.505,
-129.675, -130.455, -131.235, -131.625, -132.015, -132.405,
-132.6 , -132.6 , -132.795, -132.795, -132.795, -132.99 ,
-132.99 , -133.185, -133.575, -133.965, -134.355, -134.55 ,
-134.94 , -135.135, -135.135, -135.135, -134.94 , -134.94 ,
-134.745, -134.745, -134.745, -134.745, -134.745, -134.94 ,
-135.33 , -135.72 , -136.5 , -137.28 , -138.06 , -139.035,
-140.205, -141.375, -142.935, -144.495, -146.25 , -148.005,
-149.955, -152.1 , -154.05 , -156. , -157.755, -159.705,
-161.46 , -163.02 , -164.775, -166.53 , -167.895, -169.455,
-170.625, -171.795, -172.77 , -173.745, -174.72 , -175.695,
-176.475, -177.255, -178.035, -179.01 , -179.985, -181.155,
-182.52 , -183.885, -185.25 , -186.42 , -187.2 , -187.98 ,
-188.37 , -188.76 , -188.76 , -188.565, -188.175, -187.59 ,
-186.81 , -185.835, -184.86 , -183.495, -182.325, -181.155,
-179.985, -178.815, -177.84 , -176.865, -175.89 , -174.915,
-173.94 , -173.16 , -172.38 , -171.6 , -170.82 , -170.04 ,
-169.26 , -168.48 , -167.505, -166.53 , -165.165, -163.8 ,
-162.24 , -160.68 , -158.925, -156.975, -154.83 , -152.49 ,
-150.15 , -147.81 , -145.275, -142.935, -140.4 , -138.06 ,
-135.72 , -133.575, -131.43 , -129.675, -127.92 , -126.555,
-125.19 , -124.02 , -123.045, -122.265, -121.68 , -121.29 ,
-121.095, -121.095, -121.29 , -121.875, -122.655, -123.63 ,
-124.8 , -125.97 , -127.14 , -128.115, -128.7 , -129.285,
-129.48 , -129.675, -129.87 , -129.87 , -130.065, -130.26 ,
-130.65 , -131.43 , -132.405, -133.575, -134.94 , -136.305,
-137.67 , -139.035, -140.01 , -140.985, -141.765, -142.155,
-142.35 , -142.35 , -142.155, -141.57 , -140.985, -140.4 ,
-139.815, -139.23 , -138.84 , -138.645, -138.645, -138.84 ,
-139.23 , -139.815, -140.4 , -141.18 , -141.96 , -142.74 ,
-143.715, -144.69 , -145.86 , -147.03 , -148.2 , -149.175,
-150.345, -151.515, -152.49 , -153.66 , -154.635, -155.805,
-156.975, -158.34 , -159.51 , -160.875, -162.045, -163.41 ,
-164.58 , -165.945, -167.31 , -168.675, -170.235, -171.795,
-173.355, -174.915, -176.475, -177.84 , -179.205, -180.18 ,
-181.155, -181.935, -182.52 , -182.91 , -183.105, -183.105,
-182.91 , -182.325, -181.35 , -180.18 , -178.62 , -176.865,
-174.915, -172.965, -170.82 , -168.675, -166.53 , -164.19 ,
-161.85 , -159.51 , -156.975, -154.44 , -151.71 , -148.785,
-145.86 , -142.935, -140.01 , -137.28 , -134.55 , -132.015,
-129.675, -127.53 , -125.775, -124.215, -122.85 , -122.07 ,
-121.68 , -121.68 , -121.875, -122.46 , -123.24 , -124.41 ,
-125.775, -127.14 , -128.505, -130.065, -131.625, -133.185,
-135.135, -137.085, -139.425, -141.765, -144.105, -146.64 ,
-148.98 , -151.125, -152.88 , -154.44 , -155.61 , -156.195,
-156.585, -156.78 , -156.585, -156.39 , -155.805, -155.025,
-153.855, -152.685, -151.32 , -149.955, -148.59 , -147.42 ,
-146.055, -145.08 , -144.105, -143.13 , -142.35 , -141.96 ,
-141.57 , -141.18 , -141.18 , -141.18 , -141.18 , -141.57 ,
-141.96 , -142.545, -143.325, -144.105, -144.885, -145.665,
-146.64 , -147.81 , -148.98 , -150.345, -151.71 , -153.465,
-155.22 , -157.17 , -159.51 , -161.85 , -164.775, -167.895,
-171.21 , -174.72 , -178.23 , -181.74 , -185.445, -189.15 ,
-193.05 , -196.755, -200.655, -204.555, -208.455, -212.55 ,
-216.645, -220.545, -224.64 , -228.735, -232.635, -236.34 ,
-239.655, -242.775, -245.31 , -247.65 , -249.6 , -251.355,
-252.525, -253.5 , -254.085, -254.28 , -254.28 , -254.085,
-253.89 , -253.305, -252.915, -252.33 , -251.55 , -250.77 ,
-249.795, -248.82 , -247.26 , -245.7 , -243.555, -241.02 ,
-238.29 , -235.17 , -231.855, -228.345, -224.835, -221.325,
-217.815, -214.695, -211.575, -208.65 , -206.115, -203.775,
-201.825, -199.875, -198.315, -197.145, -196.17 , -195.39 ,
-195. , -194.61 , -194.61 , -194.61 , -194.61 , -194.805,
-194.805, -194.805, -194.805, -194.61 , -194.22 , -193.83 ,
-193.05 , -192.465, -191.49 , -190.32 , -188.955, -187.59 ,
-185.835, -183.885, -181.74 , -179.595, -177.45 , -175.305,
-173.16 , -171.405, -169.845, -168.48 , -167.505, -166.725,
-166.335, -166.14 , -166.14 , -166.335, -166.725, -167.31 ,
-167.7 , -168.285, -168.87 , -169.65 , -170.235, -170.625,
-171.015, -171.405, -171.405, -171.21 , -170.82 , -170.04 ,
-169.065, -167.895, -166.53 , -164.97 , -163.215, -161.265,
-159.315, -157.17 , -155.22 , -153.075, -150.93 , -148.785,
-146.445, -144.3 , -142.155, -140.205, -138.645, -137.085,
-135.915, -134.94 , -134.16 , -133.77 , -133.77 , -133.77 ,
-133.965, -134.355, -134.745, -135.33 , -135.915, -136.5 ,
-136.89 , -137.28 , -137.475, -137.67 , -137.865, -138.06 ,
-138.645, -139.425, -140.79 , -142.35 , -144.3 , -146.64 ,
-149.175, -151.71 , -154.245, -156.78 , -158.925, -161.07 ,
-162.63 , -163.995, -165.165, -166.14 , -166.725, -167.31 ,
-167.7 , -168.09 , -168.285, -168.48 , -168.48 , -168.48 ,
-168.09 , -167.7 , -166.92 , -166.14 , -165.165, -164.19 ,
-163.215, -162.24 , -161.265, -160.485, -159.705, -159.315,
-158.925, -158.925, -158.925, -159.12 , -159.315, -159.51 ,
-159.705, -159.9 , -160.095, -160.095, -160.095, -160.095,
-159.9 , -159.51 , -159.12 , -158.925, -158.535, -158.535,
-158.535, -158.73 , -159.315, -160.095, -161.07 , -162.24 ,
-163.605, -164.97 , -166.725, -168.48 , -170.235, -172.185,
-173.94 , -175.695, -177.06 , -177.84 , -178.425, -178.23 ,
-177.645, -176.475, -174.525, -172.38 , -169.455, -166.335,
-163.02 , -159.315, -155.805, -152.295, -148.785, -145.665,
-142.74 , -140.205, -137.865, -136.11 , -134.55 , -133.575,
-132.795, -132.21 , -132.015, -131.625, -131.43 , -131.235,
-130.845, -130.26 , -129.87 , -129.285, -128.7 , -127.92 ,
-127.14 , -126.165, -124.8 , -123.435, -121.875, -120.12 ,
-117.975, -115.635, -112.905, -109.785, -106.47 , -102.96 ,
-99.45 , -95.94 , -92.43 , -89.31 , -86.19 , -83.265,
-80.34 , -77.415, -74.685, -71.955, -69.42 , -66.885,
-64.35 , -62.01 , -60.06 , -58.5 , -57.135, -56.16 ,
-55.185, -54.6 , -53.82 , -53.235, -52.845, -52.26 ,
-51.87 , -51.675, -51.675, -51.675, -51.87 , -52.065,
-52.26 , -52.455, -52.455, -52.455, -52.455, -52.455,
-52.065, -51.87 , -51.48 , -51.09 , -50.505, -50.115,
-49.725, -49.14 , -48.75 , -48.165, -47.58 , -46.8 ,
-46.02 , -44.85 , -43.875, -43.095, -42.51 , -42.12 ,
-42.12 , -42.51 , -43.29 , -44.265, -45.63 , -46.995,
-48.555, -49.92 , -51.285, -52.26 , -53.235, -53.82 ,
-54.405, -54.6 , -54.795, -54.99 , -54.795, -54.795,
-54.795, -54.795, -54.99 , -55.185, -55.185, -55.185,
-54.99 , -54.6 , -54.015, -53.43 , -52.455, -51.675,
-50.7 , -49.92 , -49.14 , -48.36 , -47.775, -47.385,
-47.385, -47.385, -47.775, -48.555, -49.725, -50.895,
-52.65 , -54.405, -56.355, -58.5 , -61.035, -63.57 ,
-66.105, -68.835, -71.76 , -74.49 , -77.025, -79.56 ,
-82.095, -84.24 , -86.19 , -87.555, -88.725, -89.115,
-89.115, -88.725, -87.75 , -86.58 , -85.215, -83.85 ,
-82.68 , -81.705, -81.12 , -80.925, -81.12 , -81.705,
-82.68 , -83.85 , -85.41 , -87.165, -89.115, -91.26 ,
-93.405, -95.94 , -98.475, -101.205, -104.13 , -107.055,
-110.37 , -113.685, -117.39 , -121.095, -124.995, -128.895,
-132.795, -136.695, -140.205, -143.52 , -146.445, -149.175,
-151.32 , -153.27 , -155.025, -156.39 , -157.56 , -158.535,
-158.925, -159.12 , -158.925, -158.34 , -157.56 , -156.585,
-155.22 , -153.66 , -151.905, -149.955]),
'fs': 30000.0,
'name': u'CH8',
'scale': 0.195}
# describe our mouse!
this_mouse = Subject(subject_id =expt_dict['mouseID'],
date_of_birth= datetime.strptime(expt_dict['mouseDOB'], '%m/%d/%y'),
genotype = expt_dict['mouseGenotype'],
sex = expt_dict['mouseSex'],
species = 'mouse')
create_date = datetime.now()
# use the first stimulus timestamp as start time
start_time = stim_log['timestamp'][0]
nwbfile = NWBFile('example nwb conversion', 'NWB123', start_time,
file_create_date=create_date,
experimenter=expt_dict['user'],
subject=this_mouse)
Set some generic metadata
nwbfile.lab = 'Wehr Lab'
nwbfile.institution = 'University of Oregon'
nwbfile.pharmacology = expt_dict['Drugs']
Add our raw timeseries from the four .continuous files we parsed above. Normally we also collect the (sound) stimulus on an unused input channel, so rather than .add_acquisition we would use .add_stimulus for those, but I'm omitting here for brevity.
Rather than explicit timestamps, we pass starting_time and rate because this data is sampled regularly.
for cont in continuous:
nwbfile.add_acquisition(
TimeSeries(cont['name'], cont['data'], 'V',
starting_time=0.0, rate=cont['fs'],
conversion=cont['scale'])
)
We add the spiketimes from the clustered cell here. We first have to use .add_unit_column to declare the params we will be saving, and then we add the unit.
# Add descriptions to our new fields
add_fields = [('tetrode', 'the tetrode number this cell was recorded from'),
('channel', 'the recording channel \(within the tetrode\) that this cell has the strongest amplitude on'),
('cluster', 'the cluster number of this cell within a tetrode')]
for field in add_fields:
nwbfile.add_unit_column(field[0], field[1])
# Then add the data -- we only have one cell in this demo
nwbfile.add_unit(id=int(out_dict['cell']),
spike_times=out_dict['spiketimes'],
tetrode = out_dict['tetrode'],
channel = out_dict['channel'],
cluster = out_dict['cluster'])
We use the .add_trial method to add discrete, trial-level information about our data in addition to the raw stim traces we should have attached.
# first make the trial columns
trial_descriptors = [
('type', 'stimulus type'),
('amplitude', 'stimulus amplitude in dB'),
('ramp', 'stimulus onset offset ramp'),
('loop_flg', 'whether the stimulus loops'),
('seamless', 'whether the loop is seamless'),
('duration', 'duration of stimulus'),
('next', 'time to next stimulus'),
( 'soa', 'stimulus onset asynchrony'),
('soaflag', 'type of soa, isi or iti'),
('gapdelay', 'delay until gap onset'),
('gapdur', 'duration of gap'),
('pulsedur', 'duration of noise pulse'),
('pulseamp', 'amplitude of noise pulse'),
('laser', 'laser on or off'),
('LaserOnOff','laser on or off again for some reason')
]
for descriptor in trial_descriptors:
nwbfile.add_trial_column(descriptor[0], descriptor[1])
# Then add the trials
# make timestamps into time deltas
time_delts = stim_df['timestamp'] - start_time
time_delts = time_delts.dt.total_seconds()
# replace all NaNs
stim_df.fillna(value='', inplace=True)
# make timedel
for i in range(stim_df.shape[0]-1):
nwbfile.add_trial(
start_time = time_delts.loc[i],
stop_time = time_delts.loc[i+1],
type = stim_df.loc[i, 'type'],
amplitude = stim_df.loc[i, 'amplitude'],
ramp = stim_df.loc[i, 'ramp'],
loop_flg = stim_df.loc[i, 'loop_flg'],
seamless = stim_df.loc[i, 'seamless'],
duration = stim_df.loc[i, 'duration'],
next = stim_df.loc[i, 'next'],
soa = stim_df.loc[i, 'soa'],
soaflag = stim_df.loc[i, 'soaflag'],
gapdelay = stim_df.loc[i, 'gapdelay'],
gapdur = stim_df.loc[i, 'gapdur'],
pulsedur = stim_df.loc[i, 'pulsedur'],
pulseamp = stim_df.loc[i, 'pulseamp'],
laser = stim_df.loc[i, 'laser'],
LaserOnOff = stim_df.loc[i, 'LaserOnOff'])
io = NWBHDF5IO('convert_example.nwb', mode='w')
io.write(nwbfile)
io.close()
Welcome to the future...