reposit.py 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200
  1. from datetime import datetime
  2. from dateutil import tz
  3. from pathlib import Path
  4. from neuroconv.converters import SpikeGLXConverterPipe
  5. import glob
  6. import json
  7. import re
  8. import pynwb
  9. import os
  10. ABSPATH = os.path.abspath(__file__)
  11. THIS_DIR = os.path.dirname(ABSPATH)
  12. os.chdir(THIS_DIR)
  13. EXPKEY_PATTERN="*_keys.m"
  14. try:
  15. SCRATCH_PATH = os.environ['SCRATCH_PATH']
  16. except KeyError:
  17. SCRATCH_PATH = "~/.local/share/mvmnda/nwbdata/"
  18. SCRATCH_PATH = os.path.abspath(os.path.expanduser(SCRATCH_PATH))
  19. os.makedirs(SCRATCH_PATH, exist_ok=True)
  20. def read_expkeys(
  21. in_file,
  22. ):
  23. """
  24. Read ExpKeys format, which is an internal convention of the Dartmouth PBS van der Meer lab.
  25. Parameters
  26. ----------
  27. in_file: str, optional
  28. Path to ExpKeys file, will have a “.m” format.
  29. """
  30. raw_fields = [
  31. 'species',
  32. 'genetics',
  33. 'subject_id',
  34. 'hemisphere',
  35. 'experimenter',
  36. 'weight',
  37. 'probeDepth',
  38. 'date',
  39. 'target',
  40. 'light_source',
  41. 'wavelength',
  42. 'probeType',
  43. 'probeID',
  44. 'HSID',
  45. 'patch_cord_dia',
  46. 'patch_cord_NA',
  47. 'probe_fiber_dia',
  48. 'probe_fiber_NA',
  49. 'max_power',
  50. 'output_power_calib',
  51. 'dial_values_calib',
  52. 'dial_value',
  53. 'light_power',
  54. 'stim_mode',
  55. 'ISI',
  56. 'short_stim_pulse_width',
  57. 'long_stim_pulse_width',
  58. 'pre_trial_stim_on',
  59. 'trial_stim_on',
  60. 'post_trial_stim_on',
  61. 'long_stim_on',
  62. 'stim_off',
  63. 'pre_baseline_times',
  64. 'pre_stim_times',
  65. 'stim_times',
  66. 'post_baseline_times',
  67. 'post_stim_times',
  68. 'long_stim_times',
  69. 'recording_times',
  70. 'notes',
  71. 'isReferenceRecordedSeparately',
  72. 'hasSpecialStim',
  73. 'LFP_channels',
  74. 'ref_channels',
  75. 'DOB',
  76. ]
  77. p1 = re.compile('ExpKeys.(.+?)=')
  78. p2 = ['=(.+?)";', '=(.+?)};', '=(.+?)];', '=(.+?);'] # The order is important
  79. p2 = [re.compile(x) for x in p2]
  80. in_file = os.path.abspath(os.path.expanduser(in_file))
  81. out_dict = {}
  82. with open(in_file, 'r') as f:
  83. for line in f:
  84. m = re.search(p1, line)
  85. if m:
  86. key = m.group(1).strip()
  87. if key in raw_fields:
  88. for iP in range(4):
  89. m = re.search(p2[iP], line)
  90. if m:
  91. if iP == 0: # This is a simple string
  92. value = m.group(1).strip(" \"")
  93. elif iP == 1: # This is a 1-D list
  94. temp = m.group(1).strip()[1:]
  95. value = [x.strip()[1:-1] for x in temp.split(',')]
  96. elif iP == 2: # This can be a nested loop, FML AARRGGGHHHHHHH
  97. temp = m.group(1).strip()[1:]
  98. value = [float(x.strip()) for x in temp.split(',')]
  99. #value = "I don't know man"
  100. else: # iP has to be 3, and this is either a number, or true/false
  101. temp = m.group(1).strip()
  102. if temp == 'true':
  103. value = True
  104. elif temp == 'false':
  105. value = False
  106. else: # definitely a number
  107. value = float(m.group(1).strip())
  108. out_dict[key] = value
  109. break
  110. else:
  111. continue
  112. #print(json.dumps(out_dict))
  113. return out_dict
  114. def get_expkeys_file(in_dir):
  115. expkeys_files = glob.glob(os.path.join(in_dir,EXPKEY_PATTERN))
  116. if len(expkeys_files) == 1:
  117. return expkeys_files[0]
  118. elif len(expkeys_files) >= 1:
  119. print("There should only be one ExpKeys file per experiment.")
  120. print(f"In the `{in_dir}` candidate directory the following ExpKeys pattern files were found:")
  121. for expkeys_file in expkeys_files:
  122. print(f"\t* `{expkeys_file}`")
  123. return False
  124. else:
  125. print(f"No ExpKeys file found under `{in_dir}`.")
  126. def convert_all(base_dir):
  127. in_dir = os.path.abspath(os.path.expanduser(base_dir))
  128. # Is this a sensible criterion?
  129. spikeglx_dirs = [y for x in os.walk(base_dir) for y in glob.glob(os.path.join(x[0], '*_g0'))]
  130. for spikeglx_dir in spikeglx_dirs:
  131. # Only convert if we have ExpKeys
  132. if get_expkeys_file(spikeglx_dir):
  133. convert_measurement(spikeglx_dir)
  134. def convert_measurement(in_dir, scratch_path=SCRATCH_PATH):
  135. expkeys_file = get_expkeys_file(in_dir)
  136. if not expkeys_file:
  137. raise ValueError("Cannot proceed without corresponding ExpKeys file.")
  138. print(f"Converting `{in_dir}` session directory to NWB.")
  139. expkeys = read_expkeys(expkeys_file)
  140. converter = SpikeGLXConverterPipe(folder_path=in_dir)
  141. m = re.match(".*?sub-(?P<subject>[a-zA-z0-9]+).*?", in_dir)
  142. # add logic to fall back on expkeys
  143. subject = m.groupdict()["subject"]
  144. # Extract what metadata we can from the source files
  145. metadata = converter.get_metadata()
  146. # Set timezone
  147. session_start_time = metadata["NWBFile"]["session_start_time"].replace(tzinfo=tz.gettz("US/Eastern"))
  148. metadata["NWBFile"].update(session_start_time=session_start_time)
  149. session = (metadata["NWBFile"]["session_start_time"]).strftime("%Y%m%d%H%M%S")
  150. # Maybe do date of birth rather than age
  151. metadata["Subject"] = dict(
  152. subject_id=subject,
  153. sex="F",
  154. species="Mus musculus",
  155. age="P90D",
  156. date_of_birth=expkeys["DOB"],
  157. )
  158. # No idea why this needs to be done via `.update()`
  159. metadata["NWBFile"].update(session_id=session)
  160. # Read in Matt's lab “keys” files.
  161. #print(metadata)
  162. #print(metadata)
  163. #print(metadata["NWBFile"])
  164. #
  165. # Choose a path for saving the nwb file and run the conversion
  166. ##nwbfile_path = "/mnt/DATA/data/studies/manish/mvmnda/rawdata/my_spikeglx_session.nwb"
  167. ##converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata)
  168. datestamp = datetime.now().isoformat()
  169. nwbfile_path = os.path.join(scratch_path,f"ses-{metadata['NWBFile']['session_id']}_sub-{metadata['Subject']['subject_id']}_{datestamp}.nwb")
  170. converter.run_conversion(nwbfile_path=nwbfile_path, metadata=metadata)