Browse Source

filtering with extended comments

Paul Pfeiffer 4 years ago
parent
commit
c1aef4e38b
4 changed files with 84 additions and 11 deletions
  1. 11 11
      makefile
  2. 5 0
      metadata/blacklist.csv
  3. 67 0
      scripts/filter_stimulations.py
  4. 1 0
      scripts/tools/definitions.py

+ 11 - 11
makefile

@@ -1,30 +1,30 @@
-all: extract_stimulations assign_protocols filter analyse_firing_rates analyse_persistent_activity plot_firing_rates plot_traces report
+all: prepare_folder_structure extract_stimulations assign_protocols filter analyse_firing_rates analyse_persistent_activity plot_firing_rates plot_traces report
 
-check_folders:
+prepare_folder_structure:
 	python scripts/check_folder_structure.py
 
-extract_stimulations: check_folders
+extract_stimulations: prepare_folder_structure
 	python scripts/extract_stimulations.py
 
-assign_protocols: check_folders
+assign_protocols: prepare_folder_structure
 	python scripts/extract_protocols.py
 
-filter:
-	python filter_stimulations.py
+filter: prepare_folder_structure
+	python scripts/filter_stimulations.py
 
-analyse_firing_rates:
+analyse_firing_rates: prepare_folder_structure
 	python analyse_firing_rates.py
 
-analyse_persistent_activity:
+analyse_persistent_activity: prepare_folder_structure
 	python analyse_persistent_activity.py
 
-plot_firing_rates:
+plot_firing_rates: prepare_folder_structure
 	python plot_traces.py
 
-plot_traces:
+plot_traces: prepare_folder_structure
 	python plot_traces.py
 
-report:
+report: prepare_folder_structure
 	python generate_report.py
 
 clean:

+ 5 - 0
metadata/blacklist.csv

@@ -0,0 +1,5 @@
+stimulation_id,comment
+20170217_S1_C1_R029, unintended network activity
+20170216_S1_C1_R009, protocol was interupted
+20170217_S2_C1_R002, protocol does not fit with metadata
+20170217_S2_C1_R014, down protocol with very short firing pulses that do not allow to look at stationary firing

+ 67 - 0
scripts/filter_stimulations.py

@@ -0,0 +1,67 @@
+import numpy
+import pandas as pd
+from tools.definitions import  METADATA_FOLDER, OUTPUT_FOLDER, HELPER_TABLE_FOLDER, STIMULATION_METADATA_EXTENDED, STIMULATION_METADATA_FILTERED, \
+    PROTOCOL_BLACKLIST
+from tools.helper import get_filters
+from tools.helper import get_pulse_times
+from tools.helper import get_trace, get_times
+
+path_to_extended_stimulus_file = OUTPUT_FOLDER+HELPER_TABLE_FOLDER+STIMULATION_METADATA_EXTENDED
+
+path_to_filtered_stimulus_file = OUTPUT_FOLDER+HELPER_TABLE_FOLDER+STIMULATION_METADATA_FILTERED
+
+print "# Filter stimulations suitable for analysis"
+print ""
+
+spiking_threshold = -10
+minimum_width_of_a_spike_in_time_points = 100
+
+stimulations = pd.read_csv(path_to_extended_stimulus_file, index_col="stimulation_id")
+
+# get protocols
+type_number_combinations = stimulations[["protocol_type", "pulse_number"]].drop_duplicates().values
+
+
+def detect_faulty_stimulations(stimulation):
+    is_correct = True
+    times = get_times(stimulation)
+    stimulus_trace = get_trace(stimulation, "Stimulus-Current-1")
+
+    pulses = get_pulse_times(stimulation)
+    expected_amplitude_difference = 2 * numpy.abs(stimulation["pulse_amplitude"])
+
+    for pulse_idx, pulse in enumerate(pulses):
+        in_pulse = pulse["start"] + (pulse["pulse_end"] - pulse["start"]) / 2.0
+        after_pulse = pulse["pulse_end"] + (pulse["end"] - pulse["pulse_end"]) / 2.0
+        actual_amplitude_difference = numpy.abs(numpy.ediff1d(stimulus_trace[times.searchsorted([in_pulse, after_pulse])]))[0]
+        correct_pulse = numpy.abs(expected_amplitude_difference -
+                                  actual_amplitude_difference) <= 0.01
+        if not correct_pulse:
+            print "Stimulation {:s}: Wrong amplitude in pulse {:d}: expected {:.1f}nA actual {:.1f}nA.".format(stimulation.name, pulse_idx, expected_amplitude_difference, actual_amplitude_difference)
+            is_correct = False
+            break
+    return is_correct
+
+
+blacklist_protocols = list(pd.read_csv(METADATA_FOLDER+PROTOCOL_BLACKLIST)["stimulation_id"].values)
+
+print "## Manual blacklist".format(METADATA_FOLDER+PROTOCOL_BLACKLIST)
+stimulations["filter_blacklist"] = [stim_id not in blacklist_protocols for stim_id in stimulations.index.values]
+print "{:d} protocols are excluded, see {:s} for justifications".format(len(blacklist_protocols), METADATA_FOLDER+PROTOCOL_BLACKLIST)
+print ""
+
+print "## Automatic detection of problematic stimulations"
+print ""
+stimulations["filter_protocol"] = stimulations.apply(detect_faulty_stimulations, axis=1)
+print ""
+print "{:d} protocol with stimulation problems.".format(len(numpy.where(numpy.array(stimulations["filter_protocol"].values)==False)[0]))
+print ""
+
+print "## Automatic detection of protocols without conductance clamp"
+stimulations["filter_zero_conductance"] = stimulations["g"] != 0
+print "{:d} protocol without conductance clamp.".format(len(numpy.where(numpy.array(stimulations["filter_zero_conductance"].values)==False)[0]))
+print ""
+
+stimulations.to_csv(path_or_buf=path_to_filtered_stimulus_file)
+
+print "{} / {} protocols suitable for analysis".format(len(stimulations[get_filters(stimulations)]), len(stimulations))

+ 1 - 0
scripts/tools/definitions.py

@@ -6,3 +6,4 @@ HELPER_TABLE_FOLDER = "tables/"
 STIMULATION_METADATA_RAW = "stimulations_raw.csv"
 STIMULATION_METADATA_EXTENDED = "stimulations_analysed.csv"
 STIMULATION_METADATA_FILTERED = "stimulations_analysed_and_filtered.csv"
+PROTOCOL_BLACKLIST = "blacklist.csv"