HTCondor submission setup

.gitignore

@@ -0,0 +1,3 @@
+logs
+dag_tmp
+.condor_datalad_lock

code/process.condor_dag

@@ -0,0 +1,43 @@
+# Processing DAG
+JOB sub-05 code/process.condor_submit
+VARS sub-05 subject="05"
+JOB sub-16 code/process.condor_submit
+VARS sub-16 subject="16"
+JOB sub-11 code/process.condor_submit
+VARS sub-11 subject="11"
+JOB sub-08 code/process.condor_submit
+VARS sub-08 subject="08"
+JOB sub-02 code/process.condor_submit
+VARS sub-02 subject="02"
+JOB sub-06 code/process.condor_submit
+VARS sub-06 subject="06"
+JOB sub-15 code/process.condor_submit
+VARS sub-15 subject="15"
+JOB sub-20 code/process.condor_submit
+VARS sub-20 subject="20"
+JOB sub-18 code/process.condor_submit
+VARS sub-18 subject="18"
+JOB sub-12 code/process.condor_submit
+VARS sub-12 subject="12"
+JOB sub-01 code/process.condor_submit
+VARS sub-01 subject="01"
+JOB sub-03 code/process.condor_submit
+VARS sub-03 subject="03"
+JOB sub-09 code/process.condor_submit
+VARS sub-09 subject="09"
+JOB sub-10 code/process.condor_submit
+VARS sub-10 subject="10"
+JOB sub-17 code/process.condor_submit
+VARS sub-17 subject="17"
+JOB sub-04 code/process.condor_submit
+VARS sub-04 subject="04"
+JOB sub-21 code/process.condor_submit
+VARS sub-21 subject="21"
+JOB sub-13 code/process.condor_submit
+VARS sub-13 subject="13"
+JOB sub-19 code/process.condor_submit
+VARS sub-19 subject="19"
+JOB sub-14 code/process.condor_submit
+VARS sub-14 subject="14"
+JOB sub-07 code/process.condor_submit
+VARS sub-07 subject="07"

code/process.condor_submit

@@ -0,0 +1,50 @@
+universe       = vanilla
+# resource requirements for each job
+request_cpus   = 1
+request_memory = 4G
+request_disk   = 5G
+
+# tell condor that a job is self contained and the executable
+# is enough to bootstrap the computation on the execute node
+should_transfer_files = yes
+# explicitly do not transfer anything back
+# we are using datalad for everything that matters
+transfer_output_files = ""
+
+# the actual job script, nothing condor-specific in it
+executable     = $ENV(PWD)/code/participant_job
+
+# the job expects these environment variables for labeling and synchronization
+# - JOBID: subject AND process specific ID to make a branch name from
+#     (must be unique across all (even multiple) submissions)
+#     including the cluster ID will enable sorting multiple computing attempts
+# - DSLOCKFILE: lock (must be accessible from all compute jobs) to synchronize
+#     write access to the output dataset
+# - DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__...:
+#     (additional) locations for datalad to locate relevant subdatasets, in case
+#     a configured URL is outdated
+# - GIT_AUTHOR_...: Identity information used to save dataset changes in compute
+#     jobs
+environment = "\
+  JOBID=$(subject).$(Cluster) \
+  DSLOCKFILE=$ENV(PWD)/.condor_datalad_lock \
+  DATALAD_GET_SUBDATASET__SOURCE__CANDIDATE__101cat='ria+http://containers.ds.inm7.de#{id}' \
+  GIT_AUTHOR_NAME='Adina Wagner' \
+  GIT_AUTHOR_EMAIL='adina.wagner@t-online.de' \
+  "
+
+# place the job logs into PWD/logs, using the same name as for the result branches
+# (JOBID)
+log    = $ENV(PWD)/logs/$(subject)_$(Cluster).log
+output = $ENV(PWD)/logs/$(subject)_$(Cluster).out
+error  = $ENV(PWD)/logs/$(subject)_$(Cluster).err
+# essential args for 'participant_job'
+# 1: where to clone the analysis dataset
+# 2: location to push the result git branch to. The 'ria+' prefix is stripped.
+# 3: ID of the subject to process
+arguments = "\
+  ria+file:///data/group/psyinf/inputstore#47a9ef9e-9fa0-49cd-873f-c51cc501da96 \
+  /data/group/psyinf/outputstore/47a/9ef9e-9fa0-49cd-873f-c51cc501da96 \
+  sub-$(subject) \
+  "
+queue