process.condor_dag.dagman.out 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304
  1. 07/11/22 00:05:26 ******************************************************
  2. 07/11/22 00:05:26 ** condor_scheduniv_exec.914125.0 (CONDOR_DAGMAN) STARTING UP
  3. 07/11/22 00:05:26 ** /usr/bin/condor_dagman
  4. 07/11/22 00:05:26 ** SubsystemInfo: name=DAGMAN type=DAGMAN(10) class=DAEMON(1)
  5. 07/11/22 00:05:26 ** Configuration: subsystem:DAGMAN local:<NONE> class:DAEMON
  6. 07/11/22 00:05:26 ** $CondorVersion: 8.8.6 Nov 19 2019 BuildID: Debian-8.8.6~dfsg.1-1 Debian-8.8.6~dfsg.1-1 $
  7. 07/11/22 00:05:26 ** $CondorPlatform: X86_64-Debian_10 $
  8. 07/11/22 00:05:26 ** PID = 3463981
  9. 07/11/22 00:05:26 ** Log last touched time unavailable (No such file or directory)
  10. 07/11/22 00:05:26 ******************************************************
  11. 07/11/22 00:05:26 Using config source: /etc/condor/condor_config
  12. 07/11/22 00:05:26 Using local config sources:
  13. 07/11/22 00:05:26 /etc/condor/config.d/50_inm7_master
  14. 07/11/22 00:05:26 /etc/condor/condor_config.local
  15. 07/11/22 00:05:26 config Macros = 81, Sorted = 81, StringBytes = 2405, TablesBytes = 2972
  16. 07/11/22 00:05:26 CLASSAD_CACHING is ENABLED
  17. 07/11/22 00:05:26 Daemon Log is logging: D_ALWAYS D_ERROR
  18. 07/11/22 00:05:26 DaemonCore: No command port requested.
  19. 07/11/22 00:05:26 DAGMAN_USE_STRICT setting: 1
  20. 07/11/22 00:05:26 DAGMAN_VERBOSITY setting: 3
  21. 07/11/22 00:05:26 DAGMAN_DEBUG_CACHE_SIZE setting: 5242880
  22. 07/11/22 00:05:26 DAGMAN_DEBUG_CACHE_ENABLE setting: False
  23. 07/11/22 00:05:26 DAGMAN_SUBMIT_DELAY setting: 0
  24. 07/11/22 00:05:26 DAGMAN_MAX_SUBMIT_ATTEMPTS setting: 6
  25. 07/11/22 00:05:26 DAGMAN_STARTUP_CYCLE_DETECT setting: False
  26. 07/11/22 00:05:26 DAGMAN_MAX_SUBMITS_PER_INTERVAL setting: 100
  27. 07/11/22 00:05:26 DAGMAN_AGGRESSIVE_SUBMIT setting: False
  28. 07/11/22 00:05:26 DAGMAN_USER_LOG_SCAN_INTERVAL setting: 5
  29. 07/11/22 00:05:26 DAGMAN_QUEUE_UPDATE_INTERVAL setting: 300
  30. 07/11/22 00:05:26 DAGMAN_DEFAULT_PRIORITY setting: 0
  31. 07/11/22 00:05:26 DAGMAN_SUPPRESS_NOTIFICATION setting: True
  32. 07/11/22 00:05:26 allow_events (DAGMAN_ALLOW_EVENTS) setting: 114
  33. 07/11/22 00:05:26 DAGMAN_RETRY_SUBMIT_FIRST setting: True
  34. 07/11/22 00:05:26 DAGMAN_RETRY_NODE_FIRST setting: False
  35. 07/11/22 00:05:26 DAGMAN_MAX_JOBS_IDLE setting: 1000
  36. 07/11/22 00:05:26 DAGMAN_MAX_JOBS_SUBMITTED setting: 0
  37. 07/11/22 00:05:26 DAGMAN_MAX_PRE_SCRIPTS setting: 20
  38. 07/11/22 00:05:26 DAGMAN_MAX_POST_SCRIPTS setting: 20
  39. 07/11/22 00:05:26 DAGMAN_MUNGE_NODE_NAMES setting: True
  40. 07/11/22 00:05:26 DAGMAN_PROHIBIT_MULTI_JOBS setting: False
  41. 07/11/22 00:05:26 DAGMAN_SUBMIT_DEPTH_FIRST setting: False
  42. 07/11/22 00:05:26 DAGMAN_ALWAYS_RUN_POST setting: False
  43. 07/11/22 00:05:26 DAGMAN_CONDOR_SUBMIT_EXE setting: /usr/bin/condor_submit
  44. 07/11/22 00:05:26 DAGMAN_USE_CONDOR_SUBMIT setting: True
  45. 07/11/22 00:05:26 DAGMAN_ABORT_DUPLICATES setting: True
  46. 07/11/22 00:05:26 DAGMAN_ABORT_ON_SCARY_SUBMIT setting: True
  47. 07/11/22 00:05:26 DAGMAN_PENDING_REPORT_INTERVAL setting: 600
  48. 07/11/22 00:05:26 DAGMAN_AUTO_RESCUE setting: True
  49. 07/11/22 00:05:26 DAGMAN_MAX_RESCUE_NUM setting: 100
  50. 07/11/22 00:05:26 DAGMAN_WRITE_PARTIAL_RESCUE setting: True
  51. 07/11/22 00:05:26 DAGMAN_DEFAULT_NODE_LOG setting: @(DAG_DIR)/@(DAG_FILE).nodes.log
  52. 07/11/22 00:05:26 DAGMAN_GENERATE_SUBDAG_SUBMITS setting: True
  53. 07/11/22 00:05:26 DAGMAN_MAX_JOB_HOLDS setting: 100
  54. 07/11/22 00:05:26 DAGMAN_HOLD_CLAIM_TIME setting: 20
  55. 07/11/22 00:05:26 ALL_DEBUG setting:
  56. 07/11/22 00:05:26 DAGMAN_DEBUG setting:
  57. 07/11/22 00:05:26 DAGMAN_SUPPRESS_JOB_LOGS setting: False
  58. 07/11/22 00:05:26 DAGMAN_REMOVE_NODE_JOBS setting: True
  59. 07/11/22 00:05:26 argv[0] == "condor_scheduniv_exec.914125.0"
  60. 07/11/22 00:05:26 argv[1] == "-Lockfile"
  61. 07/11/22 00:05:26 argv[2] == "code/process.condor_dag.lock"
  62. 07/11/22 00:05:26 argv[3] == "-AutoRescue"
  63. 07/11/22 00:05:26 argv[4] == "1"
  64. 07/11/22 00:05:26 argv[5] == "-DoRescueFrom"
  65. 07/11/22 00:05:26 argv[6] == "0"
  66. 07/11/22 00:05:26 argv[7] == "-Dag"
  67. 07/11/22 00:05:26 argv[8] == "code/process.condor_dag"
  68. 07/11/22 00:05:26 argv[9] == "-Suppress_notification"
  69. 07/11/22 00:05:26 argv[10] == "-CsdVersion"
  70. 07/11/22 00:05:26 argv[11] == "$CondorVersion: 8.8.6 Nov 19 2019 BuildID: Debian-8.8.6~dfsg.1-1 Debian-8.8.6~dfsg.1-1 $"
  71. 07/11/22 00:05:26 argv[12] == "-Dagman"
  72. 07/11/22 00:05:26 argv[13] == "/usr/bin/condor_dagman"
  73. 07/11/22 00:05:26 Workflow batch-name: <process.condor_dag+914125>
  74. 07/11/22 00:05:26 Workflow accounting_group: <>
  75. 07/11/22 00:05:26 Workflow accounting_group_user: <>
  76. 07/11/22 00:05:26 Warning: failed to get attribute DAGNodeName
  77. 07/11/22 00:05:26 DAGMAN_LOG_ON_NFS_IS_ERROR setting: False
  78. 07/11/22 00:05:26 Default node log file is: </data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log>
  79. 07/11/22 00:05:26 DAG Lockfile will be written to code/process.condor_dag.lock
  80. 07/11/22 00:05:26 DAG Input file is code/process.condor_dag
  81. 07/11/22 00:05:26 Parsing 1 dagfiles
  82. 07/11/22 00:05:26 Parsing code/process.condor_dag ...
  83. 07/11/22 00:05:26 Dag contains 6 total jobs
  84. 07/11/22 00:05:26 Sleeping for 3 seconds to ensure ProcessId uniqueness
  85. 07/11/22 00:05:29 Bootstrapping...
  86. 07/11/22 00:05:29 Number of pre-completed nodes: 0
  87. 07/11/22 00:05:29 MultiLogFiles: truncating log file /data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log
  88. 07/11/22 00:05:29 DAG status: 0 (DAG_STATUS_OK)
  89. 07/11/22 00:05:29 Of 6 nodes total:
  90. 07/11/22 00:05:29 Done Pre Queued Post Ready Un-Ready Failed
  91. 07/11/22 00:05:29 === === === === === === ===
  92. 07/11/22 00:05:29 0 0 0 0 6 0 0
  93. 07/11/22 00:05:29 0 job proc(s) currently held
  94. 07/11/22 00:05:29 DAGMan Runtime Statistics: [ EventCycleTimeSum = 0.0; EventCycleTimeCount = 0.0; LogProcessCycleTimeSum = 0.0; SleepCycleTimeSum = 0.0; LogProcessCycleTimeCount = 0.0; SleepCycleTimeCount = 0.0; SubmitCycleTimeCount = 0.0; SubmitCycleTimeSum = 0.0; ]
  95. 07/11/22 00:05:29 Registering condor_event_timer...
  96. 07/11/22 00:05:30 Submitting HTCondor Node sub-2 job(s)...
  97. 07/11/22 00:05:30 Adding a DAGMan workflow log /data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log
  98. 07/11/22 00:05:30 Masking the events recorded in the DAGMAN workflow log
  99. 07/11/22 00:05:30 Mask for workflow log is 0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36
  100. 07/11/22 00:05:30 submitting: /usr/bin/condor_submit -a dag_node_name' '=' 'sub-2 -a +DAGManJobId' '=' '914125 -a DAGManJobId' '=' '914125 -batch-name process.condor_dag+914125 -a submit_event_notes' '=' 'DAG' 'Node:' 'sub-2 -a dagman_log' '=' '/data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log -a +DAGManNodesMask' '=' '"0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36" -a subject' '=' 'sub-2 -a DAG_STATUS' '=' '0 -a FAILED_COUNT' '=' '0 -a notification' '=' 'never -a +DAGParentNodeNames' '=' '"" code/process.condor_submit
  101. 07/11/22 00:05:30 From submit: Submitting job(s).
  102. 07/11/22 00:05:30 From submit: 1 job(s) submitted to cluster 914191.
  103. 07/11/22 00:05:30 assigned HTCondor ID (914191.0.0)
  104. 07/11/22 00:05:30 Submitting HTCondor Node sub-5 job(s)...
  105. 07/11/22 00:05:30 Adding a DAGMan workflow log /data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log
  106. 07/11/22 00:05:30 Masking the events recorded in the DAGMAN workflow log
  107. 07/11/22 00:05:30 Mask for workflow log is 0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36
  108. 07/11/22 00:05:30 submitting: /usr/bin/condor_submit -a dag_node_name' '=' 'sub-5 -a +DAGManJobId' '=' '914125 -a DAGManJobId' '=' '914125 -batch-name process.condor_dag+914125 -a submit_event_notes' '=' 'DAG' 'Node:' 'sub-5 -a dagman_log' '=' '/data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log -a +DAGManNodesMask' '=' '"0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36" -a subject' '=' 'sub-5 -a DAG_STATUS' '=' '0 -a FAILED_COUNT' '=' '0 -a notification' '=' 'never -a +DAGParentNodeNames' '=' '"" code/process.condor_submit
  109. 07/11/22 00:05:30 From submit: Submitting job(s).
  110. 07/11/22 00:05:30 From submit: 1 job(s) submitted to cluster 914193.
  111. 07/11/22 00:05:30 assigned HTCondor ID (914193.0.0)
  112. 07/11/22 00:05:30 Submitting HTCondor Node sub-1 job(s)...
  113. 07/11/22 00:05:30 Adding a DAGMan workflow log /data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log
  114. 07/11/22 00:05:30 Masking the events recorded in the DAGMAN workflow log
  115. 07/11/22 00:05:30 Mask for workflow log is 0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36
  116. 07/11/22 00:05:30 submitting: /usr/bin/condor_submit -a dag_node_name' '=' 'sub-1 -a +DAGManJobId' '=' '914125 -a DAGManJobId' '=' '914125 -batch-name process.condor_dag+914125 -a submit_event_notes' '=' 'DAG' 'Node:' 'sub-1 -a dagman_log' '=' '/data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log -a +DAGManNodesMask' '=' '"0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36" -a subject' '=' 'sub-1 -a DAG_STATUS' '=' '0 -a FAILED_COUNT' '=' '0 -a notification' '=' 'never -a +DAGParentNodeNames' '=' '"" code/process.condor_submit
  117. 07/11/22 00:05:30 From submit: Submitting job(s).
  118. 07/11/22 00:05:30 From submit: 1 job(s) submitted to cluster 914196.
  119. 07/11/22 00:05:30 assigned HTCondor ID (914196.0.0)
  120. 07/11/22 00:05:30 Submitting HTCondor Node sub-6 job(s)...
  121. 07/11/22 00:05:30 Adding a DAGMan workflow log /data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log
  122. 07/11/22 00:05:30 Masking the events recorded in the DAGMAN workflow log
  123. 07/11/22 00:05:30 Mask for workflow log is 0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36
  124. 07/11/22 00:05:30 submitting: /usr/bin/condor_submit -a dag_node_name' '=' 'sub-6 -a +DAGManJobId' '=' '914125 -a DAGManJobId' '=' '914125 -batch-name process.condor_dag+914125 -a submit_event_notes' '=' 'DAG' 'Node:' 'sub-6 -a dagman_log' '=' '/data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log -a +DAGManNodesMask' '=' '"0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36" -a subject' '=' 'sub-6 -a DAG_STATUS' '=' '0 -a FAILED_COUNT' '=' '0 -a notification' '=' 'never -a +DAGParentNodeNames' '=' '"" code/process.condor_submit
  125. 07/11/22 00:05:30 From submit: Submitting job(s).
  126. 07/11/22 00:05:30 From submit: 1 job(s) submitted to cluster 914199.
  127. 07/11/22 00:05:30 assigned HTCondor ID (914199.0.0)
  128. 07/11/22 00:05:30 Submitting HTCondor Node sub-4 job(s)...
  129. 07/11/22 00:05:30 Adding a DAGMan workflow log /data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log
  130. 07/11/22 00:05:30 Masking the events recorded in the DAGMAN workflow log
  131. 07/11/22 00:05:30 Mask for workflow log is 0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36
  132. 07/11/22 00:05:30 submitting: /usr/bin/condor_submit -a dag_node_name' '=' 'sub-4 -a +DAGManJobId' '=' '914125 -a DAGManJobId' '=' '914125 -batch-name process.condor_dag+914125 -a submit_event_notes' '=' 'DAG' 'Node:' 'sub-4 -a dagman_log' '=' '/data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log -a +DAGManNodesMask' '=' '"0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36" -a subject' '=' 'sub-4 -a DAG_STATUS' '=' '0 -a FAILED_COUNT' '=' '0 -a notification' '=' 'never -a +DAGParentNodeNames' '=' '"" code/process.condor_submit
  133. 07/11/22 00:05:30 From submit: Submitting job(s).
  134. 07/11/22 00:05:30 From submit: 1 job(s) submitted to cluster 914202.
  135. 07/11/22 00:05:30 assigned HTCondor ID (914202.0.0)
  136. 07/11/22 00:05:30 Submitting HTCondor Node sub-3 job(s)...
  137. 07/11/22 00:05:30 Adding a DAGMan workflow log /data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log
  138. 07/11/22 00:05:30 Masking the events recorded in the DAGMAN workflow log
  139. 07/11/22 00:05:30 Mask for workflow log is 0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36
  140. 07/11/22 00:05:30 submitting: /usr/bin/condor_submit -a dag_node_name' '=' 'sub-3 -a +DAGManJobId' '=' '914125 -a DAGManJobId' '=' '914125 -batch-name process.condor_dag+914125 -a submit_event_notes' '=' 'DAG' 'Node:' 'sub-3 -a dagman_log' '=' '/data/project/infrasound/ReproVBM/ds000105_ReproVBM/code/process.condor_dag.nodes.log -a +DAGManNodesMask' '=' '"0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36" -a subject' '=' 'sub-3 -a DAG_STATUS' '=' '0 -a FAILED_COUNT' '=' '0 -a notification' '=' 'never -a +DAGParentNodeNames' '=' '"" code/process.condor_submit
  141. 07/11/22 00:05:30 From submit: Submitting job(s).
  142. 07/11/22 00:05:30 From submit: 1 job(s) submitted to cluster 914205.
  143. 07/11/22 00:05:30 assigned HTCondor ID (914205.0.0)
  144. 07/11/22 00:05:30 Just submitted 6 jobs this cycle...
  145. 07/11/22 00:05:30 DAG status: 0 (DAG_STATUS_OK)
  146. 07/11/22 00:05:30 Of 6 nodes total:
  147. 07/11/22 00:05:30 Done Pre Queued Post Ready Un-Ready Failed
  148. 07/11/22 00:05:30 === === === === === === ===
  149. 07/11/22 00:05:30 0 0 6 0 0 0 0
  150. 07/11/22 00:05:30 0 job proc(s) currently held
  151. 07/11/22 00:05:30 DAGMan Runtime Statistics: [ EventCycleTimeSum = 0.0; SleepCycleTimeCount = 0.0; EventCycleTimeCount = 0.0; SubmitCycleTimeStd = 0.2291338443756104; SubmitCycleTimeMax = 0.2291338443756104; LogProcessCycleTimeSum = 0.0; LogProcessCycleTimeCount = 0.0; SubmitCycleTimeMin = 0.2291338443756104; SubmitCycleTimeCount = 1.0; SubmitCycleTimeSum = 0.2291338443756104; SleepCycleTimeSum = 0.0; SubmitCycleTimeAvg = 0.2291338443756104; ]
  152. 07/11/22 00:05:35 Currently monitoring 1 HTCondor log file(s)
  153. 07/11/22 00:05:35 Reassigning the id of job sub-2 from (914191.0.0) to (914191.0.0)
  154. 07/11/22 00:05:35 Event: ULOG_SUBMIT for HTCondor Node sub-2 (914191.0.0) {07/11/22 00:05:30}
  155. 07/11/22 00:05:35 Number of idle job procs: 1
  156. 07/11/22 00:05:35 Reassigning the id of job sub-5 from (914193.0.0) to (914193.0.0)
  157. 07/11/22 00:05:35 Event: ULOG_SUBMIT for HTCondor Node sub-5 (914193.0.0) {07/11/22 00:05:30}
  158. 07/11/22 00:05:35 Number of idle job procs: 2
  159. 07/11/22 00:05:35 Reassigning the id of job sub-1 from (914196.0.0) to (914196.0.0)
  160. 07/11/22 00:05:35 Event: ULOG_SUBMIT for HTCondor Node sub-1 (914196.0.0) {07/11/22 00:05:30}
  161. 07/11/22 00:05:35 Number of idle job procs: 3
  162. 07/11/22 00:05:35 Reassigning the id of job sub-6 from (914199.0.0) to (914199.0.0)
  163. 07/11/22 00:05:35 Event: ULOG_SUBMIT for HTCondor Node sub-6 (914199.0.0) {07/11/22 00:05:30}
  164. 07/11/22 00:05:35 Number of idle job procs: 4
  165. 07/11/22 00:05:35 Reassigning the id of job sub-4 from (914202.0.0) to (914202.0.0)
  166. 07/11/22 00:05:35 Event: ULOG_SUBMIT for HTCondor Node sub-4 (914202.0.0) {07/11/22 00:05:30}
  167. 07/11/22 00:05:35 Number of idle job procs: 5
  168. 07/11/22 00:05:35 Reassigning the id of job sub-3 from (914205.0.0) to (914205.0.0)
  169. 07/11/22 00:05:35 Event: ULOG_SUBMIT for HTCondor Node sub-3 (914205.0.0) {07/11/22 00:05:30}
  170. 07/11/22 00:05:35 Number of idle job procs: 6
  171. 07/11/22 00:05:40 Currently monitoring 1 HTCondor log file(s)
  172. 07/11/22 00:05:40 Event: ULOG_EXECUTE for HTCondor Node sub-2 (914191.0.0) {07/11/22 00:05:37}
  173. 07/11/22 00:05:40 Number of idle job procs: 5
  174. 07/11/22 00:05:40 Event: ULOG_EXECUTE for HTCondor Node sub-1 (914196.0.0) {07/11/22 00:05:37}
  175. 07/11/22 00:05:40 Number of idle job procs: 4
  176. 07/11/22 00:05:40 Event: ULOG_EXECUTE for HTCondor Node sub-5 (914193.0.0) {07/11/22 00:05:37}
  177. 07/11/22 00:05:40 Number of idle job procs: 3
  178. 07/11/22 00:05:40 Event: ULOG_EXECUTE for HTCondor Node sub-3 (914205.0.0) {07/11/22 00:05:37}
  179. 07/11/22 00:05:40 Number of idle job procs: 2
  180. 07/11/22 00:05:40 Event: ULOG_EXECUTE for HTCondor Node sub-4 (914202.0.0) {07/11/22 00:05:37}
  181. 07/11/22 00:05:40 Number of idle job procs: 1
  182. 07/11/22 00:05:40 Event: ULOG_EXECUTE for HTCondor Node sub-6 (914199.0.0) {07/11/22 00:05:37}
  183. 07/11/22 00:05:40 Number of idle job procs: 0
  184. 07/11/22 00:15:41 601 seconds since last log event
  185. 07/11/22 00:15:41 Pending DAG nodes:
  186. 07/11/22 00:15:41 Node sub-2, HTCondor ID 914191, status STATUS_SUBMITTED
  187. 07/11/22 00:15:41 Node sub-5, HTCondor ID 914193, status STATUS_SUBMITTED
  188. 07/11/22 00:15:41 Node sub-1, HTCondor ID 914196, status STATUS_SUBMITTED
  189. 07/11/22 00:15:41 Node sub-6, HTCondor ID 914199, status STATUS_SUBMITTED
  190. 07/11/22 00:15:41 Node sub-4, HTCondor ID 914202, status STATUS_SUBMITTED
  191. 07/11/22 00:15:41 Node sub-3, HTCondor ID 914205, status STATUS_SUBMITTED
  192. 07/11/22 00:25:41 1201 seconds since last log event
  193. 07/11/22 00:25:41 Pending DAG nodes:
  194. 07/11/22 00:25:41 Node sub-2, HTCondor ID 914191, status STATUS_SUBMITTED
  195. 07/11/22 00:25:41 Node sub-5, HTCondor ID 914193, status STATUS_SUBMITTED
  196. 07/11/22 00:25:41 Node sub-1, HTCondor ID 914196, status STATUS_SUBMITTED
  197. 07/11/22 00:25:41 Node sub-6, HTCondor ID 914199, status STATUS_SUBMITTED
  198. 07/11/22 00:25:41 Node sub-4, HTCondor ID 914202, status STATUS_SUBMITTED
  199. 07/11/22 00:25:41 Node sub-3, HTCondor ID 914205, status STATUS_SUBMITTED
  200. 07/11/22 00:35:42 1802 seconds since last log event
  201. 07/11/22 00:35:42 Pending DAG nodes:
  202. 07/11/22 00:35:42 Node sub-2, HTCondor ID 914191, status STATUS_SUBMITTED
  203. 07/11/22 00:35:42 Node sub-5, HTCondor ID 914193, status STATUS_SUBMITTED
  204. 07/11/22 00:35:42 Node sub-1, HTCondor ID 914196, status STATUS_SUBMITTED
  205. 07/11/22 00:35:42 Node sub-6, HTCondor ID 914199, status STATUS_SUBMITTED
  206. 07/11/22 00:35:42 Node sub-4, HTCondor ID 914202, status STATUS_SUBMITTED
  207. 07/11/22 00:35:42 Node sub-3, HTCondor ID 914205, status STATUS_SUBMITTED
  208. 07/11/22 00:45:43 2403 seconds since last log event
  209. 07/11/22 00:45:43 Pending DAG nodes:
  210. 07/11/22 00:45:43 Node sub-2, HTCondor ID 914191, status STATUS_SUBMITTED
  211. 07/11/22 00:45:43 Node sub-5, HTCondor ID 914193, status STATUS_SUBMITTED
  212. 07/11/22 00:45:43 Node sub-1, HTCondor ID 914196, status STATUS_SUBMITTED
  213. 07/11/22 00:45:43 Node sub-6, HTCondor ID 914199, status STATUS_SUBMITTED
  214. 07/11/22 00:45:43 Node sub-4, HTCondor ID 914202, status STATUS_SUBMITTED
  215. 07/11/22 00:45:43 Node sub-3, HTCondor ID 914205, status STATUS_SUBMITTED
  216. 07/11/22 00:50:09 Currently monitoring 1 HTCondor log file(s)
  217. 07/11/22 00:50:09 Event: ULOG_JOB_TERMINATED for HTCondor Node sub-3 (914205.0.0) {07/11/22 00:50:08}
  218. 07/11/22 00:50:09 Number of idle job procs: 0
  219. 07/11/22 00:50:09 Node sub-3 job proc (914205.0.0) completed successfully.
  220. 07/11/22 00:50:09 Node sub-3 job completed
  221. 07/11/22 00:50:09 DAG status: 0 (DAG_STATUS_OK)
  222. 07/11/22 00:50:09 Of 6 nodes total:
  223. 07/11/22 00:50:09 Done Pre Queued Post Ready Un-Ready Failed
  224. 07/11/22 00:50:09 === === === === === === ===
  225. 07/11/22 00:50:09 1 0 5 0 0 0 0
  226. 07/11/22 00:50:09 0 job proc(s) currently held
  227. 07/11/22 00:50:09 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01429052389071023; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002280910438466295; EventCycleTimeSum = 1.220287084579468; EventCycleTimeCount = 535.0; SleepCycleTimeStd = 0.00603064614026856; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.0005781650543212891; SubmitCycleTimeCount = 536.0; LogProcessCycleTimeAvg = 0.0004664262135823568; LogProcessCycleTimeCount = 3.0; LogProcessCycleTimeStd = 0.0001430710805144886; LogProcessCycleTimeSum = 0.00139927864074707; SubmitCycleTimeSum = 0.6528966426849365; SubmitCycleTimeAvg = 0.001218090751277867; SleepCycleTimeSum = 2677.496150016785; SubmitCycleTimeMin = 8.106231689453125E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.004665700965953; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 535.0; SubmitCycleTimeStd = 0.01207443792723733; SleepCycleTimeMin = 5.000365972518921; ]
  228. 07/11/22 00:55:19 Currently monitoring 1 HTCondor log file(s)
  229. 07/11/22 00:55:19 Event: ULOG_JOB_TERMINATED for HTCondor Node sub-4 (914202.0.0) {07/11/22 00:55:18}
  230. 07/11/22 00:55:19 Number of idle job procs: 0
  231. 07/11/22 00:55:19 Node sub-4 job proc (914202.0.0) completed successfully.
  232. 07/11/22 00:55:19 Node sub-4 job completed
  233. 07/11/22 00:55:19 DAG status: 0 (DAG_STATUS_OK)
  234. 07/11/22 00:55:19 Of 6 nodes total:
  235. 07/11/22 00:55:19 Done Pre Queued Post Ready Un-Ready Failed
  236. 07/11/22 00:55:19 === === === === === === ===
  237. 07/11/22 00:55:19 2 0 4 0 0 0 0
  238. 07/11/22 00:55:19 0 job proc(s) currently held
  239. 07/11/22 00:55:19 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01421145475929785; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002371185388996373; EventCycleTimeSum = 1.415597677230835; EventCycleTimeCount = 597.0; SleepCycleTimeStd = 0.005718060286566557; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.0005781650543212891; SubmitCycleTimeCount = 598.0; LogProcessCycleTimeAvg = 0.0004755258560180664; LogProcessCycleTimeCount = 4.0; LogProcessCycleTimeStd = 0.000118226209824866; LogProcessCycleTimeSum = 0.001902103424072266; SubmitCycleTimeSum = 0.7642848491668701; SubmitCycleTimeAvg = 0.001278068309643596; SleepCycleTimeSum = 2987.741161346436; SubmitCycleTimeMin = 5.006790161132812E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.004591560044281; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 597.0; SubmitCycleTimeStd = 0.0118433075973188; SleepCycleTimeMin = 5.000365972518921; ]
  240. 07/11/22 01:02:44 Currently monitoring 1 HTCondor log file(s)
  241. 07/11/22 01:02:44 Event: ULOG_JOB_TERMINATED for HTCondor Node sub-2 (914191.0.0) {07/11/22 01:02:40}
  242. 07/11/22 01:02:44 Number of idle job procs: 0
  243. 07/11/22 01:02:44 Node sub-2 job proc (914191.0.0) completed successfully.
  244. 07/11/22 01:02:44 Node sub-2 job completed
  245. 07/11/22 01:02:44 DAG status: 0 (DAG_STATUS_OK)
  246. 07/11/22 01:02:44 Of 6 nodes total:
  247. 07/11/22 01:02:44 Done Pre Queued Post Ready Un-Ready Failed
  248. 07/11/22 01:02:44 === === === === === === ===
  249. 07/11/22 01:02:44 3 0 3 0 0 0 0
  250. 07/11/22 01:02:44 0 job proc(s) currently held
  251. 07/11/22 01:02:44 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01346077939913752; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002181932808011336; EventCycleTimeSum = 1.496805906295776; EventCycleTimeCount = 686.0; SleepCycleTimeStd = 0.005476549242023766; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.07325100898742676; SubmitCycleTimeCount = 687.0; LogProcessCycleTimeAvg = 0.0150306224822998; LogProcessCycleTimeCount = 5.0; LogProcessCycleTimeStd = 0.03254634652429528; LogProcessCycleTimeSum = 0.07515311241149902; SubmitCycleTimeSum = 0.7780146598815918; SubmitCycleTimeAvg = 0.001132481309871313; SleepCycleTimeSum = 3433.131620168686; SubmitCycleTimeMin = 5.006790161132812E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.004565043977676; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 686.0; SubmitCycleTimeStd = 0.01106075477865907; SleepCycleTimeMin = 5.000365972518921; ]
  252. 07/11/22 01:06:50 Currently monitoring 1 HTCondor log file(s)
  253. 07/11/22 01:06:50 Event: ULOG_JOB_TERMINATED for HTCondor Node sub-1 (914196.0.0) {07/11/22 01:06:46}
  254. 07/11/22 01:06:50 Number of idle job procs: 0
  255. 07/11/22 01:06:50 Node sub-1 job proc (914196.0.0) completed successfully.
  256. 07/11/22 01:06:50 Node sub-1 job completed
  257. 07/11/22 01:06:50 DAG status: 0 (DAG_STATUS_OK)
  258. 07/11/22 01:06:50 Of 6 nodes total:
  259. 07/11/22 01:06:50 Done Pre Queued Post Ready Un-Ready Failed
  260. 07/11/22 01:06:50 === === === === === === ===
  261. 07/11/22 01:06:50 4 0 2 0 0 0 0
  262. 07/11/22 01:06:50 0 job proc(s) currently held
  263. 07/11/22 01:06:50 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01348535553479444; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002229989629213502; EventCycleTimeSum = 1.639042377471924; EventCycleTimeCount = 735.0; SleepCycleTimeStd = 0.005296946669027134; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.07325100898742676; SubmitCycleTimeCount = 736.0; LogProcessCycleTimeAvg = 0.01262434323628744; LogProcessCycleTimeCount = 6.0; LogProcessCycleTimeStd = 0.02970105749838455; LogProcessCycleTimeSum = 0.07574605941772461; SubmitCycleTimeSum = 0.83945631980896; SubmitCycleTimeAvg = 0.001140565651914348; SleepCycleTimeSum = 3678.349071741104; SubmitCycleTimeMin = 5.006790161132812E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.004556560191978; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 735.0; SubmitCycleTimeStd = 0.01090152545705376; SleepCycleTimeMin = 5.000365972518921; ]
  264. 07/11/22 01:08:00 Currently monitoring 1 HTCondor log file(s)
  265. 07/11/22 01:08:00 Event: ULOG_JOB_TERMINATED for HTCondor Node sub-5 (914193.0.0) {07/11/22 01:07:58}
  266. 07/11/22 01:08:00 Number of idle job procs: 0
  267. 07/11/22 01:08:00 Node sub-5 job proc (914193.0.0) completed successfully.
  268. 07/11/22 01:08:00 Node sub-5 job completed
  269. 07/11/22 01:08:00 DAG status: 0 (DAG_STATUS_OK)
  270. 07/11/22 01:08:00 Of 6 nodes total:
  271. 07/11/22 01:08:00 Done Pre Queued Post Ready Un-Ready Failed
  272. 07/11/22 01:08:00 === === === === === === ===
  273. 07/11/22 01:08:00 5 0 1 0 0 0 0
  274. 07/11/22 01:08:00 0 job proc(s) currently held
  275. 07/11/22 01:08:00 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01336157586870457; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002191054327624821; EventCycleTimeSum = 1.641099691390991; EventCycleTimeCount = 749.0; SleepCycleTimeStd = 0.005248949102583846; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.07325100898742676; SubmitCycleTimeCount = 750.0; LogProcessCycleTimeAvg = 0.01088629450116839; LogProcessCycleTimeCount = 7.0; LogProcessCycleTimeStd = 0.02750041892301818; LogProcessCycleTimeSum = 0.07620406150817871; SubmitCycleTimeSum = 0.8402936458587646; SubmitCycleTimeAvg = 0.001120391527811686; SleepCycleTimeSum = 3748.407328367233; SubmitCycleTimeMin = 5.006790161132812E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.00454917004971; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 749.0; SubmitCycleTimeStd = 0.01080016205887083; SleepCycleTimeMin = 5.000365972518921; ]
  276. 07/11/22 01:11:35 Currently monitoring 1 HTCondor log file(s)
  277. 07/11/22 01:11:35 Event: ULOG_JOB_TERMINATED for HTCondor Node sub-6 (914199.0.0) {07/11/22 01:11:32}
  278. 07/11/22 01:11:35 Number of idle job procs: 0
  279. 07/11/22 01:11:35 Node sub-6 job proc (914199.0.0) completed successfully.
  280. 07/11/22 01:11:35 Node sub-6 job completed
  281. 07/11/22 01:11:35 DAG status: 0 (DAG_STATUS_OK)
  282. 07/11/22 01:11:35 Of 6 nodes total:
  283. 07/11/22 01:11:35 Done Pre Queued Post Ready Un-Ready Failed
  284. 07/11/22 01:11:35 === === === === === === ===
  285. 07/11/22 01:11:35 6 0 0 0 0 0 0
  286. 07/11/22 01:11:35 0 job proc(s) currently held
  287. 07/11/22 01:11:35 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01303404984057569; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002118428247143524; EventCycleTimeSum = 1.677795171737671; EventCycleTimeCount = 792.0; SleepCycleTimeStd = 0.00511097820812103; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.07325100898742676; SubmitCycleTimeCount = 793.0; LogProcessCycleTimeAvg = 0.009602636098861694; LogProcessCycleTimeCount = 8.0; LogProcessCycleTimeStd = 0.02571801443041846; LogProcessCycleTimeSum = 0.07682108879089355; SubmitCycleTimeSum = 0.8704862594604492; SubmitCycleTimeAvg = 0.001097712811425535; SleepCycleTimeSum = 3963.587275981903; SubmitCycleTimeMin = 5.006790161132812E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.00452938886604; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 792.0; SubmitCycleTimeStd = 0.01054824235174276; SleepCycleTimeMin = 5.000365972518921; ]
  288. 07/11/22 01:11:35 All jobs Completed!
  289. 07/11/22 01:11:35 Note: 0 total job deferrals because of -MaxJobs limit (0)
  290. 07/11/22 01:11:35 Note: 0 total job deferrals because of -MaxIdle limit (1000)
  291. 07/11/22 01:11:35 Note: 0 total job deferrals because of node category throttles
  292. 07/11/22 01:11:35 Note: 0 total PRE script deferrals because of -MaxPre limit (20) or DEFER
  293. 07/11/22 01:11:35 Note: 0 total POST script deferrals because of -MaxPost limit (20) or DEFER
  294. 07/11/22 01:11:35 DAG status: 0 (DAG_STATUS_OK)
  295. 07/11/22 01:11:35 Of 6 nodes total:
  296. 07/11/22 01:11:35 Done Pre Queued Post Ready Un-Ready Failed
  297. 07/11/22 01:11:35 === === === === === === ===
  298. 07/11/22 01:11:35 6 0 0 0 0 0 0
  299. 07/11/22 01:11:35 0 job proc(s) currently held
  300. 07/11/22 01:11:35 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01303404984057569; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002118428247143524; EventCycleTimeSum = 1.677795171737671; EventCycleTimeCount = 792.0; SleepCycleTimeStd = 0.00511097820812103; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.07325100898742676; SubmitCycleTimeCount = 793.0; LogProcessCycleTimeAvg = 0.009602636098861694; LogProcessCycleTimeCount = 8.0; LogProcessCycleTimeStd = 0.02571801443041846; LogProcessCycleTimeSum = 0.07682108879089355; SubmitCycleTimeSum = 0.8704862594604492; SubmitCycleTimeAvg = 0.001097712811425535; SleepCycleTimeSum = 3963.587275981903; SubmitCycleTimeMin = 5.006790161132812E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.00452938886604; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 792.0; SubmitCycleTimeStd = 0.01054824235174276; SleepCycleTimeMin = 5.000365972518921; ]
  301. 07/11/22 01:11:35 Wrote metrics file code/process.condor_dag.metrics.
  302. 07/11/22 01:11:35 Metrics not sent because of PEGASUS_METRICS or CONDOR_DEVELOPERS setting.
  303. 07/11/22 01:11:35 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.01303404984057569; EventCycleTimeMax = 0.2293319702148438; EventCycleTimeMin = 1.406669616699219E-05; EventCycleTimeAvg = 0.002118428247143524; EventCycleTimeSum = 1.677795171737671; EventCycleTimeCount = 792.0; SleepCycleTimeStd = 0.00511097820812103; LogProcessCycleTimeMin = 0.00030517578125; LogProcessCycleTimeMax = 0.07325100898742676; SubmitCycleTimeCount = 793.0; LogProcessCycleTimeAvg = 0.009602636098861694; LogProcessCycleTimeCount = 8.0; LogProcessCycleTimeStd = 0.02571801443041846; LogProcessCycleTimeSum = 0.07682108879089355; SubmitCycleTimeSum = 0.8704862594604492; SubmitCycleTimeAvg = 0.001097712811425535; SleepCycleTimeSum = 3963.587275981903; SubmitCycleTimeMin = 5.006790161132812E-06; SubmitCycleTimeMax = 0.2291338443756104; SleepCycleTimeAvg = 5.00452938886604; SleepCycleTimeMax = 5.072962045669556; SleepCycleTimeCount = 792.0; SubmitCycleTimeStd = 0.01054824235174276; SleepCycleTimeMin = 5.000365972518921; ]
  304. 07/11/22 01:11:35 **** condor_scheduniv_exec.914125.0 (condor_DAGMAN) pid 3463981 EXITING WITH STATUS 0