process.condor_dag.dagman.out 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266
  1. 07/11/22 06:34:24 ******************************************************
  2. 07/11/22 06:34:24 ** condor_scheduniv_exec.920949.0 (CONDOR_DAGMAN) STARTING UP
  3. 07/11/22 06:34:24 ** /usr/bin/condor_dagman
  4. 07/11/22 06:34:24 ** SubsystemInfo: name=DAGMAN type=DAGMAN(10) class=DAEMON(1)
  5. 07/11/22 06:34:24 ** Configuration: subsystem:DAGMAN local:<NONE> class:DAEMON
  6. 07/11/22 06:34:24 ** $CondorVersion: 8.8.6 Nov 19 2019 BuildID: Debian-8.8.6~dfsg.1-1 Debian-8.8.6~dfsg.1-1 $
  7. 07/11/22 06:34:24 ** $CondorPlatform: X86_64-Debian_10 $
  8. 07/11/22 06:34:24 ** PID = 1042631
  9. 07/11/22 06:34:24 ** Log last touched time unavailable (No such file or directory)
  10. 07/11/22 06:34:24 ******************************************************
  11. 07/11/22 06:34:24 Using config source: /etc/condor/condor_config
  12. 07/11/22 06:34:24 Using local config sources:
  13. 07/11/22 06:34:24 /etc/condor/config.d/50_inm7_master
  14. 07/11/22 06:34:24 /etc/condor/condor_config.local
  15. 07/11/22 06:34:24 config Macros = 81, Sorted = 81, StringBytes = 2405, TablesBytes = 2972
  16. 07/11/22 06:34:24 CLASSAD_CACHING is ENABLED
  17. 07/11/22 06:34:24 Daemon Log is logging: D_ALWAYS D_ERROR
  18. 07/11/22 06:34:24 DaemonCore: No command port requested.
  19. 07/11/22 06:34:24 DAGMAN_USE_STRICT setting: 1
  20. 07/11/22 06:34:24 DAGMAN_VERBOSITY setting: 3
  21. 07/11/22 06:34:24 DAGMAN_DEBUG_CACHE_SIZE setting: 5242880
  22. 07/11/22 06:34:24 DAGMAN_DEBUG_CACHE_ENABLE setting: False
  23. 07/11/22 06:34:24 DAGMAN_SUBMIT_DELAY setting: 0
  24. 07/11/22 06:34:24 DAGMAN_MAX_SUBMIT_ATTEMPTS setting: 6
  25. 07/11/22 06:34:24 DAGMAN_STARTUP_CYCLE_DETECT setting: False
  26. 07/11/22 06:34:24 DAGMAN_MAX_SUBMITS_PER_INTERVAL setting: 100
  27. 07/11/22 06:34:24 DAGMAN_AGGRESSIVE_SUBMIT setting: False
  28. 07/11/22 06:34:24 DAGMAN_USER_LOG_SCAN_INTERVAL setting: 5
  29. 07/11/22 06:34:24 DAGMAN_QUEUE_UPDATE_INTERVAL setting: 300
  30. 07/11/22 06:34:24 DAGMAN_DEFAULT_PRIORITY setting: 0
  31. 07/11/22 06:34:24 DAGMAN_SUPPRESS_NOTIFICATION setting: True
  32. 07/11/22 06:34:24 allow_events (DAGMAN_ALLOW_EVENTS) setting: 114
  33. 07/11/22 06:34:24 DAGMAN_RETRY_SUBMIT_FIRST setting: True
  34. 07/11/22 06:34:24 DAGMAN_RETRY_NODE_FIRST setting: False
  35. 07/11/22 06:34:24 DAGMAN_MAX_JOBS_IDLE setting: 1000
  36. 07/11/22 06:34:24 DAGMAN_MAX_JOBS_SUBMITTED setting: 0
  37. 07/11/22 06:34:24 DAGMAN_MAX_PRE_SCRIPTS setting: 20
  38. 07/11/22 06:34:24 DAGMAN_MAX_POST_SCRIPTS setting: 20
  39. 07/11/22 06:34:24 DAGMAN_MUNGE_NODE_NAMES setting: True
  40. 07/11/22 06:34:24 DAGMAN_PROHIBIT_MULTI_JOBS setting: False
  41. 07/11/22 06:34:24 DAGMAN_SUBMIT_DEPTH_FIRST setting: False
  42. 07/11/22 06:34:24 DAGMAN_ALWAYS_RUN_POST setting: False
  43. 07/11/22 06:34:24 DAGMAN_CONDOR_SUBMIT_EXE setting: /usr/bin/condor_submit
  44. 07/11/22 06:34:24 DAGMAN_USE_CONDOR_SUBMIT setting: True
  45. 07/11/22 06:34:24 DAGMAN_ABORT_DUPLICATES setting: True
  46. 07/11/22 06:34:24 DAGMAN_ABORT_ON_SCARY_SUBMIT setting: True
  47. 07/11/22 06:34:24 DAGMAN_PENDING_REPORT_INTERVAL setting: 600
  48. 07/11/22 06:34:24 DAGMAN_AUTO_RESCUE setting: True
  49. 07/11/22 06:34:24 DAGMAN_MAX_RESCUE_NUM setting: 100
  50. 07/11/22 06:34:24 DAGMAN_WRITE_PARTIAL_RESCUE setting: True
  51. 07/11/22 06:34:24 DAGMAN_DEFAULT_NODE_LOG setting: @(DAG_DIR)/@(DAG_FILE).nodes.log
  52. 07/11/22 06:34:24 DAGMAN_GENERATE_SUBDAG_SUBMITS setting: True
  53. 07/11/22 06:34:24 DAGMAN_MAX_JOB_HOLDS setting: 100
  54. 07/11/22 06:34:24 DAGMAN_HOLD_CLAIM_TIME setting: 20
  55. 07/11/22 06:34:24 ALL_DEBUG setting:
  56. 07/11/22 06:34:24 DAGMAN_DEBUG setting:
  57. 07/11/22 06:34:24 DAGMAN_SUPPRESS_JOB_LOGS setting: False
  58. 07/11/22 06:34:24 DAGMAN_REMOVE_NODE_JOBS setting: True
  59. 07/11/22 06:34:24 argv[0] == "condor_scheduniv_exec.920949.0"
  60. 07/11/22 06:34:24 argv[1] == "-Lockfile"
  61. 07/11/22 06:34:24 argv[2] == "code/process.condor_dag.lock"
  62. 07/11/22 06:34:24 argv[3] == "-AutoRescue"
  63. 07/11/22 06:34:24 argv[4] == "1"
  64. 07/11/22 06:34:24 argv[5] == "-DoRescueFrom"
  65. 07/11/22 06:34:24 argv[6] == "0"
  66. 07/11/22 06:34:24 argv[7] == "-Dag"
  67. 07/11/22 06:34:24 argv[8] == "code/process.condor_dag"
  68. 07/11/22 06:34:24 argv[9] == "-Suppress_notification"
  69. 07/11/22 06:34:24 argv[10] == "-CsdVersion"
  70. 07/11/22 06:34:24 argv[11] == "$CondorVersion: 8.8.6 Nov 19 2019 BuildID: Debian-8.8.6~dfsg.1-1 Debian-8.8.6~dfsg.1-1 $"
  71. 07/11/22 06:34:24 argv[12] == "-Dagman"
  72. 07/11/22 06:34:24 argv[13] == "/usr/bin/condor_dagman"
  73. 07/11/22 06:34:24 Workflow batch-name: <process.condor_dag+920949>
  74. 07/11/22 06:34:24 Workflow accounting_group: <>
  75. 07/11/22 06:34:24 Workflow accounting_group_user: <>
  76. 07/11/22 06:34:24 Warning: failed to get attribute DAGNodeName
  77. 07/11/22 06:34:24 DAGMAN_LOG_ON_NFS_IS_ERROR setting: False
  78. 07/11/22 06:34:24 Default node log file is: </data/project/infrasound/ReproVBM/ds003455_ReproVBM/code/process.condor_dag.nodes.log>
  79. 07/11/22 06:34:24 DAG Lockfile will be written to code/process.condor_dag.lock
  80. 07/11/22 06:34:24 DAG Input file is code/process.condor_dag
  81. 07/11/22 06:34:24 Parsing 1 dagfiles
  82. 07/11/22 06:34:24 Parsing code/process.condor_dag ...
  83. 07/11/22 06:34:24 Dag contains 1 total jobs
  84. 07/11/22 06:34:24 Sleeping for 3 seconds to ensure ProcessId uniqueness
  85. 07/11/22 06:34:27 Bootstrapping...
  86. 07/11/22 06:34:27 Number of pre-completed nodes: 0
  87. 07/11/22 06:34:27 MultiLogFiles: truncating log file /data/project/infrasound/ReproVBM/ds003455_ReproVBM/code/process.condor_dag.nodes.log
  88. 07/11/22 06:34:27 DAG status: 0 (DAG_STATUS_OK)
  89. 07/11/22 06:34:27 Of 1 nodes total:
  90. 07/11/22 06:34:27 Done Pre Queued Post Ready Un-Ready Failed
  91. 07/11/22 06:34:27 === === === === === === ===
  92. 07/11/22 06:34:27 0 0 0 0 1 0 0
  93. 07/11/22 06:34:27 0 job proc(s) currently held
  94. 07/11/22 06:34:27 DAGMan Runtime Statistics: [ EventCycleTimeSum = 0.0; EventCycleTimeCount = 0.0; LogProcessCycleTimeSum = 0.0; SleepCycleTimeSum = 0.0; LogProcessCycleTimeCount = 0.0; SleepCycleTimeCount = 0.0; SubmitCycleTimeCount = 0.0; SubmitCycleTimeSum = 0.0; ]
  95. 07/11/22 06:34:27 Registering condor_event_timer...
  96. 07/11/22 06:34:28 Submitting HTCondor Node sub-001 job(s)...
  97. 07/11/22 06:34:28 Adding a DAGMan workflow log /data/project/infrasound/ReproVBM/ds003455_ReproVBM/code/process.condor_dag.nodes.log
  98. 07/11/22 06:34:28 Masking the events recorded in the DAGMAN workflow log
  99. 07/11/22 06:34:28 Mask for workflow log is 0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36
  100. 07/11/22 06:34:28 submitting: /usr/bin/condor_submit -a dag_node_name' '=' 'sub-001 -a +DAGManJobId' '=' '920949 -a DAGManJobId' '=' '920949 -batch-name process.condor_dag+920949 -a submit_event_notes' '=' 'DAG' 'Node:' 'sub-001 -a dagman_log' '=' '/data/project/infrasound/ReproVBM/ds003455_ReproVBM/code/process.condor_dag.nodes.log -a +DAGManNodesMask' '=' '"0,1,2,4,5,7,9,10,11,12,13,16,17,24,27,35,36" -a subject' '=' 'sub-001 -a DAG_STATUS' '=' '0 -a FAILED_COUNT' '=' '0 -a notification' '=' 'never -a +DAGParentNodeNames' '=' '"" code/process.condor_submit
  101. 07/11/22 06:34:28 From submit: Submitting job(s).
  102. 07/11/22 06:34:28 From submit: 1 job(s) submitted to cluster 923183.
  103. 07/11/22 06:34:28 assigned HTCondor ID (923183.0.0)
  104. 07/11/22 06:34:28 Just submitted 1 job this cycle...
  105. 07/11/22 06:34:28 DAG status: 0 (DAG_STATUS_OK)
  106. 07/11/22 06:34:28 Of 1 nodes total:
  107. 07/11/22 06:34:28 Done Pre Queued Post Ready Un-Ready Failed
  108. 07/11/22 06:34:28 === === === === === === ===
  109. 07/11/22 06:34:28 0 0 1 0 0 0 0
  110. 07/11/22 06:34:28 0 job proc(s) currently held
  111. 07/11/22 06:34:28 DAGMan Runtime Statistics: [ EventCycleTimeSum = 0.0; SleepCycleTimeCount = 0.0; EventCycleTimeCount = 0.0; SubmitCycleTimeStd = 0.0312049388885498; SubmitCycleTimeMax = 0.0312049388885498; LogProcessCycleTimeSum = 0.0; LogProcessCycleTimeCount = 0.0; SubmitCycleTimeMin = 0.0312049388885498; SubmitCycleTimeCount = 1.0; SubmitCycleTimeSum = 0.0312049388885498; SleepCycleTimeSum = 0.0; SubmitCycleTimeAvg = 0.0312049388885498; ]
  112. 07/11/22 06:34:33 Currently monitoring 1 HTCondor log file(s)
  113. 07/11/22 06:34:33 Reassigning the id of job sub-001 from (923183.0.0) to (923183.0.0)
  114. 07/11/22 06:34:33 Event: ULOG_SUBMIT for HTCondor Node sub-001 (923183.0.0) {07/11/22 06:34:28}
  115. 07/11/22 06:34:33 Number of idle job procs: 1
  116. 07/11/22 06:44:33 600 seconds since last log event
  117. 07/11/22 06:44:33 Pending DAG nodes:
  118. 07/11/22 06:44:33 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  119. 07/11/22 06:54:34 1201 seconds since last log event
  120. 07/11/22 06:54:34 Pending DAG nodes:
  121. 07/11/22 06:54:34 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  122. 07/11/22 07:04:35 1802 seconds since last log event
  123. 07/11/22 07:04:35 Pending DAG nodes:
  124. 07/11/22 07:04:35 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  125. 07/11/22 07:14:35 2402 seconds since last log event
  126. 07/11/22 07:14:35 Pending DAG nodes:
  127. 07/11/22 07:14:35 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  128. 07/11/22 07:24:36 3003 seconds since last log event
  129. 07/11/22 07:24:36 Pending DAG nodes:
  130. 07/11/22 07:24:36 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  131. 07/11/22 07:34:36 3603 seconds since last log event
  132. 07/11/22 07:34:36 Pending DAG nodes:
  133. 07/11/22 07:34:36 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  134. 07/11/22 07:44:37 4204 seconds since last log event
  135. 07/11/22 07:44:37 Pending DAG nodes:
  136. 07/11/22 07:44:37 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  137. 07/11/22 07:54:37 4804 seconds since last log event
  138. 07/11/22 07:54:37 Pending DAG nodes:
  139. 07/11/22 07:54:37 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  140. 07/11/22 08:04:38 5405 seconds since last log event
  141. 07/11/22 08:04:38 Pending DAG nodes:
  142. 07/11/22 08:04:38 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  143. 07/11/22 08:14:38 6005 seconds since last log event
  144. 07/11/22 08:14:38 Pending DAG nodes:
  145. 07/11/22 08:14:38 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  146. 07/11/22 08:24:39 6606 seconds since last log event
  147. 07/11/22 08:24:39 Pending DAG nodes:
  148. 07/11/22 08:24:39 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  149. 07/11/22 08:34:40 7207 seconds since last log event
  150. 07/11/22 08:34:40 Pending DAG nodes:
  151. 07/11/22 08:34:40 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  152. 07/11/22 08:44:40 7807 seconds since last log event
  153. 07/11/22 08:44:40 Pending DAG nodes:
  154. 07/11/22 08:44:40 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  155. 07/11/22 08:54:41 8408 seconds since last log event
  156. 07/11/22 08:54:41 Pending DAG nodes:
  157. 07/11/22 08:54:41 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  158. 07/11/22 09:04:41 9008 seconds since last log event
  159. 07/11/22 09:04:41 Pending DAG nodes:
  160. 07/11/22 09:04:41 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  161. 07/11/22 09:14:42 9609 seconds since last log event
  162. 07/11/22 09:14:42 Pending DAG nodes:
  163. 07/11/22 09:14:42 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  164. 07/11/22 09:24:43 10210 seconds since last log event
  165. 07/11/22 09:24:43 Pending DAG nodes:
  166. 07/11/22 09:24:43 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  167. 07/11/22 09:34:43 10810 seconds since last log event
  168. 07/11/22 09:34:43 Pending DAG nodes:
  169. 07/11/22 09:34:43 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  170. 07/11/22 09:44:44 11411 seconds since last log event
  171. 07/11/22 09:44:44 Pending DAG nodes:
  172. 07/11/22 09:44:44 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  173. 07/11/22 09:54:44 12011 seconds since last log event
  174. 07/11/22 09:54:44 Pending DAG nodes:
  175. 07/11/22 09:54:44 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  176. 07/11/22 10:04:45 12612 seconds since last log event
  177. 07/11/22 10:04:45 Pending DAG nodes:
  178. 07/11/22 10:04:45 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  179. 07/11/22 10:14:45 13212 seconds since last log event
  180. 07/11/22 10:14:45 Pending DAG nodes:
  181. 07/11/22 10:14:45 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  182. 07/11/22 10:24:46 13813 seconds since last log event
  183. 07/11/22 10:24:46 Pending DAG nodes:
  184. 07/11/22 10:24:46 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  185. 07/11/22 10:34:46 14413 seconds since last log event
  186. 07/11/22 10:34:46 Pending DAG nodes:
  187. 07/11/22 10:34:46 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  188. 07/11/22 10:44:47 15014 seconds since last log event
  189. 07/11/22 10:44:47 Pending DAG nodes:
  190. 07/11/22 10:44:47 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  191. 07/11/22 10:54:47 15614 seconds since last log event
  192. 07/11/22 10:54:47 Pending DAG nodes:
  193. 07/11/22 10:54:47 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  194. 07/11/22 11:04:48 16215 seconds since last log event
  195. 07/11/22 11:04:48 Pending DAG nodes:
  196. 07/11/22 11:04:48 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  197. 07/11/22 11:14:48 16815 seconds since last log event
  198. 07/11/22 11:14:48 Pending DAG nodes:
  199. 07/11/22 11:14:48 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  200. 07/11/22 11:24:49 17416 seconds since last log event
  201. 07/11/22 11:24:49 Pending DAG nodes:
  202. 07/11/22 11:24:49 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  203. 07/11/22 11:34:49 18016 seconds since last log event
  204. 07/11/22 11:34:49 Pending DAG nodes:
  205. 07/11/22 11:34:49 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  206. 07/11/22 11:44:50 18617 seconds since last log event
  207. 07/11/22 11:44:50 Pending DAG nodes:
  208. 07/11/22 11:44:50 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  209. 07/11/22 11:54:50 19217 seconds since last log event
  210. 07/11/22 11:54:50 Pending DAG nodes:
  211. 07/11/22 11:54:50 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  212. 07/11/22 12:04:50 19817 seconds since last log event
  213. 07/11/22 12:04:50 Pending DAG nodes:
  214. 07/11/22 12:04:50 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  215. 07/11/22 12:14:50 20417 seconds since last log event
  216. 07/11/22 12:14:50 Pending DAG nodes:
  217. 07/11/22 12:14:50 Node sub-001, HTCondor ID 923183, status STATUS_SUBMITTED
  218. 07/11/22 12:15:15 Currently monitoring 1 HTCondor log file(s)
  219. 07/11/22 12:15:15 Event: ULOG_EXECUTE for HTCondor Node sub-001 (923183.0.0) {07/11/22 12:15:11}
  220. 07/11/22 12:15:15 Number of idle job procs: 0
  221. 07/11/22 12:15:25 Currently monitoring 1 HTCondor log file(s)
  222. 07/11/22 12:15:25 Event: ULOG_JOB_TERMINATED for HTCondor Node sub-001 (923183.0.0) {07/11/22 12:15:21}
  223. 07/11/22 12:15:25 Number of idle job procs: 0
  224. 07/11/22 12:15:25 Node sub-001 job proc (923183.0.0) failed with status 1.
  225. 07/11/22 12:15:25 DAG status: 2 (DAG_STATUS_NODE_FAILED)
  226. 07/11/22 12:15:25 Of 1 nodes total:
  227. 07/11/22 12:15:25 Done Pre Queued Post Ready Un-Ready Failed
  228. 07/11/22 12:15:25 === === === === === === ===
  229. 07/11/22 12:15:25 0 0 0 0 0 0 1
  230. 07/11/22 12:15:25 0 job proc(s) currently held
  231. 07/11/22 12:15:25 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.0005250350649968265; EventCycleTimeMax = 0.03152704238891602; EventCycleTimeMin = 1.502037048339844E-05; EventCycleTimeAvg = 0.0001080640374797664; EventCycleTimeSum = 0.4417657852172852; EventCycleTimeCount = 4088.0; SleepCycleTimeStd = 0.01566966046893907; LogProcessCycleTimeMin = 0.0003068447113037109; LogProcessCycleTimeMax = 0.0004479885101318359; SubmitCycleTimeCount = 4089.0; LogProcessCycleTimeAvg = 0.0003805955251057943; LogProcessCycleTimeCount = 3.0; LogProcessCycleTimeStd = 7.078636504982907E-05; LogProcessCycleTimeSum = 0.001141786575317383; SubmitCycleTimeSum = 0.2684481143951416; SubmitCycleTimeAvg = 6.565128745295711E-05; SleepCycleTimeSum = 20457.25686216354; SubmitCycleTimeMin = 6.914138793945312E-06; SubmitCycleTimeMax = 0.0312049388885498; SleepCycleTimeAvg = 5.004221345930417; SleepCycleTimeMax = 5.009541034698486; SleepCycleTimeCount = 4088.0; SubmitCycleTimeStd = 0.0005153460139559114; SleepCycleTimeMin = 4.004539966583252; ]
  232. 07/11/22 12:15:25 ERROR: the following job(s) failed:
  233. 07/11/22 12:15:25 ---------------------- Job ----------------------
  234. 07/11/22 12:15:25 Node Name: sub-001
  235. 07/11/22 12:15:25 Noop: false
  236. 07/11/22 12:15:25 NodeID: 0
  237. 07/11/22 12:15:25 Node Status: STATUS_ERROR
  238. 07/11/22 12:15:25 Node return val: 1
  239. 07/11/22 12:15:25 Error: Job proc (923183.0.0) failed with status 1
  240. 07/11/22 12:15:25 Job Submit File: code/process.condor_submit
  241. 07/11/22 12:15:25 HTCondor Job ID: (923183.0.0)
  242. 07/11/22 12:15:25 Q_PARENTS: <END>
  243. 07/11/22 12:15:25 Q_WAITING: <END>
  244. 07/11/22 12:15:25 Q_CHILDREN: <END>
  245. 07/11/22 12:15:25 --------------------------------------- <END>
  246. 07/11/22 12:15:25 Aborting DAG...
  247. 07/11/22 12:15:25 Writing Rescue DAG to code/process.condor_dag.rescue001...
  248. 07/11/22 12:15:25 Removing submitted jobs...
  249. 07/11/22 12:15:25 Removing any/all submitted HTCondor jobs...
  250. 07/11/22 12:15:25 Running: /usr/bin/condor_rm -const DAGManJobId' '=?=' '920949
  251. 07/11/22 12:15:26 Note: 0 total job deferrals because of -MaxJobs limit (0)
  252. 07/11/22 12:15:26 Note: 0 total job deferrals because of -MaxIdle limit (1000)
  253. 07/11/22 12:15:26 Note: 0 total job deferrals because of node category throttles
  254. 07/11/22 12:15:26 Note: 0 total PRE script deferrals because of -MaxPre limit (20) or DEFER
  255. 07/11/22 12:15:26 Note: 0 total POST script deferrals because of -MaxPost limit (20) or DEFER
  256. 07/11/22 12:15:26 DAG status: 2 (DAG_STATUS_NODE_FAILED)
  257. 07/11/22 12:15:26 Of 1 nodes total:
  258. 07/11/22 12:15:26 Done Pre Queued Post Ready Un-Ready Failed
  259. 07/11/22 12:15:26 === === === === === === ===
  260. 07/11/22 12:15:26 0 0 0 0 0 0 1
  261. 07/11/22 12:15:26 0 job proc(s) currently held
  262. 07/11/22 12:15:26 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.0005250350649968265; EventCycleTimeMax = 0.03152704238891602; EventCycleTimeMin = 1.502037048339844E-05; EventCycleTimeAvg = 0.0001080640374797664; EventCycleTimeSum = 0.4417657852172852; EventCycleTimeCount = 4088.0; SleepCycleTimeStd = 0.01566966046893907; LogProcessCycleTimeMin = 0.0003068447113037109; LogProcessCycleTimeMax = 0.0004479885101318359; SubmitCycleTimeCount = 4089.0; LogProcessCycleTimeAvg = 0.0003805955251057943; LogProcessCycleTimeCount = 3.0; LogProcessCycleTimeStd = 7.078636504982907E-05; LogProcessCycleTimeSum = 0.001141786575317383; SubmitCycleTimeSum = 0.2684481143951416; SubmitCycleTimeAvg = 6.565128745295711E-05; SleepCycleTimeSum = 20457.25686216354; SubmitCycleTimeMin = 6.914138793945312E-06; SubmitCycleTimeMax = 0.0312049388885498; SleepCycleTimeAvg = 5.004221345930417; SleepCycleTimeMax = 5.009541034698486; SleepCycleTimeCount = 4088.0; SubmitCycleTimeStd = 0.0005153460139559114; SleepCycleTimeMin = 4.004539966583252; ]
  263. 07/11/22 12:15:26 Wrote metrics file code/process.condor_dag.metrics.
  264. 07/11/22 12:15:26 Metrics not sent because of PEGASUS_METRICS or CONDOR_DEVELOPERS setting.
  265. 07/11/22 12:15:26 DAGMan Runtime Statistics: [ EventCycleTimeStd = 0.0005250350649968265; EventCycleTimeMax = 0.03152704238891602; EventCycleTimeMin = 1.502037048339844E-05; EventCycleTimeAvg = 0.0001080640374797664; EventCycleTimeSum = 0.4417657852172852; EventCycleTimeCount = 4088.0; SleepCycleTimeStd = 0.01566966046893907; LogProcessCycleTimeMin = 0.0003068447113037109; LogProcessCycleTimeMax = 0.0004479885101318359; SubmitCycleTimeCount = 4089.0; LogProcessCycleTimeAvg = 0.0003805955251057943; LogProcessCycleTimeCount = 3.0; LogProcessCycleTimeStd = 7.078636504982907E-05; LogProcessCycleTimeSum = 0.001141786575317383; SubmitCycleTimeSum = 0.2684481143951416; SubmitCycleTimeAvg = 6.565128745295711E-05; SleepCycleTimeSum = 20457.25686216354; SubmitCycleTimeMin = 6.914138793945312E-06; SubmitCycleTimeMax = 0.0312049388885498; SleepCycleTimeAvg = 5.004221345930417; SleepCycleTimeMax = 5.009541034698486; SleepCycleTimeCount = 4088.0; SubmitCycleTimeStd = 0.0005153460139559114; SleepCycleTimeMin = 4.004539966583252; ]
  266. 07/11/22 12:15:26 **** condor_scheduniv_exec.920949.0 (condor_DAGMAN) pid 1042631 EXITING WITH STATUS 1