log.txt 8.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191
  1. 2020-11-06 13:09:31 Config:
  2. {'all_joints': [[0],
  3. [1],
  4. [2],
  5. [3],
  6. [4],
  7. [5],
  8. [6],
  9. [7],
  10. [8],
  11. [9],
  12. [10],
  13. [11],
  14. [12],
  15. [13],
  16. [14],
  17. [15],
  18. [16],
  19. [17],
  20. [18]],
  21. 'all_joints_names': ['snout',
  22. 'neck',
  23. 'upper_back',
  24. 'middle_back',
  25. 'tailbase',
  26. 'shoulder_right',
  27. 'elbow_right',
  28. 'wrist_right',
  29. 'forepaw_right',
  30. 'forepaw_left',
  31. 'hip_right',
  32. 'knee_right',
  33. 'ankle_right',
  34. 'hindpaw_right',
  35. 'hindpaw_left',
  36. 'beam_left_top',
  37. 'beam_left_bottom',
  38. 'beam_right_top',
  39. 'beam_right_bottom'],
  40. 'batch_size': 1,
  41. 'bottomheight': 400,
  42. 'crop': True,
  43. 'crop_pad': 0,
  44. 'cropratio': 0.4,
  45. 'dataset': 'training-datasets/iteration-0/UnaugmentedDataSet_RotatingBeam3.1Jul12/RotatingBeam3.1_Sebastian_Kloubert95shuffle4.mat',
  46. 'dataset_type': 'imgaug',
  47. 'deterministic': False,
  48. 'display_iters': 1000,
  49. 'fg_fraction': 0.25,
  50. 'global_scale': 0.8,
  51. 'init_weights': '/mnt/Install/DLC-GPU/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt',
  52. 'intermediate_supervision': False,
  53. 'intermediate_supervision_layer': 12,
  54. 'leftwidth': 400,
  55. 'location_refinement': True,
  56. 'locref_huber_loss': True,
  57. 'locref_loss_weight': 0.05,
  58. 'locref_stdev': 7.2801,
  59. 'log_dir': 'log',
  60. 'max_input_size': 1500,
  61. 'mean_pixel': [123.68, 116.779, 103.939],
  62. 'metadataset': 'training-datasets/iteration-0/UnaugmentedDataSet_RotatingBeam3.1Jul12/Documentation_data-RotatingBeam3.1_95shuffle4.pickle',
  63. 'min_input_size': 64,
  64. 'minsize': 100,
  65. 'mirror': False,
  66. 'multi_step': [[0.005, 10000],
  67. [0.02, 430000],
  68. [0.002, 730000],
  69. [0.001, 1030000]],
  70. 'net_type': 'resnet_50',
  71. 'num_joints': 19,
  72. 'optimizer': 'sgd',
  73. 'pairwise_huber_loss': False,
  74. 'pairwise_predict': False,
  75. 'partaffinityfield_predict': False,
  76. 'pos_dist_thresh': 17,
  77. 'project_path': '/mnt/DLC/RotatingBeam3.1-Sebastian_Kloubert-2020-07-12',
  78. 'regularize': False,
  79. 'rightwidth': 400,
  80. 'save_iters': 50000,
  81. 'scale_jitter_lo': 0.5,
  82. 'scale_jitter_up': 1.25,
  83. 'scoremap_dir': 'test',
  84. 'shuffle': True,
  85. 'snapshot_prefix': '/mnt/DLC/RotatingBeam3.1-Sebastian_Kloubert-2020-07-12/dlc-models/iteration-0/RotatingBeam3.1Jul12-trainset95shuffle4/train/snapshot',
  86. 'stride': 8.0,
  87. 'topheight': 400,
  88. 'weigh_negatives': False,
  89. 'weigh_only_present_joints': False,
  90. 'weigh_part_predictions': False,
  91. 'weight_decay': 0.0001}
  92. 2020-11-06 13:11:10 iteration: 1000 loss: 0.0263 lr: 0.005
  93. 2020-11-06 13:12:37 iteration: 2000 loss: 0.0117 lr: 0.005
  94. 2020-11-06 13:14:06 iteration: 3000 loss: 0.0095 lr: 0.005
  95. 2020-11-06 13:15:32 iteration: 4000 loss: 0.0087 lr: 0.005
  96. 2020-11-06 13:16:58 iteration: 5000 loss: 0.0080 lr: 0.005
  97. 2020-11-06 13:18:25 iteration: 6000 loss: 0.0074 lr: 0.005
  98. 2020-11-06 13:19:52 iteration: 7000 loss: 0.0074 lr: 0.005
  99. 2020-11-06 13:21:20 iteration: 8000 loss: 0.0069 lr: 0.005
  100. 2020-11-06 13:22:48 iteration: 9000 loss: 0.0067 lr: 0.005
  101. 2020-11-06 13:24:14 iteration: 10000 loss: 0.0064 lr: 0.005
  102. 2020-11-06 13:25:40 iteration: 11000 loss: 0.0076 lr: 0.02
  103. 2020-11-06 13:27:06 iteration: 12000 loss: 0.0072 lr: 0.02
  104. 2020-11-06 13:28:32 iteration: 13000 loss: 0.0063 lr: 0.02
  105. 2020-11-06 13:29:58 iteration: 14000 loss: 0.0060 lr: 0.02
  106. 2020-11-06 13:31:26 iteration: 15000 loss: 0.0058 lr: 0.02
  107. 2020-11-06 13:32:52 iteration: 16000 loss: 0.0053 lr: 0.02
  108. 2020-11-06 13:34:19 iteration: 17000 loss: 0.0053 lr: 0.02
  109. 2020-11-06 13:35:45 iteration: 18000 loss: 0.0050 lr: 0.02
  110. 2020-11-06 13:37:12 iteration: 19000 loss: 0.0047 lr: 0.02
  111. 2020-11-06 13:38:40 iteration: 20000 loss: 0.0048 lr: 0.02
  112. 2020-11-06 13:40:07 iteration: 21000 loss: 0.0046 lr: 0.02
  113. 2020-11-06 13:41:34 iteration: 22000 loss: 0.0045 lr: 0.02
  114. 2020-11-06 13:43:00 iteration: 23000 loss: 0.0044 lr: 0.02
  115. 2020-11-06 13:44:23 iteration: 24000 loss: 0.0043 lr: 0.02
  116. 2020-11-06 13:45:48 iteration: 25000 loss: 0.0042 lr: 0.02
  117. 2020-11-06 13:47:15 iteration: 26000 loss: 0.0042 lr: 0.02
  118. 2020-11-06 13:48:42 iteration: 27000 loss: 0.0041 lr: 0.02
  119. 2020-11-06 13:50:07 iteration: 28000 loss: 0.0040 lr: 0.02
  120. 2020-11-06 13:51:34 iteration: 29000 loss: 0.0040 lr: 0.02
  121. 2020-11-06 13:53:00 iteration: 30000 loss: 0.0038 lr: 0.02
  122. 2020-11-06 13:54:27 iteration: 31000 loss: 0.0037 lr: 0.02
  123. 2020-11-06 13:55:55 iteration: 32000 loss: 0.0037 lr: 0.02
  124. 2020-11-06 13:57:24 iteration: 33000 loss: 0.0037 lr: 0.02
  125. 2020-11-06 13:58:52 iteration: 34000 loss: 0.0037 lr: 0.02
  126. 2020-11-06 14:00:20 iteration: 35000 loss: 0.0035 lr: 0.02
  127. 2020-11-06 14:01:48 iteration: 36000 loss: 0.0036 lr: 0.02
  128. 2020-11-06 14:03:17 iteration: 37000 loss: 0.0035 lr: 0.02
  129. 2020-11-06 14:04:44 iteration: 38000 loss: 0.0035 lr: 0.02
  130. 2020-11-06 14:06:11 iteration: 39000 loss: 0.0034 lr: 0.02
  131. 2020-11-06 14:07:37 iteration: 40000 loss: 0.0034 lr: 0.02
  132. 2020-11-06 14:09:05 iteration: 41000 loss: 0.0033 lr: 0.02
  133. 2020-11-06 14:10:33 iteration: 42000 loss: 0.0033 lr: 0.02
  134. 2020-11-06 14:12:00 iteration: 43000 loss: 0.0033 lr: 0.02
  135. 2020-11-06 14:13:26 iteration: 44000 loss: 0.0033 lr: 0.02
  136. 2020-11-06 14:14:53 iteration: 45000 loss: 0.0032 lr: 0.02
  137. 2020-11-06 14:16:19 iteration: 46000 loss: 0.0033 lr: 0.02
  138. 2020-11-06 14:17:46 iteration: 47000 loss: 0.0034 lr: 0.02
  139. 2020-11-06 14:19:15 iteration: 48000 loss: 0.0032 lr: 0.02
  140. 2020-11-06 14:20:42 iteration: 49000 loss: 0.0032 lr: 0.02
  141. 2020-11-06 14:22:07 iteration: 50000 loss: 0.0031 lr: 0.02
  142. 2020-11-06 14:23:33 iteration: 51000 loss: 0.0032 lr: 0.02
  143. 2020-11-06 14:25:00 iteration: 52000 loss: 0.0031 lr: 0.02
  144. 2020-11-06 14:26:26 iteration: 53000 loss: 0.0030 lr: 0.02
  145. 2020-11-06 14:27:53 iteration: 54000 loss: 0.0031 lr: 0.02
  146. 2020-11-06 14:29:20 iteration: 55000 loss: 0.0031 lr: 0.02
  147. 2020-11-06 14:30:47 iteration: 56000 loss: 0.0030 lr: 0.02
  148. 2020-11-06 14:32:12 iteration: 57000 loss: 0.0032 lr: 0.02
  149. 2020-11-06 14:33:39 iteration: 58000 loss: 0.0031 lr: 0.02
  150. 2020-11-06 14:35:06 iteration: 59000 loss: 0.0030 lr: 0.02
  151. 2020-11-06 14:36:32 iteration: 60000 loss: 0.0030 lr: 0.02
  152. 2020-11-06 14:37:59 iteration: 61000 loss: 0.0030 lr: 0.02
  153. 2020-11-06 14:39:26 iteration: 62000 loss: 0.0030 lr: 0.02
  154. 2020-11-06 14:40:51 iteration: 63000 loss: 0.0030 lr: 0.02
  155. 2020-11-06 14:42:17 iteration: 64000 loss: 0.0029 lr: 0.02
  156. 2020-11-06 14:43:44 iteration: 65000 loss: 0.0030 lr: 0.02
  157. 2020-11-06 14:45:10 iteration: 66000 loss: 0.0030 lr: 0.02
  158. 2020-11-06 14:46:38 iteration: 67000 loss: 0.0028 lr: 0.02
  159. 2020-11-06 14:48:06 iteration: 68000 loss: 0.0028 lr: 0.02
  160. 2020-11-06 14:49:33 iteration: 69000 loss: 0.0029 lr: 0.02
  161. 2020-11-06 14:51:01 iteration: 70000 loss: 0.0029 lr: 0.02
  162. 2020-11-06 14:52:29 iteration: 71000 loss: 0.0028 lr: 0.02
  163. 2020-11-06 14:53:56 iteration: 72000 loss: 0.0028 lr: 0.02
  164. 2020-11-06 14:55:22 iteration: 73000 loss: 0.0028 lr: 0.02
  165. 2020-11-06 14:56:47 iteration: 74000 loss: 0.0027 lr: 0.02
  166. 2020-11-06 14:58:14 iteration: 75000 loss: 0.0027 lr: 0.02
  167. 2020-11-06 14:59:40 iteration: 76000 loss: 0.0027 lr: 0.02
  168. 2020-11-06 15:01:06 iteration: 77000 loss: 0.0028 lr: 0.02
  169. 2020-11-06 15:02:34 iteration: 78000 loss: 0.0027 lr: 0.02
  170. 2020-11-06 15:04:02 iteration: 79000 loss: 0.0027 lr: 0.02
  171. 2020-11-06 15:05:29 iteration: 80000 loss: 0.0027 lr: 0.02
  172. 2020-11-06 15:06:55 iteration: 81000 loss: 0.0026 lr: 0.02
  173. 2020-11-06 15:08:22 iteration: 82000 loss: 0.0027 lr: 0.02
  174. 2020-11-06 15:09:47 iteration: 83000 loss: 0.0026 lr: 0.02
  175. 2020-11-06 15:11:14 iteration: 84000 loss: 0.0026 lr: 0.02
  176. 2020-11-06 15:12:41 iteration: 85000 loss: 0.0027 lr: 0.02
  177. 2020-11-06 15:14:08 iteration: 86000 loss: 0.0026 lr: 0.02
  178. 2020-11-06 15:15:36 iteration: 87000 loss: 0.0026 lr: 0.02
  179. 2020-11-06 15:17:01 iteration: 88000 loss: 0.0026 lr: 0.02
  180. 2020-11-06 15:18:26 iteration: 89000 loss: 0.0026 lr: 0.02
  181. 2020-11-06 15:19:52 iteration: 90000 loss: 0.0025 lr: 0.02
  182. 2020-11-06 15:21:18 iteration: 91000 loss: 0.0025 lr: 0.02
  183. 2020-11-06 15:22:39 iteration: 92000 loss: 0.0026 lr: 0.02
  184. 2020-11-06 15:24:01 iteration: 93000 loss: 0.0025 lr: 0.02
  185. 2020-11-06 15:25:25 iteration: 94000 loss: 0.0025 lr: 0.02
  186. 2020-11-06 15:26:48 iteration: 95000 loss: 0.0025 lr: 0.02
  187. 2020-11-06 15:28:11 iteration: 96000 loss: 0.0025 lr: 0.02
  188. 2020-11-06 15:29:34 iteration: 97000 loss: 0.0025 lr: 0.02
  189. 2020-11-06 15:30:57 iteration: 98000 loss: 0.0025 lr: 0.02
  190. 2020-11-06 15:32:19 iteration: 99000 loss: 0.0025 lr: 0.02
  191. 2020-11-06 15:33:41 iteration: 100000 loss: 0.0024 lr: 0.02