Automated_Rotating_Beam_Analysis/Networks/RotatingBeam3.1-Sebastian_Kloubert-2020-07-12/dlc-models/iteration-0/RotatingBeam3.1Jul12-trainset95shuffle6/train/log.txt

2020-11-09 13:36:59 Config:
{'all_joints': [[0],
                [1],
                [2],
                [3],
                [4],
                [5],
                [6],
                [7],
                [8],
                [9],
                [10],
                [11],
                [12],
                [13],
                [14],
                [15],
                [16],
                [17],
                [18]],
 'all_joints_names': ['snout',
                      'neck',
                      'upper_back',
                      'middle_back',
                      'tailbase',
                      'shoulder_right',
                      'elbow_right',
                      'wrist_right',
                      'forepaw_right',
                      'forepaw_left',
                      'hip_right',
                      'knee_right',
                      'ankle_right',
                      'hindpaw_right',
                      'hindpaw_left',
                      'beam_left_top',
                      'beam_left_bottom',
                      'beam_right_top',
                      'beam_right_bottom'],
 'batch_size': 1,
 'bottomheight': 400,
 'crop': True,
 'crop_pad': 0,
 'cropratio': 0.4,
 'dataset': 'training-datasets/iteration-0/UnaugmentedDataSet_RotatingBeam3.1Jul12/RotatingBeam3.1_Sebastian_Kloubert95shuffle6.mat',
 'dataset_type': 'imgaug',
 'deterministic': False,
 'display_iters': 1000,
 'fg_fraction': 0.25,
 'global_scale': 0.8,
 'init_weights': '/mnt/Install/DLC-GPU/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt',
 'intermediate_supervision': False,
 'intermediate_supervision_layer': 12,
 'leftwidth': 400,
 'location_refinement': True,
 'locref_huber_loss': True,
 'locref_loss_weight': 0.05,
 'locref_stdev': 7.2801,
 'log_dir': 'log',
 'max_input_size': 1500,
 'mean_pixel': [123.68, 116.779, 103.939],
 'metadataset': 'training-datasets/iteration-0/UnaugmentedDataSet_RotatingBeam3.1Jul12/Documentation_data-RotatingBeam3.1_95shuffle6.pickle',
 'min_input_size': 64,
 'minsize': 100,
 'mirror': False,
 'multi_step': [[0.005, 10000],
                [0.02, 430000],
                [0.002, 730000],
                [0.001, 1030000]],
 'net_type': 'resnet_50',
 'num_joints': 19,
 'optimizer': 'sgd',
 'pairwise_huber_loss': False,
 'pairwise_predict': False,
 'partaffinityfield_predict': False,
 'pos_dist_thresh': 17,
 'project_path': '/mnt/DLC/RotatingBeam3.1-Sebastian_Kloubert-2020-07-12',
 'regularize': False,
 'rightwidth': 400,
 'save_iters': 50000,
 'scale_jitter_lo': 0.5,
 'scale_jitter_up': 1.25,
 'scoremap_dir': 'test',
 'shuffle': True,
 'snapshot_prefix': '/mnt/DLC/RotatingBeam3.1-Sebastian_Kloubert-2020-07-12/dlc-models/iteration-0/RotatingBeam3.1Jul12-trainset95shuffle6/train/snapshot',
 'stride': 8.0,
 'topheight': 400,
 'weigh_negatives': False,
 'weigh_only_present_joints': False,
 'weigh_part_predictions': False,
 'weight_decay': 0.0001}
2020-11-09 13:38:40 iteration: 1000 loss: 0.0268 lr: 0.005
2020-11-09 13:40:07 iteration: 2000 loss: 0.0125 lr: 0.005
2020-11-09 13:41:32 iteration: 3000 loss: 0.0099 lr: 0.005
2020-11-09 13:42:59 iteration: 4000 loss: 0.0088 lr: 0.005
2020-11-09 13:44:25 iteration: 5000 loss: 0.0082 lr: 0.005
2020-11-09 13:45:51 iteration: 6000 loss: 0.0077 lr: 0.005
2020-11-09 13:47:18 iteration: 7000 loss: 0.0073 lr: 0.005
2020-11-09 13:48:43 iteration: 8000 loss: 0.0072 lr: 0.005
2020-11-09 13:50:08 iteration: 9000 loss: 0.0066 lr: 0.005
2020-11-09 13:51:35 iteration: 10000 loss: 0.0066 lr: 0.005
2020-11-09 13:53:01 iteration: 11000 loss: 0.0081 lr: 0.02
2020-11-09 13:54:28 iteration: 12000 loss: 0.0071 lr: 0.02
2020-11-09 13:55:55 iteration: 13000 loss: 0.0065 lr: 0.02
2020-11-09 13:57:21 iteration: 14000 loss: 0.0059 lr: 0.02
2020-11-09 13:58:48 iteration: 15000 loss: 0.0059 lr: 0.02
2020-11-09 14:00:14 iteration: 16000 loss: 0.0056 lr: 0.02
2020-11-09 14:01:40 iteration: 17000 loss: 0.0054 lr: 0.02
2020-11-09 14:03:09 iteration: 18000 loss: 0.0051 lr: 0.02
2020-11-09 14:04:34 iteration: 19000 loss: 0.0050 lr: 0.02
2020-11-09 14:06:02 iteration: 20000 loss: 0.0048 lr: 0.02
2020-11-09 14:07:28 iteration: 21000 loss: 0.0047 lr: 0.02
2020-11-09 14:08:55 iteration: 22000 loss: 0.0047 lr: 0.02
2020-11-09 14:10:22 iteration: 23000 loss: 0.0045 lr: 0.02
2020-11-09 14:11:50 iteration: 24000 loss: 0.0045 lr: 0.02
2020-11-09 14:13:17 iteration: 25000 loss: 0.0043 lr: 0.02
2020-11-09 14:14:44 iteration: 26000 loss: 0.0042 lr: 0.02
2020-11-09 14:16:12 iteration: 27000 loss: 0.0041 lr: 0.02
2020-11-09 14:17:39 iteration: 28000 loss: 0.0041 lr: 0.02
2020-11-09 14:19:06 iteration: 29000 loss: 0.0040 lr: 0.02
2020-11-09 14:20:32 iteration: 30000 loss: 0.0039 lr: 0.02
2020-11-09 14:22:00 iteration: 31000 loss: 0.0039 lr: 0.02
2020-11-09 14:23:27 iteration: 32000 loss: 0.0039 lr: 0.02
2020-11-09 14:24:55 iteration: 33000 loss: 0.0038 lr: 0.02
2020-11-09 14:26:22 iteration: 34000 loss: 0.0039 lr: 0.02
2020-11-09 14:27:50 iteration: 35000 loss: 0.0037 lr: 0.02
2020-11-09 14:29:17 iteration: 36000 loss: 0.0037 lr: 0.02
2020-11-09 14:30:45 iteration: 37000 loss: 0.0037 lr: 0.02
2020-11-09 14:32:11 iteration: 38000 loss: 0.0036 lr: 0.02
2020-11-09 14:33:37 iteration: 39000 loss: 0.0036 lr: 0.02
2020-11-09 14:35:03 iteration: 40000 loss: 0.0035 lr: 0.02
2020-11-09 14:36:31 iteration: 41000 loss: 0.0035 lr: 0.02
2020-11-09 14:37:59 iteration: 42000 loss: 0.0035 lr: 0.02
2020-11-09 14:39:26 iteration: 43000 loss: 0.0035 lr: 0.02
2020-11-09 14:40:53 iteration: 44000 loss: 0.0034 lr: 0.02
2020-11-09 14:42:20 iteration: 45000 loss: 0.0034 lr: 0.02
2020-11-09 14:43:48 iteration: 46000 loss: 0.0034 lr: 0.02
2020-11-09 14:45:15 iteration: 47000 loss: 0.0033 lr: 0.02
2020-11-09 14:46:44 iteration: 48000 loss: 0.0032 lr: 0.02
2020-11-09 14:48:11 iteration: 49000 loss: 0.0032 lr: 0.02
2020-11-09 14:49:38 iteration: 50000 loss: 0.0032 lr: 0.02
2020-11-09 14:51:04 iteration: 51000 loss: 0.0032 lr: 0.02
2020-11-09 14:52:32 iteration: 52000 loss: 0.0035 lr: 0.02
2020-11-09 14:53:59 iteration: 53000 loss: 0.0032 lr: 0.02
2020-11-09 14:55:26 iteration: 54000 loss: 0.0033 lr: 0.02
2020-11-09 14:56:54 iteration: 55000 loss: 0.0032 lr: 0.02
2020-11-09 14:58:22 iteration: 56000 loss: 0.0032 lr: 0.02
2020-11-09 14:59:50 iteration: 57000 loss: 0.0032 lr: 0.02
2020-11-09 15:01:16 iteration: 58000 loss: 0.0032 lr: 0.02
2020-11-09 15:02:43 iteration: 59000 loss: 0.0031 lr: 0.02
2020-11-09 15:04:09 iteration: 60000 loss: 0.0030 lr: 0.02
2020-11-09 15:05:32 iteration: 61000 loss: 0.0030 lr: 0.02
2020-11-09 15:06:57 iteration: 62000 loss: 0.0029 lr: 0.02
2020-11-09 15:08:22 iteration: 63000 loss: 0.0030 lr: 0.02
2020-11-09 15:09:49 iteration: 64000 loss: 0.0031 lr: 0.02
2020-11-09 15:11:14 iteration: 65000 loss: 0.0030 lr: 0.02
2020-11-09 15:12:38 iteration: 66000 loss: 0.0028 lr: 0.02
2020-11-09 15:14:06 iteration: 67000 loss: 0.0029 lr: 0.02
2020-11-09 15:15:33 iteration: 68000 loss: 0.0029 lr: 0.02
2020-11-09 15:17:00 iteration: 69000 loss: 0.0029 lr: 0.02
2020-11-09 15:18:26 iteration: 70000 loss: 0.0029 lr: 0.02
2020-11-09 15:19:54 iteration: 71000 loss: 0.0029 lr: 0.02
2020-11-09 15:21:20 iteration: 72000 loss: 0.0029 lr: 0.02
2020-11-09 15:22:43 iteration: 73000 loss: 0.0028 lr: 0.02
2020-11-09 15:24:07 iteration: 74000 loss: 0.0027 lr: 0.02
2020-11-09 15:25:29 iteration: 75000 loss: 0.0027 lr: 0.02
2020-11-09 15:26:54 iteration: 76000 loss: 0.0028 lr: 0.02
2020-11-09 15:28:18 iteration: 77000 loss: 0.0028 lr: 0.02
2020-11-09 15:29:42 iteration: 78000 loss: 0.0029 lr: 0.02
2020-11-09 15:31:05 iteration: 79000 loss: 0.0027 lr: 0.02
2020-11-09 15:32:30 iteration: 80000 loss: 0.0027 lr: 0.02
2020-11-09 15:33:54 iteration: 81000 loss: 0.0027 lr: 0.02
2020-11-09 15:35:17 iteration: 82000 loss: 0.0027 lr: 0.02
2020-11-09 15:36:40 iteration: 83000 loss: 0.0027 lr: 0.02
2020-11-09 15:38:06 iteration: 84000 loss: 0.0028 lr: 0.02
2020-11-09 15:39:34 iteration: 85000 loss: 0.0027 lr: 0.02
2020-11-09 15:41:00 iteration: 86000 loss: 0.0026 lr: 0.02
2020-11-09 15:42:27 iteration: 87000 loss: 0.0026 lr: 0.02
2020-11-09 15:43:51 iteration: 88000 loss: 0.0026 lr: 0.02
2020-11-09 15:45:19 iteration: 89000 loss: 0.0026 lr: 0.02
2020-11-09 15:46:45 iteration: 90000 loss: 0.0027 lr: 0.02
2020-11-09 15:48:11 iteration: 91000 loss: 0.0025 lr: 0.02
2020-11-09 15:49:39 iteration: 92000 loss: 0.0027 lr: 0.02
2020-11-09 15:51:05 iteration: 93000 loss: 0.0026 lr: 0.02
2020-11-09 15:52:30 iteration: 94000 loss: 0.0026 lr: 0.02
2020-11-09 15:53:58 iteration: 95000 loss: 0.0025 lr: 0.02
2020-11-09 15:55:25 iteration: 96000 loss: 0.0027 lr: 0.02
2020-11-09 15:56:51 iteration: 97000 loss: 0.0026 lr: 0.02
2020-11-09 15:58:18 iteration: 98000 loss: 0.0026 lr: 0.02
2020-11-09 15:59:45 iteration: 99000 loss: 0.0026 lr: 0.02
2020-11-09 16:01:12 iteration: 100000 loss: 0.0026 lr: 0.02
2020-11-09 16:02:39 iteration: 101000 loss: 0.0025 lr: 0.02
2020-11-09 16:04:06 iteration: 102000 loss: 0.0026 lr: 0.02
2020-11-09 16:05:34 iteration: 103000 loss: 0.0025 lr: 0.02
2020-11-09 16:07:01 iteration: 104000 loss: 0.0025 lr: 0.02
2020-11-09 16:08:29 iteration: 105000 loss: 0.0025 lr: 0.02
2020-11-09 16:09:56 iteration: 106000 loss: 0.0024 lr: 0.02
2020-11-09 16:11:23 iteration: 107000 loss: 0.0025 lr: 0.02
2020-11-09 16:12:49 iteration: 108000 loss: 0.0025 lr: 0.02
2020-11-09 16:14:13 iteration: 109000 loss: 0.0025 lr: 0.02
2020-11-09 16:15:38 iteration: 110000 loss: 0.0024 lr: 0.02
2020-11-09 16:17:05 iteration: 111000 loss: 0.0025 lr: 0.02
2020-11-09 16:18:30 iteration: 112000 loss: 0.0025 lr: 0.02
2020-11-09 16:19:55 iteration: 113000 loss: 0.0024 lr: 0.02
2020-11-09 16:21:21 iteration: 114000 loss: 0.0025 lr: 0.02
2020-11-09 16:22:48 iteration: 115000 loss: 0.0024 lr: 0.02
2020-11-09 16:24:15 iteration: 116000 loss: 0.0024 lr: 0.02
2020-11-09 16:25:42 iteration: 117000 loss: 0.0024 lr: 0.02
2020-11-09 16:27:10 iteration: 118000 loss: 0.0024 lr: 0.02
2020-11-09 16:28:38 iteration: 119000 loss: 0.0023 lr: 0.02
2020-11-09 16:30:06 iteration: 120000 loss: 0.0025 lr: 0.02
2020-11-09 16:31:33 iteration: 121000 loss: 0.0024 lr: 0.02
2020-11-09 16:33:00 iteration: 122000 loss: 0.0023 lr: 0.02
2020-11-09 16:34:28 iteration: 123000 loss: 0.0025 lr: 0.02
2020-11-09 16:35:56 iteration: 124000 loss: 0.0024 lr: 0.02
2020-11-09 16:37:22 iteration: 125000 loss: 0.0024 lr: 0.02
2020-11-09 16:38:49 iteration: 126000 loss: 0.0024 lr: 0.02
2020-11-09 16:40:15 iteration: 127000 loss: 0.0024 lr: 0.02
2020-11-09 16:41:43 iteration: 128000 loss: 0.0024 lr: 0.02
2020-11-09 16:43:09 iteration: 129000 loss: 0.0023 lr: 0.02
2020-11-09 16:44:37 iteration: 130000 loss: 0.0023 lr: 0.02
2020-11-09 16:46:04 iteration: 131000 loss: 0.0023 lr: 0.02
2020-11-09 16:47:30 iteration: 132000 loss: 0.0023 lr: 0.02
2020-11-09 16:48:56 iteration: 133000 loss: 0.0023 lr: 0.02
2020-11-09 16:50:26 iteration: 134000 loss: 0.0023 lr: 0.02
2020-11-09 16:51:52 iteration: 135000 loss: 0.0023 lr: 0.02
2020-11-09 16:53:19 iteration: 136000 loss: 0.0024 lr: 0.02
2020-11-09 16:54:44 iteration: 137000 loss: 0.0023 lr: 0.02
2020-11-09 16:56:09 iteration: 138000 loss: 0.0023 lr: 0.02
2020-11-09 16:57:36 iteration: 139000 loss: 0.0023 lr: 0.02
2020-11-09 16:59:03 iteration: 140000 loss: 0.0023 lr: 0.02
2020-11-09 17:00:30 iteration: 141000 loss: 0.0022 lr: 0.02
2020-11-09 17:01:56 iteration: 142000 loss: 0.0023 lr: 0.02
2020-11-09 17:03:23 iteration: 143000 loss: 0.0023 lr: 0.02
2020-11-09 17:04:50 iteration: 144000 loss: 0.0023 lr: 0.02
2020-11-09 17:06:19 iteration: 145000 loss: 0.0023 lr: 0.02
2020-11-09 17:07:46 iteration: 146000 loss: 0.0023 lr: 0.02
2020-11-09 17:09:13 iteration: 147000 loss: 0.0022 lr: 0.02
2020-11-09 17:10:39 iteration: 148000 loss: 0.0022 lr: 0.02
2020-11-09 17:12:06 iteration: 149000 loss: 0.0022 lr: 0.02
2020-11-09 17:13:34 iteration: 150000 loss: 0.0022 lr: 0.02