Upload files to ''

data/experiments/true_batch_002/config.json

@@ -0,0 +1,117 @@
+{
+    "image_size": [
+        64,
+        64
+    ],
+    "batch_size": 24,
+    "base_lr": 0.0003,
+    "min_base_lr": 2.9999999999999997e-06,
+    "ema_decay": 0.999,
+    "train_epoch_size": 72000,
+    "num_epochs": [
+        3,
+        18
+    ],
+    "stages": [
+        "warmup",
+        "train"
+    ],
+    "num_dataloader_workers": 8,
+    "init_weights": true,
+    "argus_params": {
+        "nn_module": [
+            "dwiseneuro",
+            {
+                "readout_outputs": [
+                    7863,
+                    7908,
+                    8202,
+                    7939,
+                    8122,
+                    7440,
+                    7928,
+                    8285,
+                    7671,
+                    7495
+                ],
+                "in_channels": 5,
+                "core_features": [
+                    64,
+                    64,
+                    64,
+                    64,
+                    128,
+                    128,
+                    128,
+                    256,
+                    256
+                ],
+                "spatial_strides": [
+                    2,
+                    1,
+                    1,
+                    1,
+                    2,
+                    1,
+                    1,
+                    2,
+                    1
+                ],
+                "spatial_kernel": 3,
+                "temporal_kernel": 5,
+                "expansion_ratio": 7,
+                "se_reduce_ratio": 32,
+                "cortex_features": [
+                    1024,
+                    2048,
+                    4096
+                ],
+                "groups": 2,
+                "softplus_beta": 0.07,
+                "drop_rate": 0.4,
+                "drop_path_rate": 0.1
+            }
+        ],
+        "loss": [
+            "mice_poisson",
+            {
+                "log_input": false,
+                "full": false,
+                "eps": 1e-08
+            }
+        ],
+        "optimizer": [
+            "AdamW",
+            {
+                "lr": 0.0018,
+                "weight_decay": 0.05
+            }
+        ],
+        "device": "cuda:0",
+        "frame_stack": {
+            "size": 16,
+            "step": 2,
+            "position": "last"
+        },
+        "inputs_processor": [
+            "stack_inputs",
+            {
+                "size": [
+                    64,
+                    64
+                ],
+                "pad_fill_value": 0.0
+            }
+        ],
+        "responses_processor": [
+            "identity",
+            {}
+        ],
+        "amp": true,
+        "iter_size": 1
+    },
+    "cutmix": {
+        "alpha": 1.0,
+        "prob": 0.5
+    }
+}

data/experiments/true_batch_002/fold_0/log.csv

@@ -0,0 +1,23 @@
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 19:41:41.713260,0,0.0006,-210979.6916529949,-262594.42268819705,0.054842055,0.048037626,0.049866844,0.048467018,0.04550785,0.061004285,0.048958283,0.05756247,0.0635027,0.06585407,0.05436032
+2024-08-10 20:11:23.161118,1,0.0012,-349537.8568854172,-400594.5088108446,0.15943079,0.19163808,0.17694487,0.19449298,0.19342037,0.18404937,0.18502909,0.20011127,0.18926974,0.18317388,0.18575604
+2024-08-10 20:41:09.796656,2,0.0018,-410906.09947916685,-463907.79296874994,0.20151736,0.24221069,0.22515959,0.24079454,0.24894816,0.22572747,0.23524852,0.2600953,0.23676613,0.22706124,0.2343529
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 21:11:46.975033,0,0.0017865,-442166.9841614573,-486652.8254530671,0.21838072,0.26519477,0.24367447,0.25812843,0.27293888,0.24356405,0.25538892,0.28526568,0.2581222,0.2461846,0.25468427
+2024-08-10 21:40:57.099796,1,0.0017463,-461804.9704062497,-500040.84552160796,0.22633512,0.27755436,0.25291345,0.26878336,0.2862762,0.2522875,0.26776215,0.29793447,0.26932555,0.25648504,0.2655657
+2024-08-10 22:10:00.160173,2,0.0016806,-475048.819020833,-508274.7602230481,0.23194394,0.2842134,0.25799957,0.27469066,0.29450572,0.2595396,0.27685928,0.30587548,0.27807307,0.26278263,0.2726483
+2024-08-10 22:39:04.905412,3,0.0015915,-484263.79493749904,-513579.2413162176,0.23486769,0.28882638,0.26270077,0.27862865,0.29949334,0.2638428,0.2813126,0.3118721,0.28279644,0.26707816,0.27714187
+2024-08-10 23:08:04.159413,4,0.0014817,-493137.1745833335,-517399.0916008367,0.2358587,0.2904635,0.26554558,0.2822512,0.30198452,0.26786262,0.2864939,0.3154179,0.28606898,0.27062067,0.28025672
+2024-08-10 23:37:03.088293,5,0.0013545,-503938.5874166665,-519725.28877207264,0.23726344,0.29210106,0.26600662,0.28367683,0.30488744,0.2696646,0.28953543,0.31877202,0.28884125,0.27199963,0.28227484
+2024-08-11 00:06:01.353010,6,0.0012137,-505846.1350052084,-521633.520852695,0.23760618,0.29452714,0.26708037,0.2844096,0.3068275,0.2711799,0.29285198,0.32164207,0.29148453,0.27440345,0.2842013
+2024-08-11 00:35:07.029091,7,0.0010637,-514596.70004166773,-523162.06395213795,0.23993531,0.2949951,0.2685264,0.28610674,0.3081961,0.27341697,0.29411831,0.32304716,0.29336417,0.27567035,0.28573763
+2024-08-11 01:04:12.986649,8,0.000909,-517452.06591666664,-524394.4939881511,0.24007608,0.2968313,0.27056175,0.2864633,0.30906323,0.27452463,0.2958922,0.3250378,0.29617965,0.27707034,0.28717
+2024-08-11 01:33:16.817856,9,0.00075428,-523045.9666354168,-525619.2302509292,0.2404011,0.29748356,0.27144882,0.28764027,0.30917305,0.2760529,0.297399,0.32575747,0.29711333,0.27880824,0.28812775
+2024-08-11 02:02:11.693022,10,0.00060426,-530782.9315260429,-526397.706145446,0.24102262,0.2990908,0.2715428,0.2892544,0.31008428,0.2770697,0.2982894,0.3272332,0.29871407,0.28005317,0.28923544
+2024-08-11 02:31:17.463850,11,0.0004635,-539025.3877083332,-527057.1887778807,0.24094957,0.29897755,0.272529,0.2889301,0.31045544,0.27738982,0.30025047,0.32881355,0.29972872,0.28155535,0.28995794
+2024-08-11 03:00:21.355841,12,0.00033628,-546479.6185208339,-527561.3806633363,0.24167034,0.30030277,0.27342227,0.2886639,0.31095603,0.2785543,0.3018437,0.32987207,0.30074203,0.28233775,0.2908365
+2024-08-11 03:29:19.676510,13,0.00022645,-550927.6877083324,-527888.4573652414,0.24093409,0.30016357,0.2740403,0.28856647,0.31103653,0.27914342,0.3020895,0.33035147,0.30208036,0.28274572,0.29111513
+2024-08-11 03:58:23.747395,14,0.00013737,-555592.7925312511,-528265.0902939126,0.2411301,0.29967034,0.27418137,0.28838328,0.31156152,0.27990624,0.30250254,0.33088258,0.30250785,0.28381413,0.29145402
+2024-08-11 04:27:29.818575,15,7.1734e-05,-558473.7864687501,-528708.5643296929,0.24092717,0.30003545,0.274586,0.28858218,0.31167957,0.28068697,0.30267322,0.33163947,0.30345955,0.2841051,0.29183748
+2024-08-11 04:56:22.485107,16,3.1536e-05,-560573.7728125016,-528646.4433956787,0.24121943,0.30028275,0.27407485,0.2888177,0.3113322,0.28088805,0.30324894,0.331802,0.30385983,0.28449735,0.2920023
+2024-08-11 05:25:27.746536,17,1.8e-05,-560239.2318020826,-528674.880140567,0.24131036,0.30046603,0.27404284,0.288637,0.31148142,0.2810987,0.30340737,0.33211794,0.3039615,0.28465825,0.29211813

data/experiments/true_batch_002/fold_0/log.txt

@@ -0,0 +1,176 @@
+[2024-08-10 15:44:57,406][ERROR]: CUDA out of memory. Tried to allocate 66.00 MiB. GPU 0 has a total capacty of 39.39 GiB of which 35.06 MiB is free. Including non-PyTorch memory, this process has 39.35 GiB memory in use. Of the allocated memory 38.51 GiB is allocated by PyTorch, and 332.36 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.11/dist-packages/argus/engine/engine.py", line 242, in run
+    self.state.step_output = self.step_method(batch, self.state)
+                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/video_reconstruction_from_sensorium2023_winner/src/argus_models.py", line 55, in train_step
+    self.grad_scaler.scale(loss).backward()
+  File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 492, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 251, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 66.00 MiB. GPU 0 has a total capacty of 39.39 GiB of which 35.06 MiB is free. Including non-PyTorch memory, this process has 39.35 GiB memory in use. Of the allocated memory 38.51 GiB is allocated by PyTorch, and 332.36 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+[2024-08-10 16:43:05,284][ERROR]: CUDA out of memory. Tried to allocate 112.00 MiB. GPU 0 has a total capacty of 39.39 GiB of which 15.06 MiB is free. Including non-PyTorch memory, this process has 39.37 GiB memory in use. Of the allocated memory 37.60 GiB is allocated by PyTorch, and 1.25 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.11/dist-packages/argus/engine/engine.py", line 242, in run
+    self.state.step_output = self.step_method(batch, self.state)
+                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/video_reconstruction_from_sensorium2023_winner/src/argus_models.py", line 52, in train_step
+    prediction = self.nn_module(input)
+                 ^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/video_reconstruction_from_sensorium2023_winner/src/models/dwiseneuro.py", line 399, in forward
+    x = self.core(x)  # (32, 256, 16, 8, 8)
+        ^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/video_reconstruction_from_sensorium2023_winner/src/models/dwiseneuro.py", line 339, in forward
+    x = self.blocks(x)
+        ^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/container.py", line 215, in forward
+    input = module(input)
+            ^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/video_reconstruction_from_sensorium2023_winner/src/models/dwiseneuro.py", line 140, in forward
+    x = self.temp_covn_dw(x)
+        ^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/container.py", line 215, in forward
+    input = module(input)
+            ^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/video_reconstruction_from_sensorium2023_winner/src/models/dwiseneuro.py", line 20, in forward
+    x = self.bn(x)
+        ^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1518, in _wrapped_call_impl
+    return self._call_impl(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/module.py", line 1527, in _call_impl
+    return forward_call(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/modules/batchnorm.py", line 171, in forward
+    return F.batch_norm(
+           ^^^^^^^^^^^^^
+  File "/usr/local/lib/python3.11/dist-packages/torch/nn/functional.py", line 2478, in batch_norm
+    return torch.batch_norm(
+           ^^^^^^^^^^^^^^^^^
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 112.00 MiB. GPU 0 has a total capacty of 39.39 GiB of which 15.06 MiB is free. Including non-PyTorch memory, this process has 39.37 GiB memory in use. Of the allocated memory 37.60 GiB is allocated by PyTorch, and 1.25 GiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+[2024-08-10 18:22:17,402][ERROR]: CUDA out of memory. Tried to allocate 64.00 MiB. GPU 0 has a total capacty of 39.39 GiB of which 51.06 MiB is free. Including non-PyTorch memory, this process has 39.33 GiB memory in use. Of the allocated memory 38.38 GiB is allocated by PyTorch, and 446.11 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+Traceback (most recent call last):
+  File "/usr/local/lib/python3.11/dist-packages/argus/engine/engine.py", line 242, in run
+    self.state.step_output = self.step_method(batch, self.state)
+                             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  File "/root/video_reconstruction_from_sensorium2023_winner/src/argus_models.py", line 55, in train_step
+    self.grad_scaler.scale(loss).backward()
+  File "/usr/local/lib/python3.11/dist-packages/torch/_tensor.py", line 492, in backward
+    torch.autograd.backward(
+  File "/usr/local/lib/python3.11/dist-packages/torch/autograd/__init__.py", line 251, in backward
+    Variable._execution_engine.run_backward(  # Calls into the C++ engine to run the backward pass
+torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 64.00 MiB. GPU 0 has a total capacty of 39.39 GiB of which 51.06 MiB is free. Including non-PyTorch memory, this process has 39.33 GiB memory in use. Of the allocated memory 38.38 GiB is allocated by PyTorch, and 446.11 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting max_split_size_mb to avoid fragmentation.  See documentation for Memory Management and PYTORCH_CUDA_ALLOC_CONF
+[2024-08-10 19:40:28,102][INFO]: train - epoch: 0, lr: 0.0006, train_loss: -210979.7
+[2024-08-10 19:41:41,526][INFO]: val - epoch: 0, val_loss: -262594.4, val_corr_mouse_0: 0.05484205, val_corr_mouse_1: 0.04803763, val_corr_mouse_2: 0.04986684, val_corr_mouse_3: 0.04846702, val_corr_mouse_4: 0.04550785, val_corr_mouse_5: 0.06100428, val_corr_mouse_6: 0.04895828, val_corr_mouse_7: 0.05756247, val_corr_mouse_8: 0.0635027, val_corr_mouse_9: 0.06585407, val_corr: 0.05436032
+[2024-08-10 20:10:10,092][INFO]: train - epoch: 1, lr: 0.0012, train_loss: -349537.9
+[2024-08-10 20:11:23,160][INFO]: val - epoch: 1, val_loss: -400594.5, val_corr_mouse_0: 0.1594308, val_corr_mouse_1: 0.1916381, val_corr_mouse_2: 0.1769449, val_corr_mouse_3: 0.194493, val_corr_mouse_4: 0.1934204, val_corr_mouse_5: 0.1840494, val_corr_mouse_6: 0.1850291, val_corr_mouse_7: 0.2001113, val_corr_mouse_8: 0.1892697, val_corr_mouse_9: 0.1831739, val_corr: 0.185756
+[2024-08-10 20:39:57,037][INFO]: train - epoch: 2, lr: 0.0018, train_loss: -410906.1
+[2024-08-10 20:41:09,767][INFO]: val - epoch: 2, val_loss: -463907.8, val_corr_mouse_0: 0.2015174, val_corr_mouse_1: 0.2422107, val_corr_mouse_2: 0.2251596, val_corr_mouse_3: 0.2407945, val_corr_mouse_4: 0.2489482, val_corr_mouse_5: 0.2257275, val_corr_mouse_6: 0.2352485, val_corr_mouse_7: 0.2600953, val_corr_mouse_8: 0.2367661, val_corr_mouse_9: 0.2270612, val_corr: 0.2343529
+[2024-08-10 21:10:37,897][INFO]: train - epoch: 0, lr: 0.0017865, train_loss: -442167
+[2024-08-10 21:11:46,974][INFO]: val - epoch: 0, val_loss: -486652.8, val_corr_mouse_0: 0.2183807, val_corr_mouse_1: 0.2651948, val_corr_mouse_2: 0.2436745, val_corr_mouse_3: 0.2581284, val_corr_mouse_4: 0.2729389, val_corr_mouse_5: 0.2435641, val_corr_mouse_6: 0.2553889, val_corr_mouse_7: 0.2852657, val_corr_mouse_8: 0.2581222, val_corr_mouse_9: 0.2461846, val_corr: 0.2546843
+[2024-08-10 21:11:47,667][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-000-0.254684.pth'
+[2024-08-10 21:39:47,765][INFO]: train - epoch: 1, lr: 0.0017463, train_loss: -461805
+[2024-08-10 21:40:57,099][INFO]: val - epoch: 1, val_loss: -500040.8, val_corr_mouse_0: 0.2263351, val_corr_mouse_1: 0.2775544, val_corr_mouse_2: 0.2529134, val_corr_mouse_3: 0.2687834, val_corr_mouse_4: 0.2862762, val_corr_mouse_5: 0.2522875, val_corr_mouse_6: 0.2677622, val_corr_mouse_7: 0.2979345, val_corr_mouse_8: 0.2693256, val_corr_mouse_9: 0.256485, val_corr: 0.2655657
+[2024-08-10 21:40:57,735][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-001-0.265566.pth'
+[2024-08-10 21:40:57,736][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-000-0.254684.pth'
+[2024-08-10 22:08:50,301][INFO]: train - epoch: 2, lr: 0.0016806, train_loss: -475048.8
+[2024-08-10 22:10:00,159][INFO]: val - epoch: 2, val_loss: -508274.8, val_corr_mouse_0: 0.2319439, val_corr_mouse_1: 0.2842134, val_corr_mouse_2: 0.2579996, val_corr_mouse_3: 0.2746907, val_corr_mouse_4: 0.2945057, val_corr_mouse_5: 0.2595396, val_corr_mouse_6: 0.2768593, val_corr_mouse_7: 0.3058755, val_corr_mouse_8: 0.2780731, val_corr_mouse_9: 0.2627826, val_corr: 0.2726483
+[2024-08-10 22:10:00,706][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-002-0.272648.pth'
+[2024-08-10 22:10:00,707][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-001-0.265566.pth'
+[2024-08-10 22:37:55,658][INFO]: train - epoch: 3, lr: 0.0015915, train_loss: -484263.8
+[2024-08-10 22:39:04,905][INFO]: val - epoch: 3, val_loss: -513579.2, val_corr_mouse_0: 0.2348677, val_corr_mouse_1: 0.2888264, val_corr_mouse_2: 0.2627008, val_corr_mouse_3: 0.2786286, val_corr_mouse_4: 0.2994933, val_corr_mouse_5: 0.2638428, val_corr_mouse_6: 0.2813126, val_corr_mouse_7: 0.3118721, val_corr_mouse_8: 0.2827964, val_corr_mouse_9: 0.2670782, val_corr: 0.2771419
+[2024-08-10 22:39:05,450][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-003-0.277142.pth'
+[2024-08-10 22:39:05,451][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-002-0.272648.pth'
+[2024-08-10 23:06:55,046][INFO]: train - epoch: 4, lr: 0.0014817, train_loss: -493137.2
+[2024-08-10 23:08:04,133][INFO]: val - epoch: 4, val_loss: -517399.1, val_corr_mouse_0: 0.2358587, val_corr_mouse_1: 0.2904635, val_corr_mouse_2: 0.2655456, val_corr_mouse_3: 0.2822512, val_corr_mouse_4: 0.3019845, val_corr_mouse_5: 0.2678626, val_corr_mouse_6: 0.2864939, val_corr_mouse_7: 0.3154179, val_corr_mouse_8: 0.286069, val_corr_mouse_9: 0.2706207, val_corr: 0.2802567
+[2024-08-10 23:08:04,759][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-004-0.280257.pth'
+[2024-08-10 23:08:04,760][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-003-0.277142.pth'
+[2024-08-10 23:35:53,340][INFO]: train - epoch: 5, lr: 0.0013545, train_loss: -503938.6
+[2024-08-10 23:37:02,992][INFO]: val - epoch: 5, val_loss: -519725.3, val_corr_mouse_0: 0.2372634, val_corr_mouse_1: 0.2921011, val_corr_mouse_2: 0.2660066, val_corr_mouse_3: 0.2836768, val_corr_mouse_4: 0.3048874, val_corr_mouse_5: 0.2696646, val_corr_mouse_6: 0.2895354, val_corr_mouse_7: 0.318772, val_corr_mouse_8: 0.2888412, val_corr_mouse_9: 0.2719996, val_corr: 0.2822748
+[2024-08-10 23:37:03,739][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-005-0.282275.pth'
+[2024-08-10 23:37:03,740][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-004-0.280257.pth'
+[2024-08-11 00:04:52,078][INFO]: train - epoch: 6, lr: 0.0012137, train_loss: -505846.1
+[2024-08-11 00:06:01,280][INFO]: val - epoch: 6, val_loss: -521633.5, val_corr_mouse_0: 0.2376062, val_corr_mouse_1: 0.2945271, val_corr_mouse_2: 0.2670804, val_corr_mouse_3: 0.2844096, val_corr_mouse_4: 0.3068275, val_corr_mouse_5: 0.2711799, val_corr_mouse_6: 0.292852, val_corr_mouse_7: 0.3216421, val_corr_mouse_8: 0.2914845, val_corr_mouse_9: 0.2744035, val_corr: 0.2842013
+[2024-08-11 00:06:02,061][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-006-0.284201.pth'
+[2024-08-11 00:06:02,062][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-005-0.282275.pth'
+[2024-08-11 00:33:57,225][INFO]: train - epoch: 7, lr: 0.0010637, train_loss: -514596.7
+[2024-08-11 00:35:07,028][INFO]: val - epoch: 7, val_loss: -523162.1, val_corr_mouse_0: 0.2399353, val_corr_mouse_1: 0.2949951, val_corr_mouse_2: 0.2685264, val_corr_mouse_3: 0.2861067, val_corr_mouse_4: 0.3081961, val_corr_mouse_5: 0.273417, val_corr_mouse_6: 0.2941183, val_corr_mouse_7: 0.3230472, val_corr_mouse_8: 0.2933642, val_corr_mouse_9: 0.2756703, val_corr: 0.2857376
+[2024-08-11 00:35:07,587][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-007-0.285738.pth'
+[2024-08-11 00:35:07,588][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-006-0.284201.pth'
+[2024-08-11 01:03:03,072][INFO]: train - epoch: 8, lr: 0.000909, train_loss: -517452.1
+[2024-08-11 01:04:12,986][INFO]: val - epoch: 8, val_loss: -524394.5, val_corr_mouse_0: 0.2400761, val_corr_mouse_1: 0.2968313, val_corr_mouse_2: 0.2705618, val_corr_mouse_3: 0.2864633, val_corr_mouse_4: 0.3090632, val_corr_mouse_5: 0.2745246, val_corr_mouse_6: 0.2958922, val_corr_mouse_7: 0.3250378, val_corr_mouse_8: 0.2961797, val_corr_mouse_9: 0.2770703, val_corr: 0.28717
+[2024-08-11 01:04:13,544][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-008-0.287170.pth'
+[2024-08-11 01:04:13,545][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-007-0.285738.pth'
+[2024-08-11 01:32:07,659][INFO]: train - epoch: 9, lr: 0.00075428, train_loss: -523046
+[2024-08-11 01:33:16,817][INFO]: val - epoch: 9, val_loss: -525619.2, val_corr_mouse_0: 0.2404011, val_corr_mouse_1: 0.2974836, val_corr_mouse_2: 0.2714488, val_corr_mouse_3: 0.2876403, val_corr_mouse_4: 0.309173, val_corr_mouse_5: 0.2760529, val_corr_mouse_6: 0.297399, val_corr_mouse_7: 0.3257575, val_corr_mouse_8: 0.2971133, val_corr_mouse_9: 0.2788082, val_corr: 0.2881278
+[2024-08-11 01:33:17,366][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-009-0.288128.pth'
+[2024-08-11 01:33:17,367][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-008-0.287170.pth'
+[2024-08-11 02:01:02,343][INFO]: train - epoch: 10, lr: 0.00060426, train_loss: -530782.9
+[2024-08-11 02:02:11,692][INFO]: val - epoch: 10, val_loss: -526397.7, val_corr_mouse_0: 0.2410226, val_corr_mouse_1: 0.2990908, val_corr_mouse_2: 0.2715428, val_corr_mouse_3: 0.2892544, val_corr_mouse_4: 0.3100843, val_corr_mouse_5: 0.2770697, val_corr_mouse_6: 0.2982894, val_corr_mouse_7: 0.3272332, val_corr_mouse_8: 0.2987141, val_corr_mouse_9: 0.2800532, val_corr: 0.2892354
+[2024-08-11 02:02:12,243][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-010-0.289235.pth'
+[2024-08-11 02:02:12,244][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-009-0.288128.pth'
+[2024-08-11 02:30:08,360][INFO]: train - epoch: 11, lr: 0.0004635, train_loss: -539025.4
+[2024-08-11 02:31:17,463][INFO]: val - epoch: 11, val_loss: -527057.2, val_corr_mouse_0: 0.2409496, val_corr_mouse_1: 0.2989776, val_corr_mouse_2: 0.272529, val_corr_mouse_3: 0.2889301, val_corr_mouse_4: 0.3104554, val_corr_mouse_5: 0.2773898, val_corr_mouse_6: 0.3002505, val_corr_mouse_7: 0.3288136, val_corr_mouse_8: 0.2997287, val_corr_mouse_9: 0.2815554, val_corr: 0.2899579
+[2024-08-11 02:31:18,016][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-011-0.289958.pth'
+[2024-08-11 02:31:18,017][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-010-0.289235.pth'
+[2024-08-11 02:59:12,528][INFO]: train - epoch: 12, lr: 0.00033628, train_loss: -546479.6
+[2024-08-11 03:00:21,355][INFO]: val - epoch: 12, val_loss: -527561.4, val_corr_mouse_0: 0.2416703, val_corr_mouse_1: 0.3003028, val_corr_mouse_2: 0.2734223, val_corr_mouse_3: 0.2886639, val_corr_mouse_4: 0.310956, val_corr_mouse_5: 0.2785543, val_corr_mouse_6: 0.3018437, val_corr_mouse_7: 0.3298721, val_corr_mouse_8: 0.300742, val_corr_mouse_9: 0.2823378, val_corr: 0.2908365
+[2024-08-11 03:00:21,971][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-012-0.290837.pth'
+[2024-08-11 03:00:21,972][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-011-0.289958.pth'
+[2024-08-11 03:28:10,443][INFO]: train - epoch: 13, lr: 0.00022645, train_loss: -550927.7
+[2024-08-11 03:29:19,676][INFO]: val - epoch: 13, val_loss: -527888.5, val_corr_mouse_0: 0.2409341, val_corr_mouse_1: 0.3001636, val_corr_mouse_2: 0.2740403, val_corr_mouse_3: 0.2885665, val_corr_mouse_4: 0.3110365, val_corr_mouse_5: 0.2791434, val_corr_mouse_6: 0.3020895, val_corr_mouse_7: 0.3303515, val_corr_mouse_8: 0.3020804, val_corr_mouse_9: 0.2827457, val_corr: 0.2911151
+[2024-08-11 03:29:20,228][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-013-0.291115.pth'
+[2024-08-11 03:29:20,229][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-012-0.290837.pth'
+[2024-08-11 03:57:14,244][INFO]: train - epoch: 14, lr: 0.00013737, train_loss: -555592.8
+[2024-08-11 03:58:23,746][INFO]: val - epoch: 14, val_loss: -528265.1, val_corr_mouse_0: 0.2411301, val_corr_mouse_1: 0.2996703, val_corr_mouse_2: 0.2741814, val_corr_mouse_3: 0.2883833, val_corr_mouse_4: 0.3115615, val_corr_mouse_5: 0.2799062, val_corr_mouse_6: 0.3025025, val_corr_mouse_7: 0.3308826, val_corr_mouse_8: 0.3025078, val_corr_mouse_9: 0.2838141, val_corr: 0.291454
+[2024-08-11 03:58:24,311][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-014-0.291454.pth'
+[2024-08-11 03:58:24,312][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-013-0.291115.pth'
+[2024-08-11 04:26:20,672][INFO]: train - epoch: 15, lr: 7.1734e-05, train_loss: -558473.8
+[2024-08-11 04:27:29,818][INFO]: val - epoch: 15, val_loss: -528708.6, val_corr_mouse_0: 0.2409272, val_corr_mouse_1: 0.3000354, val_corr_mouse_2: 0.274586, val_corr_mouse_3: 0.2885822, val_corr_mouse_4: 0.3116796, val_corr_mouse_5: 0.280687, val_corr_mouse_6: 0.3026732, val_corr_mouse_7: 0.3316395, val_corr_mouse_8: 0.3034596, val_corr_mouse_9: 0.2841051, val_corr: 0.2918375
+[2024-08-11 04:27:30,375][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-015-0.291837.pth'
+[2024-08-11 04:27:30,377][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-014-0.291454.pth'
+[2024-08-11 04:55:12,698][INFO]: train - epoch: 16, lr: 3.1536e-05, train_loss: -560573.8
+[2024-08-11 04:56:22,484][INFO]: val - epoch: 16, val_loss: -528646.4, val_corr_mouse_0: 0.2412194, val_corr_mouse_1: 0.3002827, val_corr_mouse_2: 0.2740749, val_corr_mouse_3: 0.2888177, val_corr_mouse_4: 0.3113322, val_corr_mouse_5: 0.2808881, val_corr_mouse_6: 0.3032489, val_corr_mouse_7: 0.331802, val_corr_mouse_8: 0.3038598, val_corr_mouse_9: 0.2844974, val_corr: 0.2920023
+[2024-08-11 04:56:23,035][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-016-0.292002.pth'
+[2024-08-11 04:56:23,036][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-015-0.291837.pth'
+[2024-08-11 05:24:19,056][INFO]: train - epoch: 17, lr: 1.8e-05, train_loss: -560239.2
+[2024-08-11 05:25:27,746][INFO]: val - epoch: 17, val_loss: -528674.9, val_corr_mouse_0: 0.2413104, val_corr_mouse_1: 0.300466, val_corr_mouse_2: 0.2740428, val_corr_mouse_3: 0.288637, val_corr_mouse_4: 0.3114814, val_corr_mouse_5: 0.2810987, val_corr_mouse_6: 0.3034074, val_corr_mouse_7: 0.3321179, val_corr_mouse_8: 0.3039615, val_corr_mouse_9: 0.2846583, val_corr: 0.2921181
+[2024-08-11 05:25:28,293][INFO]: Model saved to 'data/experiments/true_batch_002/fold_0/model-017-0.292118.pth'
+[2024-08-11 05:25:28,294][INFO]: Model removed 'data/experiments/true_batch_002/fold_0/model-016-0.292002.pth'

data/experiments/true_batch_002/fold_0/model-017-0.292118.pth

@@ -0,0 +1 @@
+/annex/objects/MD5-s683073203--03d62fadfaa2734ac968e765e9ccc0c2

data/experiments/true_batch_002/fold_1/log.csv

@@ -0,0 +1,23 @@
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 20:11:59.154492,0,0.0006,-210346.01542993175,-262137.24029246054,0.055358507,0.04829586,0.047355443,0.04669303,0.04660933,0.057747833,0.053566433,0.06034189,0.06300194,0.063963495,0.054293375
+2024-08-10 20:41:39.529571,1,0.0012,-354670.39246354194,-403583.75218546693,0.16288109,0.19337186,0.18046711,0.19520435,0.19467525,0.18468544,0.18832755,0.20457307,0.19093394,0.1851706,0.18802902
+2024-08-10 21:11:05.075543,2,0.0018,-411576.4728906252,-464080.6958497909,0.20141748,0.24247596,0.22709246,0.2384417,0.24856754,0.22642049,0.23584253,0.26113173,0.23864694,0.22754666,0.23475835
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 21:41:14.391327,0,0.0017865,-444820.2046822914,-486708.8935437965,0.21874002,0.2640634,0.24519803,0.25804502,0.27215415,0.24422719,0.25622424,0.28502795,0.2586259,0.24701932,0.25493252
+2024-08-10 22:10:00.458954,1,0.0017463,-467094.3219895832,-500568.6668796468,0.22762923,0.27762544,0.25349632,0.26782882,0.28740597,0.25360203,0.26834983,0.29751825,0.26986596,0.25675213,0.26600742
+2024-08-10 22:38:47.835846,2,0.0016806,-474607.41154166666,-508857.06421352207,0.23136254,0.28463444,0.25807115,0.27284816,0.29472727,0.26084924,0.27615064,0.30436578,0.27761942,0.26346406,0.27240926
+2024-08-10 23:07:31.979231,3,0.0015915,-486074.1473541672,-514211.8276312734,0.23362063,0.28744602,0.26073235,0.2782524,0.29977426,0.26460013,0.2818425,0.30963105,0.28151885,0.26745248,0.27648708
+2024-08-10 23:36:19.249241,4,0.0014817,-493138.66603125026,-516931.31305181247,0.23547293,0.29102156,0.2631662,0.28110945,0.30240816,0.267734,0.28633574,0.31294802,0.28659078,0.26923898,0.2796026
+2024-08-11 00:05:06.821507,5,0.0013545,-500839.861145833,-519235.2102114313,0.23685382,0.2938365,0.26490808,0.28302374,0.30422652,0.26943162,0.28867412,0.31660098,0.28921834,0.27285632,0.281963
+2024-08-11 00:33:52.408486,6,0.0012137,-504808.53267187555,-521394.3060815519,0.23840521,0.29430854,0.26770923,0.28356043,0.30593586,0.27124164,0.29155475,0.3198195,0.29084376,0.27442223,0.28378013
+2024-08-11 01:02:36.326255,7,0.0010637,-513246.7864583334,-522949.2272885684,0.23876534,0.2963343,0.26854274,0.28475755,0.30789128,0.27310356,0.2934407,0.3222229,0.29344404,0.2763204,0.28548226
+2024-08-11 01:31:21.452419,8,0.000909,-521810.5228697917,-524673.5106006039,0.23970903,0.29673055,0.2695138,0.2869682,0.30969232,0.2752596,0.2957301,0.32510167,0.2958267,0.27734125,0.28718728
+2024-08-11 02:00:10.353299,9,0.00075428,-528190.49534375,-525601.3460153348,0.23887016,0.29773113,0.27065045,0.2878983,0.31068388,0.27639544,0.29714698,0.3250394,0.2958784,0.27967277,0.2879967
+2024-08-11 02:28:56.080986,10,0.00060426,-534639.6158229177,-526508.2824407534,0.23981085,0.299028,0.2706966,0.28810543,0.3112427,0.2775513,0.2992545,0.3262218,0.29779273,0.28022987,0.28899336
+2024-08-11 02:57:40.969565,11,0.0004635,-540015.4583020845,-527229.6552915896,0.24003233,0.3001254,0.270614,0.28896827,0.31167066,0.2788725,0.29949674,0.32798946,0.29903424,0.28131333,0.2898117
+2024-08-11 03:26:29.287178,12,0.00033628,-543717.2090833328,-527629.3351533461,0.23972575,0.30110863,0.27156165,0.28853986,0.31168312,0.27939957,0.30054152,0.32899165,0.30024663,0.28162408,0.29034227
+2024-08-11 03:55:15.586964,13,0.00022645,-553570.5080729178,-528097.0165688891,0.24005622,0.30122155,0.27228945,0.28811628,0.31158277,0.2805543,0.30132246,0.32999006,0.30141753,0.2820639,0.29086146
+2024-08-11 04:24:02.026897,14,0.00013737,-554804.017937501,-528097.1689852463,0.23969807,0.30064625,0.2718084,0.28791207,0.31158718,0.2808043,0.30244797,0.331011,0.30184042,0.2826256,0.29103813
+2024-08-11 04:52:45.033436,15,7.1734e-05,-558072.7207187506,-528713.3597235125,0.2396932,0.30148268,0.27164853,0.28811952,0.31140536,0.281097,0.30304912,0.3319732,0.3026665,0.28341198,0.2914547
+2024-08-11 05:21:30.123551,16,3.1536e-05,-564221.6546250008,-528800.6206290661,0.23976795,0.3016397,0.27192378,0.2883844,0.3112444,0.28117493,0.3033576,0.332,0.30272317,0.2834592,0.2915675
+2024-08-11 05:50:17.287709,17,1.8e-05,-564328.7728854176,-529034.7099355253,0.23983891,0.3013826,0.27213442,0.28833422,0.31112954,0.28134516,0.30360633,0.33241493,0.3031252,0.28376687,0.2917078

data/experiments/true_batch_002/fold_1/log.txt

@@ -0,0 +1,77 @@
+[2024-08-10 20:10:48,386][INFO]: train - epoch: 0, lr: 0.0006, train_loss: -210346
+[2024-08-10 20:11:59,154][INFO]: val - epoch: 0, val_loss: -262137.2, val_corr_mouse_0: 0.05535851, val_corr_mouse_1: 0.04829586, val_corr_mouse_2: 0.04735544, val_corr_mouse_3: 0.04669303, val_corr_mouse_4: 0.04660933, val_corr_mouse_5: 0.05774783, val_corr_mouse_6: 0.05356643, val_corr_mouse_7: 0.06034189, val_corr_mouse_8: 0.06300194, val_corr_mouse_9: 0.0639635, val_corr: 0.05429338
+[2024-08-10 20:40:28,700][INFO]: train - epoch: 1, lr: 0.0012, train_loss: -354670.4
+[2024-08-10 20:41:39,529][INFO]: val - epoch: 1, val_loss: -403583.8, val_corr_mouse_0: 0.1628811, val_corr_mouse_1: 0.1933719, val_corr_mouse_2: 0.1804671, val_corr_mouse_3: 0.1952043, val_corr_mouse_4: 0.1946753, val_corr_mouse_5: 0.1846854, val_corr_mouse_6: 0.1883276, val_corr_mouse_7: 0.2045731, val_corr_mouse_8: 0.1909339, val_corr_mouse_9: 0.1851706, val_corr: 0.188029
+[2024-08-10 21:09:53,933][INFO]: train - epoch: 2, lr: 0.0018, train_loss: -411576.5
+[2024-08-10 21:11:05,074][INFO]: val - epoch: 2, val_loss: -464080.7, val_corr_mouse_0: 0.2014175, val_corr_mouse_1: 0.242476, val_corr_mouse_2: 0.2270925, val_corr_mouse_3: 0.2384417, val_corr_mouse_4: 0.2485675, val_corr_mouse_5: 0.2264205, val_corr_mouse_6: 0.2358425, val_corr_mouse_7: 0.2611317, val_corr_mouse_8: 0.2386469, val_corr_mouse_9: 0.2275467, val_corr: 0.2347583
+[2024-08-10 21:40:06,518][INFO]: train - epoch: 0, lr: 0.0017865, train_loss: -444820.2
+[2024-08-10 21:41:14,391][INFO]: val - epoch: 0, val_loss: -486708.9, val_corr_mouse_0: 0.21874, val_corr_mouse_1: 0.2640634, val_corr_mouse_2: 0.245198, val_corr_mouse_3: 0.258045, val_corr_mouse_4: 0.2721542, val_corr_mouse_5: 0.2442272, val_corr_mouse_6: 0.2562242, val_corr_mouse_7: 0.285028, val_corr_mouse_8: 0.2586259, val_corr_mouse_9: 0.2470193, val_corr: 0.2549325
+[2024-08-10 21:41:14,931][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-000-0.254933.pth'
+[2024-08-10 22:08:52,529][INFO]: train - epoch: 1, lr: 0.0017463, train_loss: -467094.3
+[2024-08-10 22:10:00,458][INFO]: val - epoch: 1, val_loss: -500568.7, val_corr_mouse_0: 0.2276292, val_corr_mouse_1: 0.2776254, val_corr_mouse_2: 0.2534963, val_corr_mouse_3: 0.2678288, val_corr_mouse_4: 0.287406, val_corr_mouse_5: 0.253602, val_corr_mouse_6: 0.2683498, val_corr_mouse_7: 0.2975183, val_corr_mouse_8: 0.269866, val_corr_mouse_9: 0.2567521, val_corr: 0.2660074
+[2024-08-10 22:10:01,069][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-001-0.266007.pth'
+[2024-08-10 22:10:01,070][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-000-0.254933.pth'
+[2024-08-10 22:37:39,647][INFO]: train - epoch: 2, lr: 0.0016806, train_loss: -474607.4
+[2024-08-10 22:38:47,835][INFO]: val - epoch: 2, val_loss: -508857.1, val_corr_mouse_0: 0.2313625, val_corr_mouse_1: 0.2846344, val_corr_mouse_2: 0.2580712, val_corr_mouse_3: 0.2728482, val_corr_mouse_4: 0.2947273, val_corr_mouse_5: 0.2608492, val_corr_mouse_6: 0.2761506, val_corr_mouse_7: 0.3043658, val_corr_mouse_8: 0.2776194, val_corr_mouse_9: 0.2634641, val_corr: 0.2724093
+[2024-08-10 22:38:48,368][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-002-0.272409.pth'
+[2024-08-10 22:38:48,369][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-001-0.266007.pth'
+[2024-08-10 23:06:23,658][INFO]: train - epoch: 3, lr: 0.0015915, train_loss: -486074.1
+[2024-08-10 23:07:31,978][INFO]: val - epoch: 3, val_loss: -514211.8, val_corr_mouse_0: 0.2336206, val_corr_mouse_1: 0.287446, val_corr_mouse_2: 0.2607324, val_corr_mouse_3: 0.2782524, val_corr_mouse_4: 0.2997743, val_corr_mouse_5: 0.2646001, val_corr_mouse_6: 0.2818425, val_corr_mouse_7: 0.309631, val_corr_mouse_8: 0.2815188, val_corr_mouse_9: 0.2674525, val_corr: 0.2764871
+[2024-08-10 23:07:32,510][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-003-0.276487.pth'
+[2024-08-10 23:07:32,510][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-002-0.272409.pth'
+[2024-08-10 23:35:10,316][INFO]: train - epoch: 4, lr: 0.0014817, train_loss: -493138.7
+[2024-08-10 23:36:19,248][INFO]: val - epoch: 4, val_loss: -516931.3, val_corr_mouse_0: 0.2354729, val_corr_mouse_1: 0.2910216, val_corr_mouse_2: 0.2631662, val_corr_mouse_3: 0.2811095, val_corr_mouse_4: 0.3024082, val_corr_mouse_5: 0.267734, val_corr_mouse_6: 0.2863357, val_corr_mouse_7: 0.312948, val_corr_mouse_8: 0.2865908, val_corr_mouse_9: 0.269239, val_corr: 0.2796026
+[2024-08-10 23:36:19,818][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-004-0.279603.pth'
+[2024-08-10 23:36:19,819][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-003-0.276487.pth'
+[2024-08-11 00:03:59,016][INFO]: train - epoch: 5, lr: 0.0013545, train_loss: -500839.9
+[2024-08-11 00:05:06,820][INFO]: val - epoch: 5, val_loss: -519235.2, val_corr_mouse_0: 0.2368538, val_corr_mouse_1: 0.2938365, val_corr_mouse_2: 0.2649081, val_corr_mouse_3: 0.2830237, val_corr_mouse_4: 0.3042265, val_corr_mouse_5: 0.2694316, val_corr_mouse_6: 0.2886741, val_corr_mouse_7: 0.316601, val_corr_mouse_8: 0.2892183, val_corr_mouse_9: 0.2728563, val_corr: 0.281963
+[2024-08-11 00:05:07,371][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-005-0.281963.pth'
+[2024-08-11 00:05:07,372][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-004-0.279603.pth'
+[2024-08-11 00:32:44,368][INFO]: train - epoch: 6, lr: 0.0012137, train_loss: -504808.5
+[2024-08-11 00:33:52,407][INFO]: val - epoch: 6, val_loss: -521394.3, val_corr_mouse_0: 0.2384052, val_corr_mouse_1: 0.2943085, val_corr_mouse_2: 0.2677092, val_corr_mouse_3: 0.2835604, val_corr_mouse_4: 0.3059359, val_corr_mouse_5: 0.2712416, val_corr_mouse_6: 0.2915547, val_corr_mouse_7: 0.3198195, val_corr_mouse_8: 0.2908438, val_corr_mouse_9: 0.2744222, val_corr: 0.2837801
+[2024-08-11 00:33:52,932][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-006-0.283780.pth'
+[2024-08-11 00:33:52,933][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-005-0.281963.pth'
+[2024-08-11 01:01:28,124][INFO]: train - epoch: 7, lr: 0.0010637, train_loss: -513246.8
+[2024-08-11 01:02:36,325][INFO]: val - epoch: 7, val_loss: -522949.2, val_corr_mouse_0: 0.2387653, val_corr_mouse_1: 0.2963343, val_corr_mouse_2: 0.2685427, val_corr_mouse_3: 0.2847576, val_corr_mouse_4: 0.3078913, val_corr_mouse_5: 0.2731036, val_corr_mouse_6: 0.2934407, val_corr_mouse_7: 0.3222229, val_corr_mouse_8: 0.293444, val_corr_mouse_9: 0.2763204, val_corr: 0.2854823
+[2024-08-11 01:02:36,871][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-007-0.285482.pth'
+[2024-08-11 01:02:36,872][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-006-0.283780.pth'
+[2024-08-11 01:30:13,513][INFO]: train - epoch: 8, lr: 0.000909, train_loss: -521810.5
+[2024-08-11 01:31:21,451][INFO]: val - epoch: 8, val_loss: -524673.5, val_corr_mouse_0: 0.239709, val_corr_mouse_1: 0.2967305, val_corr_mouse_2: 0.2695138, val_corr_mouse_3: 0.2869682, val_corr_mouse_4: 0.3096923, val_corr_mouse_5: 0.2752596, val_corr_mouse_6: 0.2957301, val_corr_mouse_7: 0.3251017, val_corr_mouse_8: 0.2958267, val_corr_mouse_9: 0.2773412, val_corr: 0.2871873
+[2024-08-11 01:31:21,990][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-008-0.287187.pth'
+[2024-08-11 01:31:21,990][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-007-0.285482.pth'
+[2024-08-11 01:59:01,827][INFO]: train - epoch: 9, lr: 0.00075428, train_loss: -528190.5
+[2024-08-11 02:00:10,351][INFO]: val - epoch: 9, val_loss: -525601.3, val_corr_mouse_0: 0.2388702, val_corr_mouse_1: 0.2977311, val_corr_mouse_2: 0.2706504, val_corr_mouse_3: 0.2878983, val_corr_mouse_4: 0.3106839, val_corr_mouse_5: 0.2763954, val_corr_mouse_6: 0.297147, val_corr_mouse_7: 0.3250394, val_corr_mouse_8: 0.2958784, val_corr_mouse_9: 0.2796728, val_corr: 0.2879967
+[2024-08-11 02:00:10,888][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-009-0.287997.pth'
+[2024-08-11 02:00:10,889][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-008-0.287187.pth'
+[2024-08-11 02:27:47,812][INFO]: train - epoch: 10, lr: 0.00060426, train_loss: -534639.6
+[2024-08-11 02:28:56,080][INFO]: val - epoch: 10, val_loss: -526508.3, val_corr_mouse_0: 0.2398109, val_corr_mouse_1: 0.299028, val_corr_mouse_2: 0.2706966, val_corr_mouse_3: 0.2881054, val_corr_mouse_4: 0.3112427, val_corr_mouse_5: 0.2775513, val_corr_mouse_6: 0.2992545, val_corr_mouse_7: 0.3262218, val_corr_mouse_8: 0.2977927, val_corr_mouse_9: 0.2802299, val_corr: 0.2889934
+[2024-08-11 02:28:56,660][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-010-0.288993.pth'
+[2024-08-11 02:28:56,660][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-009-0.287997.pth'
+[2024-08-11 02:56:32,034][INFO]: train - epoch: 11, lr: 0.0004635, train_loss: -540015.5
+[2024-08-11 02:57:40,969][INFO]: val - epoch: 11, val_loss: -527229.7, val_corr_mouse_0: 0.2400323, val_corr_mouse_1: 0.3001254, val_corr_mouse_2: 0.270614, val_corr_mouse_3: 0.2889683, val_corr_mouse_4: 0.3116707, val_corr_mouse_5: 0.2788725, val_corr_mouse_6: 0.2994967, val_corr_mouse_7: 0.3279895, val_corr_mouse_8: 0.2990342, val_corr_mouse_9: 0.2813133, val_corr: 0.2898117
+[2024-08-11 02:57:41,501][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-011-0.289812.pth'
+[2024-08-11 02:57:41,502][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-010-0.288993.pth'
+[2024-08-11 03:25:20,953][INFO]: train - epoch: 12, lr: 0.00033628, train_loss: -543717.2
+[2024-08-11 03:26:29,286][INFO]: val - epoch: 12, val_loss: -527629.3, val_corr_mouse_0: 0.2397258, val_corr_mouse_1: 0.3011086, val_corr_mouse_2: 0.2715617, val_corr_mouse_3: 0.2885399, val_corr_mouse_4: 0.3116831, val_corr_mouse_5: 0.2793996, val_corr_mouse_6: 0.3005415, val_corr_mouse_7: 0.3289917, val_corr_mouse_8: 0.3002466, val_corr_mouse_9: 0.2816241, val_corr: 0.2903423
+[2024-08-11 03:26:29,860][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-012-0.290342.pth'
+[2024-08-11 03:26:29,861][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-011-0.289812.pth'
+[2024-08-11 03:54:07,004][INFO]: train - epoch: 13, lr: 0.00022645, train_loss: -553570.5
+[2024-08-11 03:55:15,586][INFO]: val - epoch: 13, val_loss: -528097, val_corr_mouse_0: 0.2400562, val_corr_mouse_1: 0.3012215, val_corr_mouse_2: 0.2722895, val_corr_mouse_3: 0.2881163, val_corr_mouse_4: 0.3115828, val_corr_mouse_5: 0.2805543, val_corr_mouse_6: 0.3013225, val_corr_mouse_7: 0.3299901, val_corr_mouse_8: 0.3014175, val_corr_mouse_9: 0.2820639, val_corr: 0.2908615
+[2024-08-11 03:55:16,119][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-013-0.290861.pth'
+[2024-08-11 03:55:16,120][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-012-0.290342.pth'
+[2024-08-11 04:22:52,608][INFO]: train - epoch: 14, lr: 0.00013737, train_loss: -554804
+[2024-08-11 04:24:02,026][INFO]: val - epoch: 14, val_loss: -528097.2, val_corr_mouse_0: 0.2396981, val_corr_mouse_1: 0.3006462, val_corr_mouse_2: 0.2718084, val_corr_mouse_3: 0.2879121, val_corr_mouse_4: 0.3115872, val_corr_mouse_5: 0.2808043, val_corr_mouse_6: 0.302448, val_corr_mouse_7: 0.331011, val_corr_mouse_8: 0.3018404, val_corr_mouse_9: 0.2826256, val_corr: 0.2910381
+[2024-08-11 04:24:02,549][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-014-0.291038.pth'
+[2024-08-11 04:24:02,550][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-013-0.290861.pth'
+[2024-08-11 04:51:35,767][INFO]: train - epoch: 15, lr: 7.1734e-05, train_loss: -558072.7
+[2024-08-11 04:52:45,032][INFO]: val - epoch: 15, val_loss: -528713.4, val_corr_mouse_0: 0.2396932, val_corr_mouse_1: 0.3014827, val_corr_mouse_2: 0.2716485, val_corr_mouse_3: 0.2881195, val_corr_mouse_4: 0.3114054, val_corr_mouse_5: 0.281097, val_corr_mouse_6: 0.3030491, val_corr_mouse_7: 0.3319732, val_corr_mouse_8: 0.3026665, val_corr_mouse_9: 0.283412, val_corr: 0.2914547
+[2024-08-11 04:52:45,574][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-015-0.291455.pth'
+[2024-08-11 04:52:45,575][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-014-0.291038.pth'
+[2024-08-11 05:20:21,705][INFO]: train - epoch: 16, lr: 3.1536e-05, train_loss: -564221.7
+[2024-08-11 05:21:30,122][INFO]: val - epoch: 16, val_loss: -528800.6, val_corr_mouse_0: 0.239768, val_corr_mouse_1: 0.3016397, val_corr_mouse_2: 0.2719238, val_corr_mouse_3: 0.2883844, val_corr_mouse_4: 0.3112444, val_corr_mouse_5: 0.2811749, val_corr_mouse_6: 0.3033576, val_corr_mouse_7: 0.332, val_corr_mouse_8: 0.3027232, val_corr_mouse_9: 0.2834592, val_corr: 0.2915675
+[2024-08-11 05:21:30,663][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-016-0.291568.pth'
+[2024-08-11 05:21:30,664][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-015-0.291455.pth'
+[2024-08-11 05:49:07,651][INFO]: train - epoch: 17, lr: 1.8e-05, train_loss: -564328.8
+[2024-08-11 05:50:17,287][INFO]: val - epoch: 17, val_loss: -529034.7, val_corr_mouse_0: 0.2398389, val_corr_mouse_1: 0.3013826, val_corr_mouse_2: 0.2721344, val_corr_mouse_3: 0.2883342, val_corr_mouse_4: 0.3111295, val_corr_mouse_5: 0.2813452, val_corr_mouse_6: 0.3036063, val_corr_mouse_7: 0.3324149, val_corr_mouse_8: 0.3031252, val_corr_mouse_9: 0.2837669, val_corr: 0.2917078
+[2024-08-11 05:50:17,807][INFO]: Model saved to 'data/experiments/true_batch_002/fold_1/model-017-0.291708.pth'
+[2024-08-11 05:50:17,808][INFO]: Model removed 'data/experiments/true_batch_002/fold_1/model-016-0.291568.pth'

data/experiments/true_batch_002/fold_1/model-017-0.291708.pth

@@ -0,0 +1 @@
+/annex/objects/MD5-s683073203--abea91a4882ed4c662fb514e5cbd7dec

data/experiments/true_batch_002/fold_2/log.csv

@@ -0,0 +1,23 @@
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 20:12:06.808667,0,0.0006,-211805.07578889994,-254615.93102709696,0.055522043,0.044118956,0.05410439,0.0371901,0.042866193,0.06627033,0.051076625,0.05689693,0.06730559,0.05906065,0.053441178
+2024-08-10 20:41:49.285991,1,0.0012,-350049.01958333235,-399541.189990416,0.15930167,0.19190373,0.17709845,0.19329065,0.19201393,0.18420085,0.1839809,0.19982165,0.18866014,0.18218856,0.18524605
+2024-08-10 21:11:23.287213,2,0.0018,-410126.6728229161,-462391.3467994887,0.19935255,0.24129295,0.22358339,0.23929058,0.24611527,0.22606356,0.23381527,0.2587152,0.23751292,0.22653413,0.23322758
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 21:41:38.094868,0,0.0017865,-441295.73564583424,-485896.2895852697,0.21761388,0.26496154,0.24440071,0.25767767,0.27189663,0.24313472,0.25516093,0.2836769,0.25733638,0.24537323,0.25412327
+2024-08-10 22:10:37.404911,1,0.0017463,-464039.95783333277,-500702.9302683551,0.22715257,0.27815422,0.25201905,0.26722172,0.28593206,0.25308838,0.26789686,0.29679084,0.26918212,0.2556039,0.26530418
+2024-08-10 22:39:30.116036,2,0.0016806,-475233.657489583,-508416.4775209112,0.23096336,0.28352797,0.25706124,0.27424493,0.2928873,0.25908706,0.2756801,0.3038193,0.27607274,0.2627009,0.27160448
+2024-08-10 23:07:43.552000,3,0.0015915,-485927.74615625007,-513732.33079693286,0.23526281,0.28792158,0.2602192,0.27791002,0.29768887,0.26292777,0.281942,0.3093092,0.2810241,0.2662747,0.276048
+2024-08-10 23:36:25.974768,4,0.0014817,-492266.1352447909,-517512.6979844331,0.23676154,0.2921088,0.26272207,0.28255588,0.30177656,0.26737088,0.2868173,0.31412128,0.2847025,0.27027848,0.27992153
+2024-08-11 00:05:27.503016,5,0.0013545,-499885.97508854227,-519319.62444818794,0.23748764,0.29260203,0.26420596,0.2837771,0.30388695,0.26994953,0.28961325,0.31694344,0.28631696,0.27222037,0.2817003
+2024-08-11 00:34:26.809959,6,0.0012137,-508444.51394791703,-520859.0920655207,0.23827663,0.29425335,0.2646321,0.28530976,0.3049751,0.2711996,0.29132265,0.31926614,0.2883491,0.2744435,0.2832028
+2024-08-11 01:03:31.377688,7,0.0010637,-508749.9977656265,-522047.16699581797,0.2383055,0.2974208,0.2663076,0.28555858,0.30676168,0.27315632,0.29287994,0.3212841,0.29096082,0.27531886,0.2847954
+2024-08-11 01:32:00.180235,8,0.000909,-518992.89330208366,-523426.4886152415,0.2399619,0.29752162,0.26643515,0.2860894,0.30729586,0.27426705,0.29484132,0.32307646,0.2923224,0.27678037,0.28585917
+2024-08-11 02:01:00.188602,9,0.00075428,-527189.2024166676,-524668.9091252326,0.23908345,0.29850546,0.26770845,0.28747636,0.30818203,0.27570912,0.29703674,0.32508636,0.2944295,0.27800685,0.28712243
+2024-08-11 02:30:02.615307,10,0.00060426,-534074.2865833334,-525231.4966600836,0.23912252,0.30045012,0.2690016,0.28866127,0.30928224,0.27644604,0.29829046,0.3254168,0.2967841,0.27839303,0.28818482
+2024-08-11 02:59:04.377942,11,0.0004635,-539549.4916458348,-525626.6554368031,0.23888071,0.3004846,0.26931015,0.28899962,0.30844808,0.27828166,0.2987246,0.32728896,0.29798388,0.27956295,0.2887965
+2024-08-11 03:27:57.194403,12,0.00033628,-544772.1563645821,-526612.0931981874,0.2391346,0.30053607,0.2697978,0.28889614,0.30898553,0.27953407,0.2996041,0.3279701,0.29927167,0.28061014,0.28943402
+2024-08-11 03:56:49.191998,13,0.00022645,-549158.9411979167,-526535.1153868499,0.2390389,0.3002037,0.270281,0.28914502,0.30862224,0.27979407,0.30106497,0.32872817,0.29941806,0.28057683,0.2896873
+2024-08-11 04:25:51.931390,14,0.00013737,-555838.370375,-526940.7391960967,0.23896217,0.3013771,0.2697656,0.28947085,0.30875504,0.28047925,0.30062872,0.32955924,0.29957274,0.28120565,0.2899776
+2024-08-11 04:54:54.946653,15,7.1734e-05,-558914.1022708322,-527177.8824494656,0.2387819,0.30167583,0.26975998,0.28946596,0.30916288,0.28063336,0.3010554,0.3300801,0.3007058,0.28160784,0.29029292
+2024-08-11 05:23:59.839042,16,3.1536e-05,-560621.6298645809,-527084.2125929369,0.23840049,0.30208498,0.26916212,0.2895932,0.3089993,0.28082153,0.30146483,0.33001012,0.30118173,0.28174657,0.2903465
+2024-08-11 05:53:05.143529,17,1.8e-05,-562023.214531248,-527009.2846044379,0.23830041,0.30196428,0.26953804,0.28951615,0.30898535,0.28095338,0.30158082,0.33017626,0.30138043,0.28188545,0.29042807

data/experiments/true_batch_002/fold_2/log.txt

@@ -0,0 +1,77 @@
+[2024-08-10 20:10:54,564][INFO]: train - epoch: 0, lr: 0.0006, train_loss: -211805.1
+[2024-08-10 20:12:06,808][INFO]: val - epoch: 0, val_loss: -254615.9, val_corr_mouse_0: 0.05552204, val_corr_mouse_1: 0.04411896, val_corr_mouse_2: 0.05410439, val_corr_mouse_3: 0.0371901, val_corr_mouse_4: 0.04286619, val_corr_mouse_5: 0.06627033, val_corr_mouse_6: 0.05107662, val_corr_mouse_7: 0.05689693, val_corr_mouse_8: 0.06730559, val_corr_mouse_9: 0.05906065, val_corr: 0.05344118
+[2024-08-10 20:40:36,921][INFO]: train - epoch: 1, lr: 0.0012, train_loss: -350049
+[2024-08-10 20:41:49,285][INFO]: val - epoch: 1, val_loss: -399541.2, val_corr_mouse_0: 0.1593017, val_corr_mouse_1: 0.1919037, val_corr_mouse_2: 0.1770985, val_corr_mouse_3: 0.1932907, val_corr_mouse_4: 0.1920139, val_corr_mouse_5: 0.1842009, val_corr_mouse_6: 0.1839809, val_corr_mouse_7: 0.1998217, val_corr_mouse_8: 0.1886601, val_corr_mouse_9: 0.1821886, val_corr: 0.1852461
+[2024-08-10 21:10:11,355][INFO]: train - epoch: 2, lr: 0.0018, train_loss: -410126.7
+[2024-08-10 21:11:23,286][INFO]: val - epoch: 2, val_loss: -462391.3, val_corr_mouse_0: 0.1993525, val_corr_mouse_1: 0.241293, val_corr_mouse_2: 0.2235834, val_corr_mouse_3: 0.2392906, val_corr_mouse_4: 0.2461153, val_corr_mouse_5: 0.2260636, val_corr_mouse_6: 0.2338153, val_corr_mouse_7: 0.2587152, val_corr_mouse_8: 0.2375129, val_corr_mouse_9: 0.2265341, val_corr: 0.2332276
+[2024-08-10 21:40:28,926][INFO]: train - epoch: 0, lr: 0.0017865, train_loss: -441295.7
+[2024-08-10 21:41:38,094][INFO]: val - epoch: 0, val_loss: -485896.3, val_corr_mouse_0: 0.2176139, val_corr_mouse_1: 0.2649615, val_corr_mouse_2: 0.2444007, val_corr_mouse_3: 0.2576777, val_corr_mouse_4: 0.2718966, val_corr_mouse_5: 0.2431347, val_corr_mouse_6: 0.2551609, val_corr_mouse_7: 0.2836769, val_corr_mouse_8: 0.2573364, val_corr_mouse_9: 0.2453732, val_corr: 0.2541233
+[2024-08-10 21:41:38,646][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-000-0.254123.pth'
+[2024-08-10 22:09:28,076][INFO]: train - epoch: 1, lr: 0.0017463, train_loss: -464040
+[2024-08-10 22:10:37,404][INFO]: val - epoch: 1, val_loss: -500702.9, val_corr_mouse_0: 0.2271526, val_corr_mouse_1: 0.2781542, val_corr_mouse_2: 0.252019, val_corr_mouse_3: 0.2672217, val_corr_mouse_4: 0.2859321, val_corr_mouse_5: 0.2530884, val_corr_mouse_6: 0.2678969, val_corr_mouse_7: 0.2967908, val_corr_mouse_8: 0.2691821, val_corr_mouse_9: 0.2556039, val_corr: 0.2653042
+[2024-08-10 22:10:37,955][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-001-0.265304.pth'
+[2024-08-10 22:10:37,956][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-000-0.254123.pth'
+[2024-08-10 22:38:20,640][INFO]: train - epoch: 2, lr: 0.0016806, train_loss: -475233.7
+[2024-08-10 22:39:30,115][INFO]: val - epoch: 2, val_loss: -508416.5, val_corr_mouse_0: 0.2309634, val_corr_mouse_1: 0.283528, val_corr_mouse_2: 0.2570612, val_corr_mouse_3: 0.2742449, val_corr_mouse_4: 0.2928873, val_corr_mouse_5: 0.2590871, val_corr_mouse_6: 0.2756801, val_corr_mouse_7: 0.3038193, val_corr_mouse_8: 0.2760727, val_corr_mouse_9: 0.2627009, val_corr: 0.2716045
+[2024-08-10 22:39:30,670][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-002-0.271604.pth'
+[2024-08-10 22:39:30,671][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-001-0.265304.pth'
+[2024-08-10 23:06:34,582][INFO]: train - epoch: 3, lr: 0.0015915, train_loss: -485927.7
+[2024-08-10 23:07:43,551][INFO]: val - epoch: 3, val_loss: -513732.3, val_corr_mouse_0: 0.2352628, val_corr_mouse_1: 0.2879216, val_corr_mouse_2: 0.2602192, val_corr_mouse_3: 0.27791, val_corr_mouse_4: 0.2976889, val_corr_mouse_5: 0.2629278, val_corr_mouse_6: 0.281942, val_corr_mouse_7: 0.3093092, val_corr_mouse_8: 0.2810241, val_corr_mouse_9: 0.2662747, val_corr: 0.276048
+[2024-08-10 23:07:44,101][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-003-0.276048.pth'
+[2024-08-10 23:07:44,102][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-002-0.271604.pth'
+[2024-08-10 23:35:15,987][INFO]: train - epoch: 4, lr: 0.0014817, train_loss: -492266.1
+[2024-08-10 23:36:25,974][INFO]: val - epoch: 4, val_loss: -517512.7, val_corr_mouse_0: 0.2367615, val_corr_mouse_1: 0.2921088, val_corr_mouse_2: 0.2627221, val_corr_mouse_3: 0.2825559, val_corr_mouse_4: 0.3017766, val_corr_mouse_5: 0.2673709, val_corr_mouse_6: 0.2868173, val_corr_mouse_7: 0.3141213, val_corr_mouse_8: 0.2847025, val_corr_mouse_9: 0.2702785, val_corr: 0.2799215
+[2024-08-10 23:36:26,552][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-004-0.279922.pth'
+[2024-08-10 23:36:26,553][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-003-0.276048.pth'
+[2024-08-11 00:04:18,296][INFO]: train - epoch: 5, lr: 0.0013545, train_loss: -499886
+[2024-08-11 00:05:27,502][INFO]: val - epoch: 5, val_loss: -519319.6, val_corr_mouse_0: 0.2374876, val_corr_mouse_1: 0.292602, val_corr_mouse_2: 0.264206, val_corr_mouse_3: 0.2837771, val_corr_mouse_4: 0.303887, val_corr_mouse_5: 0.2699495, val_corr_mouse_6: 0.2896132, val_corr_mouse_7: 0.3169434, val_corr_mouse_8: 0.286317, val_corr_mouse_9: 0.2722204, val_corr: 0.2817003
+[2024-08-11 00:05:28,107][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-005-0.281700.pth'
+[2024-08-11 00:05:28,108][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-004-0.279922.pth'
+[2024-08-11 00:33:17,147][INFO]: train - epoch: 6, lr: 0.0012137, train_loss: -508444.5
+[2024-08-11 00:34:26,809][INFO]: val - epoch: 6, val_loss: -520859.1, val_corr_mouse_0: 0.2382766, val_corr_mouse_1: 0.2942533, val_corr_mouse_2: 0.2646321, val_corr_mouse_3: 0.2853098, val_corr_mouse_4: 0.3049751, val_corr_mouse_5: 0.2711996, val_corr_mouse_6: 0.2913226, val_corr_mouse_7: 0.3192661, val_corr_mouse_8: 0.2883491, val_corr_mouse_9: 0.2744435, val_corr: 0.2832028
+[2024-08-11 00:34:27,368][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-006-0.283203.pth'
+[2024-08-11 00:34:27,369][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-005-0.281700.pth'
+[2024-08-11 01:02:21,648][INFO]: train - epoch: 7, lr: 0.0010637, train_loss: -508750
+[2024-08-11 01:03:31,377][INFO]: val - epoch: 7, val_loss: -522047.2, val_corr_mouse_0: 0.2383055, val_corr_mouse_1: 0.2974208, val_corr_mouse_2: 0.2663076, val_corr_mouse_3: 0.2855586, val_corr_mouse_4: 0.3067617, val_corr_mouse_5: 0.2731563, val_corr_mouse_6: 0.2928799, val_corr_mouse_7: 0.3212841, val_corr_mouse_8: 0.2909608, val_corr_mouse_9: 0.2753189, val_corr: 0.2847954
+[2024-08-11 01:03:31,928][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-007-0.284795.pth'
+[2024-08-11 01:03:31,929][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-006-0.283203.pth'
+[2024-08-11 01:30:51,048][INFO]: train - epoch: 8, lr: 0.000909, train_loss: -518992.9
+[2024-08-11 01:32:00,180][INFO]: val - epoch: 8, val_loss: -523426.5, val_corr_mouse_0: 0.2399619, val_corr_mouse_1: 0.2975216, val_corr_mouse_2: 0.2664351, val_corr_mouse_3: 0.2860894, val_corr_mouse_4: 0.3072959, val_corr_mouse_5: 0.274267, val_corr_mouse_6: 0.2948413, val_corr_mouse_7: 0.3230765, val_corr_mouse_8: 0.2923224, val_corr_mouse_9: 0.2767804, val_corr: 0.2858592
+[2024-08-11 01:32:00,733][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-008-0.285859.pth'
+[2024-08-11 01:32:00,734][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-007-0.284795.pth'
+[2024-08-11 01:59:50,959][INFO]: train - epoch: 9, lr: 0.00075428, train_loss: -527189.2
+[2024-08-11 02:01:00,188][INFO]: val - epoch: 9, val_loss: -524668.9, val_corr_mouse_0: 0.2390835, val_corr_mouse_1: 0.2985055, val_corr_mouse_2: 0.2677085, val_corr_mouse_3: 0.2874764, val_corr_mouse_4: 0.308182, val_corr_mouse_5: 0.2757091, val_corr_mouse_6: 0.2970367, val_corr_mouse_7: 0.3250864, val_corr_mouse_8: 0.2944295, val_corr_mouse_9: 0.2780069, val_corr: 0.2871224
+[2024-08-11 02:01:00,799][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-009-0.287122.pth'
+[2024-08-11 02:01:00,800][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-008-0.285859.pth'
+[2024-08-11 02:28:53,632][INFO]: train - epoch: 10, lr: 0.00060426, train_loss: -534074.3
+[2024-08-11 02:30:02,614][INFO]: val - epoch: 10, val_loss: -525231.5, val_corr_mouse_0: 0.2391225, val_corr_mouse_1: 0.3004501, val_corr_mouse_2: 0.2690016, val_corr_mouse_3: 0.2886613, val_corr_mouse_4: 0.3092822, val_corr_mouse_5: 0.276446, val_corr_mouse_6: 0.2982905, val_corr_mouse_7: 0.3254168, val_corr_mouse_8: 0.2967841, val_corr_mouse_9: 0.278393, val_corr: 0.2881848
+[2024-08-11 02:30:03,159][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-010-0.288185.pth'
+[2024-08-11 02:30:03,160][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-009-0.287122.pth'
+[2024-08-11 02:57:54,847][INFO]: train - epoch: 11, lr: 0.0004635, train_loss: -539549.5
+[2024-08-11 02:59:04,377][INFO]: val - epoch: 11, val_loss: -525626.7, val_corr_mouse_0: 0.2388807, val_corr_mouse_1: 0.3004846, val_corr_mouse_2: 0.2693101, val_corr_mouse_3: 0.2889996, val_corr_mouse_4: 0.3084481, val_corr_mouse_5: 0.2782817, val_corr_mouse_6: 0.2987246, val_corr_mouse_7: 0.327289, val_corr_mouse_8: 0.2979839, val_corr_mouse_9: 0.279563, val_corr: 0.2887965
+[2024-08-11 02:59:04,929][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-011-0.288797.pth'
+[2024-08-11 02:59:04,930][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-010-0.288185.pth'
+[2024-08-11 03:26:47,943][INFO]: train - epoch: 12, lr: 0.00033628, train_loss: -544772.2
+[2024-08-11 03:27:57,193][INFO]: val - epoch: 12, val_loss: -526612.1, val_corr_mouse_0: 0.2391346, val_corr_mouse_1: 0.3005361, val_corr_mouse_2: 0.2697978, val_corr_mouse_3: 0.2888961, val_corr_mouse_4: 0.3089855, val_corr_mouse_5: 0.2795341, val_corr_mouse_6: 0.2996041, val_corr_mouse_7: 0.3279701, val_corr_mouse_8: 0.2992717, val_corr_mouse_9: 0.2806101, val_corr: 0.289434
+[2024-08-11 03:27:57,749][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-012-0.289434.pth'
+[2024-08-11 03:27:57,750][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-011-0.288797.pth'
+[2024-08-11 03:55:39,601][INFO]: train - epoch: 13, lr: 0.00022645, train_loss: -549158.9
+[2024-08-11 03:56:49,191][INFO]: val - epoch: 13, val_loss: -526535.1, val_corr_mouse_0: 0.2390389, val_corr_mouse_1: 0.3002037, val_corr_mouse_2: 0.270281, val_corr_mouse_3: 0.289145, val_corr_mouse_4: 0.3086222, val_corr_mouse_5: 0.2797941, val_corr_mouse_6: 0.301065, val_corr_mouse_7: 0.3287282, val_corr_mouse_8: 0.2994181, val_corr_mouse_9: 0.2805768, val_corr: 0.2896873
+[2024-08-11 03:56:49,788][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-013-0.289687.pth'
+[2024-08-11 03:56:49,789][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-012-0.289434.pth'
+[2024-08-11 04:24:42,764][INFO]: train - epoch: 14, lr: 0.00013737, train_loss: -555838.4
+[2024-08-11 04:25:51,930][INFO]: val - epoch: 14, val_loss: -526940.7, val_corr_mouse_0: 0.2389622, val_corr_mouse_1: 0.3013771, val_corr_mouse_2: 0.2697656, val_corr_mouse_3: 0.2894709, val_corr_mouse_4: 0.308755, val_corr_mouse_5: 0.2804793, val_corr_mouse_6: 0.3006287, val_corr_mouse_7: 0.3295592, val_corr_mouse_8: 0.2995727, val_corr_mouse_9: 0.2812057, val_corr: 0.2899776
+[2024-08-11 04:25:52,479][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-014-0.289978.pth'
+[2024-08-11 04:25:52,480][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-013-0.289687.pth'
+[2024-08-11 04:53:45,678][INFO]: train - epoch: 15, lr: 7.1734e-05, train_loss: -558914.1
+[2024-08-11 04:54:54,946][INFO]: val - epoch: 15, val_loss: -527177.9, val_corr_mouse_0: 0.2387819, val_corr_mouse_1: 0.3016758, val_corr_mouse_2: 0.26976, val_corr_mouse_3: 0.289466, val_corr_mouse_4: 0.3091629, val_corr_mouse_5: 0.2806334, val_corr_mouse_6: 0.3010554, val_corr_mouse_7: 0.3300801, val_corr_mouse_8: 0.3007058, val_corr_mouse_9: 0.2816078, val_corr: 0.2902929
+[2024-08-11 04:54:55,622][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-015-0.290293.pth'
+[2024-08-11 04:54:55,623][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-014-0.289978.pth'
+[2024-08-11 05:22:50,425][INFO]: train - epoch: 16, lr: 3.1536e-05, train_loss: -560621.6
+[2024-08-11 05:23:59,838][INFO]: val - epoch: 16, val_loss: -527084.2, val_corr_mouse_0: 0.2384005, val_corr_mouse_1: 0.302085, val_corr_mouse_2: 0.2691621, val_corr_mouse_3: 0.2895932, val_corr_mouse_4: 0.3089993, val_corr_mouse_5: 0.2808215, val_corr_mouse_6: 0.3014648, val_corr_mouse_7: 0.3300101, val_corr_mouse_8: 0.3011817, val_corr_mouse_9: 0.2817466, val_corr: 0.2903465
+[2024-08-11 05:24:00,407][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-016-0.290347.pth'
+[2024-08-11 05:24:00,408][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-015-0.290293.pth'
+[2024-08-11 05:51:56,680][INFO]: train - epoch: 17, lr: 1.8e-05, train_loss: -562023.2
+[2024-08-11 05:53:05,143][INFO]: val - epoch: 17, val_loss: -527009.3, val_corr_mouse_0: 0.2383004, val_corr_mouse_1: 0.3019643, val_corr_mouse_2: 0.269538, val_corr_mouse_3: 0.2895162, val_corr_mouse_4: 0.3089854, val_corr_mouse_5: 0.2809534, val_corr_mouse_6: 0.3015808, val_corr_mouse_7: 0.3301763, val_corr_mouse_8: 0.3013804, val_corr_mouse_9: 0.2818854, val_corr: 0.2904281
+[2024-08-11 05:53:05,698][INFO]: Model saved to 'data/experiments/true_batch_002/fold_2/model-017-0.290428.pth'
+[2024-08-11 05:53:05,699][INFO]: Model removed 'data/experiments/true_batch_002/fold_2/model-016-0.290347.pth'

data/experiments/true_batch_002/fold_2/model-017-0.290428.pth

@@ -0,0 +1 @@
+/annex/objects/MD5-s683073203--e5d29c10004331637929ebfef0770d1e

data/experiments/true_batch_002/fold_3/log.csv

@@ -0,0 +1,23 @@
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 20:23:08.152022,0,0.0006,-209745.04758528637,-256773.8576360652,0.054771688,0.052918784,0.05036089,0.041403092,0.043799233,0.065000385,0.05546981,0.05986646,0.07045413,0.06492054,0.055896502
+2024-08-10 20:51:59.594006,1,0.0012,-352008.31914843677,-405403.4734912292,0.16004382,0.19378404,0.1792269,0.19672947,0.19533734,0.18529786,0.1877908,0.20347205,0.19056024,0.18594144,0.1878184
+2024-08-10 21:20:32.698585,2,0.0018,-413109.1888593752,-465894.7864341309,0.20160812,0.24511401,0.22656071,0.24185848,0.248674,0.2276167,0.23507726,0.2615384,0.23886205,0.22859287,0.23555025
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 21:50:31.159342,0,0.0017865,-445413.44092187437,-488391.04813835974,0.21913978,0.26568323,0.24450709,0.25819924,0.27224,0.24474192,0.25701502,0.28576887,0.25854504,0.24673879,0.2552579
+2024-08-10 22:19:24.775682,1,0.0017463,-462395.6913333342,-501237.7030959573,0.2261951,0.2770174,0.2543667,0.26796177,0.2845974,0.25285447,0.26921338,0.29778144,0.2692038,0.25676906,0.26559606
+2024-08-10 22:48:18.934346,2,0.0016806,-474119.86174479156,-508326.40654042753,0.23119596,0.2835315,0.2579478,0.27303216,0.2921713,0.26041552,0.2765214,0.30440232,0.27549022,0.26243106,0.27171394
+2024-08-10 23:17:15.616955,3,0.0015915,-485061.7550833341,-513345.9158631505,0.23370363,0.28745982,0.26169884,0.27649063,0.29818553,0.26380756,0.28279546,0.30981702,0.2795172,0.2666313,0.2760107
+2024-08-10 23:46:05.609435,4,0.0014817,-494769.3534895835,-516696.01344679337,0.23500338,0.28974146,0.26495525,0.2796558,0.30192873,0.2667777,0.28706238,0.3133998,0.28354728,0.2693446,0.27914163
+2024-08-11 00:14:59.107083,5,0.0013545,-500782.5070052087,-519299.05198652425,0.23653668,0.29227737,0.2664139,0.28191978,0.30385366,0.26998192,0.29014876,0.3172281,0.28768924,0.2717387,0.2817788
+2024-08-11 00:43:56.564398,6,0.0012137,-505320.4871354179,-521248.0490241637,0.23765492,0.2938733,0.26725772,0.28307518,0.30568528,0.27170312,0.2928056,0.3193054,0.29000995,0.27409503,0.28354654
+2024-08-11 01:12:54.830269,7,0.0010637,-512174.3574270834,-522946.84589916334,0.23813844,0.2958747,0.26883516,0.28507283,0.30788544,0.2730324,0.29457232,0.32114953,0.29163578,0.27563533,0.28518316
+2024-08-11 01:41:51.801122,8,0.000909,-520844.8578645838,-524626.5669435407,0.23898649,0.297325,0.27053776,0.2855534,0.30966374,0.27507105,0.29607347,0.322736,0.29376268,0.277513,0.28672227
+2024-08-11 02:10:45.360263,9,0.00075428,-527021.1465729174,-525505.1290950279,0.23901822,0.2991721,0.2708564,0.28677043,0.3098714,0.27597123,0.29691914,0.32454702,0.2955845,0.2784869,0.28771973
+2024-08-11 02:39:39.204996,10,0.00060426,-530996.8129583317,-526097.5027300183,0.2390089,0.30022553,0.27185696,0.28805825,0.31000277,0.27722603,0.29848662,0.32606485,0.29742873,0.27898037,0.28873387
+2024-08-11 03:08:31.980776,11,0.0004635,-537265.898270834,-526871.7761675189,0.23958258,0.30080134,0.27267644,0.28865504,0.31031486,0.27828842,0.29937172,0.32733166,0.29873574,0.27969363,0.28954515
+2024-08-11 03:37:28.558343,12,0.00033628,-545149.6659270849,-527476.6650789966,0.23900725,0.30096728,0.27260518,0.28913927,0.3103845,0.27947196,0.30041617,0.32815173,0.29960227,0.2801268,0.28998724
+2024-08-11 04:06:22.888012,13,0.00022645,-549114.9732083337,-528086.4977056234,0.23980744,0.30135456,0.2736322,0.2887431,0.31120193,0.28024155,0.3018319,0.32896477,0.3009107,0.28103948,0.29077277
+2024-08-11 04:35:23.915043,14,0.00013737,-556350.3995937501,-528497.7782876397,0.23931624,0.30184445,0.27411908,0.2883899,0.31114513,0.28079775,0.30251002,0.32958114,0.30197302,0.2819852,0.29116622
+2024-08-11 05:04:18.635671,15,7.1734e-05,-558189.4224791677,-528713.4568715147,0.23937047,0.30227202,0.2747453,0.28863245,0.31083155,0.28138202,0.3028467,0.33001217,0.30237523,0.28255913,0.2915027
+2024-08-11 05:33:17.031637,16,3.1536e-05,-561137.1739322923,-528573.0259351765,0.23930916,0.3023642,0.27449352,0.28822896,0.3106224,0.2816494,0.30331573,0.33062598,0.3029332,0.28259054,0.2916133
+2024-08-11 06:02:15.860495,17,1.8e-05,-562082.1786354183,-528999.1283108734,0.2394058,0.30264148,0.2749245,0.28853688,0.31096637,0.28189132,0.30354434,0.33085722,0.30318934,0.2825689,0.29185262

data/experiments/true_batch_002/fold_3/log.txt

@@ -0,0 +1,77 @@
+[2024-08-10 20:21:57,672][INFO]: train - epoch: 0, lr: 0.0006, train_loss: -209745
+[2024-08-10 20:23:08,151][INFO]: val - epoch: 0, val_loss: -256773.9, val_corr_mouse_0: 0.05477169, val_corr_mouse_1: 0.05291878, val_corr_mouse_2: 0.05036089, val_corr_mouse_3: 0.04140309, val_corr_mouse_4: 0.04379923, val_corr_mouse_5: 0.06500039, val_corr_mouse_6: 0.05546981, val_corr_mouse_7: 0.05986646, val_corr_mouse_8: 0.07045413, val_corr_mouse_9: 0.06492054, val_corr: 0.0558965
+[2024-08-10 20:50:49,444][INFO]: train - epoch: 1, lr: 0.0012, train_loss: -352008.3
+[2024-08-10 20:51:59,593][INFO]: val - epoch: 1, val_loss: -405403.5, val_corr_mouse_0: 0.1600438, val_corr_mouse_1: 0.193784, val_corr_mouse_2: 0.1792269, val_corr_mouse_3: 0.1967295, val_corr_mouse_4: 0.1953373, val_corr_mouse_5: 0.1852979, val_corr_mouse_6: 0.1877908, val_corr_mouse_7: 0.203472, val_corr_mouse_8: 0.1905602, val_corr_mouse_9: 0.1859414, val_corr: 0.1878184
+[2024-08-10 21:19:22,278][INFO]: train - epoch: 2, lr: 0.0018, train_loss: -413109.2
+[2024-08-10 21:20:32,697][INFO]: val - epoch: 2, val_loss: -465894.8, val_corr_mouse_0: 0.2016081, val_corr_mouse_1: 0.245114, val_corr_mouse_2: 0.2265607, val_corr_mouse_3: 0.2418585, val_corr_mouse_4: 0.248674, val_corr_mouse_5: 0.2276167, val_corr_mouse_6: 0.2350773, val_corr_mouse_7: 0.2615384, val_corr_mouse_8: 0.2388621, val_corr_mouse_9: 0.2285929, val_corr: 0.2355503
+[2024-08-10 21:49:22,556][INFO]: train - epoch: 0, lr: 0.0017865, train_loss: -445413.4
+[2024-08-10 21:50:31,159][INFO]: val - epoch: 0, val_loss: -488391, val_corr_mouse_0: 0.2191398, val_corr_mouse_1: 0.2656832, val_corr_mouse_2: 0.2445071, val_corr_mouse_3: 0.2581992, val_corr_mouse_4: 0.27224, val_corr_mouse_5: 0.2447419, val_corr_mouse_6: 0.257015, val_corr_mouse_7: 0.2857689, val_corr_mouse_8: 0.258545, val_corr_mouse_9: 0.2467388, val_corr: 0.2552579
+[2024-08-10 21:50:31,793][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-000-0.255258.pth'
+[2024-08-10 22:18:15,779][INFO]: train - epoch: 1, lr: 0.0017463, train_loss: -462395.7
+[2024-08-10 22:19:24,775][INFO]: val - epoch: 1, val_loss: -501237.7, val_corr_mouse_0: 0.2261951, val_corr_mouse_1: 0.2770174, val_corr_mouse_2: 0.2543667, val_corr_mouse_3: 0.2679618, val_corr_mouse_4: 0.2845974, val_corr_mouse_5: 0.2528545, val_corr_mouse_6: 0.2692134, val_corr_mouse_7: 0.2977814, val_corr_mouse_8: 0.2692038, val_corr_mouse_9: 0.2567691, val_corr: 0.2655961
+[2024-08-10 22:19:25,444][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-001-0.265596.pth'
+[2024-08-10 22:19:25,445][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-000-0.255258.pth'
+[2024-08-10 22:47:10,829][INFO]: train - epoch: 2, lr: 0.0016806, train_loss: -474119.9
+[2024-08-10 22:48:18,934][INFO]: val - epoch: 2, val_loss: -508326.4, val_corr_mouse_0: 0.231196, val_corr_mouse_1: 0.2835315, val_corr_mouse_2: 0.2579478, val_corr_mouse_3: 0.2730322, val_corr_mouse_4: 0.2921713, val_corr_mouse_5: 0.2604155, val_corr_mouse_6: 0.2765214, val_corr_mouse_7: 0.3044023, val_corr_mouse_8: 0.2754902, val_corr_mouse_9: 0.2624311, val_corr: 0.2717139
+[2024-08-10 22:48:19,557][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-002-0.271714.pth'
+[2024-08-10 22:48:19,558][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-001-0.265596.pth'
+[2024-08-10 23:16:07,012][INFO]: train - epoch: 3, lr: 0.0015915, train_loss: -485061.8
+[2024-08-10 23:17:15,616][INFO]: val - epoch: 3, val_loss: -513345.9, val_corr_mouse_0: 0.2337036, val_corr_mouse_1: 0.2874598, val_corr_mouse_2: 0.2616988, val_corr_mouse_3: 0.2764906, val_corr_mouse_4: 0.2981855, val_corr_mouse_5: 0.2638076, val_corr_mouse_6: 0.2827955, val_corr_mouse_7: 0.309817, val_corr_mouse_8: 0.2795172, val_corr_mouse_9: 0.2666313, val_corr: 0.2760107
+[2024-08-10 23:17:16,261][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-003-0.276011.pth'
+[2024-08-10 23:17:16,262][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-002-0.271714.pth'
+[2024-08-10 23:44:56,644][INFO]: train - epoch: 4, lr: 0.0014817, train_loss: -494769.4
+[2024-08-10 23:46:05,609][INFO]: val - epoch: 4, val_loss: -516696, val_corr_mouse_0: 0.2350034, val_corr_mouse_1: 0.2897415, val_corr_mouse_2: 0.2649553, val_corr_mouse_3: 0.2796558, val_corr_mouse_4: 0.3019287, val_corr_mouse_5: 0.2667777, val_corr_mouse_6: 0.2870624, val_corr_mouse_7: 0.3133998, val_corr_mouse_8: 0.2835473, val_corr_mouse_9: 0.2693446, val_corr: 0.2791416
+[2024-08-10 23:46:06,273][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-004-0.279142.pth'
+[2024-08-10 23:46:06,273][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-003-0.276011.pth'
+[2024-08-11 00:13:50,911][INFO]: train - epoch: 5, lr: 0.0013545, train_loss: -500782.5
+[2024-08-11 00:14:59,106][INFO]: val - epoch: 5, val_loss: -519299.1, val_corr_mouse_0: 0.2365367, val_corr_mouse_1: 0.2922774, val_corr_mouse_2: 0.2664139, val_corr_mouse_3: 0.2819198, val_corr_mouse_4: 0.3038537, val_corr_mouse_5: 0.2699819, val_corr_mouse_6: 0.2901488, val_corr_mouse_7: 0.3172281, val_corr_mouse_8: 0.2876892, val_corr_mouse_9: 0.2717387, val_corr: 0.2817788
+[2024-08-11 00:14:59,728][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-005-0.281779.pth'
+[2024-08-11 00:14:59,728][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-004-0.279142.pth'
+[2024-08-11 00:42:48,055][INFO]: train - epoch: 6, lr: 0.0012137, train_loss: -505320.5
+[2024-08-11 00:43:56,563][INFO]: val - epoch: 6, val_loss: -521248, val_corr_mouse_0: 0.2376549, val_corr_mouse_1: 0.2938733, val_corr_mouse_2: 0.2672577, val_corr_mouse_3: 0.2830752, val_corr_mouse_4: 0.3056853, val_corr_mouse_5: 0.2717031, val_corr_mouse_6: 0.2928056, val_corr_mouse_7: 0.3193054, val_corr_mouse_8: 0.2900099, val_corr_mouse_9: 0.274095, val_corr: 0.2835465
+[2024-08-11 00:43:57,211][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-006-0.283547.pth'
+[2024-08-11 00:43:57,212][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-005-0.281779.pth'
+[2024-08-11 01:11:46,342][INFO]: train - epoch: 7, lr: 0.0010637, train_loss: -512174.4
+[2024-08-11 01:12:54,830][INFO]: val - epoch: 7, val_loss: -522946.8, val_corr_mouse_0: 0.2381384, val_corr_mouse_1: 0.2958747, val_corr_mouse_2: 0.2688352, val_corr_mouse_3: 0.2850728, val_corr_mouse_4: 0.3078854, val_corr_mouse_5: 0.2730324, val_corr_mouse_6: 0.2945723, val_corr_mouse_7: 0.3211495, val_corr_mouse_8: 0.2916358, val_corr_mouse_9: 0.2756353, val_corr: 0.2851832
+[2024-08-11 01:12:55,494][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-007-0.285183.pth'
+[2024-08-11 01:12:55,495][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-006-0.283547.pth'
+[2024-08-11 01:40:43,630][INFO]: train - epoch: 8, lr: 0.000909, train_loss: -520844.9
+[2024-08-11 01:41:51,800][INFO]: val - epoch: 8, val_loss: -524626.6, val_corr_mouse_0: 0.2389865, val_corr_mouse_1: 0.297325, val_corr_mouse_2: 0.2705378, val_corr_mouse_3: 0.2855534, val_corr_mouse_4: 0.3096637, val_corr_mouse_5: 0.2750711, val_corr_mouse_6: 0.2960735, val_corr_mouse_7: 0.322736, val_corr_mouse_8: 0.2937627, val_corr_mouse_9: 0.277513, val_corr: 0.2867223
+[2024-08-11 01:41:52,436][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-008-0.286722.pth'
+[2024-08-11 01:41:52,437][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-007-0.285183.pth'
+[2024-08-11 02:09:37,274][INFO]: train - epoch: 9, lr: 0.00075428, train_loss: -527021.1
+[2024-08-11 02:10:45,359][INFO]: val - epoch: 9, val_loss: -525505.1, val_corr_mouse_0: 0.2390182, val_corr_mouse_1: 0.2991721, val_corr_mouse_2: 0.2708564, val_corr_mouse_3: 0.2867704, val_corr_mouse_4: 0.3098714, val_corr_mouse_5: 0.2759712, val_corr_mouse_6: 0.2969191, val_corr_mouse_7: 0.324547, val_corr_mouse_8: 0.2955845, val_corr_mouse_9: 0.2784869, val_corr: 0.2877197
+[2024-08-11 02:10:45,987][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-009-0.287720.pth'
+[2024-08-11 02:10:45,987][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-008-0.286722.pth'
+[2024-08-11 02:38:30,844][INFO]: train - epoch: 10, lr: 0.00060426, train_loss: -530996.8
+[2024-08-11 02:39:39,204][INFO]: val - epoch: 10, val_loss: -526097.5, val_corr_mouse_0: 0.2390089, val_corr_mouse_1: 0.3002255, val_corr_mouse_2: 0.271857, val_corr_mouse_3: 0.2880583, val_corr_mouse_4: 0.3100028, val_corr_mouse_5: 0.277226, val_corr_mouse_6: 0.2984866, val_corr_mouse_7: 0.3260649, val_corr_mouse_8: 0.2974287, val_corr_mouse_9: 0.2789804, val_corr: 0.2887339
+[2024-08-11 02:39:39,831][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-010-0.288734.pth'
+[2024-08-11 02:39:39,832][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-009-0.287720.pth'
+[2024-08-11 03:07:24,018][INFO]: train - epoch: 11, lr: 0.0004635, train_loss: -537265.9
+[2024-08-11 03:08:31,980][INFO]: val - epoch: 11, val_loss: -526871.8, val_corr_mouse_0: 0.2395826, val_corr_mouse_1: 0.3008013, val_corr_mouse_2: 0.2726764, val_corr_mouse_3: 0.288655, val_corr_mouse_4: 0.3103149, val_corr_mouse_5: 0.2782884, val_corr_mouse_6: 0.2993717, val_corr_mouse_7: 0.3273317, val_corr_mouse_8: 0.2987357, val_corr_mouse_9: 0.2796936, val_corr: 0.2895451
+[2024-08-11 03:08:32,601][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-011-0.289545.pth'
+[2024-08-11 03:08:32,602][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-010-0.288734.pth'
+[2024-08-11 03:36:20,397][INFO]: train - epoch: 12, lr: 0.00033628, train_loss: -545149.7
+[2024-08-11 03:37:28,557][INFO]: val - epoch: 12, val_loss: -527476.7, val_corr_mouse_0: 0.2390072, val_corr_mouse_1: 0.3009673, val_corr_mouse_2: 0.2726052, val_corr_mouse_3: 0.2891393, val_corr_mouse_4: 0.3103845, val_corr_mouse_5: 0.279472, val_corr_mouse_6: 0.3004162, val_corr_mouse_7: 0.3281517, val_corr_mouse_8: 0.2996023, val_corr_mouse_9: 0.2801268, val_corr: 0.2899872
+[2024-08-11 03:37:29,232][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-012-0.289987.pth'
+[2024-08-11 03:37:29,233][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-011-0.289545.pth'
+[2024-08-11 04:05:14,462][INFO]: train - epoch: 13, lr: 0.00022645, train_loss: -549115
+[2024-08-11 04:06:22,887][INFO]: val - epoch: 13, val_loss: -528086.5, val_corr_mouse_0: 0.2398074, val_corr_mouse_1: 0.3013546, val_corr_mouse_2: 0.2736322, val_corr_mouse_3: 0.2887431, val_corr_mouse_4: 0.3112019, val_corr_mouse_5: 0.2802415, val_corr_mouse_6: 0.3018319, val_corr_mouse_7: 0.3289648, val_corr_mouse_8: 0.3009107, val_corr_mouse_9: 0.2810395, val_corr: 0.2907728
+[2024-08-11 04:06:23,516][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-013-0.290773.pth'
+[2024-08-11 04:06:23,517][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-012-0.289987.pth'
+[2024-08-11 04:34:15,088][INFO]: train - epoch: 14, lr: 0.00013737, train_loss: -556350.4
+[2024-08-11 04:35:23,914][INFO]: val - epoch: 14, val_loss: -528497.8, val_corr_mouse_0: 0.2393162, val_corr_mouse_1: 0.3018444, val_corr_mouse_2: 0.2741191, val_corr_mouse_3: 0.2883899, val_corr_mouse_4: 0.3111451, val_corr_mouse_5: 0.2807977, val_corr_mouse_6: 0.30251, val_corr_mouse_7: 0.3295811, val_corr_mouse_8: 0.301973, val_corr_mouse_9: 0.2819852, val_corr: 0.2911662
+[2024-08-11 04:35:24,584][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-014-0.291166.pth'
+[2024-08-11 04:35:24,584][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-013-0.290773.pth'
+[2024-08-11 05:03:09,694][INFO]: train - epoch: 15, lr: 7.1734e-05, train_loss: -558189.4
+[2024-08-11 05:04:18,635][INFO]: val - epoch: 15, val_loss: -528713.5, val_corr_mouse_0: 0.2393705, val_corr_mouse_1: 0.302272, val_corr_mouse_2: 0.2747453, val_corr_mouse_3: 0.2886325, val_corr_mouse_4: 0.3108315, val_corr_mouse_5: 0.281382, val_corr_mouse_6: 0.3028467, val_corr_mouse_7: 0.3300122, val_corr_mouse_8: 0.3023752, val_corr_mouse_9: 0.2825591, val_corr: 0.2915027
+[2024-08-11 05:04:19,281][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-015-0.291503.pth'
+[2024-08-11 05:04:19,282][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-014-0.291166.pth'
+[2024-08-11 05:32:08,830][INFO]: train - epoch: 16, lr: 3.1536e-05, train_loss: -561137.2
+[2024-08-11 05:33:17,031][INFO]: val - epoch: 16, val_loss: -528573, val_corr_mouse_0: 0.2393092, val_corr_mouse_1: 0.3023642, val_corr_mouse_2: 0.2744935, val_corr_mouse_3: 0.288229, val_corr_mouse_4: 0.3106224, val_corr_mouse_5: 0.2816494, val_corr_mouse_6: 0.3033157, val_corr_mouse_7: 0.330626, val_corr_mouse_8: 0.3029332, val_corr_mouse_9: 0.2825905, val_corr: 0.2916133
+[2024-08-11 05:33:17,759][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-016-0.291613.pth'
+[2024-08-11 05:33:17,760][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-015-0.291503.pth'
+[2024-08-11 06:01:08,093][INFO]: train - epoch: 17, lr: 1.8e-05, train_loss: -562082.2
+[2024-08-11 06:02:15,859][INFO]: val - epoch: 17, val_loss: -528999.1, val_corr_mouse_0: 0.2394058, val_corr_mouse_1: 0.3026415, val_corr_mouse_2: 0.2749245, val_corr_mouse_3: 0.2885369, val_corr_mouse_4: 0.3109664, val_corr_mouse_5: 0.2818913, val_corr_mouse_6: 0.3035443, val_corr_mouse_7: 0.3308572, val_corr_mouse_8: 0.3031893, val_corr_mouse_9: 0.2825689, val_corr: 0.2918526
+[2024-08-11 06:02:16,481][INFO]: Model saved to 'data/experiments/true_batch_002/fold_3/model-017-0.291853.pth'
+[2024-08-11 06:02:16,482][INFO]: Model removed 'data/experiments/true_batch_002/fold_3/model-016-0.291613.pth'

data/experiments/true_batch_002/fold_3/model-017-0.291853.pth

@@ -0,0 +1 @@
+/annex/objects/MD5-s683073203--cb9b5ffcef1a138b103d89e2c173eada

data/experiments/true_batch_002/fold_4/log.csv

@@ -0,0 +1,23 @@
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 20:23:07.081853,0,0.0006,-211524.90528548192,-254176.3206464916,0.05188047,0.04359711,0.049231164,0.034293983,0.04136453,0.062261656,0.048150443,0.052500997,0.06338521,0.06241119,0.05090768
+2024-08-10 20:51:51.197098,1,0.0012,-349569.4714270836,-400974.47087738157,0.15814568,0.19271494,0.17735562,0.19380476,0.19149959,0.18408154,0.18405491,0.19799192,0.18962798,0.18362723,0.18529043
+2024-08-10 21:20:36.046387,2,0.0018,-409849.77164062473,-463626.04643935873,0.2005451,0.24202937,0.2247283,0.23874293,0.24787104,0.22602887,0.23338157,0.2565623,0.23774993,0.22734122,0.23349805
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 21:50:24.782591,0,0.0017865,-441770.1636718745,-486409.8988731415,0.21762338,0.26414767,0.24395435,0.2578326,0.2721128,0.24408467,0.25434944,0.28300944,0.25850308,0.2456382,0.25412557
+2024-08-10 22:19:04.960999,1,0.0017463,-460837.931109375,-499061.95071445114,0.22516637,0.27609977,0.2531711,0.26790586,0.28553575,0.25322336,0.26653576,0.29583105,0.26872107,0.25569943,0.26478893
+2024-08-10 22:47:46.430527,2,0.0016806,-475549.7319635403,-507741.5851243027,0.23089798,0.28254777,0.25675806,0.27428985,0.29402566,0.25961396,0.27561763,0.30424103,0.2759558,0.262308,0.27162558
+2024-08-10 23:16:29.199683,3,0.0015915,-484194.43940104236,-512364.3283573421,0.23384173,0.28709522,0.26033252,0.27815166,0.29760605,0.26452607,0.28142285,0.30990973,0.27994213,0.26597637,0.27588043
+2024-08-10 23:45:10.997131,4,0.0014817,-490332.36015625024,-516747.1960385686,0.23708247,0.29057202,0.26465368,0.28241435,0.3019749,0.26771215,0.28545398,0.31431124,0.28340498,0.2697987,0.27973786
+2024-08-11 00:13:52.735788,5,0.0013545,-498957.9897552085,-519080.0857342009,0.23734538,0.2935584,0.26570475,0.28325683,0.30436036,0.26871547,0.28823498,0.3162827,0.28699833,0.27287853,0.28173357
+2024-08-11 00:42:35.819481,6,0.0012137,-505423.3455156251,-520897.8271375465,0.23862813,0.29504946,0.26700816,0.2848481,0.30548406,0.27068332,0.29081732,0.31783316,0.2896988,0.2738787,0.2833929
+2024-08-11 01:11:19.710062,7,0.0010637,-511707.3419687502,-522281.4461547397,0.23849443,0.29676953,0.26776594,0.2856853,0.30673954,0.27168664,0.29313594,0.32047406,0.29176578,0.2748304,0.28473473
+2024-08-11 01:40:01.700742,8,0.000909,-519891.4043645823,-523184.7943192378,0.23894948,0.29659015,0.269273,0.28649348,0.308182,0.27291974,0.29502147,0.32306176,0.29294446,0.27645454,0.285989
+2024-08-11 02:08:45.292419,9,0.00075428,-524095.77290104114,-524492.7261849439,0.2401348,0.29839808,0.2696922,0.28751287,0.30852255,0.27455673,0.2960474,0.32411,0.29527628,0.2776394,0.287189
+2024-08-11 02:37:27.093802,10,0.00060426,-531726.3863125005,-525527.3275441447,0.23995832,0.2982711,0.27069804,0.28799558,0.3098898,0.27608293,0.29747263,0.3259782,0.29626712,0.27881426,0.2881428
+2024-08-11 03:06:12.207921,11,0.0004635,-539623.9470624983,-526366.7531947028,0.24027933,0.29923356,0.27187744,0.28842515,0.31007263,0.27719948,0.29909006,0.32771513,0.29797325,0.27972862,0.28915948
+2024-08-11 03:35:00.156044,12,0.00033628,-545831.0913749996,-527191.6374012547,0.240309,0.29941833,0.27168462,0.28784725,0.31026,0.2785848,0.3003701,0.32934353,0.2990566,0.28065687,0.2897531
+2024-08-11 04:03:43.524383,13,0.00022645,-548487.2329062488,-527605.7287697496,0.24032265,0.29948682,0.2721522,0.28792295,0.31048408,0.27936617,0.30097458,0.3303338,0.29972914,0.28132695,0.29020995
+2024-08-11 04:32:28.182746,14,0.00013737,-553188.6202187501,-527715.2568831319,0.23996061,0.29917538,0.27198982,0.2880767,0.31025657,0.2799322,0.30147207,0.3306856,0.29979423,0.28246066,0.2903804
+2024-08-11 05:01:13.817950,15,7.1734e-05,-558060.2418020844,-527859.7514521376,0.23930702,0.29918507,0.27258155,0.28796583,0.31039828,0.28007478,0.30258158,0.33118737,0.30080998,0.28287947,0.29069704
+2024-08-11 05:29:59.618723,16,3.1536e-05,-559005.7175937508,-528180.494133364,0.23936333,0.29923657,0.27311978,0.2882713,0.31046343,0.28068936,0.30313945,0.3312319,0.30120936,0.28298688,0.29097116
+2024-08-11 05:58:47.950254,17,1.8e-05,-562280.5660937483,-528427.8397711429,0.2394542,0.29941726,0.27347642,0.288276,0.3106945,0.2806709,0.3031371,0.33142218,0.30150676,0.28297427,0.29110295

data/experiments/true_batch_002/fold_4/log.txt

@@ -0,0 +1,77 @@
+[2024-08-10 20:21:55,003][INFO]: train - epoch: 0, lr: 0.0006, train_loss: -211524.9
+[2024-08-10 20:23:07,081][INFO]: val - epoch: 0, val_loss: -254176.3, val_corr_mouse_0: 0.05188047, val_corr_mouse_1: 0.04359711, val_corr_mouse_2: 0.04923116, val_corr_mouse_3: 0.03429398, val_corr_mouse_4: 0.04136453, val_corr_mouse_5: 0.06226166, val_corr_mouse_6: 0.04815044, val_corr_mouse_7: 0.052501, val_corr_mouse_8: 0.06338521, val_corr_mouse_9: 0.06241119, val_corr: 0.05090768
+[2024-08-10 20:50:42,179][INFO]: train - epoch: 1, lr: 0.0012, train_loss: -349569.5
+[2024-08-10 20:51:51,196][INFO]: val - epoch: 1, val_loss: -400974.5, val_corr_mouse_0: 0.1581457, val_corr_mouse_1: 0.1927149, val_corr_mouse_2: 0.1773556, val_corr_mouse_3: 0.1938048, val_corr_mouse_4: 0.1914996, val_corr_mouse_5: 0.1840815, val_corr_mouse_6: 0.1840549, val_corr_mouse_7: 0.1979919, val_corr_mouse_8: 0.189628, val_corr_mouse_9: 0.1836272, val_corr: 0.1852904
+[2024-08-10 21:19:26,819][INFO]: train - epoch: 2, lr: 0.0018, train_loss: -409849.8
+[2024-08-10 21:20:36,046][INFO]: val - epoch: 2, val_loss: -463626, val_corr_mouse_0: 0.2005451, val_corr_mouse_1: 0.2420294, val_corr_mouse_2: 0.2247283, val_corr_mouse_3: 0.2387429, val_corr_mouse_4: 0.247871, val_corr_mouse_5: 0.2260289, val_corr_mouse_6: 0.2333816, val_corr_mouse_7: 0.2565623, val_corr_mouse_8: 0.2377499, val_corr_mouse_9: 0.2273412, val_corr: 0.2334981
+[2024-08-10 21:49:17,371][INFO]: train - epoch: 0, lr: 0.0017865, train_loss: -441770.2
+[2024-08-10 21:50:24,736][INFO]: val - epoch: 0, val_loss: -486409.9, val_corr_mouse_0: 0.2176234, val_corr_mouse_1: 0.2641477, val_corr_mouse_2: 0.2439543, val_corr_mouse_3: 0.2578326, val_corr_mouse_4: 0.2721128, val_corr_mouse_5: 0.2440847, val_corr_mouse_6: 0.2543494, val_corr_mouse_7: 0.2830094, val_corr_mouse_8: 0.2585031, val_corr_mouse_9: 0.2456382, val_corr: 0.2541256
+[2024-08-10 21:50:25,467][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-000-0.254126.pth'
+[2024-08-10 22:17:57,476][INFO]: train - epoch: 1, lr: 0.0017463, train_loss: -460837.9
+[2024-08-10 22:19:04,915][INFO]: val - epoch: 1, val_loss: -499062, val_corr_mouse_0: 0.2251664, val_corr_mouse_1: 0.2760998, val_corr_mouse_2: 0.2531711, val_corr_mouse_3: 0.2679059, val_corr_mouse_4: 0.2855358, val_corr_mouse_5: 0.2532234, val_corr_mouse_6: 0.2665358, val_corr_mouse_7: 0.2958311, val_corr_mouse_8: 0.2687211, val_corr_mouse_9: 0.2556994, val_corr: 0.2647889
+[2024-08-10 22:19:05,663][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-001-0.264789.pth'
+[2024-08-10 22:19:05,664][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-000-0.254126.pth'
+[2024-08-10 22:46:39,365][INFO]: train - epoch: 2, lr: 0.0016806, train_loss: -475549.7
+[2024-08-10 22:47:46,429][INFO]: val - epoch: 2, val_loss: -507741.6, val_corr_mouse_0: 0.230898, val_corr_mouse_1: 0.2825478, val_corr_mouse_2: 0.2567581, val_corr_mouse_3: 0.2742898, val_corr_mouse_4: 0.2940257, val_corr_mouse_5: 0.259614, val_corr_mouse_6: 0.2756176, val_corr_mouse_7: 0.304241, val_corr_mouse_8: 0.2759558, val_corr_mouse_9: 0.262308, val_corr: 0.2716256
+[2024-08-10 22:47:47,071][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-002-0.271626.pth'
+[2024-08-10 22:47:47,072][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-001-0.264789.pth'
+[2024-08-10 23:15:22,270][INFO]: train - epoch: 3, lr: 0.0015915, train_loss: -484194.4
+[2024-08-10 23:16:29,160][INFO]: val - epoch: 3, val_loss: -512364.3, val_corr_mouse_0: 0.2338417, val_corr_mouse_1: 0.2870952, val_corr_mouse_2: 0.2603325, val_corr_mouse_3: 0.2781517, val_corr_mouse_4: 0.2976061, val_corr_mouse_5: 0.2645261, val_corr_mouse_6: 0.2814229, val_corr_mouse_7: 0.3099097, val_corr_mouse_8: 0.2799421, val_corr_mouse_9: 0.2659764, val_corr: 0.2758804
+[2024-08-10 23:16:29,965][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-003-0.275880.pth'
+[2024-08-10 23:16:29,965][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-002-0.271626.pth'
+[2024-08-10 23:44:04,880][INFO]: train - epoch: 4, lr: 0.0014817, train_loss: -490332.4
+[2024-08-10 23:45:10,996][INFO]: val - epoch: 4, val_loss: -516747.2, val_corr_mouse_0: 0.2370825, val_corr_mouse_1: 0.290572, val_corr_mouse_2: 0.2646537, val_corr_mouse_3: 0.2824143, val_corr_mouse_4: 0.3019749, val_corr_mouse_5: 0.2677121, val_corr_mouse_6: 0.285454, val_corr_mouse_7: 0.3143112, val_corr_mouse_8: 0.283405, val_corr_mouse_9: 0.2697987, val_corr: 0.2797379
+[2024-08-10 23:45:11,678][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-004-0.279738.pth'
+[2024-08-10 23:45:11,679][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-003-0.275880.pth'
+[2024-08-11 00:12:45,889][INFO]: train - epoch: 5, lr: 0.0013545, train_loss: -498958
+[2024-08-11 00:13:52,735][INFO]: val - epoch: 5, val_loss: -519080.1, val_corr_mouse_0: 0.2373454, val_corr_mouse_1: 0.2935584, val_corr_mouse_2: 0.2657048, val_corr_mouse_3: 0.2832568, val_corr_mouse_4: 0.3043604, val_corr_mouse_5: 0.2687155, val_corr_mouse_6: 0.288235, val_corr_mouse_7: 0.3162827, val_corr_mouse_8: 0.2869983, val_corr_mouse_9: 0.2728785, val_corr: 0.2817336
+[2024-08-11 00:13:53,467][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-005-0.281734.pth'
+[2024-08-11 00:13:53,468][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-004-0.279738.pth'
+[2024-08-11 00:41:28,204][INFO]: train - epoch: 6, lr: 0.0012137, train_loss: -505423.3
+[2024-08-11 00:42:35,818][INFO]: val - epoch: 6, val_loss: -520897.8, val_corr_mouse_0: 0.2386281, val_corr_mouse_1: 0.2950495, val_corr_mouse_2: 0.2670082, val_corr_mouse_3: 0.2848481, val_corr_mouse_4: 0.3054841, val_corr_mouse_5: 0.2706833, val_corr_mouse_6: 0.2908173, val_corr_mouse_7: 0.3178332, val_corr_mouse_8: 0.2896988, val_corr_mouse_9: 0.2738787, val_corr: 0.2833929
+[2024-08-11 00:42:36,515][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-006-0.283393.pth'
+[2024-08-11 00:42:36,516][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-005-0.281734.pth'
+[2024-08-11 01:10:12,795][INFO]: train - epoch: 7, lr: 0.0010637, train_loss: -511707.3
+[2024-08-11 01:11:19,709][INFO]: val - epoch: 7, val_loss: -522281.4, val_corr_mouse_0: 0.2384944, val_corr_mouse_1: 0.2967695, val_corr_mouse_2: 0.2677659, val_corr_mouse_3: 0.2856853, val_corr_mouse_4: 0.3067395, val_corr_mouse_5: 0.2716866, val_corr_mouse_6: 0.2931359, val_corr_mouse_7: 0.3204741, val_corr_mouse_8: 0.2917658, val_corr_mouse_9: 0.2748304, val_corr: 0.2847347
+[2024-08-11 01:11:20,320][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-007-0.284735.pth'
+[2024-08-11 01:11:20,321][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-006-0.283393.pth'
+[2024-08-11 01:38:55,037][INFO]: train - epoch: 8, lr: 0.000909, train_loss: -519891.4
+[2024-08-11 01:40:01,700][INFO]: val - epoch: 8, val_loss: -523184.8, val_corr_mouse_0: 0.2389495, val_corr_mouse_1: 0.2965901, val_corr_mouse_2: 0.269273, val_corr_mouse_3: 0.2864935, val_corr_mouse_4: 0.308182, val_corr_mouse_5: 0.2729197, val_corr_mouse_6: 0.2950215, val_corr_mouse_7: 0.3230618, val_corr_mouse_8: 0.2929445, val_corr_mouse_9: 0.2764545, val_corr: 0.285989
+[2024-08-11 01:40:02,319][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-008-0.285989.pth'
+[2024-08-11 01:40:02,320][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-007-0.284735.pth'
+[2024-08-11 02:07:38,036][INFO]: train - epoch: 9, lr: 0.00075428, train_loss: -524095.8
+[2024-08-11 02:08:45,291][INFO]: val - epoch: 9, val_loss: -524492.7, val_corr_mouse_0: 0.2401348, val_corr_mouse_1: 0.2983981, val_corr_mouse_2: 0.2696922, val_corr_mouse_3: 0.2875129, val_corr_mouse_4: 0.3085226, val_corr_mouse_5: 0.2745567, val_corr_mouse_6: 0.2960474, val_corr_mouse_7: 0.32411, val_corr_mouse_8: 0.2952763, val_corr_mouse_9: 0.2776394, val_corr: 0.287189
+[2024-08-11 02:08:45,906][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-009-0.287189.pth'
+[2024-08-11 02:08:45,907][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-008-0.285989.pth'
+[2024-08-11 02:36:20,371][INFO]: train - epoch: 10, lr: 0.00060426, train_loss: -531726.4
+[2024-08-11 02:37:27,093][INFO]: val - epoch: 10, val_loss: -525527.3, val_corr_mouse_0: 0.2399583, val_corr_mouse_1: 0.2982711, val_corr_mouse_2: 0.270698, val_corr_mouse_3: 0.2879956, val_corr_mouse_4: 0.3098898, val_corr_mouse_5: 0.2760829, val_corr_mouse_6: 0.2974726, val_corr_mouse_7: 0.3259782, val_corr_mouse_8: 0.2962671, val_corr_mouse_9: 0.2788143, val_corr: 0.2881428
+[2024-08-11 02:37:27,728][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-010-0.288143.pth'
+[2024-08-11 02:37:27,729][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-009-0.287189.pth'
+[2024-08-11 03:05:04,737][INFO]: train - epoch: 11, lr: 0.0004635, train_loss: -539623.9
+[2024-08-11 03:06:12,207][INFO]: val - epoch: 11, val_loss: -526366.8, val_corr_mouse_0: 0.2402793, val_corr_mouse_1: 0.2992336, val_corr_mouse_2: 0.2718774, val_corr_mouse_3: 0.2884251, val_corr_mouse_4: 0.3100726, val_corr_mouse_5: 0.2771995, val_corr_mouse_6: 0.2990901, val_corr_mouse_7: 0.3277151, val_corr_mouse_8: 0.2979732, val_corr_mouse_9: 0.2797286, val_corr: 0.2891595
+[2024-08-11 03:06:12,856][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-011-0.289159.pth'
+[2024-08-11 03:06:12,857][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-010-0.288143.pth'
+[2024-08-11 03:33:53,035][INFO]: train - epoch: 12, lr: 0.00033628, train_loss: -545831.1
+[2024-08-11 03:35:00,155][INFO]: val - epoch: 12, val_loss: -527191.6, val_corr_mouse_0: 0.240309, val_corr_mouse_1: 0.2994183, val_corr_mouse_2: 0.2716846, val_corr_mouse_3: 0.2878473, val_corr_mouse_4: 0.31026, val_corr_mouse_5: 0.2785848, val_corr_mouse_6: 0.3003701, val_corr_mouse_7: 0.3293435, val_corr_mouse_8: 0.2990566, val_corr_mouse_9: 0.2806569, val_corr: 0.2897531
+[2024-08-11 03:35:00,858][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-012-0.289753.pth'
+[2024-08-11 03:35:00,858][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-011-0.289159.pth'
+[2024-08-11 04:02:36,352][INFO]: train - epoch: 13, lr: 0.00022645, train_loss: -548487.2
+[2024-08-11 04:03:43,523][INFO]: val - epoch: 13, val_loss: -527605.7, val_corr_mouse_0: 0.2403226, val_corr_mouse_1: 0.2994868, val_corr_mouse_2: 0.2721522, val_corr_mouse_3: 0.2879229, val_corr_mouse_4: 0.3104841, val_corr_mouse_5: 0.2793662, val_corr_mouse_6: 0.3009746, val_corr_mouse_7: 0.3303338, val_corr_mouse_8: 0.2997291, val_corr_mouse_9: 0.2813269, val_corr: 0.2902099
+[2024-08-11 04:03:44,133][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-013-0.290210.pth'
+[2024-08-11 04:03:44,133][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-012-0.289753.pth'
+[2024-08-11 04:31:21,089][INFO]: train - epoch: 14, lr: 0.00013737, train_loss: -553188.6
+[2024-08-11 04:32:28,182][INFO]: val - epoch: 14, val_loss: -527715.3, val_corr_mouse_0: 0.2399606, val_corr_mouse_1: 0.2991754, val_corr_mouse_2: 0.2719898, val_corr_mouse_3: 0.2880767, val_corr_mouse_4: 0.3102566, val_corr_mouse_5: 0.2799322, val_corr_mouse_6: 0.3014721, val_corr_mouse_7: 0.3306856, val_corr_mouse_8: 0.2997942, val_corr_mouse_9: 0.2824607, val_corr: 0.2903804
+[2024-08-11 04:32:28,798][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-014-0.290380.pth'
+[2024-08-11 04:32:28,799][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-013-0.290210.pth'
+[2024-08-11 05:00:06,841][INFO]: train - epoch: 15, lr: 7.1734e-05, train_loss: -558060.2
+[2024-08-11 05:01:13,817][INFO]: val - epoch: 15, val_loss: -527859.8, val_corr_mouse_0: 0.239307, val_corr_mouse_1: 0.2991851, val_corr_mouse_2: 0.2725815, val_corr_mouse_3: 0.2879658, val_corr_mouse_4: 0.3103983, val_corr_mouse_5: 0.2800748, val_corr_mouse_6: 0.3025816, val_corr_mouse_7: 0.3311874, val_corr_mouse_8: 0.30081, val_corr_mouse_9: 0.2828795, val_corr: 0.290697
+[2024-08-11 05:01:14,450][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-015-0.290697.pth'
+[2024-08-11 05:01:14,451][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-014-0.290380.pth'
+[2024-08-11 05:28:52,741][INFO]: train - epoch: 16, lr: 3.1536e-05, train_loss: -559005.7
+[2024-08-11 05:29:59,618][INFO]: val - epoch: 16, val_loss: -528180.5, val_corr_mouse_0: 0.2393633, val_corr_mouse_1: 0.2992366, val_corr_mouse_2: 0.2731198, val_corr_mouse_3: 0.2882713, val_corr_mouse_4: 0.3104634, val_corr_mouse_5: 0.2806894, val_corr_mouse_6: 0.3031394, val_corr_mouse_7: 0.3312319, val_corr_mouse_8: 0.3012094, val_corr_mouse_9: 0.2829869, val_corr: 0.2909712
+[2024-08-11 05:30:00,250][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-016-0.290971.pth'
+[2024-08-11 05:30:00,251][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-015-0.290697.pth'
+[2024-08-11 05:57:39,986][INFO]: train - epoch: 17, lr: 1.8e-05, train_loss: -562280.6
+[2024-08-11 05:58:47,949][INFO]: val - epoch: 17, val_loss: -528427.8, val_corr_mouse_0: 0.2394542, val_corr_mouse_1: 0.2994173, val_corr_mouse_2: 0.2734764, val_corr_mouse_3: 0.288276, val_corr_mouse_4: 0.3106945, val_corr_mouse_5: 0.2806709, val_corr_mouse_6: 0.3031371, val_corr_mouse_7: 0.3314222, val_corr_mouse_8: 0.3015068, val_corr_mouse_9: 0.2829743, val_corr: 0.2911029
+[2024-08-11 05:58:48,589][INFO]: Model saved to 'data/experiments/true_batch_002/fold_4/model-017-0.291103.pth'
+[2024-08-11 05:58:48,590][INFO]: Model removed 'data/experiments/true_batch_002/fold_4/model-016-0.290971.pth'

data/experiments/true_batch_002/fold_4/model-017-0.291103.pth

@@ -0,0 +1 @@
+/annex/objects/MD5-s683073203--e8d78718d1f43b57724823d9d79d5ec7

data/experiments/true_batch_002/fold_5/log.csv

@@ -0,0 +1,23 @@
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 20:23:05.561416,0,0.0006,-209466.76060986295,-259170.43745280532,0.0538681,0.04498461,0.056552853,0.042546578,0.04507382,0.06775033,0.05121694,0.059123565,0.06996018,0.06644741,0.055752434
+2024-08-10 20:51:46.757646,1,0.0012,-352894.4083229158,-401938.6854960503,0.15982367,0.19372012,0.17962594,0.19539307,0.195416,0.18462545,0.1861334,0.20153528,0.18960802,0.18506594,0.18709469
+2024-08-10 21:20:33.465031,2,0.0018,-416778.4068854158,-464664.0386559014,0.20198187,0.24416189,0.22733691,0.23989242,0.24812332,0.2275495,0.23605892,0.25999847,0.23931377,0.22832294,0.235274
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-10 21:50:29.719780,0,0.0017865,-444193.7170104169,-487726.50548908004,0.21871537,0.2669426,0.2431386,0.25971088,0.2724399,0.24589022,0.25737685,0.28439856,0.25793087,0.24736585,0.25539097
+2024-08-10 22:19:18.932378,1,0.0017463,-462214.5890208324,-501318.50145213766,0.22648373,0.2781843,0.25167343,0.26926,0.28667128,0.25510317,0.2692951,0.29631108,0.26979467,0.2577094,0.2660486
+2024-08-10 22:48:11.545930,2,0.0016806,-474870.6747291666,-508819.7281598514,0.2304336,0.28346962,0.25717923,0.27539408,0.29470918,0.260522,0.278219,0.30444336,0.2773193,0.2634719,0.27251613
+2024-08-10 23:17:05.068707,3,0.0015915,-484696.8786250001,-514646.9749651487,0.23265004,0.28837973,0.2601456,0.279347,0.2994862,0.26494184,0.28401813,0.31034747,0.2822721,0.2685069,0.27700952
+2024-08-10 23:45:55.870429,4,0.0014817,-491605.28157291654,-517851.9474326212,0.2355604,0.29097036,0.26259664,0.28170094,0.30271748,0.2678215,0.28645894,0.31419832,0.28541622,0.27118134,0.2798622
+2024-08-11 00:14:38.874484,5,0.0013545,-501395.75880208274,-520299.4776951673,0.23752868,0.29283005,0.26558062,0.2844605,0.30513465,0.27051434,0.2894271,0.3172181,0.2889462,0.27329242,0.28249326
+2024-08-11 00:43:33.311219,6,0.0012137,-507883.28578124975,-521961.57580157986,0.23845625,0.29437208,0.26638284,0.2864276,0.3064099,0.27131772,0.29220417,0.3193092,0.29086348,0.2744949,0.28402382
+2024-08-11 01:12:29.633450,7,0.0010637,-514583.8499895832,-523031.1178264403,0.23858985,0.29550886,0.26645005,0.28653985,0.30730438,0.2726713,0.2940889,0.32243136,0.2922195,0.27589393,0.28516978
+2024-08-11 01:41:25.826198,8,0.000909,-519037.6471093739,-524565.1860769051,0.2399532,0.29651433,0.26806298,0.2874244,0.30870864,0.27588087,0.2958955,0.32462606,0.29483452,0.27752042,0.2869421
+2024-08-11 02:09:33.832709,9,0.00075428,-526139.3926145823,-525617.4568134291,0.2398507,0.29720658,0.2693681,0.28797477,0.31043723,0.27687898,0.29707968,0.32569376,0.2968463,0.27882773,0.28801638
+2024-08-11 02:38:00.896901,10,0.00060426,-531147.6428854183,-526506.098687267,0.2392108,0.29852152,0.27018547,0.28837577,0.31063128,0.2782201,0.2980537,0.32795358,0.29789904,0.28005043,0.28891015
+2024-08-11 03:06:23.849120,11,0.0004635,-541534.4800729176,-527061.8607980947,0.23936088,0.29889908,0.27109092,0.2887091,0.3112813,0.27921405,0.2994941,0.32873222,0.29935268,0.2803477,0.2896482
+2024-08-11 03:34:48.929199,12,0.00033628,-547246.6284166685,-528030.8128775562,0.24030188,0.29912624,0.2711585,0.28955477,0.3121042,0.28031164,0.30075163,0.3298567,0.30039647,0.2810751,0.2904637
+2024-08-11 04:03:44.180653,13,0.00022645,-551243.9814999987,-528503.6331029276,0.24009751,0.29958025,0.2718224,0.28907973,0.31237662,0.28149277,0.30124182,0.33083215,0.30122444,0.28233385,0.29100817
+2024-08-11 04:32:38.911626,14,0.00013737,-555015.967666667,-528697.6173472351,0.23987772,0.29928252,0.27183115,0.28981906,0.31172606,0.28143212,0.30186552,0.3309513,0.30183113,0.28311324,0.291173
+2024-08-11 05:01:34.994719,15,7.1734e-05,-557065.6102291676,-528857.3140392662,0.2405891,0.29996738,0.27202153,0.29005507,0.31186694,0.28207618,0.3022963,0.33132377,0.30221996,0.28314722,0.2915563
+2024-08-11 05:30:32.101593,16,3.1536e-05,-562224.8860000008,-528982.4508887081,0.24049743,0.2999359,0.27208954,0.28945017,0.3116232,0.28233105,0.30262727,0.33189878,0.30267373,0.28348762,0.29166147
+2024-08-11 05:59:32.048742,17,1.8e-05,-562118.0363020832,-529117.0859374999,0.2406528,0.29975912,0.27197945,0.2897569,0.31141883,0.28250864,0.30288517,0.33188048,0.302841,0.28366163,0.2917344

data/experiments/true_batch_002/fold_5/log.txt

@@ -0,0 +1,77 @@
+[2024-08-10 20:21:54,794][INFO]: train - epoch: 0, lr: 0.0006, train_loss: -209466.8
+[2024-08-10 20:23:05,560][INFO]: val - epoch: 0, val_loss: -259170.4, val_corr_mouse_0: 0.0538681, val_corr_mouse_1: 0.04498461, val_corr_mouse_2: 0.05655285, val_corr_mouse_3: 0.04254658, val_corr_mouse_4: 0.04507382, val_corr_mouse_5: 0.06775033, val_corr_mouse_6: 0.05121694, val_corr_mouse_7: 0.05912356, val_corr_mouse_8: 0.06996018, val_corr_mouse_9: 0.06644741, val_corr: 0.05575243
+[2024-08-10 20:50:37,432][INFO]: train - epoch: 1, lr: 0.0012, train_loss: -352894.4
+[2024-08-10 20:51:46,756][INFO]: val - epoch: 1, val_loss: -401938.7, val_corr_mouse_0: 0.1598237, val_corr_mouse_1: 0.1937201, val_corr_mouse_2: 0.1796259, val_corr_mouse_3: 0.1953931, val_corr_mouse_4: 0.195416, val_corr_mouse_5: 0.1846254, val_corr_mouse_6: 0.1861334, val_corr_mouse_7: 0.2015353, val_corr_mouse_8: 0.189608, val_corr_mouse_9: 0.1850659, val_corr: 0.1870947
+[2024-08-10 21:19:23,462][INFO]: train - epoch: 2, lr: 0.0018, train_loss: -416778.4
+[2024-08-10 21:20:33,464][INFO]: val - epoch: 2, val_loss: -464664, val_corr_mouse_0: 0.2019819, val_corr_mouse_1: 0.2441619, val_corr_mouse_2: 0.2273369, val_corr_mouse_3: 0.2398924, val_corr_mouse_4: 0.2481233, val_corr_mouse_5: 0.2275495, val_corr_mouse_6: 0.2360589, val_corr_mouse_7: 0.2599985, val_corr_mouse_8: 0.2393138, val_corr_mouse_9: 0.2283229, val_corr: 0.235274
+[2024-08-10 21:49:21,704][INFO]: train - epoch: 0, lr: 0.0017865, train_loss: -444193.7
+[2024-08-10 21:50:29,719][INFO]: val - epoch: 0, val_loss: -487726.5, val_corr_mouse_0: 0.2187154, val_corr_mouse_1: 0.2669426, val_corr_mouse_2: 0.2431386, val_corr_mouse_3: 0.2597109, val_corr_mouse_4: 0.2724399, val_corr_mouse_5: 0.2458902, val_corr_mouse_6: 0.2573768, val_corr_mouse_7: 0.2843986, val_corr_mouse_8: 0.2579309, val_corr_mouse_9: 0.2473658, val_corr: 0.255391
+[2024-08-10 21:50:30,427][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-000-0.255391.pth'
+[2024-08-10 22:18:10,979][INFO]: train - epoch: 1, lr: 0.0017463, train_loss: -462214.6
+[2024-08-10 22:19:18,932][INFO]: val - epoch: 1, val_loss: -501318.5, val_corr_mouse_0: 0.2264837, val_corr_mouse_1: 0.2781843, val_corr_mouse_2: 0.2516734, val_corr_mouse_3: 0.26926, val_corr_mouse_4: 0.2866713, val_corr_mouse_5: 0.2551032, val_corr_mouse_6: 0.2692951, val_corr_mouse_7: 0.2963111, val_corr_mouse_8: 0.2697947, val_corr_mouse_9: 0.2577094, val_corr: 0.2660486
+[2024-08-10 22:19:19,585][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-001-0.266049.pth'
+[2024-08-10 22:19:19,585][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-000-0.255391.pth'
+[2024-08-10 22:47:03,254][INFO]: train - epoch: 2, lr: 0.0016806, train_loss: -474870.7
+[2024-08-10 22:48:11,545][INFO]: val - epoch: 2, val_loss: -508819.7, val_corr_mouse_0: 0.2304336, val_corr_mouse_1: 0.2834696, val_corr_mouse_2: 0.2571792, val_corr_mouse_3: 0.2753941, val_corr_mouse_4: 0.2947092, val_corr_mouse_5: 0.260522, val_corr_mouse_6: 0.278219, val_corr_mouse_7: 0.3044434, val_corr_mouse_8: 0.2773193, val_corr_mouse_9: 0.2634719, val_corr: 0.2725161
+[2024-08-10 22:48:12,226][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-002-0.272516.pth'
+[2024-08-10 22:48:12,227][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-001-0.266049.pth'
+[2024-08-10 23:15:57,374][INFO]: train - epoch: 3, lr: 0.0015915, train_loss: -484696.9
+[2024-08-10 23:17:05,068][INFO]: val - epoch: 3, val_loss: -514647, val_corr_mouse_0: 0.23265, val_corr_mouse_1: 0.2883797, val_corr_mouse_2: 0.2601456, val_corr_mouse_3: 0.279347, val_corr_mouse_4: 0.2994862, val_corr_mouse_5: 0.2649418, val_corr_mouse_6: 0.2840181, val_corr_mouse_7: 0.3103475, val_corr_mouse_8: 0.2822721, val_corr_mouse_9: 0.2685069, val_corr: 0.2770095
+[2024-08-10 23:17:05,723][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-003-0.277010.pth'
+[2024-08-10 23:17:05,724][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-002-0.272516.pth'
+[2024-08-10 23:44:47,817][INFO]: train - epoch: 4, lr: 0.0014817, train_loss: -491605.3
+[2024-08-10 23:45:55,870][INFO]: val - epoch: 4, val_loss: -517851.9, val_corr_mouse_0: 0.2355604, val_corr_mouse_1: 0.2909704, val_corr_mouse_2: 0.2625966, val_corr_mouse_3: 0.2817009, val_corr_mouse_4: 0.3027175, val_corr_mouse_5: 0.2678215, val_corr_mouse_6: 0.2864589, val_corr_mouse_7: 0.3141983, val_corr_mouse_8: 0.2854162, val_corr_mouse_9: 0.2711813, val_corr: 0.2798622
+[2024-08-10 23:45:56,542][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-004-0.279862.pth'
+[2024-08-10 23:45:56,542][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-003-0.277010.pth'
+[2024-08-11 00:13:30,110][INFO]: train - epoch: 5, lr: 0.0013545, train_loss: -501395.8
+[2024-08-11 00:14:38,874][INFO]: val - epoch: 5, val_loss: -520299.5, val_corr_mouse_0: 0.2375287, val_corr_mouse_1: 0.29283, val_corr_mouse_2: 0.2655806, val_corr_mouse_3: 0.2844605, val_corr_mouse_4: 0.3051347, val_corr_mouse_5: 0.2705143, val_corr_mouse_6: 0.2894271, val_corr_mouse_7: 0.3172181, val_corr_mouse_8: 0.2889462, val_corr_mouse_9: 0.2732924, val_corr: 0.2824933
+[2024-08-11 00:14:39,509][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-005-0.282493.pth'
+[2024-08-11 00:14:39,510][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-004-0.279862.pth'
+[2024-08-11 00:42:25,243][INFO]: train - epoch: 6, lr: 0.0012137, train_loss: -507883.3
+[2024-08-11 00:43:33,311][INFO]: val - epoch: 6, val_loss: -521961.6, val_corr_mouse_0: 0.2384562, val_corr_mouse_1: 0.2943721, val_corr_mouse_2: 0.2663828, val_corr_mouse_3: 0.2864276, val_corr_mouse_4: 0.3064099, val_corr_mouse_5: 0.2713177, val_corr_mouse_6: 0.2922042, val_corr_mouse_7: 0.3193092, val_corr_mouse_8: 0.2908635, val_corr_mouse_9: 0.2744949, val_corr: 0.2840238
+[2024-08-11 00:43:33,976][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-006-0.284024.pth'
+[2024-08-11 00:43:33,977][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-005-0.282493.pth'
+[2024-08-11 01:11:20,999][INFO]: train - epoch: 7, lr: 0.0010637, train_loss: -514583.8
+[2024-08-11 01:12:29,632][INFO]: val - epoch: 7, val_loss: -523031.1, val_corr_mouse_0: 0.2385899, val_corr_mouse_1: 0.2955089, val_corr_mouse_2: 0.26645, val_corr_mouse_3: 0.2865399, val_corr_mouse_4: 0.3073044, val_corr_mouse_5: 0.2726713, val_corr_mouse_6: 0.2940889, val_corr_mouse_7: 0.3224314, val_corr_mouse_8: 0.2922195, val_corr_mouse_9: 0.2758939, val_corr: 0.2851698
+[2024-08-11 01:12:30,251][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-007-0.285170.pth'
+[2024-08-11 01:12:30,252][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-006-0.284024.pth'
+[2024-08-11 01:40:16,857][INFO]: train - epoch: 8, lr: 0.000909, train_loss: -519037.6
+[2024-08-11 01:41:25,825][INFO]: val - epoch: 8, val_loss: -524565.2, val_corr_mouse_0: 0.2399532, val_corr_mouse_1: 0.2965143, val_corr_mouse_2: 0.268063, val_corr_mouse_3: 0.2874244, val_corr_mouse_4: 0.3087086, val_corr_mouse_5: 0.2758809, val_corr_mouse_6: 0.2958955, val_corr_mouse_7: 0.3246261, val_corr_mouse_8: 0.2948345, val_corr_mouse_9: 0.2775204, val_corr: 0.2869421
+[2024-08-11 01:41:26,531][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-008-0.286942.pth'
+[2024-08-11 01:41:26,532][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-007-0.285170.pth'
+[2024-08-11 02:08:25,998][INFO]: train - epoch: 9, lr: 0.00075428, train_loss: -526139.4
+[2024-08-11 02:09:33,832][INFO]: val - epoch: 9, val_loss: -525617.5, val_corr_mouse_0: 0.2398507, val_corr_mouse_1: 0.2972066, val_corr_mouse_2: 0.2693681, val_corr_mouse_3: 0.2879748, val_corr_mouse_4: 0.3104372, val_corr_mouse_5: 0.276879, val_corr_mouse_6: 0.2970797, val_corr_mouse_7: 0.3256938, val_corr_mouse_8: 0.2968463, val_corr_mouse_9: 0.2788277, val_corr: 0.2880164
+[2024-08-11 02:09:34,482][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-009-0.288016.pth'
+[2024-08-11 02:09:34,482][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-008-0.286942.pth'
+[2024-08-11 02:36:53,169][INFO]: train - epoch: 10, lr: 0.00060426, train_loss: -531147.6
+[2024-08-11 02:38:00,896][INFO]: val - epoch: 10, val_loss: -526506.1, val_corr_mouse_0: 0.2392108, val_corr_mouse_1: 0.2985215, val_corr_mouse_2: 0.2701855, val_corr_mouse_3: 0.2883758, val_corr_mouse_4: 0.3106313, val_corr_mouse_5: 0.2782201, val_corr_mouse_6: 0.2980537, val_corr_mouse_7: 0.3279536, val_corr_mouse_8: 0.297899, val_corr_mouse_9: 0.2800504, val_corr: 0.2889102
+[2024-08-11 02:38:01,503][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-010-0.288910.pth'
+[2024-08-11 02:38:01,504][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-009-0.288016.pth'
+[2024-08-11 03:05:15,618][INFO]: train - epoch: 11, lr: 0.0004635, train_loss: -541534.5
+[2024-08-11 03:06:23,848][INFO]: val - epoch: 11, val_loss: -527061.9, val_corr_mouse_0: 0.2393609, val_corr_mouse_1: 0.2988991, val_corr_mouse_2: 0.2710909, val_corr_mouse_3: 0.2887091, val_corr_mouse_4: 0.3112813, val_corr_mouse_5: 0.2792141, val_corr_mouse_6: 0.2994941, val_corr_mouse_7: 0.3287322, val_corr_mouse_8: 0.2993527, val_corr_mouse_9: 0.2803477, val_corr: 0.2896482
+[2024-08-11 03:06:24,513][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-011-0.289648.pth'
+[2024-08-11 03:06:24,514][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-010-0.288910.pth'
+[2024-08-11 03:33:40,911][INFO]: train - epoch: 12, lr: 0.00033628, train_loss: -547246.6
+[2024-08-11 03:34:48,928][INFO]: val - epoch: 12, val_loss: -528030.8, val_corr_mouse_0: 0.2403019, val_corr_mouse_1: 0.2991262, val_corr_mouse_2: 0.2711585, val_corr_mouse_3: 0.2895548, val_corr_mouse_4: 0.3121042, val_corr_mouse_5: 0.2803116, val_corr_mouse_6: 0.3007516, val_corr_mouse_7: 0.3298567, val_corr_mouse_8: 0.3003965, val_corr_mouse_9: 0.2810751, val_corr: 0.2904637
+[2024-08-11 03:34:49,553][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-012-0.290464.pth'
+[2024-08-11 03:34:49,554][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-011-0.289648.pth'
+[2024-08-11 04:02:36,217][INFO]: train - epoch: 13, lr: 0.00022645, train_loss: -551244
+[2024-08-11 04:03:44,180][INFO]: val - epoch: 13, val_loss: -528503.6, val_corr_mouse_0: 0.2400975, val_corr_mouse_1: 0.2995802, val_corr_mouse_2: 0.2718224, val_corr_mouse_3: 0.2890797, val_corr_mouse_4: 0.3123766, val_corr_mouse_5: 0.2814928, val_corr_mouse_6: 0.3012418, val_corr_mouse_7: 0.3308322, val_corr_mouse_8: 0.3012244, val_corr_mouse_9: 0.2823339, val_corr: 0.2910082
+[2024-08-11 04:03:44,897][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-013-0.291008.pth'
+[2024-08-11 04:03:44,898][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-012-0.290464.pth'
+[2024-08-11 04:31:30,819][INFO]: train - epoch: 14, lr: 0.00013737, train_loss: -555016
+[2024-08-11 04:32:38,911][INFO]: val - epoch: 14, val_loss: -528697.6, val_corr_mouse_0: 0.2398777, val_corr_mouse_1: 0.2992825, val_corr_mouse_2: 0.2718312, val_corr_mouse_3: 0.2898191, val_corr_mouse_4: 0.3117261, val_corr_mouse_5: 0.2814321, val_corr_mouse_6: 0.3018655, val_corr_mouse_7: 0.3309513, val_corr_mouse_8: 0.3018311, val_corr_mouse_9: 0.2831132, val_corr: 0.291173
+[2024-08-11 04:32:39,557][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-014-0.291173.pth'
+[2024-08-11 04:32:39,558][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-013-0.291008.pth'
+[2024-08-11 05:00:26,842][INFO]: train - epoch: 15, lr: 7.1734e-05, train_loss: -557065.6
+[2024-08-11 05:01:34,994][INFO]: val - epoch: 15, val_loss: -528857.3, val_corr_mouse_0: 0.2405891, val_corr_mouse_1: 0.2999674, val_corr_mouse_2: 0.2720215, val_corr_mouse_3: 0.2900551, val_corr_mouse_4: 0.3118669, val_corr_mouse_5: 0.2820762, val_corr_mouse_6: 0.3022963, val_corr_mouse_7: 0.3313238, val_corr_mouse_8: 0.30222, val_corr_mouse_9: 0.2831472, val_corr: 0.2915563
+[2024-08-11 05:01:35,628][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-015-0.291556.pth'
+[2024-08-11 05:01:35,629][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-014-0.291173.pth'
+[2024-08-11 05:29:23,399][INFO]: train - epoch: 16, lr: 3.1536e-05, train_loss: -562224.9
+[2024-08-11 05:30:32,101][INFO]: val - epoch: 16, val_loss: -528982.5, val_corr_mouse_0: 0.2404974, val_corr_mouse_1: 0.2999359, val_corr_mouse_2: 0.2720895, val_corr_mouse_3: 0.2894502, val_corr_mouse_4: 0.3116232, val_corr_mouse_5: 0.282331, val_corr_mouse_6: 0.3026273, val_corr_mouse_7: 0.3318988, val_corr_mouse_8: 0.3026737, val_corr_mouse_9: 0.2834876, val_corr: 0.2916615
+[2024-08-11 05:30:32,802][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-016-0.291661.pth'
+[2024-08-11 05:30:32,803][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-015-0.291556.pth'
+[2024-08-11 05:58:23,616][INFO]: train - epoch: 17, lr: 1.8e-05, train_loss: -562118
+[2024-08-11 05:59:32,048][INFO]: val - epoch: 17, val_loss: -529117.1, val_corr_mouse_0: 0.2406528, val_corr_mouse_1: 0.2997591, val_corr_mouse_2: 0.2719795, val_corr_mouse_3: 0.2897569, val_corr_mouse_4: 0.3114188, val_corr_mouse_5: 0.2825086, val_corr_mouse_6: 0.3028852, val_corr_mouse_7: 0.3318805, val_corr_mouse_8: 0.302841, val_corr_mouse_9: 0.2836616, val_corr: 0.2917344
+[2024-08-11 05:59:32,666][INFO]: Model saved to 'data/experiments/true_batch_002/fold_5/model-017-0.291734.pth'
+[2024-08-11 05:59:32,667][INFO]: Model removed 'data/experiments/true_batch_002/fold_5/model-016-0.291661.pth'

data/experiments/true_batch_002/fold_5/model-017-0.291734.pth

@@ -0,0 +1 @@
+/annex/objects/MD5-s683073203--ba2c9c2058b418a2af876ff8163bfc9a

data/experiments/true_batch_002/fold_6/log.csv

@@ -0,0 +1,23 @@
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-11 06:21:31.509189,0,0.0006,-209360.9370445963,-251804.34310742916,0.05283319,0.042753737,0.051521428,0.03605264,0.04396081,0.05961714,0.04743705,0.05922406,0.06554778,0.06836244,0.052731026
+2024-08-11 06:50:16.432069,1,0.0012,-350388.7793177086,-401502.3529275094,0.1581684,0.19274445,0.17628685,0.19481795,0.1936388,0.18475609,0.1865251,0.20174922,0.18747893,0.18564986,0.18618156
+2024-08-11 07:19:05.152929,2,0.0018,-412683.9470677087,-463144.42139579484,0.20018032,0.242965,0.22553942,0.23923928,0.24751894,0.22657198,0.23519759,0.25991794,0.23816384,0.2270359,0.23423305
+time,epoch,lr,train_loss,val_loss,val_corr_mouse_0,val_corr_mouse_1,val_corr_mouse_2,val_corr_mouse_3,val_corr_mouse_4,val_corr_mouse_5,val_corr_mouse_6,val_corr_mouse_7,val_corr_mouse_8,val_corr_mouse_9,val_corr
+2024-08-11 07:49:02.403250,0,0.0017865,-443871.03065624955,-487345.53434305306,0.21842273,0.2656082,0.24359809,0.25885144,0.2727116,0.24370661,0.25498617,0.28497496,0.25962725,0.246124,0.25486112
+2024-08-11 08:17:50.122866,1,0.0017463,-463173.08287500055,-499559.1425418216,0.22611713,0.27669185,0.25101668,0.2696255,0.285724,0.25454098,0.26754585,0.29676485,0.27054334,0.25601515,0.26545852
+2024-08-11 08:46:37.905218,2,0.0016806,-473502.4823385412,-508393.2253427046,0.23175365,0.28345495,0.25653824,0.27517873,0.2941452,0.26069933,0.27737918,0.30561677,0.27662122,0.26356423,0.27249512
+2024-08-11 09:15:26.895781,3,0.0015915,-485375.31017708394,-513485.6228508362,0.23397465,0.28905436,0.2608696,0.27920488,0.29861638,0.26361343,0.28376624,0.3110012,0.2817431,0.26658762,0.27684313
+2024-08-11 09:44:16.127686,4,0.0014817,-494283.2493437503,-517006.000029043,0.23616767,0.29158026,0.26422125,0.28165495,0.3013743,0.26713938,0.28744626,0.31391868,0.28631148,0.2704286,0.28002426
+2024-08-11 10:13:07.789640,5,0.0013545,-501525.04970833345,-520110.20591310394,0.23721625,0.2939899,0.266617,0.28333282,0.30402264,0.2699954,0.29053998,0.3169924,0.2892311,0.2740092,0.28259465
+2024-08-11 10:42:42.770545,6,0.0012137,-508432.59533333377,-521723.64835037163,0.23829845,0.2948411,0.26816905,0.2839588,0.30628827,0.27167496,0.2925375,0.31943125,0.2917431,0.27470082,0.28416434
+2024-08-11 11:12:40.438845,7,0.0010637,-515348.1095416663,-522548.3426754183,0.238756,0.29683557,0.26880887,0.2854631,0.30616623,0.27325448,0.29317668,0.32116717,0.29285863,0.27561632,0.2852103
+2024-08-11 11:42:36.995150,8,0.000909,-519409.2073958331,-524022.35623838304,0.23897706,0.29756212,0.27124655,0.28749064,0.30753058,0.2747028,0.2957963,0.32297808,0.2948427,0.27693117,0.2868058
+2024-08-11 12:11:55.270997,9,0.00075428,-525566.8616249986,-524911.7586256971,0.23966338,0.29896346,0.27124605,0.28777838,0.30922347,0.27534953,0.2970433,0.3246071,0.29689813,0.27846506,0.28792378
+2024-08-11 12:40:41.548170,10,0.00060426,-533562.7218541688,-525781.3815927053,0.23944917,0.29980716,0.27222982,0.2895506,0.3097351,0.27661055,0.29793036,0.32569492,0.2971885,0.2795231,0.28877193
+2024-08-11 13:09:28.548224,11,0.0004635,-538476.9483645819,-526975.6559595726,0.24028242,0.29997128,0.27269837,0.29030496,0.31041437,0.27765617,0.29962227,0.3276785,0.2986768,0.28065878,0.28979638
+2024-08-11 13:38:11.983225,12,0.00033628,-544858.9851249992,-527612.2167170073,0.2398146,0.29993528,0.27390403,0.29050332,0.31056306,0.27841583,0.3008913,0.32948688,0.30001608,0.28143436,0.29049647
+2024-08-11 14:06:57.998027,13,0.00022645,-551561.4681354169,-527848.0277939127,0.23852946,0.30026135,0.27441195,0.29110807,0.3106026,0.2798401,0.30155295,0.3296109,0.30139318,0.2819835,0.2909294
+2024-08-11 14:35:43.919349,14,0.00013737,-554185.4790937495,-528174.8873722118,0.2391154,0.30007926,0.274391,0.29059765,0.31000918,0.2804359,0.30241546,0.33024424,0.3025887,0.2822514,0.2912128
+2024-08-11 15:04:27.869836,15,7.1734e-05,-559456.462458334,-528683.8332074811,0.23900855,0.30049866,0.2747424,0.29105124,0.31013957,0.28097057,0.30339938,0.33085746,0.30323884,0.2828888,0.29167956
+2024-08-11 15:33:12.972570,16,3.1536e-05,-561614.2009479168,-528746.0820167284,0.23879695,0.30113167,0.27445894,0.29148293,0.31033137,0.2811162,0.30377406,0.33141187,0.30384177,0.28305787,0.2919404
+2024-08-11 16:02:00.016072,17,1.8e-05,-563026.3260104168,-528694.202486059,0.23855731,0.30083176,0.2745545,0.29121903,0.31063867,0.28117183,0.30378953,0.33158046,0.30411318,0.2832873,0.29197437

data/experiments/true_batch_002/fold_6/log.txt

@@ -0,0 +1,77 @@
+[2024-08-11 06:20:21,250][INFO]: train - epoch: 0, lr: 0.0006, train_loss: -209360.9
+[2024-08-11 06:21:31,508][INFO]: val - epoch: 0, val_loss: -251804.3, val_corr_mouse_0: 0.05283319, val_corr_mouse_1: 0.04275374, val_corr_mouse_2: 0.05152143, val_corr_mouse_3: 0.03605264, val_corr_mouse_4: 0.04396081, val_corr_mouse_5: 0.05961714, val_corr_mouse_6: 0.04743705, val_corr_mouse_7: 0.05922406, val_corr_mouse_8: 0.06554778, val_corr_mouse_9: 0.06836244, val_corr: 0.05273103
+[2024-08-11 06:49:06,495][INFO]: train - epoch: 1, lr: 0.0012, train_loss: -350388.8
+[2024-08-11 06:50:16,431][INFO]: val - epoch: 1, val_loss: -401502.4, val_corr_mouse_0: 0.1581684, val_corr_mouse_1: 0.1927444, val_corr_mouse_2: 0.1762868, val_corr_mouse_3: 0.1948179, val_corr_mouse_4: 0.1936388, val_corr_mouse_5: 0.1847561, val_corr_mouse_6: 0.1865251, val_corr_mouse_7: 0.2017492, val_corr_mouse_8: 0.1874789, val_corr_mouse_9: 0.1856499, val_corr: 0.1861816
+[2024-08-11 07:17:53,944][INFO]: train - epoch: 2, lr: 0.0018, train_loss: -412683.9
+[2024-08-11 07:19:05,152][INFO]: val - epoch: 2, val_loss: -463144.4, val_corr_mouse_0: 0.2001803, val_corr_mouse_1: 0.242965, val_corr_mouse_2: 0.2255394, val_corr_mouse_3: 0.2392393, val_corr_mouse_4: 0.2475189, val_corr_mouse_5: 0.226572, val_corr_mouse_6: 0.2351976, val_corr_mouse_7: 0.2599179, val_corr_mouse_8: 0.2381638, val_corr_mouse_9: 0.2270359, val_corr: 0.2342331
+[2024-08-11 07:47:52,911][INFO]: train - epoch: 0, lr: 0.0017865, train_loss: -443871
+[2024-08-11 07:49:02,402][INFO]: val - epoch: 0, val_loss: -487345.5, val_corr_mouse_0: 0.2184227, val_corr_mouse_1: 0.2656082, val_corr_mouse_2: 0.2435981, val_corr_mouse_3: 0.2588514, val_corr_mouse_4: 0.2727116, val_corr_mouse_5: 0.2437066, val_corr_mouse_6: 0.2549862, val_corr_mouse_7: 0.284975, val_corr_mouse_8: 0.2596273, val_corr_mouse_9: 0.246124, val_corr: 0.2548611
+[2024-08-11 07:49:02,922][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-000-0.254861.pth'
+[2024-08-11 08:16:40,049][INFO]: train - epoch: 1, lr: 0.0017463, train_loss: -463173.1
+[2024-08-11 08:17:50,122][INFO]: val - epoch: 1, val_loss: -499559.1, val_corr_mouse_0: 0.2261171, val_corr_mouse_1: 0.2766919, val_corr_mouse_2: 0.2510167, val_corr_mouse_3: 0.2696255, val_corr_mouse_4: 0.285724, val_corr_mouse_5: 0.254541, val_corr_mouse_6: 0.2675458, val_corr_mouse_7: 0.2967649, val_corr_mouse_8: 0.2705433, val_corr_mouse_9: 0.2560152, val_corr: 0.2654585
+[2024-08-11 08:17:50,646][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-001-0.265459.pth'
+[2024-08-11 08:17:50,647][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-000-0.254861.pth'
+[2024-08-11 08:45:27,488][INFO]: train - epoch: 2, lr: 0.0016806, train_loss: -473502.5
+[2024-08-11 08:46:37,904][INFO]: val - epoch: 2, val_loss: -508393.2, val_corr_mouse_0: 0.2317536, val_corr_mouse_1: 0.283455, val_corr_mouse_2: 0.2565382, val_corr_mouse_3: 0.2751787, val_corr_mouse_4: 0.2941452, val_corr_mouse_5: 0.2606993, val_corr_mouse_6: 0.2773792, val_corr_mouse_7: 0.3056168, val_corr_mouse_8: 0.2766212, val_corr_mouse_9: 0.2635642, val_corr: 0.2724951
+[2024-08-11 08:46:38,429][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-002-0.272495.pth'
+[2024-08-11 08:46:38,430][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-001-0.265459.pth'
+[2024-08-11 09:14:18,012][INFO]: train - epoch: 3, lr: 0.0015915, train_loss: -485375.3
+[2024-08-11 09:15:26,895][INFO]: val - epoch: 3, val_loss: -513485.6, val_corr_mouse_0: 0.2339747, val_corr_mouse_1: 0.2890544, val_corr_mouse_2: 0.2608696, val_corr_mouse_3: 0.2792049, val_corr_mouse_4: 0.2986164, val_corr_mouse_5: 0.2636134, val_corr_mouse_6: 0.2837662, val_corr_mouse_7: 0.3110012, val_corr_mouse_8: 0.2817431, val_corr_mouse_9: 0.2665876, val_corr: 0.2768431
+[2024-08-11 09:15:27,500][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-003-0.276843.pth'
+[2024-08-11 09:15:27,500][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-002-0.272495.pth'
+[2024-08-11 09:43:06,763][INFO]: train - epoch: 4, lr: 0.0014817, train_loss: -494283.2
+[2024-08-11 09:44:16,127][INFO]: val - epoch: 4, val_loss: -517006, val_corr_mouse_0: 0.2361677, val_corr_mouse_1: 0.2915803, val_corr_mouse_2: 0.2642213, val_corr_mouse_3: 0.281655, val_corr_mouse_4: 0.3013743, val_corr_mouse_5: 0.2671394, val_corr_mouse_6: 0.2874463, val_corr_mouse_7: 0.3139187, val_corr_mouse_8: 0.2863115, val_corr_mouse_9: 0.2704286, val_corr: 0.2800243
+[2024-08-11 09:44:16,659][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-004-0.280024.pth'
+[2024-08-11 09:44:16,660][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-003-0.276843.pth'
+[2024-08-11 10:11:58,293][INFO]: train - epoch: 5, lr: 0.0013545, train_loss: -501525
+[2024-08-11 10:13:07,789][INFO]: val - epoch: 5, val_loss: -520110.2, val_corr_mouse_0: 0.2372162, val_corr_mouse_1: 0.2939899, val_corr_mouse_2: 0.266617, val_corr_mouse_3: 0.2833328, val_corr_mouse_4: 0.3040226, val_corr_mouse_5: 0.2699954, val_corr_mouse_6: 0.29054, val_corr_mouse_7: 0.3169924, val_corr_mouse_8: 0.2892311, val_corr_mouse_9: 0.2740092, val_corr: 0.2825947
+[2024-08-11 10:13:08,314][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-005-0.282595.pth'
+[2024-08-11 10:13:08,315][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-004-0.280024.pth'
+[2024-08-11 10:41:30,553][INFO]: train - epoch: 6, lr: 0.0012137, train_loss: -508432.6
+[2024-08-11 10:42:42,769][INFO]: val - epoch: 6, val_loss: -521723.6, val_corr_mouse_0: 0.2382984, val_corr_mouse_1: 0.2948411, val_corr_mouse_2: 0.268169, val_corr_mouse_3: 0.2839588, val_corr_mouse_4: 0.3062883, val_corr_mouse_5: 0.271675, val_corr_mouse_6: 0.2925375, val_corr_mouse_7: 0.3194312, val_corr_mouse_8: 0.2917431, val_corr_mouse_9: 0.2747008, val_corr: 0.2841643
+[2024-08-11 10:42:43,290][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-006-0.284164.pth'
+[2024-08-11 10:42:43,291][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-005-0.282595.pth'
+[2024-08-11 11:11:29,025][INFO]: train - epoch: 7, lr: 0.0010637, train_loss: -515348.1
+[2024-08-11 11:12:40,438][INFO]: val - epoch: 7, val_loss: -522548.3, val_corr_mouse_0: 0.238756, val_corr_mouse_1: 0.2968356, val_corr_mouse_2: 0.2688089, val_corr_mouse_3: 0.2854631, val_corr_mouse_4: 0.3061662, val_corr_mouse_5: 0.2732545, val_corr_mouse_6: 0.2931767, val_corr_mouse_7: 0.3211672, val_corr_mouse_8: 0.2928586, val_corr_mouse_9: 0.2756163, val_corr: 0.2852103
+[2024-08-11 11:12:40,963][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-007-0.285210.pth'
+[2024-08-11 11:12:40,965][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-006-0.284164.pth'
+[2024-08-11 11:41:27,208][INFO]: train - epoch: 8, lr: 0.000909, train_loss: -519409.2
+[2024-08-11 11:42:36,994][INFO]: val - epoch: 8, val_loss: -524022.4, val_corr_mouse_0: 0.2389771, val_corr_mouse_1: 0.2975621, val_corr_mouse_2: 0.2712466, val_corr_mouse_3: 0.2874906, val_corr_mouse_4: 0.3075306, val_corr_mouse_5: 0.2747028, val_corr_mouse_6: 0.2957963, val_corr_mouse_7: 0.3229781, val_corr_mouse_8: 0.2948427, val_corr_mouse_9: 0.2769312, val_corr: 0.2868058
+[2024-08-11 11:42:37,573][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-008-0.286806.pth'
+[2024-08-11 11:42:37,574][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-007-0.285210.pth'
+[2024-08-11 12:10:44,741][INFO]: train - epoch: 9, lr: 0.00075428, train_loss: -525566.9
+[2024-08-11 12:11:55,270][INFO]: val - epoch: 9, val_loss: -524911.8, val_corr_mouse_0: 0.2396634, val_corr_mouse_1: 0.2989635, val_corr_mouse_2: 0.271246, val_corr_mouse_3: 0.2877784, val_corr_mouse_4: 0.3092235, val_corr_mouse_5: 0.2753495, val_corr_mouse_6: 0.2970433, val_corr_mouse_7: 0.3246071, val_corr_mouse_8: 0.2968981, val_corr_mouse_9: 0.2784651, val_corr: 0.2879238
+[2024-08-11 12:11:55,981][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-009-0.287924.pth'
+[2024-08-11 12:11:55,982][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-008-0.286806.pth'
+[2024-08-11 12:39:32,427][INFO]: train - epoch: 10, lr: 0.00060426, train_loss: -533562.7
+[2024-08-11 12:40:41,547][INFO]: val - epoch: 10, val_loss: -525781.4, val_corr_mouse_0: 0.2394492, val_corr_mouse_1: 0.2998072, val_corr_mouse_2: 0.2722298, val_corr_mouse_3: 0.2895506, val_corr_mouse_4: 0.3097351, val_corr_mouse_5: 0.2766106, val_corr_mouse_6: 0.2979304, val_corr_mouse_7: 0.3256949, val_corr_mouse_8: 0.2971885, val_corr_mouse_9: 0.2795231, val_corr: 0.2887719
+[2024-08-11 12:40:42,209][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-010-0.288772.pth'
+[2024-08-11 12:40:42,211][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-009-0.287924.pth'
+[2024-08-11 13:08:19,137][INFO]: train - epoch: 11, lr: 0.0004635, train_loss: -538476.9
+[2024-08-11 13:09:28,548][INFO]: val - epoch: 11, val_loss: -526975.7, val_corr_mouse_0: 0.2402824, val_corr_mouse_1: 0.2999713, val_corr_mouse_2: 0.2726984, val_corr_mouse_3: 0.290305, val_corr_mouse_4: 0.3104144, val_corr_mouse_5: 0.2776562, val_corr_mouse_6: 0.2996223, val_corr_mouse_7: 0.3276785, val_corr_mouse_8: 0.2986768, val_corr_mouse_9: 0.2806588, val_corr: 0.2897964
+[2024-08-11 13:09:29,083][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-011-0.289796.pth'
+[2024-08-11 13:09:29,084][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-010-0.288772.pth'
+[2024-08-11 13:37:03,268][INFO]: train - epoch: 12, lr: 0.00033628, train_loss: -544859
+[2024-08-11 13:38:11,983][INFO]: val - epoch: 12, val_loss: -527612.2, val_corr_mouse_0: 0.2398146, val_corr_mouse_1: 0.2999353, val_corr_mouse_2: 0.273904, val_corr_mouse_3: 0.2905033, val_corr_mouse_4: 0.3105631, val_corr_mouse_5: 0.2784158, val_corr_mouse_6: 0.3008913, val_corr_mouse_7: 0.3294869, val_corr_mouse_8: 0.3000161, val_corr_mouse_9: 0.2814344, val_corr: 0.2904965
+[2024-08-11 13:38:12,578][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-012-0.290496.pth'
+[2024-08-11 13:38:12,579][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-011-0.289796.pth'
+[2024-08-11 14:05:49,659][INFO]: train - epoch: 13, lr: 0.00022645, train_loss: -551561.5
+[2024-08-11 14:06:57,997][INFO]: val - epoch: 13, val_loss: -527848, val_corr_mouse_0: 0.2385295, val_corr_mouse_1: 0.3002613, val_corr_mouse_2: 0.2744119, val_corr_mouse_3: 0.2911081, val_corr_mouse_4: 0.3106026, val_corr_mouse_5: 0.2798401, val_corr_mouse_6: 0.301553, val_corr_mouse_7: 0.3296109, val_corr_mouse_8: 0.3013932, val_corr_mouse_9: 0.2819835, val_corr: 0.2909294
+[2024-08-11 14:06:58,642][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-013-0.290929.pth'
+[2024-08-11 14:06:58,643][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-012-0.290496.pth'
+[2024-08-11 14:34:36,710][INFO]: train - epoch: 14, lr: 0.00013737, train_loss: -554185.5
+[2024-08-11 14:35:43,919][INFO]: val - epoch: 14, val_loss: -528174.9, val_corr_mouse_0: 0.2391154, val_corr_mouse_1: 0.3000793, val_corr_mouse_2: 0.274391, val_corr_mouse_3: 0.2905976, val_corr_mouse_4: 0.3100092, val_corr_mouse_5: 0.2804359, val_corr_mouse_6: 0.3024155, val_corr_mouse_7: 0.3302442, val_corr_mouse_8: 0.3025887, val_corr_mouse_9: 0.2822514, val_corr: 0.2912128
+[2024-08-11 14:35:44,573][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-014-0.291213.pth'
+[2024-08-11 14:35:44,574][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-013-0.290929.pth'
+[2024-08-11 15:03:20,637][INFO]: train - epoch: 15, lr: 7.1734e-05, train_loss: -559456.5
+[2024-08-11 15:04:27,869][INFO]: val - epoch: 15, val_loss: -528683.8, val_corr_mouse_0: 0.2390085, val_corr_mouse_1: 0.3004987, val_corr_mouse_2: 0.2747424, val_corr_mouse_3: 0.2910512, val_corr_mouse_4: 0.3101396, val_corr_mouse_5: 0.2809706, val_corr_mouse_6: 0.3033994, val_corr_mouse_7: 0.3308575, val_corr_mouse_8: 0.3032388, val_corr_mouse_9: 0.2828888, val_corr: 0.2916796
+[2024-08-11 15:04:28,399][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-015-0.291680.pth'
+[2024-08-11 15:04:28,400][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-014-0.291213.pth'
+[2024-08-11 15:32:05,395][INFO]: train - epoch: 16, lr: 3.1536e-05, train_loss: -561614.2
+[2024-08-11 15:33:12,972][INFO]: val - epoch: 16, val_loss: -528746.1, val_corr_mouse_0: 0.2387969, val_corr_mouse_1: 0.3011317, val_corr_mouse_2: 0.2744589, val_corr_mouse_3: 0.2914829, val_corr_mouse_4: 0.3103314, val_corr_mouse_5: 0.2811162, val_corr_mouse_6: 0.3037741, val_corr_mouse_7: 0.3314119, val_corr_mouse_8: 0.3038418, val_corr_mouse_9: 0.2830579, val_corr: 0.2919404
+[2024-08-11 15:33:13,530][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-016-0.291940.pth'
+[2024-08-11 15:33:13,532][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-015-0.291680.pth'
+[2024-08-11 16:00:51,952][INFO]: train - epoch: 17, lr: 1.8e-05, train_loss: -563026.3
+[2024-08-11 16:02:00,015][INFO]: val - epoch: 17, val_loss: -528694.2, val_corr_mouse_0: 0.2385573, val_corr_mouse_1: 0.3008318, val_corr_mouse_2: 0.2745545, val_corr_mouse_3: 0.291219, val_corr_mouse_4: 0.3106387, val_corr_mouse_5: 0.2811718, val_corr_mouse_6: 0.3037895, val_corr_mouse_7: 0.3315805, val_corr_mouse_8: 0.3041132, val_corr_mouse_9: 0.2832873, val_corr: 0.2919744
+[2024-08-11 16:02:00,731][INFO]: Model saved to 'data/experiments/true_batch_002/fold_6/model-017-0.291974.pth'
+[2024-08-11 16:02:00,732][INFO]: Model removed 'data/experiments/true_batch_002/fold_6/model-016-0.291940.pth'

data/experiments/true_batch_002/fold_6/model-017-0.291974.pth

@@ -0,0 +1 @@
+/annex/objects/MD5-s683073203--7efcef68d6da760a18cf8befa5a64ac2

data/experiments/true_batch_002/train.py

@@ -0,0 +1,190 @@
+import time
+import copy
+import json
+import argparse
+from pathlib import Path
+from pprint import pprint
+from importlib.machinery import SourceFileLoader
+
+import torch
+from torch.utils.data import DataLoader
+
+from argus import load_model
+from argus.callbacks import (
+    LoggingToFile,
+    LoggingToCSV,
+    CosineAnnealingLR,
+    Checkpoint,
+    LambdaLR,
+)
+
+from src.datasets import TrainMouseVideoDataset, ValMouseVideoDataset, ConcatMiceVideoDataset
+from src.utils import get_lr, init_weights, get_best_model_path
+from src.responses import get_responses_processor
+from src.ema import ModelEma, EmaCheckpoint
+from src.inputs import get_inputs_processor
+from src.metrics import CorrelationMetric
+from src.indexes import IndexesGenerator
+from src.argus_models import MouseModel
+from src.data import get_mouse_data
+from src.mixers import CutMix
+from src import constants
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-e", "--experiment", required=True, type=str)
+    parser.add_argument("-f", "--folds", default="all", type=str)
+    return parser.parse_args()
+
+
+def train_mouse(config: dict, save_dir: Path, train_splits: list[str], val_splits: list[str]):
+    config = copy.deepcopy(config)
+    argus_params = config["argus_params"]
+
+    model = MouseModel(argus_params)
+
+    if config["init_weights"]:
+        print("Weight initialization")
+        init_weights(model.nn_module)
+
+    if config["ema_decay"]:
+        print("EMA decay:", config["ema_decay"])
+        model.model_ema = ModelEma(model.nn_module, decay=config["ema_decay"])
+        checkpoint_class = EmaCheckpoint
+    else:
+        checkpoint_class = Checkpoint
+
+    if "distill" in config:
+        distill_params = config["distill"]
+        distill_experiment_dir = constants.experiments_dir / distill_params["experiment"] / val_splits[0]
+        distill_model_path = get_best_model_path(distill_experiment_dir)
+        distill_model = load_model(distill_model_path, device=argus_params["device"])
+        distill_model.eval()
+        model.distill_model = distill_model.nn_module
+        model.distill_ratio = distill_params["ratio"]
+        print(f"Distillation model {str(distill_model_path)}, ratio {model.distill_ratio}")
+
+    indexes_generator = IndexesGenerator(**argus_params["frame_stack"])
+    inputs_processor = get_inputs_processor(*argus_params["inputs_processor"])
+    responses_processor = get_responses_processor(*argus_params["responses_processor"])
+
+    cutmix = CutMix(**config["cutmix"])
+    train_datasets = []
+    mouse_epoch_size = config["train_epoch_size"] // constants.num_mice
+    for mouse in constants.mice:
+        train_datasets += [
+            TrainMouseVideoDataset(
+                mouse_data=get_mouse_data(mouse=mouse, splits=train_splits),
+                indexes_generator=indexes_generator,
+                inputs_processor=inputs_processor,
+                responses_processor=responses_processor,
+                epoch_size=mouse_epoch_size,
+                mixer=cutmix,
+            )
+        ]
+    train_dataset = ConcatMiceVideoDataset(train_datasets)
+    print("Train dataset len:", len(train_dataset))
+    val_datasets = []
+    for mouse in constants.mice:
+        val_datasets += [
+            ValMouseVideoDataset(
+                mouse_data=get_mouse_data(mouse=mouse, splits=val_splits),
+                indexes_generator=indexes_generator,
+                inputs_processor=inputs_processor,
+                responses_processor=responses_processor,
+            )
+        ]
+    val_dataset = ConcatMiceVideoDataset(val_datasets)
+    print("Val dataset len:", len(val_dataset))
+
+    train_loader = DataLoader(
+        train_dataset,
+        batch_size=config["batch_size"],
+        num_workers=config["num_dataloader_workers"],
+        shuffle=True,
+    )
+    val_loader = DataLoader(
+        val_dataset,
+        batch_size=config["batch_size"] // argus_params["iter_size"],
+        num_workers=config["num_dataloader_workers"],
+        shuffle=False,
+    )
+
+    for num_epochs, stage in zip(config["num_epochs"], config["stages"]):
+        callbacks = [
+            LoggingToFile(save_dir / "log.txt", append=True),
+            LoggingToCSV(save_dir / "log.csv", append=True),
+        ]
+
+        num_iterations = (len(train_dataset) // config["batch_size"]) * num_epochs
+        if stage == "warmup":
+            callbacks += [
+                LambdaLR(lambda x: x / num_iterations,
+                         step_on_iteration=True),
+            ]
+        elif stage == "train":
+            checkpoint_format = "model-{epoch:03d}-{val_corr:.6f}.pth"
+            callbacks += [
+                checkpoint_class(save_dir, file_format=checkpoint_format, max_saves=1),
+                CosineAnnealingLR(
+                    T_max=num_iterations,
+                    eta_min=get_lr(config["min_base_lr"], config["batch_size"]),
+                    step_on_iteration=True,
+                ),
+            ]
+
+        metrics = [
+            CorrelationMetric(),
+        ]
+
+        model.fit(train_loader,
+                  val_loader=val_loader,
+                  num_epochs=num_epochs,
+                  callbacks=callbacks,
+                  metrics=metrics)
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    print("Experiment:", args.experiment)
+
+    config_path = constants.configs_dir / f"{args.experiment}.py"
+    if not config_path.exists():
+        raise RuntimeError(f"Config '{config_path}' is not exists")
+
+    train_config = SourceFileLoader(args.experiment, str(config_path)).load_module().config
+    print("Experiment config:")
+    pprint(train_config, sort_dicts=False)
+
+    experiment_dir = constants.experiments_dir / args.experiment
+    print("Experiment dir:", experiment_dir)
+    if not experiment_dir.exists():
+        experiment_dir.mkdir(parents=True, exist_ok=True)
+    else:
+        print(f"Folder '{experiment_dir}' already exists.")
+
+    with open(experiment_dir / "train.py", "w") as outfile:
+        outfile.write(open(__file__).read())
+
+    with open(experiment_dir / "config.json", "w") as outfile:
+        json.dump(train_config, outfile, indent=4)
+
+    if args.folds == "all":
+        folds_splits = constants.folds_splits
+    else:
+        folds_splits = [f"fold_{fold}" for fold in args.folds.split(",")]
+
+    for fold_split in folds_splits:
+        fold_experiment_dir = experiment_dir / fold_split
+
+        # val_folds_splits = [fold_split]
+        val_folds_splits = ['fold_0'] # in this version the held out data fold is always 0
+        train_folds_splits = sorted(set(constants.folds_splits) - set(val_folds_splits))
+
+        print(f"Val fold: {val_folds_splits}, train folds: {train_folds_splits}")
+        print(f"Fold experiment dir: {fold_experiment_dir}")
+        train_mouse(train_config, fold_experiment_dir, train_folds_splits, val_folds_splits)
+
+        torch.cuda.empty_cache()
+        time.sleep(12)

utils_reconstruction/gaussian_noise_movies.npz

@@ -0,0 +1 @@
+/annex/objects/MD5-s152409864--fafbef9d2d3c4ac2350c0902832fcf98

utils_reconstruction/grating_movies.npz

@@ -0,0 +1 @@
+/annex/objects/MD5-s15484392--d7b418afff392e5194c11ebf7c64071a

utils_reconstruction/grating_movies.tiff

@@ -0,0 +1 @@
+/annex/objects/MD5-s16598634--21f4ad012202eedde53ad376de4503e6

utils_reconstruction/grating_stim_hyperstack.tiff

@@ -0,0 +1 @@
+/annex/objects/MD5-s49145274--fe5db509c728822d1f8a49e3b98aef3c