{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n", "/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n", "/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n", "/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n", "/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n", "/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n", " np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n" ] } ], "source": [ "import deeplabcut as dlc\n", "from pathlib import Path\n", "from time import time as now" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Project setup" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Created \"/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/videos\"\n", "Created \"/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/labeled-data\"\n", "Created \"/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/training-datasets\"\n", "Created \"/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models\"\n", "Attempting to create a symbolic link of the video ...\n", "Created the symlink of /home/mouse/ks/Projects/whiskers-realtime/videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz to /home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/videos/S005-19_2020-09-05_run124520.npz\n", "/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/videos/S005-19_2020-09-05_run124520.npz\n", "Generated \"/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/config.yaml\"\n", "\n", "A new project with name S005-tips-Keisuke-2020-09-05 is created at /home/mouse/ks/Projects/whiskers-realtime/dlc_projects and a configurable file (config.yaml) is stored there. Change the parameters in this file to adapt to your project's needs.\n", " Once you have changed the configuration file, use the function 'extract_frames' to select frames for labeling.\n", ". [OPTIONAL] Use the function 'add_new_videos' to add new videos to your project (at any stage).\n" ] } ], "source": [ "cfg = dlc.create_new_project(\"S005-tips\", \"Keisuke\",\n", " [\"../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz\"],\n", " videotype=\".npz\", driver=\"numpy\")" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/config.yaml'" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "cfg" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Iteration 0" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Config file read successfully.\n", "Do you want to extract (perhaps additional) frames for video: /home/mouse/ks/Projects/whiskers-realtime/videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz ?\n", "yes/noyes\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "114it [00:00, 1136.55it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Extracting frames based on kmeans ...\n", "Kmeans-quantization based extracting of frames from 0.0 seconds to 37.84 seconds.\n", "Extracting and downsampling 3781 frames from the video.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "3781it [00:03, 987.93it/s] \n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Kmeans clustering ... (this might take a while)\n", "\n", "Frames were selected.\n", "You can now label the frames using the function 'label_frames' (if you extracted enough frames for all videos).\n" ] } ], "source": [ "dlc.extract_frames(cfg, driver=\"numpy\", cluster_resizewidth=640)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Image conversion\n", "\n", "8 bit images: $[0, 255]$ to $[12, 60]$" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "You can now check the labels, using 'check_labels' before proceeding. Then, you can use the function 'create_training_dataset' to create the training dataset.\n" ] } ], "source": [ "dlc.label_frames(cfg)" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Creating images with labels by Keisuke.\n", "They are stored in the following folder: /home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/labeled-data/S005-19_2020-09-05_run124520_labeled.\n", "If all the labels are ok, then use the function 'create_training_dataset' to create the training dataset!\n" ] } ], "source": [ "dlc.check_labels(cfg)" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!\n" ] }, { "data": { "text/plain": [ "[(0.95,\n", " 1,\n", " (array([ 6, 19, 2, 0, 3, 26, 9, 11, 5, 17, 23, 22, 10, 16, 4, 18, 28,\n", " 21, 20, 8, 27, 29, 7, 24, 1, 12, 15, 13]),\n", " array([14, 25])))]" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dlc.create_training_dataset(cfg)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Config:\n", "{'all_joints': [[0], [1], [2]],\n", " 'all_joints_names': ['Tip1', 'Tip2', 'Tip3'],\n", " 'batch_size': 1,\n", " 'bottomheight': 400,\n", " 'crop': True,\n", " 'crop_pad': 0,\n", " 'cropratio': 0.4,\n", " 'dataset': 'training-datasets/iteration-0/UnaugmentedDataSet_S005-tipsSep5/S005-tips_Keisuke95shuffle1.mat',\n", " 'dataset_type': 'default',\n", " 'deconvolutionstride': 2,\n", " 'deterministic': False,\n", " 'display_iters': 1000,\n", " 'fg_fraction': 0.25,\n", " 'global_scale': 0.8,\n", " 'init_weights': '/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt',\n", " 'intermediate_supervision': False,\n", " 'intermediate_supervision_layer': 12,\n", " 'leftwidth': 400,\n", " 'location_refinement': True,\n", " 'locref_huber_loss': True,\n", " 'locref_loss_weight': 0.05,\n", " 'locref_stdev': 7.2801,\n", " 'log_dir': 'log',\n", " 'max_input_size': 1500,\n", " 'mean_pixel': [123.68, 116.779, 103.939],\n", " 'metadataset': 'training-datasets/iteration-0/UnaugmentedDataSet_S005-tipsSep5/Documentation_data-S005-tips_95shuffle1.pickle',\n", " 'min_input_size': 64,\n", " 'minsize': 100,\n", " 'mirror': False,\n", " 'multi_step': [[0.005, 10000],\n", " [0.02, 430000],\n", " [0.002, 730000],\n", " [0.001, 1030000]],\n", " 'net_type': 'resnet_50',\n", " 'num_joints': 3,\n", " 'optimizer': 'sgd',\n", " 'output_stride': 16,\n", " 'pos_dist_thresh': 17,\n", " 'project_path': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05',\n", " 'regularize': False,\n", " 'rightwidth': 400,\n", " 'save_iters': 50000,\n", " 'scale_jitter_lo': 0.5,\n", " 'scale_jitter_up': 1.25,\n", " 'scoremap_dir': 'test',\n", " 'shuffle': True,\n", " 'snapshot_prefix': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-0/S005-tipsSep5-trainset95shuffle1/train/snapshot',\n", " 'stride': 8.0,\n", " 'topheight': 400,\n", " 'weigh_negatives': False,\n", " 'weigh_only_present_joints': False,\n", " 'weigh_part_predictions': False,\n", " 'weight_decay': 0.0001}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Switching batchsize to 1, as default/tensorpack/deterministic loaders do not support batches >1. Use imgaug loader.\n", "Starting with standard pose-dataset loader.\n", "Initializing ResNet\n", "Loading ImageNet-pretrained resnet_50\n", "INFO:tensorflow:Restoring parameters from /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt\n", "Training parameter:\n", "{'stride': 8.0, 'weigh_part_predictions': False, 'weigh_negatives': False, 'fg_fraction': 0.25, 'weigh_only_present_joints': False, 'mean_pixel': [123.68, 116.779, 103.939], 'shuffle': True, 'snapshot_prefix': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-0/S005-tipsSep5-trainset95shuffle1/train/snapshot', 'log_dir': 'log', 'global_scale': 0.8, 'location_refinement': True, 'locref_stdev': 7.2801, 'locref_loss_weight': 0.05, 'locref_huber_loss': True, 'optimizer': 'sgd', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'regularize': False, 'weight_decay': 0.0001, 'mirror': False, 'crop_pad': 0, 'scoremap_dir': 'test', 'batch_size': 1, 'dataset_type': 'default', 'deterministic': False, 'crop': True, 'cropratio': 0.4, 'minsize': 100, 'leftwidth': 400, 'rightwidth': 400, 'topheight': 400, 'bottomheight': 400, 'all_joints': [[0], [1], [2]], 'all_joints_names': ['Tip1', 'Tip2', 'Tip3'], 'dataset': 'training-datasets/iteration-0/UnaugmentedDataSet_S005-tipsSep5/S005-tips_Keisuke95shuffle1.mat', 'display_iters': 1000, 'init_weights': '/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt', 'max_input_size': 1500, 'metadataset': 'training-datasets/iteration-0/UnaugmentedDataSet_S005-tipsSep5/Documentation_data-S005-tips_95shuffle1.pickle', 'min_input_size': 64, 'multi_step': [[0.005, 10000], [0.02, 430000], [0.002, 730000], [0.001, 1030000]], 'net_type': 'resnet_50', 'num_joints': 3, 'pos_dist_thresh': 17, 'project_path': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05', 'save_iters': 50000, 'scale_jitter_lo': 0.5, 'scale_jitter_up': 1.25, 'output_stride': 16, 'deconvolutionstride': 2}\n", "Starting training....\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 1000 loss: 0.0159 lr: 0.005\n", "iteration: 2000 loss: 0.0060 lr: 0.005\n", "iteration: 3000 loss: 0.0046 lr: 0.005\n", "iteration: 4000 loss: 0.0040 lr: 0.005\n", "iteration: 5000 loss: 0.0036 lr: 0.005\n", "iteration: 6000 loss: 0.0034 lr: 0.005\n", "iteration: 7000 loss: 0.0031 lr: 0.005\n", "iteration: 8000 loss: 0.0029 lr: 0.005\n", "iteration: 9000 loss: 0.0028 lr: 0.005\n", "iteration: 10000 loss: 0.0028 lr: 0.005\n", "iteration: 11000 loss: 0.0046 lr: 0.02\n", "iteration: 12000 loss: 0.0037 lr: 0.02\n", "iteration: 13000 loss: 0.0033 lr: 0.02\n", "iteration: 14000 loss: 0.0031 lr: 0.02\n", "iteration: 15000 loss: 0.0027 lr: 0.02\n", "iteration: 16000 loss: 0.0027 lr: 0.02\n", "iteration: 17000 loss: 0.0025 lr: 0.02\n", "iteration: 18000 loss: 0.0024 lr: 0.02\n", "iteration: 19000 loss: 0.0024 lr: 0.02\n", "iteration: 20000 loss: 0.0023 lr: 0.02\n", "iteration: 21000 loss: 0.0023 lr: 0.02\n", "iteration: 22000 loss: 0.0022 lr: 0.02\n", "iteration: 23000 loss: 0.0022 lr: 0.02\n", "iteration: 24000 loss: 0.0021 lr: 0.02\n", "iteration: 25000 loss: 0.0020 lr: 0.02\n", "iteration: 26000 loss: 0.0020 lr: 0.02\n", "iteration: 27000 loss: 0.0020 lr: 0.02\n", "iteration: 28000 loss: 0.0020 lr: 0.02\n", "iteration: 29000 loss: 0.0020 lr: 0.02\n", "iteration: 30000 loss: 0.0019 lr: 0.02\n", "iteration: 31000 loss: 0.0020 lr: 0.02\n", "iteration: 32000 loss: 0.0019 lr: 0.02\n", "iteration: 33000 loss: 0.0019 lr: 0.02\n", "iteration: 34000 loss: 0.0018 lr: 0.02\n", "iteration: 35000 loss: 0.0019 lr: 0.02\n", "iteration: 36000 loss: 0.0018 lr: 0.02\n", "iteration: 37000 loss: 0.0018 lr: 0.02\n", "iteration: 38000 loss: 0.0017 lr: 0.02\n", "iteration: 39000 loss: 0.0017 lr: 0.02\n", "iteration: 40000 loss: 0.0018 lr: 0.02\n", "iteration: 41000 loss: 0.0017 lr: 0.02\n", "iteration: 42000 loss: 0.0017 lr: 0.02\n", "iteration: 43000 loss: 0.0017 lr: 0.02\n", "iteration: 44000 loss: 0.0017 lr: 0.02\n", "iteration: 45000 loss: 0.0017 lr: 0.02\n", "iteration: 46000 loss: 0.0016 lr: 0.02\n", "iteration: 47000 loss: 0.0016 lr: 0.02\n", "iteration: 48000 loss: 0.0017 lr: 0.02\n", "iteration: 49000 loss: 0.0017 lr: 0.02\n", "iteration: 50000 loss: 0.0016 lr: 0.02\n", "iteration: 51000 loss: 0.0017 lr: 0.02\n", "iteration: 52000 loss: 0.0015 lr: 0.02\n", "iteration: 53000 loss: 0.0016 lr: 0.02\n", "iteration: 54000 loss: 0.0016 lr: 0.02\n", "iteration: 55000 loss: 0.0016 lr: 0.02\n", "iteration: 56000 loss: 0.0016 lr: 0.02\n", "iteration: 57000 loss: 0.0016 lr: 0.02\n", "iteration: 58000 loss: 0.0015 lr: 0.02\n", "iteration: 59000 loss: 0.0015 lr: 0.02\n", "iteration: 60000 loss: 0.0016 lr: 0.02\n", "iteration: 61000 loss: 0.0016 lr: 0.02\n", "iteration: 62000 loss: 0.0016 lr: 0.02\n", "iteration: 63000 loss: 0.0016 lr: 0.02\n", "iteration: 64000 loss: 0.0015 lr: 0.02\n", "iteration: 65000 loss: 0.0016 lr: 0.02\n", "iteration: 66000 loss: 0.0015 lr: 0.02\n", "iteration: 67000 loss: 0.0015 lr: 0.02\n", "iteration: 68000 loss: 0.0015 lr: 0.02\n", "iteration: 69000 loss: 0.0015 lr: 0.02\n", "iteration: 70000 loss: 0.0014 lr: 0.02\n", "iteration: 71000 loss: 0.0015 lr: 0.02\n", "iteration: 72000 loss: 0.0014 lr: 0.02\n", "iteration: 73000 loss: 0.0015 lr: 0.02\n", "iteration: 74000 loss: 0.0015 lr: 0.02\n", "iteration: 75000 loss: 0.0014 lr: 0.02\n", "iteration: 76000 loss: 0.0014 lr: 0.02\n", "iteration: 77000 loss: 0.0014 lr: 0.02\n", "iteration: 78000 loss: 0.0014 lr: 0.02\n", "iteration: 79000 loss: 0.0014 lr: 0.02\n", "iteration: 80000 loss: 0.0013 lr: 0.02\n", "iteration: 81000 loss: 0.0013 lr: 0.02\n", "iteration: 82000 loss: 0.0013 lr: 0.02\n", "iteration: 83000 loss: 0.0014 lr: 0.02\n", "iteration: 84000 loss: 0.0013 lr: 0.02\n", "iteration: 85000 loss: 0.0013 lr: 0.02\n", "iteration: 86000 loss: 0.0013 lr: 0.02\n", "iteration: 87000 loss: 0.0012 lr: 0.02\n", "iteration: 88000 loss: 0.0013 lr: 0.02\n", "iteration: 89000 loss: 0.0012 lr: 0.02\n", "iteration: 90000 loss: 0.0013 lr: 0.02\n", "iteration: 91000 loss: 0.0013 lr: 0.02\n", "iteration: 92000 loss: 0.0012 lr: 0.02\n", "iteration: 93000 loss: 0.0013 lr: 0.02\n", "iteration: 94000 loss: 0.0012 lr: 0.02\n", "iteration: 95000 loss: 0.0012 lr: 0.02\n", "iteration: 96000 loss: 0.0012 lr: 0.02\n", "iteration: 97000 loss: 0.0012 lr: 0.02\n", "iteration: 98000 loss: 0.0012 lr: 0.02\n", "iteration: 99000 loss: 0.0012 lr: 0.02\n", "iteration: 100000 loss: 0.0012 lr: 0.02\n", "iteration: 101000 loss: 0.0011 lr: 0.02\n", "iteration: 102000 loss: 0.0012 lr: 0.02\n", "iteration: 103000 loss: 0.0012 lr: 0.02\n", "iteration: 104000 loss: 0.0012 lr: 0.02\n", "iteration: 105000 loss: 0.0011 lr: 0.02\n", "iteration: 106000 loss: 0.0011 lr: 0.02\n", "iteration: 107000 loss: 0.0012 lr: 0.02\n", "iteration: 108000 loss: 0.0011 lr: 0.02\n", "iteration: 109000 loss: 0.0012 lr: 0.02\n", "iteration: 110000 loss: 0.0011 lr: 0.02\n", "iteration: 111000 loss: 0.0011 lr: 0.02\n", "iteration: 112000 loss: 0.0011 lr: 0.02\n", "iteration: 113000 loss: 0.0011 lr: 0.02\n", "iteration: 114000 loss: 0.0011 lr: 0.02\n", "iteration: 115000 loss: 0.0011 lr: 0.02\n", "iteration: 116000 loss: 0.0011 lr: 0.02\n", "iteration: 117000 loss: 0.0010 lr: 0.02\n", "iteration: 118000 loss: 0.0011 lr: 0.02\n", "iteration: 119000 loss: 0.0011 lr: 0.02\n", "iteration: 120000 loss: 0.0011 lr: 0.02\n", "iteration: 121000 loss: 0.0011 lr: 0.02\n", "iteration: 122000 loss: 0.0011 lr: 0.02\n", "iteration: 123000 loss: 0.0011 lr: 0.02\n", "iteration: 124000 loss: 0.0011 lr: 0.02\n", "iteration: 125000 loss: 0.0010 lr: 0.02\n", "iteration: 126000 loss: 0.0010 lr: 0.02\n", "iteration: 127000 loss: 0.0011 lr: 0.02\n", "iteration: 128000 loss: 0.0010 lr: 0.02\n", "iteration: 129000 loss: 0.0010 lr: 0.02\n", "iteration: 130000 loss: 0.0010 lr: 0.02\n", "iteration: 131000 loss: 0.0010 lr: 0.02\n", "iteration: 132000 loss: 0.0010 lr: 0.02\n", "iteration: 133000 loss: 0.0010 lr: 0.02\n", "iteration: 134000 loss: 0.0010 lr: 0.02\n", "iteration: 135000 loss: 0.0009 lr: 0.02\n", "iteration: 136000 loss: 0.0010 lr: 0.02\n", "iteration: 137000 loss: 0.0010 lr: 0.02\n", "iteration: 138000 loss: 0.0010 lr: 0.02\n", "iteration: 139000 loss: 0.0009 lr: 0.02\n", "iteration: 140000 loss: 0.0010 lr: 0.02\n", "iteration: 141000 loss: 0.0010 lr: 0.02\n", "iteration: 142000 loss: 0.0010 lr: 0.02\n", "iteration: 143000 loss: 0.0010 lr: 0.02\n", "iteration: 144000 loss: 0.0010 lr: 0.02\n", "iteration: 145000 loss: 0.0009 lr: 0.02\n", "iteration: 146000 loss: 0.0009 lr: 0.02\n", "iteration: 147000 loss: 0.0010 lr: 0.02\n", "iteration: 148000 loss: 0.0009 lr: 0.02\n", "iteration: 149000 loss: 0.0010 lr: 0.02\n", "iteration: 150000 loss: 0.0009 lr: 0.02\n", "iteration: 151000 loss: 0.0009 lr: 0.02\n", "iteration: 152000 loss: 0.0009 lr: 0.02\n", "iteration: 153000 loss: 0.0009 lr: 0.02\n", "iteration: 154000 loss: 0.0010 lr: 0.02\n", "iteration: 155000 loss: 0.0009 lr: 0.02\n", "iteration: 156000 loss: 0.0010 lr: 0.02\n", "iteration: 157000 loss: 0.0009 lr: 0.02\n", "iteration: 158000 loss: 0.0009 lr: 0.02\n", "iteration: 159000 loss: 0.0009 lr: 0.02\n", "iteration: 160000 loss: 0.0010 lr: 0.02\n", "iteration: 161000 loss: 0.0009 lr: 0.02\n", "iteration: 162000 loss: 0.0009 lr: 0.02\n", "iteration: 163000 loss: 0.0009 lr: 0.02\n", "iteration: 164000 loss: 0.0009 lr: 0.02\n", "iteration: 165000 loss: 0.0009 lr: 0.02\n", "iteration: 166000 loss: 0.0009 lr: 0.02\n", "iteration: 167000 loss: 0.0009 lr: 0.02\n", "iteration: 168000 loss: 0.0009 lr: 0.02\n", "iteration: 169000 loss: 0.0009 lr: 0.02\n", "iteration: 170000 loss: 0.0009 lr: 0.02\n", "iteration: 171000 loss: 0.0009 lr: 0.02\n", "iteration: 172000 loss: 0.0008 lr: 0.02\n", "iteration: 173000 loss: 0.0009 lr: 0.02\n", "iteration: 174000 loss: 0.0009 lr: 0.02\n", "iteration: 175000 loss: 0.0009 lr: 0.02\n", "iteration: 176000 loss: 0.0009 lr: 0.02\n", "iteration: 177000 loss: 0.0010 lr: 0.02\n", "iteration: 178000 loss: 0.0009 lr: 0.02\n", "iteration: 179000 loss: 0.0009 lr: 0.02\n", "iteration: 180000 loss: 0.0009 lr: 0.02\n", "iteration: 181000 loss: 0.0008 lr: 0.02\n", "iteration: 182000 loss: 0.0008 lr: 0.02\n", "iteration: 183000 loss: 0.0009 lr: 0.02\n", "iteration: 184000 loss: 0.0009 lr: 0.02\n", "iteration: 185000 loss: 0.0009 lr: 0.02\n", "iteration: 186000 loss: 0.0009 lr: 0.02\n", "iteration: 187000 loss: 0.0009 lr: 0.02\n", "iteration: 188000 loss: 0.0009 lr: 0.02\n", "iteration: 189000 loss: 0.0009 lr: 0.02\n", "iteration: 190000 loss: 0.0009 lr: 0.02\n", "iteration: 191000 loss: 0.0009 lr: 0.02\n", "iteration: 192000 loss: 0.0009 lr: 0.02\n", "iteration: 193000 loss: 0.0009 lr: 0.02\n", "iteration: 194000 loss: 0.0009 lr: 0.02\n", "iteration: 195000 loss: 0.0009 lr: 0.02\n", "iteration: 196000 loss: 0.0009 lr: 0.02\n", "iteration: 197000 loss: 0.0008 lr: 0.02\n", "iteration: 198000 loss: 0.0009 lr: 0.02\n", "iteration: 199000 loss: 0.0008 lr: 0.02\n", "iteration: 200000 loss: 0.0009 lr: 0.02\n", "iteration: 201000 loss: 0.0009 lr: 0.02\n", "iteration: 202000 loss: 0.0008 lr: 0.02\n", "iteration: 203000 loss: 0.0008 lr: 0.02\n", "iteration: 204000 loss: 0.0009 lr: 0.02\n", "iteration: 205000 loss: 0.0009 lr: 0.02\n", "iteration: 206000 loss: 0.0008 lr: 0.02\n", "iteration: 207000 loss: 0.0008 lr: 0.02\n", "iteration: 208000 loss: 0.0008 lr: 0.02\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 209000 loss: 0.0009 lr: 0.02\n", "iteration: 210000 loss: 0.0009 lr: 0.02\n", "iteration: 211000 loss: 0.0009 lr: 0.02\n", "iteration: 212000 loss: 0.0009 lr: 0.02\n", "iteration: 213000 loss: 0.0008 lr: 0.02\n", "iteration: 214000 loss: 0.0008 lr: 0.02\n", "iteration: 215000 loss: 0.0008 lr: 0.02\n", "iteration: 216000 loss: 0.0008 lr: 0.02\n", "iteration: 217000 loss: 0.0008 lr: 0.02\n", "iteration: 218000 loss: 0.0009 lr: 0.02\n", "iteration: 219000 loss: 0.0008 lr: 0.02\n", "iteration: 220000 loss: 0.0009 lr: 0.02\n", "iteration: 221000 loss: 0.0008 lr: 0.02\n", "iteration: 222000 loss: 0.0008 lr: 0.02\n", "iteration: 223000 loss: 0.0008 lr: 0.02\n", "iteration: 224000 loss: 0.0008 lr: 0.02\n", "iteration: 225000 loss: 0.0008 lr: 0.02\n", "iteration: 226000 loss: 0.0008 lr: 0.02\n", "iteration: 227000 loss: 0.0008 lr: 0.02\n", "iteration: 228000 loss: 0.0009 lr: 0.02\n", "iteration: 229000 loss: 0.0008 lr: 0.02\n", "iteration: 230000 loss: 0.0008 lr: 0.02\n", "iteration: 231000 loss: 0.0008 lr: 0.02\n", "iteration: 232000 loss: 0.0008 lr: 0.02\n", "iteration: 233000 loss: 0.0008 lr: 0.02\n", "iteration: 234000 loss: 0.0008 lr: 0.02\n", "iteration: 235000 loss: 0.0008 lr: 0.02\n", "iteration: 236000 loss: 0.0008 lr: 0.02\n", "iteration: 237000 loss: 0.0008 lr: 0.02\n", "iteration: 238000 loss: 0.0008 lr: 0.02\n", "iteration: 239000 loss: 0.0008 lr: 0.02\n", "iteration: 240000 loss: 0.0008 lr: 0.02\n", "iteration: 241000 loss: 0.0008 lr: 0.02\n", "iteration: 242000 loss: 0.0008 lr: 0.02\n", "iteration: 243000 loss: 0.0007 lr: 0.02\n", "iteration: 244000 loss: 0.0008 lr: 0.02\n", "iteration: 245000 loss: 0.0008 lr: 0.02\n", "iteration: 246000 loss: 0.0008 lr: 0.02\n", "iteration: 247000 loss: 0.0008 lr: 0.02\n", "iteration: 248000 loss: 0.0007 lr: 0.02\n", "iteration: 249000 loss: 0.0008 lr: 0.02\n", "iteration: 250000 loss: 0.0008 lr: 0.02\n", "iteration: 251000 loss: 0.0008 lr: 0.02\n", "iteration: 252000 loss: 0.0007 lr: 0.02\n", "iteration: 253000 loss: 0.0008 lr: 0.02\n", "iteration: 254000 loss: 0.0008 lr: 0.02\n", "iteration: 255000 loss: 0.0007 lr: 0.02\n", "iteration: 256000 loss: 0.0007 lr: 0.02\n", "iteration: 257000 loss: 0.0008 lr: 0.02\n", "iteration: 258000 loss: 0.0008 lr: 0.02\n", "iteration: 259000 loss: 0.0008 lr: 0.02\n", "iteration: 260000 loss: 0.0008 lr: 0.02\n", "iteration: 261000 loss: 0.0008 lr: 0.02\n", "iteration: 262000 loss: 0.0007 lr: 0.02\n", "iteration: 263000 loss: 0.0007 lr: 0.02\n", "iteration: 264000 loss: 0.0008 lr: 0.02\n", "iteration: 265000 loss: 0.0008 lr: 0.02\n", "iteration: 266000 loss: 0.0007 lr: 0.02\n", "iteration: 267000 loss: 0.0007 lr: 0.02\n", "iteration: 268000 loss: 0.0007 lr: 0.02\n", "iteration: 269000 loss: 0.0008 lr: 0.02\n", "iteration: 270000 loss: 0.0008 lr: 0.02\n", "iteration: 271000 loss: 0.0008 lr: 0.02\n", "iteration: 272000 loss: 0.0007 lr: 0.02\n", "iteration: 273000 loss: 0.0008 lr: 0.02\n", "iteration: 274000 loss: 0.0007 lr: 0.02\n", "iteration: 275000 loss: 0.0008 lr: 0.02\n", "iteration: 276000 loss: 0.0008 lr: 0.02\n", "iteration: 277000 loss: 0.0007 lr: 0.02\n", "iteration: 278000 loss: 0.0007 lr: 0.02\n", "iteration: 279000 loss: 0.0008 lr: 0.02\n", "iteration: 280000 loss: 0.0008 lr: 0.02\n", "iteration: 281000 loss: 0.0008 lr: 0.02\n", "iteration: 282000 loss: 0.0007 lr: 0.02\n", "iteration: 283000 loss: 0.0007 lr: 0.02\n", "iteration: 284000 loss: 0.0007 lr: 0.02\n", "iteration: 285000 loss: 0.0007 lr: 0.02\n", "iteration: 286000 loss: 0.0008 lr: 0.02\n", "iteration: 287000 loss: 0.0008 lr: 0.02\n", "iteration: 288000 loss: 0.0007 lr: 0.02\n", "iteration: 289000 loss: 0.0007 lr: 0.02\n", "iteration: 290000 loss: 0.0008 lr: 0.02\n", "iteration: 291000 loss: 0.0007 lr: 0.02\n", "iteration: 292000 loss: 0.0008 lr: 0.02\n", "iteration: 293000 loss: 0.0007 lr: 0.02\n", "iteration: 294000 loss: 0.0007 lr: 0.02\n", "iteration: 295000 loss: 0.0007 lr: 0.02\n", "iteration: 296000 loss: 0.0007 lr: 0.02\n", "iteration: 297000 loss: 0.0007 lr: 0.02\n", "iteration: 298000 loss: 0.0007 lr: 0.02\n", "iteration: 299000 loss: 0.0007 lr: 0.02\n", "iteration: 300000 loss: 0.0007 lr: 0.02\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:From /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use standard file APIs to delete files with this prefix.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 301000 loss: 0.0007 lr: 0.02\n", "iteration: 302000 loss: 0.0007 lr: 0.02\n", "iteration: 303000 loss: 0.0008 lr: 0.02\n", "iteration: 304000 loss: 0.0007 lr: 0.02\n", "iteration: 305000 loss: 0.0007 lr: 0.02\n", "iteration: 306000 loss: 0.0007 lr: 0.02\n", "iteration: 307000 loss: 0.0007 lr: 0.02\n", "iteration: 308000 loss: 0.0007 lr: 0.02\n", "iteration: 309000 loss: 0.0008 lr: 0.02\n", "iteration: 310000 loss: 0.0007 lr: 0.02\n", "iteration: 311000 loss: 0.0007 lr: 0.02\n", "iteration: 312000 loss: 0.0007 lr: 0.02\n", "iteration: 313000 loss: 0.0007 lr: 0.02\n", "iteration: 314000 loss: 0.0007 lr: 0.02\n", "iteration: 315000 loss: 0.0007 lr: 0.02\n", "iteration: 316000 loss: 0.0007 lr: 0.02\n", "iteration: 317000 loss: 0.0007 lr: 0.02\n", "iteration: 318000 loss: 0.0007 lr: 0.02\n", "iteration: 319000 loss: 0.0007 lr: 0.02\n", "iteration: 320000 loss: 0.0007 lr: 0.02\n", "iteration: 321000 loss: 0.0007 lr: 0.02\n", "iteration: 322000 loss: 0.0007 lr: 0.02\n", "iteration: 323000 loss: 0.0007 lr: 0.02\n", "iteration: 324000 loss: 0.0007 lr: 0.02\n", "iteration: 325000 loss: 0.0007 lr: 0.02\n", "iteration: 326000 loss: 0.0007 lr: 0.02\n", "iteration: 327000 loss: 0.0007 lr: 0.02\n", "iteration: 328000 loss: 0.0007 lr: 0.02\n", "iteration: 329000 loss: 0.0007 lr: 0.02\n", "iteration: 330000 loss: 0.0007 lr: 0.02\n", "iteration: 331000 loss: 0.0007 lr: 0.02\n", "iteration: 332000 loss: 0.0007 lr: 0.02\n", "iteration: 333000 loss: 0.0007 lr: 0.02\n", "iteration: 334000 loss: 0.0007 lr: 0.02\n", "iteration: 335000 loss: 0.0007 lr: 0.02\n", "iteration: 336000 loss: 0.0007 lr: 0.02\n", "iteration: 337000 loss: 0.0007 lr: 0.02\n", "iteration: 338000 loss: 0.0007 lr: 0.02\n", "iteration: 339000 loss: 0.0007 lr: 0.02\n", "iteration: 340000 loss: 0.0007 lr: 0.02\n", "iteration: 341000 loss: 0.0007 lr: 0.02\n", "iteration: 342000 loss: 0.0007 lr: 0.02\n", "iteration: 343000 loss: 0.0007 lr: 0.02\n", "iteration: 344000 loss: 0.0007 lr: 0.02\n", "iteration: 345000 loss: 0.0007 lr: 0.02\n", "iteration: 346000 loss: 0.0007 lr: 0.02\n", "iteration: 347000 loss: 0.0007 lr: 0.02\n", "iteration: 348000 loss: 0.0007 lr: 0.02\n", "iteration: 349000 loss: 0.0007 lr: 0.02\n", "iteration: 350000 loss: 0.0006 lr: 0.02\n", "iteration: 351000 loss: 0.0007 lr: 0.02\n", "iteration: 352000 loss: 0.0007 lr: 0.02\n", "iteration: 353000 loss: 0.0007 lr: 0.02\n", "iteration: 354000 loss: 0.0007 lr: 0.02\n", "iteration: 355000 loss: 0.0006 lr: 0.02\n", "iteration: 356000 loss: 0.0007 lr: 0.02\n", "iteration: 357000 loss: 0.0007 lr: 0.02\n", "iteration: 358000 loss: 0.0007 lr: 0.02\n", "iteration: 359000 loss: 0.0007 lr: 0.02\n", "iteration: 360000 loss: 0.0007 lr: 0.02\n", "iteration: 361000 loss: 0.0007 lr: 0.02\n", "iteration: 362000 loss: 0.0006 lr: 0.02\n", "iteration: 363000 loss: 0.0006 lr: 0.02\n", "iteration: 364000 loss: 0.0007 lr: 0.02\n", "iteration: 365000 loss: 0.0007 lr: 0.02\n", "iteration: 366000 loss: 0.0006 lr: 0.02\n", "iteration: 367000 loss: 0.0006 lr: 0.02\n", "iteration: 368000 loss: 0.0007 lr: 0.02\n", "iteration: 369000 loss: 0.0006 lr: 0.02\n", "iteration: 370000 loss: 0.0007 lr: 0.02\n", "iteration: 371000 loss: 0.0007 lr: 0.02\n", "iteration: 372000 loss: 0.0007 lr: 0.02\n", "iteration: 373000 loss: 0.0007 lr: 0.02\n", "iteration: 374000 loss: 0.0007 lr: 0.02\n", "iteration: 375000 loss: 0.0007 lr: 0.02\n", "iteration: 376000 loss: 0.0006 lr: 0.02\n", "iteration: 377000 loss: 0.0006 lr: 0.02\n", "iteration: 378000 loss: 0.0006 lr: 0.02\n", "iteration: 379000 loss: 0.0007 lr: 0.02\n", "iteration: 380000 loss: 0.0007 lr: 0.02\n", "iteration: 381000 loss: 0.0006 lr: 0.02\n", "iteration: 382000 loss: 0.0006 lr: 0.02\n", "iteration: 383000 loss: 0.0006 lr: 0.02\n", "iteration: 384000 loss: 0.0007 lr: 0.02\n", "iteration: 385000 loss: 0.0007 lr: 0.02\n", "iteration: 386000 loss: 0.0006 lr: 0.02\n", "iteration: 387000 loss: 0.0006 lr: 0.02\n", "iteration: 388000 loss: 0.0006 lr: 0.02\n", "iteration: 389000 loss: 0.0007 lr: 0.02\n", "iteration: 390000 loss: 0.0006 lr: 0.02\n", "iteration: 391000 loss: 0.0006 lr: 0.02\n", "iteration: 392000 loss: 0.0006 lr: 0.02\n", "iteration: 393000 loss: 0.0007 lr: 0.02\n", "iteration: 394000 loss: 0.0006 lr: 0.02\n", "iteration: 395000 loss: 0.0006 lr: 0.02\n", "iteration: 396000 loss: 0.0007 lr: 0.02\n", "iteration: 397000 loss: 0.0006 lr: 0.02\n", "iteration: 398000 loss: 0.0006 lr: 0.02\n", "iteration: 399000 loss: 0.0006 lr: 0.02\n", "iteration: 400000 loss: 0.0006 lr: 0.02\n", "iteration: 401000 loss: 0.0006 lr: 0.02\n", "iteration: 402000 loss: 0.0006 lr: 0.02\n", "iteration: 403000 loss: 0.0006 lr: 0.02\n", "iteration: 404000 loss: 0.0006 lr: 0.02\n", "iteration: 405000 loss: 0.0006 lr: 0.02\n", "iteration: 406000 loss: 0.0006 lr: 0.02\n", "iteration: 407000 loss: 0.0006 lr: 0.02\n", "iteration: 408000 loss: 0.0007 lr: 0.02\n", "iteration: 409000 loss: 0.0006 lr: 0.02\n", "iteration: 410000 loss: 0.0007 lr: 0.02\n", "iteration: 411000 loss: 0.0006 lr: 0.02\n", "iteration: 412000 loss: 0.0006 lr: 0.02\n", "iteration: 413000 loss: 0.0007 lr: 0.02\n", "iteration: 414000 loss: 0.0006 lr: 0.02\n", "iteration: 415000 loss: 0.0006 lr: 0.02\n", "iteration: 416000 loss: 0.0006 lr: 0.02\n", "iteration: 417000 loss: 0.0006 lr: 0.02\n", "iteration: 418000 loss: 0.0007 lr: 0.02\n", "iteration: 419000 loss: 0.0007 lr: 0.02\n", "iteration: 420000 loss: 0.0007 lr: 0.02\n", "iteration: 421000 loss: 0.0006 lr: 0.02\n", "iteration: 422000 loss: 0.0006 lr: 0.02\n", "iteration: 423000 loss: 0.0006 lr: 0.02\n", "iteration: 424000 loss: 0.0006 lr: 0.02\n", "iteration: 425000 loss: 0.0006 lr: 0.02\n", "iteration: 426000 loss: 0.0007 lr: 0.02\n", "iteration: 427000 loss: 0.0006 lr: 0.02\n", "iteration: 428000 loss: 0.0006 lr: 0.02\n", "iteration: 429000 loss: 0.0006 lr: 0.02\n", "iteration: 430000 loss: 0.0007 lr: 0.02\n", "iteration: 431000 loss: 0.0005 lr: 0.002\n", "iteration: 432000 loss: 0.0005 lr: 0.002\n", "iteration: 433000 loss: 0.0005 lr: 0.002\n", "iteration: 434000 loss: 0.0005 lr: 0.002\n", "iteration: 435000 loss: 0.0005 lr: 0.002\n", "iteration: 436000 loss: 0.0005 lr: 0.002\n", "iteration: 437000 loss: 0.0005 lr: 0.002\n", "iteration: 438000 loss: 0.0005 lr: 0.002\n", "iteration: 439000 loss: 0.0004 lr: 0.002\n", "iteration: 440000 loss: 0.0004 lr: 0.002\n", "iteration: 441000 loss: 0.0005 lr: 0.002\n", "iteration: 442000 loss: 0.0004 lr: 0.002\n", "iteration: 443000 loss: 0.0005 lr: 0.002\n", "iteration: 444000 loss: 0.0004 lr: 0.002\n", "iteration: 445000 loss: 0.0004 lr: 0.002\n", "iteration: 446000 loss: 0.0005 lr: 0.002\n", "iteration: 447000 loss: 0.0005 lr: 0.002\n", "iteration: 448000 loss: 0.0004 lr: 0.002\n", "iteration: 449000 loss: 0.0004 lr: 0.002\n", "iteration: 450000 loss: 0.0004 lr: 0.002\n", "iteration: 451000 loss: 0.0004 lr: 0.002\n", "iteration: 452000 loss: 0.0005 lr: 0.002\n", "iteration: 453000 loss: 0.0005 lr: 0.002\n", "iteration: 454000 loss: 0.0004 lr: 0.002\n", "iteration: 455000 loss: 0.0004 lr: 0.002\n", "iteration: 456000 loss: 0.0004 lr: 0.002\n", "iteration: 457000 loss: 0.0004 lr: 0.002\n", "iteration: 458000 loss: 0.0004 lr: 0.002\n", "iteration: 459000 loss: 0.0004 lr: 0.002\n", "iteration: 460000 loss: 0.0004 lr: 0.002\n", "iteration: 461000 loss: 0.0004 lr: 0.002\n", "iteration: 462000 loss: 0.0004 lr: 0.002\n", "iteration: 463000 loss: 0.0005 lr: 0.002\n", "iteration: 464000 loss: 0.0005 lr: 0.002\n", "iteration: 465000 loss: 0.0004 lr: 0.002\n", "iteration: 466000 loss: 0.0004 lr: 0.002\n", "iteration: 467000 loss: 0.0004 lr: 0.002\n", "iteration: 468000 loss: 0.0004 lr: 0.002\n", "iteration: 469000 loss: 0.0005 lr: 0.002\n", "iteration: 470000 loss: 0.0004 lr: 0.002\n", "iteration: 471000 loss: 0.0004 lr: 0.002\n", "iteration: 472000 loss: 0.0004 lr: 0.002\n", "iteration: 473000 loss: 0.0004 lr: 0.002\n", "iteration: 474000 loss: 0.0004 lr: 0.002\n", "iteration: 475000 loss: 0.0004 lr: 0.002\n", "iteration: 476000 loss: 0.0004 lr: 0.002\n", "iteration: 477000 loss: 0.0004 lr: 0.002\n", "iteration: 478000 loss: 0.0004 lr: 0.002\n", "iteration: 479000 loss: 0.0004 lr: 0.002\n", "iteration: 480000 loss: 0.0004 lr: 0.002\n", "iteration: 481000 loss: 0.0004 lr: 0.002\n", "iteration: 482000 loss: 0.0004 lr: 0.002\n", "iteration: 483000 loss: 0.0004 lr: 0.002\n", "iteration: 484000 loss: 0.0004 lr: 0.002\n", "iteration: 485000 loss: 0.0004 lr: 0.002\n", "iteration: 486000 loss: 0.0004 lr: 0.002\n", "iteration: 487000 loss: 0.0004 lr: 0.002\n", "iteration: 488000 loss: 0.0004 lr: 0.002\n", "iteration: 489000 loss: 0.0004 lr: 0.002\n", "iteration: 490000 loss: 0.0004 lr: 0.002\n", "iteration: 491000 loss: 0.0004 lr: 0.002\n", "iteration: 492000 loss: 0.0004 lr: 0.002\n", "iteration: 493000 loss: 0.0004 lr: 0.002\n", "iteration: 494000 loss: 0.0004 lr: 0.002\n", "iteration: 495000 loss: 0.0004 lr: 0.002\n", "iteration: 496000 loss: 0.0004 lr: 0.002\n", "iteration: 497000 loss: 0.0004 lr: 0.002\n", "iteration: 498000 loss: 0.0004 lr: 0.002\n", "iteration: 499000 loss: 0.0004 lr: 0.002\n", "iteration: 500000 loss: 0.0004 lr: 0.002\n", "iteration: 501000 loss: 0.0004 lr: 0.002\n", "iteration: 502000 loss: 0.0004 lr: 0.002\n", "iteration: 503000 loss: 0.0004 lr: 0.002\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 504000 loss: 0.0004 lr: 0.002\n", "iteration: 505000 loss: 0.0004 lr: 0.002\n", "iteration: 506000 loss: 0.0004 lr: 0.002\n", "iteration: 507000 loss: 0.0004 lr: 0.002\n", "iteration: 508000 loss: 0.0004 lr: 0.002\n", "iteration: 509000 loss: 0.0004 lr: 0.002\n", "iteration: 510000 loss: 0.0004 lr: 0.002\n", "iteration: 511000 loss: 0.0004 lr: 0.002\n", "iteration: 512000 loss: 0.0004 lr: 0.002\n", "iteration: 513000 loss: 0.0004 lr: 0.002\n", "iteration: 514000 loss: 0.0004 lr: 0.002\n", "iteration: 515000 loss: 0.0004 lr: 0.002\n", "iteration: 516000 loss: 0.0004 lr: 0.002\n", "iteration: 517000 loss: 0.0004 lr: 0.002\n", "iteration: 518000 loss: 0.0004 lr: 0.002\n", "iteration: 519000 loss: 0.0004 lr: 0.002\n", "iteration: 520000 loss: 0.0004 lr: 0.002\n", "iteration: 521000 loss: 0.0004 lr: 0.002\n", "iteration: 522000 loss: 0.0004 lr: 0.002\n", "iteration: 523000 loss: 0.0004 lr: 0.002\n", "iteration: 524000 loss: 0.0004 lr: 0.002\n", "iteration: 525000 loss: 0.0004 lr: 0.002\n", "iteration: 526000 loss: 0.0004 lr: 0.002\n", "iteration: 527000 loss: 0.0004 lr: 0.002\n", "iteration: 528000 loss: 0.0004 lr: 0.002\n", "iteration: 529000 loss: 0.0004 lr: 0.002\n", "iteration: 530000 loss: 0.0004 lr: 0.002\n", "iteration: 531000 loss: 0.0004 lr: 0.002\n", "iteration: 532000 loss: 0.0004 lr: 0.002\n", "iteration: 533000 loss: 0.0004 lr: 0.002\n", "iteration: 534000 loss: 0.0004 lr: 0.002\n", "iteration: 535000 loss: 0.0004 lr: 0.002\n", "iteration: 536000 loss: 0.0004 lr: 0.002\n", "iteration: 537000 loss: 0.0004 lr: 0.002\n", "iteration: 538000 loss: 0.0004 lr: 0.002\n", "iteration: 539000 loss: 0.0004 lr: 0.002\n", "iteration: 540000 loss: 0.0004 lr: 0.002\n", "iteration: 541000 loss: 0.0004 lr: 0.002\n", "iteration: 542000 loss: 0.0004 lr: 0.002\n", "iteration: 543000 loss: 0.0004 lr: 0.002\n", "iteration: 544000 loss: 0.0004 lr: 0.002\n", "iteration: 545000 loss: 0.0004 lr: 0.002\n", "iteration: 546000 loss: 0.0004 lr: 0.002\n", "iteration: 547000 loss: 0.0004 lr: 0.002\n", "iteration: 548000 loss: 0.0004 lr: 0.002\n", "iteration: 549000 loss: 0.0004 lr: 0.002\n", "iteration: 550000 loss: 0.0004 lr: 0.002\n", "iteration: 551000 loss: 0.0004 lr: 0.002\n", "iteration: 552000 loss: 0.0004 lr: 0.002\n", "iteration: 553000 loss: 0.0004 lr: 0.002\n", "iteration: 554000 loss: 0.0004 lr: 0.002\n", "iteration: 555000 loss: 0.0004 lr: 0.002\n", "iteration: 556000 loss: 0.0004 lr: 0.002\n", "iteration: 557000 loss: 0.0004 lr: 0.002\n", "iteration: 558000 loss: 0.0004 lr: 0.002\n", "iteration: 559000 loss: 0.0004 lr: 0.002\n", "iteration: 560000 loss: 0.0004 lr: 0.002\n", "iteration: 561000 loss: 0.0004 lr: 0.002\n", "iteration: 562000 loss: 0.0004 lr: 0.002\n", "iteration: 563000 loss: 0.0004 lr: 0.002\n", "iteration: 564000 loss: 0.0004 lr: 0.002\n", "iteration: 565000 loss: 0.0004 lr: 0.002\n", "iteration: 566000 loss: 0.0004 lr: 0.002\n", "iteration: 567000 loss: 0.0004 lr: 0.002\n", "iteration: 568000 loss: 0.0004 lr: 0.002\n", "iteration: 569000 loss: 0.0004 lr: 0.002\n", "iteration: 570000 loss: 0.0004 lr: 0.002\n", "iteration: 571000 loss: 0.0004 lr: 0.002\n", "iteration: 572000 loss: 0.0004 lr: 0.002\n", "iteration: 573000 loss: 0.0004 lr: 0.002\n", "iteration: 574000 loss: 0.0004 lr: 0.002\n", "iteration: 575000 loss: 0.0004 lr: 0.002\n", "iteration: 576000 loss: 0.0004 lr: 0.002\n", "iteration: 577000 loss: 0.0004 lr: 0.002\n", "iteration: 578000 loss: 0.0004 lr: 0.002\n", "iteration: 579000 loss: 0.0004 lr: 0.002\n", "iteration: 580000 loss: 0.0004 lr: 0.002\n", "iteration: 581000 loss: 0.0004 lr: 0.002\n", "iteration: 582000 loss: 0.0004 lr: 0.002\n", "iteration: 583000 loss: 0.0004 lr: 0.002\n", "iteration: 584000 loss: 0.0004 lr: 0.002\n", "iteration: 585000 loss: 0.0004 lr: 0.002\n", "iteration: 586000 loss: 0.0004 lr: 0.002\n", "iteration: 587000 loss: 0.0004 lr: 0.002\n", "iteration: 588000 loss: 0.0004 lr: 0.002\n", "iteration: 589000 loss: 0.0004 lr: 0.002\n", "iteration: 590000 loss: 0.0004 lr: 0.002\n", "iteration: 591000 loss: 0.0004 lr: 0.002\n", "iteration: 592000 loss: 0.0004 lr: 0.002\n", "iteration: 593000 loss: 0.0004 lr: 0.002\n", "iteration: 594000 loss: 0.0004 lr: 0.002\n", "iteration: 595000 loss: 0.0004 lr: 0.002\n", "iteration: 596000 loss: 0.0004 lr: 0.002\n", "iteration: 597000 loss: 0.0004 lr: 0.002\n", "iteration: 598000 loss: 0.0004 lr: 0.002\n", "iteration: 599000 loss: 0.0004 lr: 0.002\n", "iteration: 600000 loss: 0.0004 lr: 0.002\n", "iteration: 601000 loss: 0.0004 lr: 0.002\n", "iteration: 602000 loss: 0.0004 lr: 0.002\n", "iteration: 603000 loss: 0.0004 lr: 0.002\n", "iteration: 604000 loss: 0.0004 lr: 0.002\n", "iteration: 605000 loss: 0.0004 lr: 0.002\n", "iteration: 606000 loss: 0.0004 lr: 0.002\n", "iteration: 607000 loss: 0.0004 lr: 0.002\n", "iteration: 608000 loss: 0.0004 lr: 0.002\n", "iteration: 609000 loss: 0.0004 lr: 0.002\n", "iteration: 610000 loss: 0.0004 lr: 0.002\n", "iteration: 611000 loss: 0.0004 lr: 0.002\n", "iteration: 612000 loss: 0.0004 lr: 0.002\n", "iteration: 613000 loss: 0.0004 lr: 0.002\n", "iteration: 614000 loss: 0.0004 lr: 0.002\n", "iteration: 615000 loss: 0.0004 lr: 0.002\n", "iteration: 616000 loss: 0.0003 lr: 0.002\n", "iteration: 617000 loss: 0.0004 lr: 0.002\n", "iteration: 618000 loss: 0.0004 lr: 0.002\n", "iteration: 619000 loss: 0.0004 lr: 0.002\n", "iteration: 620000 loss: 0.0004 lr: 0.002\n", "iteration: 621000 loss: 0.0004 lr: 0.002\n", "iteration: 622000 loss: 0.0004 lr: 0.002\n", "iteration: 623000 loss: 0.0004 lr: 0.002\n", "iteration: 624000 loss: 0.0004 lr: 0.002\n", "iteration: 625000 loss: 0.0004 lr: 0.002\n", "iteration: 626000 loss: 0.0003 lr: 0.002\n", "iteration: 627000 loss: 0.0004 lr: 0.002\n", "iteration: 628000 loss: 0.0003 lr: 0.002\n", "iteration: 629000 loss: 0.0004 lr: 0.002\n", "iteration: 630000 loss: 0.0003 lr: 0.002\n", "iteration: 631000 loss: 0.0004 lr: 0.002\n", "iteration: 632000 loss: 0.0004 lr: 0.002\n", "iteration: 633000 loss: 0.0004 lr: 0.002\n", "iteration: 634000 loss: 0.0004 lr: 0.002\n", "iteration: 635000 loss: 0.0004 lr: 0.002\n", "iteration: 636000 loss: 0.0004 lr: 0.002\n", "iteration: 637000 loss: 0.0004 lr: 0.002\n", "iteration: 638000 loss: 0.0004 lr: 0.002\n", "iteration: 639000 loss: 0.0004 lr: 0.002\n", "iteration: 640000 loss: 0.0004 lr: 0.002\n", "iteration: 641000 loss: 0.0004 lr: 0.002\n", "iteration: 642000 loss: 0.0004 lr: 0.002\n", "iteration: 643000 loss: 0.0004 lr: 0.002\n", "iteration: 644000 loss: 0.0004 lr: 0.002\n", "iteration: 645000 loss: 0.0004 lr: 0.002\n", "iteration: 646000 loss: 0.0004 lr: 0.002\n", "iteration: 647000 loss: 0.0004 lr: 0.002\n", "iteration: 648000 loss: 0.0004 lr: 0.002\n", "iteration: 649000 loss: 0.0004 lr: 0.002\n", "iteration: 650000 loss: 0.0004 lr: 0.002\n", "iteration: 651000 loss: 0.0004 lr: 0.002\n", "iteration: 652000 loss: 0.0004 lr: 0.002\n", "iteration: 653000 loss: 0.0004 lr: 0.002\n", "iteration: 654000 loss: 0.0004 lr: 0.002\n", "iteration: 655000 loss: 0.0004 lr: 0.002\n", "iteration: 656000 loss: 0.0004 lr: 0.002\n", "iteration: 657000 loss: 0.0004 lr: 0.002\n", "iteration: 658000 loss: 0.0004 lr: 0.002\n", "iteration: 659000 loss: 0.0004 lr: 0.002\n", "iteration: 660000 loss: 0.0004 lr: 0.002\n", "iteration: 661000 loss: 0.0003 lr: 0.002\n", "iteration: 662000 loss: 0.0004 lr: 0.002\n", "iteration: 663000 loss: 0.0003 lr: 0.002\n", "iteration: 664000 loss: 0.0004 lr: 0.002\n", "iteration: 665000 loss: 0.0003 lr: 0.002\n", "iteration: 666000 loss: 0.0004 lr: 0.002\n", "iteration: 667000 loss: 0.0004 lr: 0.002\n", "iteration: 668000 loss: 0.0004 lr: 0.002\n", "iteration: 669000 loss: 0.0004 lr: 0.002\n", "iteration: 670000 loss: 0.0004 lr: 0.002\n", "iteration: 671000 loss: 0.0004 lr: 0.002\n", "iteration: 672000 loss: 0.0004 lr: 0.002\n", "iteration: 673000 loss: 0.0004 lr: 0.002\n", "iteration: 674000 loss: 0.0003 lr: 0.002\n", "iteration: 675000 loss: 0.0004 lr: 0.002\n", "iteration: 676000 loss: 0.0004 lr: 0.002\n", "iteration: 677000 loss: 0.0004 lr: 0.002\n", "iteration: 678000 loss: 0.0003 lr: 0.002\n", "iteration: 679000 loss: 0.0004 lr: 0.002\n", "iteration: 680000 loss: 0.0003 lr: 0.002\n", "iteration: 681000 loss: 0.0004 lr: 0.002\n", "iteration: 682000 loss: 0.0003 lr: 0.002\n", "iteration: 683000 loss: 0.0004 lr: 0.002\n", "iteration: 684000 loss: 0.0004 lr: 0.002\n", "iteration: 685000 loss: 0.0004 lr: 0.002\n", "iteration: 686000 loss: 0.0003 lr: 0.002\n", "iteration: 687000 loss: 0.0004 lr: 0.002\n", "iteration: 688000 loss: 0.0003 lr: 0.002\n", "iteration: 689000 loss: 0.0004 lr: 0.002\n", "iteration: 690000 loss: 0.0004 lr: 0.002\n", "iteration: 691000 loss: 0.0004 lr: 0.002\n", "iteration: 692000 loss: 0.0004 lr: 0.002\n", "iteration: 693000 loss: 0.0004 lr: 0.002\n", "iteration: 694000 loss: 0.0003 lr: 0.002\n", "iteration: 695000 loss: 0.0004 lr: 0.002\n", "iteration: 696000 loss: 0.0004 lr: 0.002\n", "iteration: 697000 loss: 0.0004 lr: 0.002\n", "iteration: 698000 loss: 0.0004 lr: 0.002\n", "iteration: 699000 loss: 0.0004 lr: 0.002\n", "iteration: 700000 loss: 0.0004 lr: 0.002\n", "iteration: 701000 loss: 0.0004 lr: 0.002\n", "iteration: 702000 loss: 0.0004 lr: 0.002\n", "iteration: 703000 loss: 0.0004 lr: 0.002\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 704000 loss: 0.0004 lr: 0.002\n", "iteration: 705000 loss: 0.0003 lr: 0.002\n", "iteration: 706000 loss: 0.0003 lr: 0.002\n", "iteration: 707000 loss: 0.0004 lr: 0.002\n", "iteration: 708000 loss: 0.0004 lr: 0.002\n", "iteration: 709000 loss: 0.0004 lr: 0.002\n", "iteration: 710000 loss: 0.0004 lr: 0.002\n", "iteration: 711000 loss: 0.0003 lr: 0.002\n", "iteration: 712000 loss: 0.0004 lr: 0.002\n", "iteration: 713000 loss: 0.0004 lr: 0.002\n", "iteration: 714000 loss: 0.0004 lr: 0.002\n", "iteration: 715000 loss: 0.0004 lr: 0.002\n", "iteration: 716000 loss: 0.0003 lr: 0.002\n", "iteration: 717000 loss: 0.0003 lr: 0.002\n", "iteration: 718000 loss: 0.0003 lr: 0.002\n", "iteration: 719000 loss: 0.0004 lr: 0.002\n", "iteration: 720000 loss: 0.0004 lr: 0.002\n", "iteration: 721000 loss: 0.0004 lr: 0.002\n", "iteration: 722000 loss: 0.0004 lr: 0.002\n", "iteration: 723000 loss: 0.0004 lr: 0.002\n", "iteration: 724000 loss: 0.0003 lr: 0.002\n", "iteration: 725000 loss: 0.0004 lr: 0.002\n", "iteration: 726000 loss: 0.0004 lr: 0.002\n", "iteration: 727000 loss: 0.0004 lr: 0.002\n", "iteration: 728000 loss: 0.0004 lr: 0.002\n", "iteration: 729000 loss: 0.0004 lr: 0.002\n", "iteration: 730000 loss: 0.0004 lr: 0.002\n", "iteration: 731000 loss: 0.0004 lr: 0.001\n", "iteration: 732000 loss: 0.0003 lr: 0.001\n", "iteration: 733000 loss: 0.0004 lr: 0.001\n", "iteration: 734000 loss: 0.0003 lr: 0.001\n", "iteration: 735000 loss: 0.0003 lr: 0.001\n", "iteration: 736000 loss: 0.0004 lr: 0.001\n", "iteration: 737000 loss: 0.0003 lr: 0.001\n", "iteration: 738000 loss: 0.0003 lr: 0.001\n", "iteration: 739000 loss: 0.0004 lr: 0.001\n", "iteration: 740000 loss: 0.0003 lr: 0.001\n", "iteration: 741000 loss: 0.0003 lr: 0.001\n", "iteration: 742000 loss: 0.0003 lr: 0.001\n", "iteration: 743000 loss: 0.0003 lr: 0.001\n", "iteration: 744000 loss: 0.0003 lr: 0.001\n", "iteration: 745000 loss: 0.0003 lr: 0.001\n", "iteration: 746000 loss: 0.0004 lr: 0.001\n", "iteration: 747000 loss: 0.0003 lr: 0.001\n", "iteration: 748000 loss: 0.0004 lr: 0.001\n", "iteration: 749000 loss: 0.0003 lr: 0.001\n", "iteration: 750000 loss: 0.0003 lr: 0.001\n", "iteration: 751000 loss: 0.0004 lr: 0.001\n", "iteration: 752000 loss: 0.0004 lr: 0.001\n", "iteration: 753000 loss: 0.0003 lr: 0.001\n", "iteration: 754000 loss: 0.0003 lr: 0.001\n", "iteration: 755000 loss: 0.0003 lr: 0.001\n", "iteration: 756000 loss: 0.0004 lr: 0.001\n", "iteration: 757000 loss: 0.0003 lr: 0.001\n", "iteration: 758000 loss: 0.0003 lr: 0.001\n", "iteration: 759000 loss: 0.0003 lr: 0.001\n", "iteration: 760000 loss: 0.0003 lr: 0.001\n", "iteration: 761000 loss: 0.0003 lr: 0.001\n", "iteration: 762000 loss: 0.0003 lr: 0.001\n", "iteration: 763000 loss: 0.0003 lr: 0.001\n", "iteration: 764000 loss: 0.0003 lr: 0.001\n", "iteration: 765000 loss: 0.0004 lr: 0.001\n", "iteration: 766000 loss: 0.0003 lr: 0.001\n", "iteration: 767000 loss: 0.0004 lr: 0.001\n", "iteration: 768000 loss: 0.0003 lr: 0.001\n", "iteration: 769000 loss: 0.0003 lr: 0.001\n", "iteration: 770000 loss: 0.0003 lr: 0.001\n", "iteration: 771000 loss: 0.0003 lr: 0.001\n", "iteration: 772000 loss: 0.0003 lr: 0.001\n", "iteration: 773000 loss: 0.0003 lr: 0.001\n", "iteration: 774000 loss: 0.0003 lr: 0.001\n", "iteration: 775000 loss: 0.0003 lr: 0.001\n", "iteration: 776000 loss: 0.0003 lr: 0.001\n", "iteration: 777000 loss: 0.0003 lr: 0.001\n", "iteration: 778000 loss: 0.0003 lr: 0.001\n", "iteration: 779000 loss: 0.0003 lr: 0.001\n", "iteration: 780000 loss: 0.0003 lr: 0.001\n", "iteration: 781000 loss: 0.0003 lr: 0.001\n", "iteration: 782000 loss: 0.0004 lr: 0.001\n", "iteration: 783000 loss: 0.0004 lr: 0.001\n", "iteration: 784000 loss: 0.0003 lr: 0.001\n", "iteration: 785000 loss: 0.0003 lr: 0.001\n", "iteration: 786000 loss: 0.0003 lr: 0.001\n", "iteration: 787000 loss: 0.0003 lr: 0.001\n", "iteration: 788000 loss: 0.0003 lr: 0.001\n", "iteration: 789000 loss: 0.0003 lr: 0.001\n", "iteration: 790000 loss: 0.0004 lr: 0.001\n", "iteration: 791000 loss: 0.0003 lr: 0.001\n", "iteration: 792000 loss: 0.0003 lr: 0.001\n", "iteration: 793000 loss: 0.0003 lr: 0.001\n", "iteration: 794000 loss: 0.0003 lr: 0.001\n", "iteration: 795000 loss: 0.0003 lr: 0.001\n", "iteration: 796000 loss: 0.0003 lr: 0.001\n", "iteration: 797000 loss: 0.0004 lr: 0.001\n", "iteration: 798000 loss: 0.0003 lr: 0.001\n", "iteration: 799000 loss: 0.0003 lr: 0.001\n", "iteration: 800000 loss: 0.0004 lr: 0.001\n", "iteration: 801000 loss: 0.0003 lr: 0.001\n", "iteration: 802000 loss: 0.0003 lr: 0.001\n", "iteration: 803000 loss: 0.0003 lr: 0.001\n", "iteration: 804000 loss: 0.0004 lr: 0.001\n", "iteration: 805000 loss: 0.0003 lr: 0.001\n", "iteration: 806000 loss: 0.0003 lr: 0.001\n", "iteration: 807000 loss: 0.0004 lr: 0.001\n", "iteration: 808000 loss: 0.0003 lr: 0.001\n", "iteration: 809000 loss: 0.0004 lr: 0.001\n", "iteration: 810000 loss: 0.0003 lr: 0.001\n", "iteration: 811000 loss: 0.0004 lr: 0.001\n", "iteration: 812000 loss: 0.0003 lr: 0.001\n", "iteration: 813000 loss: 0.0003 lr: 0.001\n", "iteration: 814000 loss: 0.0003 lr: 0.001\n", "iteration: 815000 loss: 0.0003 lr: 0.001\n", "iteration: 816000 loss: 0.0003 lr: 0.001\n", "iteration: 817000 loss: 0.0003 lr: 0.001\n", "iteration: 818000 loss: 0.0003 lr: 0.001\n", "iteration: 819000 loss: 0.0003 lr: 0.001\n", "iteration: 820000 loss: 0.0004 lr: 0.001\n", "iteration: 821000 loss: 0.0003 lr: 0.001\n", "iteration: 822000 loss: 0.0003 lr: 0.001\n", "iteration: 823000 loss: 0.0003 lr: 0.001\n", "iteration: 824000 loss: 0.0003 lr: 0.001\n", "iteration: 825000 loss: 0.0003 lr: 0.001\n", "iteration: 826000 loss: 0.0003 lr: 0.001\n", "iteration: 827000 loss: 0.0003 lr: 0.001\n", "iteration: 828000 loss: 0.0003 lr: 0.001\n", "iteration: 829000 loss: 0.0003 lr: 0.001\n", "iteration: 830000 loss: 0.0003 lr: 0.001\n", "iteration: 831000 loss: 0.0003 lr: 0.001\n", "iteration: 832000 loss: 0.0003 lr: 0.001\n", "iteration: 833000 loss: 0.0003 lr: 0.001\n", "iteration: 834000 loss: 0.0003 lr: 0.001\n", "iteration: 835000 loss: 0.0003 lr: 0.001\n", "iteration: 836000 loss: 0.0003 lr: 0.001\n", "iteration: 837000 loss: 0.0003 lr: 0.001\n", "iteration: 838000 loss: 0.0003 lr: 0.001\n", "iteration: 839000 loss: 0.0003 lr: 0.001\n", "iteration: 840000 loss: 0.0003 lr: 0.001\n", "iteration: 841000 loss: 0.0003 lr: 0.001\n", "iteration: 842000 loss: 0.0003 lr: 0.001\n", "iteration: 843000 loss: 0.0003 lr: 0.001\n", "iteration: 844000 loss: 0.0003 lr: 0.001\n", "iteration: 845000 loss: 0.0004 lr: 0.001\n", "iteration: 846000 loss: 0.0003 lr: 0.001\n", "iteration: 847000 loss: 0.0003 lr: 0.001\n", "iteration: 848000 loss: 0.0003 lr: 0.001\n", "iteration: 849000 loss: 0.0004 lr: 0.001\n", "iteration: 850000 loss: 0.0003 lr: 0.001\n", "iteration: 851000 loss: 0.0003 lr: 0.001\n", "iteration: 852000 loss: 0.0004 lr: 0.001\n", "iteration: 853000 loss: 0.0003 lr: 0.001\n", "iteration: 854000 loss: 0.0003 lr: 0.001\n", "iteration: 855000 loss: 0.0003 lr: 0.001\n", "iteration: 856000 loss: 0.0003 lr: 0.001\n", "iteration: 857000 loss: 0.0003 lr: 0.001\n", "iteration: 858000 loss: 0.0003 lr: 0.001\n", "iteration: 859000 loss: 0.0003 lr: 0.001\n", "iteration: 860000 loss: 0.0003 lr: 0.001\n", "iteration: 861000 loss: 0.0003 lr: 0.001\n", "iteration: 862000 loss: 0.0003 lr: 0.001\n", "iteration: 863000 loss: 0.0003 lr: 0.001\n", "iteration: 864000 loss: 0.0003 lr: 0.001\n", "iteration: 865000 loss: 0.0004 lr: 0.001\n", "iteration: 866000 loss: 0.0004 lr: 0.001\n", "iteration: 867000 loss: 0.0003 lr: 0.001\n", "iteration: 868000 loss: 0.0003 lr: 0.001\n", "iteration: 869000 loss: 0.0003 lr: 0.001\n", "iteration: 870000 loss: 0.0003 lr: 0.001\n", "iteration: 871000 loss: 0.0003 lr: 0.001\n", "iteration: 872000 loss: 0.0003 lr: 0.001\n", "iteration: 873000 loss: 0.0003 lr: 0.001\n", "iteration: 874000 loss: 0.0003 lr: 0.001\n", "iteration: 875000 loss: 0.0003 lr: 0.001\n", "iteration: 876000 loss: 0.0003 lr: 0.001\n", "iteration: 877000 loss: 0.0003 lr: 0.001\n", "iteration: 878000 loss: 0.0003 lr: 0.001\n", "iteration: 879000 loss: 0.0003 lr: 0.001\n", "iteration: 880000 loss: 0.0004 lr: 0.001\n", "iteration: 881000 loss: 0.0003 lr: 0.001\n", "iteration: 882000 loss: 0.0003 lr: 0.001\n", "iteration: 883000 loss: 0.0003 lr: 0.001\n", "iteration: 884000 loss: 0.0003 lr: 0.001\n", "iteration: 885000 loss: 0.0004 lr: 0.001\n", "iteration: 886000 loss: 0.0003 lr: 0.001\n", "iteration: 887000 loss: 0.0003 lr: 0.001\n", "iteration: 888000 loss: 0.0003 lr: 0.001\n", "iteration: 889000 loss: 0.0003 lr: 0.001\n", "iteration: 890000 loss: 0.0003 lr: 0.001\n", "iteration: 891000 loss: 0.0003 lr: 0.001\n", "iteration: 892000 loss: 0.0003 lr: 0.001\n", "iteration: 893000 loss: 0.0003 lr: 0.001\n", "iteration: 894000 loss: 0.0003 lr: 0.001\n", "iteration: 895000 loss: 0.0003 lr: 0.001\n", "iteration: 896000 loss: 0.0003 lr: 0.001\n", "iteration: 897000 loss: 0.0003 lr: 0.001\n", "iteration: 898000 loss: 0.0003 lr: 0.001\n", "iteration: 899000 loss: 0.0003 lr: 0.001\n", "iteration: 900000 loss: 0.0003 lr: 0.001\n", "iteration: 901000 loss: 0.0003 lr: 0.001\n", "iteration: 902000 loss: 0.0003 lr: 0.001\n", "iteration: 903000 loss: 0.0003 lr: 0.001\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 904000 loss: 0.0004 lr: 0.001\n", "iteration: 905000 loss: 0.0003 lr: 0.001\n", "iteration: 906000 loss: 0.0003 lr: 0.001\n", "iteration: 907000 loss: 0.0003 lr: 0.001\n", "iteration: 908000 loss: 0.0003 lr: 0.001\n", "iteration: 909000 loss: 0.0003 lr: 0.001\n", "iteration: 910000 loss: 0.0003 lr: 0.001\n", "iteration: 911000 loss: 0.0004 lr: 0.001\n", "iteration: 912000 loss: 0.0003 lr: 0.001\n", "iteration: 913000 loss: 0.0003 lr: 0.001\n", "iteration: 914000 loss: 0.0003 lr: 0.001\n", "iteration: 915000 loss: 0.0003 lr: 0.001\n", "iteration: 916000 loss: 0.0003 lr: 0.001\n", "iteration: 917000 loss: 0.0003 lr: 0.001\n", "iteration: 918000 loss: 0.0003 lr: 0.001\n", "iteration: 919000 loss: 0.0003 lr: 0.001\n", "iteration: 920000 loss: 0.0003 lr: 0.001\n", "iteration: 921000 loss: 0.0003 lr: 0.001\n", "iteration: 922000 loss: 0.0003 lr: 0.001\n", "iteration: 923000 loss: 0.0004 lr: 0.001\n", "iteration: 924000 loss: 0.0003 lr: 0.001\n", "iteration: 925000 loss: 0.0003 lr: 0.001\n", "iteration: 926000 loss: 0.0003 lr: 0.001\n", "iteration: 927000 loss: 0.0003 lr: 0.001\n", "iteration: 928000 loss: 0.0003 lr: 0.001\n", "iteration: 929000 loss: 0.0003 lr: 0.001\n", "iteration: 930000 loss: 0.0003 lr: 0.001\n", "iteration: 931000 loss: 0.0003 lr: 0.001\n", "iteration: 932000 loss: 0.0003 lr: 0.001\n", "iteration: 933000 loss: 0.0003 lr: 0.001\n", "iteration: 934000 loss: 0.0003 lr: 0.001\n", "iteration: 935000 loss: 0.0003 lr: 0.001\n", "iteration: 936000 loss: 0.0003 lr: 0.001\n", "iteration: 937000 loss: 0.0003 lr: 0.001\n", "iteration: 938000 loss: 0.0003 lr: 0.001\n", "iteration: 939000 loss: 0.0003 lr: 0.001\n", "iteration: 940000 loss: 0.0003 lr: 0.001\n", "iteration: 941000 loss: 0.0003 lr: 0.001\n", "iteration: 942000 loss: 0.0003 lr: 0.001\n", "iteration: 943000 loss: 0.0003 lr: 0.001\n", "iteration: 944000 loss: 0.0003 lr: 0.001\n", "iteration: 945000 loss: 0.0003 lr: 0.001\n", "iteration: 946000 loss: 0.0003 lr: 0.001\n", "iteration: 947000 loss: 0.0003 lr: 0.001\n", "iteration: 948000 loss: 0.0003 lr: 0.001\n", "iteration: 949000 loss: 0.0004 lr: 0.001\n", "iteration: 950000 loss: 0.0003 lr: 0.001\n", "iteration: 951000 loss: 0.0003 lr: 0.001\n", "iteration: 952000 loss: 0.0003 lr: 0.001\n", "iteration: 953000 loss: 0.0003 lr: 0.001\n", "iteration: 954000 loss: 0.0003 lr: 0.001\n", "iteration: 955000 loss: 0.0003 lr: 0.001\n", "iteration: 956000 loss: 0.0003 lr: 0.001\n", "iteration: 957000 loss: 0.0003 lr: 0.001\n", "iteration: 958000 loss: 0.0003 lr: 0.001\n", "iteration: 959000 loss: 0.0003 lr: 0.001\n", "iteration: 960000 loss: 0.0003 lr: 0.001\n", "iteration: 961000 loss: 0.0003 lr: 0.001\n", "iteration: 962000 loss: 0.0003 lr: 0.001\n", "iteration: 963000 loss: 0.0003 lr: 0.001\n", "iteration: 964000 loss: 0.0003 lr: 0.001\n", "iteration: 965000 loss: 0.0003 lr: 0.001\n", "iteration: 966000 loss: 0.0003 lr: 0.001\n", "iteration: 967000 loss: 0.0003 lr: 0.001\n", "iteration: 968000 loss: 0.0003 lr: 0.001\n", "iteration: 969000 loss: 0.0003 lr: 0.001\n", "iteration: 970000 loss: 0.0003 lr: 0.001\n", "iteration: 971000 loss: 0.0003 lr: 0.001\n", "iteration: 972000 loss: 0.0003 lr: 0.001\n", "iteration: 973000 loss: 0.0003 lr: 0.001\n", "iteration: 974000 loss: 0.0003 lr: 0.001\n", "iteration: 975000 loss: 0.0003 lr: 0.001\n", "iteration: 976000 loss: 0.0003 lr: 0.001\n", "iteration: 977000 loss: 0.0003 lr: 0.001\n", "iteration: 978000 loss: 0.0003 lr: 0.001\n", "iteration: 979000 loss: 0.0003 lr: 0.001\n", "iteration: 980000 loss: 0.0003 lr: 0.001\n", "iteration: 981000 loss: 0.0003 lr: 0.001\n", "iteration: 982000 loss: 0.0003 lr: 0.001\n", "iteration: 983000 loss: 0.0003 lr: 0.001\n", "iteration: 984000 loss: 0.0003 lr: 0.001\n", "iteration: 985000 loss: 0.0003 lr: 0.001\n", "iteration: 986000 loss: 0.0003 lr: 0.001\n", "iteration: 987000 loss: 0.0003 lr: 0.001\n", "iteration: 988000 loss: 0.0003 lr: 0.001\n", "iteration: 989000 loss: 0.0003 lr: 0.001\n", "iteration: 990000 loss: 0.0003 lr: 0.001\n", "iteration: 991000 loss: 0.0003 lr: 0.001\n", "iteration: 992000 loss: 0.0003 lr: 0.001\n", "iteration: 993000 loss: 0.0003 lr: 0.001\n", "iteration: 994000 loss: 0.0003 lr: 0.001\n", "iteration: 995000 loss: 0.0003 lr: 0.001\n", "iteration: 996000 loss: 0.0003 lr: 0.001\n", "iteration: 997000 loss: 0.0003 lr: 0.001\n", "iteration: 998000 loss: 0.0003 lr: 0.001\n", "iteration: 999000 loss: 0.0003 lr: 0.001\n", "iteration: 1000000 loss: 0.0003 lr: 0.001\n", "iteration: 1001000 loss: 0.0003 lr: 0.001\n", "iteration: 1002000 loss: 0.0003 lr: 0.001\n", "iteration: 1003000 loss: 0.0003 lr: 0.001\n", "iteration: 1004000 loss: 0.0003 lr: 0.001\n", "iteration: 1005000 loss: 0.0003 lr: 0.001\n", "iteration: 1006000 loss: 0.0003 lr: 0.001\n", "iteration: 1007000 loss: 0.0003 lr: 0.001\n", "iteration: 1008000 loss: 0.0003 lr: 0.001\n", "iteration: 1009000 loss: 0.0003 lr: 0.001\n", "iteration: 1010000 loss: 0.0003 lr: 0.001\n", "iteration: 1011000 loss: 0.0003 lr: 0.001\n", "iteration: 1012000 loss: 0.0003 lr: 0.001\n", "iteration: 1013000 loss: 0.0003 lr: 0.001\n", "iteration: 1014000 loss: 0.0003 lr: 0.001\n", "iteration: 1015000 loss: 0.0003 lr: 0.001\n", "iteration: 1016000 loss: 0.0003 lr: 0.001\n", "iteration: 1017000 loss: 0.0003 lr: 0.001\n", "iteration: 1018000 loss: 0.0003 lr: 0.001\n", "iteration: 1019000 loss: 0.0003 lr: 0.001\n", "iteration: 1020000 loss: 0.0003 lr: 0.001\n", "iteration: 1021000 loss: 0.0003 lr: 0.001\n", "iteration: 1022000 loss: 0.0003 lr: 0.001\n", "iteration: 1023000 loss: 0.0003 lr: 0.001\n", "iteration: 1024000 loss: 0.0003 lr: 0.001\n", "iteration: 1025000 loss: 0.0003 lr: 0.001\n", "iteration: 1026000 loss: 0.0003 lr: 0.001\n", "iteration: 1027000 loss: 0.0003 lr: 0.001\n", "iteration: 1028000 loss: 0.0003 lr: 0.001\n", "iteration: 1029000 loss: 0.0003 lr: 0.001\n", "iteration: 1030000 loss: 0.0003 lr: 0.001\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "The network is now trained and ready to evaluate. Use the function 'evaluate_network' to evaluate the network.\n", "--> training took 11.5 hours\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Exception in thread Thread-10:\n", "Traceback (most recent call last):\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1334, in _do_call\n", " return fn(*args)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1319, in _run_fn\n", " options, feed_dict, fetch_list, target_list, run_metadata)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1407, in _call_tf_sessionrun\n", " run_metadata)\n", "tensorflow.python.framework.errors_impl.CancelledError: Enqueue operation was cancelled\n", "\t [[{{node fifo_queue_enqueue}}]]\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\n", " self.run()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/threading.py\", line 870, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py\", line 81, in load_and_enqueue\n", " sess.run(enqueue_op, feed_dict=food)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 929, in run\n", " run_metadata_ptr)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1152, in _run\n", " feed_dict_tensor, options, run_metadata)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1328, in _do_run\n", " run_metadata)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1348, in _do_call\n", " raise type(e)(node_def, op, message)\n", "tensorflow.python.framework.errors_impl.CancelledError: Enqueue operation was cancelled\n", "\t [[node fifo_queue_enqueue (defined at /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py:67) ]]\n", "\n", "Caused by op 'fifo_queue_enqueue', defined at:\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n", " \"__main__\", mod_spec)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/runpy.py\", line 85, in _run_code\n", " exec(code, run_globals)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel_launcher.py\", line 16, in \n", " app.launch_new_instance()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/traitlets/config/application.py\", line 664, in launch_instance\n", " app.start()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 612, in start\n", " self.io_loop.start()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 149, in start\n", " self.asyncio_loop.run_forever()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n", " self._run_once()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n", " handle._run()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/asyncio/events.py\", line 88, in _run\n", " self._context.run(self._callback, *self._args)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/ioloop.py\", line 690, in \n", " lambda f: self._run_callback(functools.partial(callback, future))\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n", " ret = callback()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 787, in inner\n", " self.run()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 748, in run\n", " yielded = self.gen.send(value)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n", " yield gen.maybe_future(dispatch(*args))\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 268, in dispatch_shell\n", " yield gen.maybe_future(handler(stream, idents, msg))\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 545, in execute_request\n", " user_expressions, allow_stdin,\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 306, in do_execute\n", " res = shell.run_cell(code, store_history=store_history, silent=silent)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n", " return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2877, in run_cell\n", " raw_cell, store_history, silent, shell_futures)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2922, in _run_cell\n", " return runner(coro)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n", " coro.send(None)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3146, in run_cell_async\n", " interactivity=interactivity, compiler=compiler, result=result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3337, in run_ast_nodes\n", " if (await self.run_code(code, result, async_=asy)):\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3417, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 2, in \n", " dlc.train_network(cfg)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/training.py\", line 132, in train_network\n", " train(str(poseconfigfile),displayiters,saveiters,maxiters,max_to_keep=max_snapshots_to_keep,keepdeconvweights=keepdeconvweights,allow_growth=allow_growth) #pass on path and file name for pose_cfg.yaml!\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py\", line 118, in train\n", " batch, enqueue_op, placeholders = setup_preloading(batch_spec)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py\", line 67, in setup_preloading\n", " enqueue_op = q.enqueue(placeholders_list)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/ops/data_flow_ops.py\", line 345, in enqueue\n", " self._queue_ref, vals, name=scope)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/ops/gen_data_flow_ops.py\", line 4158, in queue_enqueue_v2\n", " timeout_ms=timeout_ms, name=name)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 788, in _apply_op_helper\n", " op_def=op_def)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py\", line 507, in new_func\n", " return func(*args, **kwargs)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\", line 3300, in create_op\n", " op_def=op_def)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\", line 1801, in __init__\n", " self._traceback = tf_stack.extract_stack()\n", "\n", "CancelledError (see above for traceback): Enqueue operation was cancelled\n", "\t [[node fifo_queue_enqueue (defined at /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py:67) ]]\n", "\n", "\n" ] } ], "source": [ "start = now()\n", "dlc.train_network(cfg)\n", "end = now()\n", "print(f\"--> training took {(end - start)/3600:.1f} hours\")" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "video = [\"../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz\"]\n", "dest = Path(\"../labeled/S005_2020-09-05/iteration-0\")\n", "if not dest.exists():\n", " dest.mkdir(parents=True)" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Config:\n", "{'all_joints': [[0], [1], [2]],\n", " 'all_joints_names': ['Tip1', 'Tip2', 'Tip3'],\n", " 'batch_size': 1,\n", " 'bottomheight': 400,\n", " 'crop': True,\n", " 'crop_pad': 0,\n", " 'cropratio': 0.4,\n", " 'dataset': 'training-datasets/iteration-0/UnaugmentedDataSet_S005-tipsSep5/S005-tips_Keisuke95shuffle1.mat',\n", " 'dataset_type': 'default',\n", " 'deconvolutionstride': 2,\n", " 'deterministic': False,\n", " 'display_iters': 1000,\n", " 'fg_fraction': 0.25,\n", " 'global_scale': 0.8,\n", " 'init_weights': '/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt',\n", " 'intermediate_supervision': False,\n", " 'intermediate_supervision_layer': 12,\n", " 'leftwidth': 400,\n", " 'location_refinement': True,\n", " 'locref_huber_loss': True,\n", " 'locref_loss_weight': 0.05,\n", " 'locref_stdev': 7.2801,\n", " 'log_dir': 'log',\n", " 'max_input_size': 1500,\n", " 'mean_pixel': [123.68, 116.779, 103.939],\n", " 'metadataset': 'training-datasets/iteration-0/UnaugmentedDataSet_S005-tipsSep5/Documentation_data-S005-tips_95shuffle1.pickle',\n", " 'min_input_size': 64,\n", " 'minsize': 100,\n", " 'mirror': False,\n", " 'multi_step': [[0.005, 10000],\n", " [0.02, 430000],\n", " [0.002, 730000],\n", " [0.001, 1030000]],\n", " 'net_type': 'resnet_50',\n", " 'num_joints': 3,\n", " 'num_outputs': 1,\n", " 'optimizer': 'sgd',\n", " 'output_stride': 16,\n", " 'pos_dist_thresh': 17,\n", " 'project_path': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05',\n", " 'regularize': False,\n", " 'rightwidth': 400,\n", " 'save_iters': 50000,\n", " 'scale_jitter_lo': 0.5,\n", " 'scale_jitter_up': 1.25,\n", " 'scoremap_dir': 'test',\n", " 'shuffle': True,\n", " 'snapshot_prefix': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-0/S005-tipsSep5-trainset95shuffle1/test/snapshot',\n", " 'stride': 8.0,\n", " 'topheight': 400,\n", " 'weigh_negatives': False,\n", " 'weigh_only_present_joints': False,\n", " 'weigh_part_predictions': False,\n", " 'weight_decay': 0.0001}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Using snapshot-1030000 for model /home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-0/S005-tipsSep5-trainset95shuffle1\n", "Initializing ResNet\n", "INFO:tensorflow:Restoring parameters from /home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-0/S005-tipsSep5-trainset95shuffle1/train/snapshot-1030000\n", "Starting to analyze: ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", " 0%| | 0/3781 [00:00iteration-0\n", "Loading ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz and data.\n", "reader=NumpyPicker, writer=SkVideoWriter\n", "opening source: ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 1/3781 [00:00<10:27, 6.02it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "opening sink: ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520DLC_resnet50_S005-tipsSep5shuffle1_1030000_labeled.mp4\n", "opened a writer to: ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520DLC_resnet50_S005-tipsSep5shuffle1_1030000_labeled.mp4\n", "Duration of video [s]: 37.84 , recorded with 99.93 fps!\n", "Overall # of frames: 3781 with cropped frame dimensions: 640 480\n", "Generating frames and creating video.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████| 3781/3781 [00:10<00:00, 356.83it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "closed: ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520DLC_resnet50_S005-tipsSep5shuffle1_1030000_labeled.mp4\n" ] } ], "source": [ "dlc.create_labeled_video(cfg, video, destfolder=str(dest),\n", " videotype=\".npz\", reader=\"numpy\", writer=\"skvideo\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Iteration 1" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "cfg = '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/config.yaml'\n", "video = [\"../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz\"]\n", "annotation = Path(\"../labeled/S005_2020-09-05/iteration-0\")" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Method jump found 1069 putative outlier frames.\n", "Do you want to proceed with extracting 30 of those?\n", "If this list is very large, perhaps consider changing the paramters (start, stop, epsilon, comparisonbodyparts) or use a different method.\n", "yes/noyes\n", "Frames from video S005-19_2020-09-05_run124520 already extracted (more will be added)!\n", "Loading video...\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "114it [00:00, 1130.93it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Duration of video [s]: 37.83732828034295 , recorded @ 99.92777428643885 fps!\n", "Overall # of frames: 3781 with (cropped) frame dimensions: (0, 640, 0, 480)\n", "Kmeans-quantization based extracting of frames from 0.0 seconds to 37.84 seconds.\n", "Extracting and downsampling 1069 frames from the video.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "1069it [00:00, 1128.18it/s]\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Kmeans clustering ... (this might take a while)\n", "Let's select frames indices: [2261, 202, 3315, 1087, 3581, 1554, 343, 1297, 3040, 532, 529, 2129, 3633, 421, 1224, 646, 2847, 2858, 3487, 11, 2191, 882, 1996, 2557, 3082, 1370, 2992, 2494, 2691, 3545]\n", "New video was added to the project! Use the function 'extract_frames' to select frames for labeling.\n", "The outlier frames are extracted. They are stored in the subdirectory labeled-data\\S005-19_2020-09-05_run124520.\n", "Once you extracted frames for all videos, use 'refine_labels' to manually correct the labels.\n" ] } ], "source": [ "dlc.extract_outlier_frames(cfg, video, videotype=\".npz\", driver=\"numpy\", destfolder=str(annotation),\n", " cluster_resizewidth=640, epsilon=100)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Notes on the results\n", "\n", "The mistakes seemed to have occurred because I used brightness-corrected images for training: as soon as I started using the raw frames, areas that used to be saturated in the corrected images came to confuse the model.\n", "\n", "So: I only use the corrected images for labeling, and not for training from now on." ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Checking labels if they are outside the image\n", "A training dataset file is already found for this video. The refined machine labels are merged to this data!\n", "Closing... The refined labels are stored in a subdirectory under labeled-data. Use the function 'merge_datasets' to augment the training dataset, and then re-train a network using create_training_dataset followed by train_network!\n" ] } ], "source": [ "dlc.refine_labels(cfg)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### trying to revert the images labeled during iteration 0" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "picker = dlc.utils.frame_pickers.NumpyPicker(video[0])" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "from imageio import imsave" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "tmp = Path(\"S005-tips-Keisuke-2020-09-05/labeled-data/S005-19_2020-09-05_run124520/origtmp\")\n", "tmp.mkdir(parents=True)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "import re\n", "pat = re.compile(r\"img(\\d{5})\\.png\")" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [], "source": [ "indexes = []\n", "for p in tmp.parent.iterdir():\n", " matches = pat.match(p.name)\n", " if matches:\n", " indexes.append(int(matches.group(1)))" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "for index in indexes:\n", " dst = tmp / f\"img{str(index).zfill(5)}.png\"\n", " imsave(str(dst), picker.pick_single(index))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Running training" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Merged data sets and updated refinement iteration to 1.\n", "Now you can create a new training set for the expanded annotated images (use create_training_dataset).\n" ] } ], "source": [ "dlc.merge_datasets(cfg)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "The training dataset is successfully created. Use the function 'train_network' to start training. Happy training!\n" ] }, { "data": { "text/plain": [ "[(0.95,\n", " 1,\n", " (array([17, 15, 36, 47, 5, 11, 31, 55, 54, 18, 40, 44, 46, 3, 56, 32, 29,\n", " 48, 6, 1, 39, 49, 45, 23, 25, 52, 24, 0, 12, 10, 13, 41, 50, 35,\n", " 26, 59, 34, 14, 19, 33, 28, 9, 51, 7, 20, 57, 30, 42, 38, 21, 4,\n", " 2, 37, 58, 8, 43, 53]),\n", " array([16, 22, 27])))]" ] }, "execution_count": 17, "metadata": {}, "output_type": "execute_result" } ], "source": [ "dlc.create_training_dataset(cfg)" ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Config:\n", "{'all_joints': [[0], [1], [2]],\n", " 'all_joints_names': ['Tip1', 'Tip2', 'Tip3'],\n", " 'batch_size': 1,\n", " 'bottomheight': 400,\n", " 'crop': True,\n", " 'crop_pad': 0,\n", " 'cropratio': 0.4,\n", " 'dataset': 'training-datasets/iteration-1/UnaugmentedDataSet_S005-tipsSep5/S005-tips_Keisuke95shuffle1.mat',\n", " 'dataset_type': 'default',\n", " 'deterministic': False,\n", " 'display_iters': 1000,\n", " 'fg_fraction': 0.25,\n", " 'global_scale': 0.8,\n", " 'init_weights': '/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt',\n", " 'intermediate_supervision': False,\n", " 'intermediate_supervision_layer': 12,\n", " 'leftwidth': 400,\n", " 'location_refinement': True,\n", " 'locref_huber_loss': True,\n", " 'locref_loss_weight': 0.05,\n", " 'locref_stdev': 7.2801,\n", " 'log_dir': 'log',\n", " 'max_input_size': 1500,\n", " 'mean_pixel': [123.68, 116.779, 103.939],\n", " 'metadataset': 'training-datasets/iteration-1/UnaugmentedDataSet_S005-tipsSep5/Documentation_data-S005-tips_95shuffle1.pickle',\n", " 'min_input_size': 64,\n", " 'minsize': 100,\n", " 'mirror': False,\n", " 'multi_step': [[0.005, 10000],\n", " [0.02, 430000],\n", " [0.002, 730000],\n", " [0.001, 1030000]],\n", " 'net_type': 'resnet_50',\n", " 'num_joints': 3,\n", " 'optimizer': 'sgd',\n", " 'pos_dist_thresh': 17,\n", " 'project_path': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05',\n", " 'regularize': False,\n", " 'rightwidth': 400,\n", " 'save_iters': 50000,\n", " 'scale_jitter_lo': 0.5,\n", " 'scale_jitter_up': 1.25,\n", " 'scoremap_dir': 'test',\n", " 'shuffle': True,\n", " 'snapshot_prefix': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-1/S005-tipsSep5-trainset95shuffle1/train/snapshot',\n", " 'stride': 8.0,\n", " 'topheight': 400,\n", " 'weigh_negatives': False,\n", " 'weigh_only_present_joints': False,\n", " 'weigh_part_predictions': False,\n", " 'weight_decay': 0.0001}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Switching batchsize to 1, as default/tensorpack/deterministic loaders do not support batches >1. Use imgaug loader.\n", "Starting with standard pose-dataset loader.\n", "Initializing ResNet\n", "WARNING:tensorflow:From /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\n", "WARNING:tensorflow:From /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/ops/losses/losses_impl.py:209: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use tf.cast instead.\n", "Loading ImageNet-pretrained resnet_50\n", "WARNING:tensorflow:From /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use standard file APIs to check for files with this prefix.\n", "INFO:tensorflow:Restoring parameters from /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt\n", "Training parameter:\n", "{'stride': 8.0, 'weigh_part_predictions': False, 'weigh_negatives': False, 'fg_fraction': 0.25, 'weigh_only_present_joints': False, 'mean_pixel': [123.68, 116.779, 103.939], 'shuffle': True, 'snapshot_prefix': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-1/S005-tipsSep5-trainset95shuffle1/train/snapshot', 'log_dir': 'log', 'global_scale': 0.8, 'location_refinement': True, 'locref_stdev': 7.2801, 'locref_loss_weight': 0.05, 'locref_huber_loss': True, 'optimizer': 'sgd', 'intermediate_supervision': False, 'intermediate_supervision_layer': 12, 'regularize': False, 'weight_decay': 0.0001, 'mirror': False, 'crop_pad': 0, 'scoremap_dir': 'test', 'batch_size': 1, 'dataset_type': 'default', 'deterministic': False, 'crop': True, 'cropratio': 0.4, 'minsize': 100, 'leftwidth': 400, 'rightwidth': 400, 'topheight': 400, 'bottomheight': 400, 'all_joints': [[0], [1], [2]], 'all_joints_names': ['Tip1', 'Tip2', 'Tip3'], 'dataset': 'training-datasets/iteration-1/UnaugmentedDataSet_S005-tipsSep5/S005-tips_Keisuke95shuffle1.mat', 'display_iters': 1000, 'init_weights': '/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt', 'max_input_size': 1500, 'metadataset': 'training-datasets/iteration-1/UnaugmentedDataSet_S005-tipsSep5/Documentation_data-S005-tips_95shuffle1.pickle', 'min_input_size': 64, 'multi_step': [[0.005, 10000], [0.02, 430000], [0.002, 730000], [0.001, 1030000]], 'net_type': 'resnet_50', 'num_joints': 3, 'pos_dist_thresh': 17, 'project_path': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05', 'save_iters': 50000, 'scale_jitter_lo': 0.5, 'scale_jitter_up': 1.25, 'output_stride': 16, 'deconvolutionstride': 2}\n", "Starting training....\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 1000 loss: 0.0247 lr: 0.005\n", "iteration: 2000 loss: 0.0094 lr: 0.005\n", "iteration: 3000 loss: 0.0071 lr: 0.005\n", "iteration: 4000 loss: 0.0062 lr: 0.005\n", "iteration: 5000 loss: 0.0054 lr: 0.005\n", "iteration: 6000 loss: 0.0050 lr: 0.005\n", "iteration: 7000 loss: 0.0046 lr: 0.005\n", "iteration: 8000 loss: 0.0044 lr: 0.005\n", "iteration: 9000 loss: 0.0043 lr: 0.005\n", "iteration: 10000 loss: 0.0041 lr: 0.005\n", "iteration: 11000 loss: 0.0069 lr: 0.02\n", "iteration: 12000 loss: 0.0055 lr: 0.02\n", "iteration: 13000 loss: 0.0051 lr: 0.02\n", "iteration: 14000 loss: 0.0043 lr: 0.02\n", "iteration: 15000 loss: 0.0041 lr: 0.02\n", "iteration: 16000 loss: 0.0039 lr: 0.02\n", "iteration: 17000 loss: 0.0036 lr: 0.02\n", "iteration: 18000 loss: 0.0033 lr: 0.02\n", "iteration: 19000 loss: 0.0034 lr: 0.02\n", "iteration: 20000 loss: 0.0033 lr: 0.02\n", "iteration: 21000 loss: 0.0029 lr: 0.02\n", "iteration: 22000 loss: 0.0031 lr: 0.02\n", "iteration: 23000 loss: 0.0031 lr: 0.02\n", "iteration: 24000 loss: 0.0029 lr: 0.02\n", "iteration: 25000 loss: 0.0029 lr: 0.02\n", "iteration: 26000 loss: 0.0028 lr: 0.02\n", "iteration: 27000 loss: 0.0026 lr: 0.02\n", "iteration: 28000 loss: 0.0026 lr: 0.02\n", "iteration: 29000 loss: 0.0027 lr: 0.02\n", "iteration: 30000 loss: 0.0025 lr: 0.02\n", "iteration: 31000 loss: 0.0025 lr: 0.02\n", "iteration: 32000 loss: 0.0025 lr: 0.02\n", "iteration: 33000 loss: 0.0024 lr: 0.02\n", "iteration: 34000 loss: 0.0024 lr: 0.02\n", "iteration: 35000 loss: 0.0023 lr: 0.02\n", "iteration: 36000 loss: 0.0023 lr: 0.02\n", "iteration: 37000 loss: 0.0023 lr: 0.02\n", "iteration: 38000 loss: 0.0023 lr: 0.02\n", "iteration: 39000 loss: 0.0023 lr: 0.02\n", "iteration: 40000 loss: 0.0023 lr: 0.02\n", "iteration: 41000 loss: 0.0023 lr: 0.02\n", "iteration: 42000 loss: 0.0021 lr: 0.02\n", "iteration: 43000 loss: 0.0022 lr: 0.02\n", "iteration: 44000 loss: 0.0022 lr: 0.02\n", "iteration: 45000 loss: 0.0022 lr: 0.02\n", "iteration: 46000 loss: 0.0022 lr: 0.02\n", "iteration: 47000 loss: 0.0021 lr: 0.02\n", "iteration: 48000 loss: 0.0020 lr: 0.02\n", "iteration: 49000 loss: 0.0022 lr: 0.02\n", "iteration: 50000 loss: 0.0022 lr: 0.02\n", "iteration: 51000 loss: 0.0021 lr: 0.02\n", "iteration: 52000 loss: 0.0020 lr: 0.02\n", "iteration: 53000 loss: 0.0021 lr: 0.02\n", "iteration: 54000 loss: 0.0020 lr: 0.02\n", "iteration: 55000 loss: 0.0020 lr: 0.02\n", "iteration: 56000 loss: 0.0021 lr: 0.02\n", "iteration: 57000 loss: 0.0020 lr: 0.02\n", "iteration: 58000 loss: 0.0020 lr: 0.02\n", "iteration: 59000 loss: 0.0020 lr: 0.02\n", "iteration: 60000 loss: 0.0018 lr: 0.02\n", "iteration: 61000 loss: 0.0019 lr: 0.02\n", "iteration: 62000 loss: 0.0019 lr: 0.02\n", "iteration: 63000 loss: 0.0020 lr: 0.02\n", "iteration: 64000 loss: 0.0019 lr: 0.02\n", "iteration: 65000 loss: 0.0019 lr: 0.02\n", "iteration: 66000 loss: 0.0018 lr: 0.02\n", "iteration: 67000 loss: 0.0018 lr: 0.02\n", "iteration: 68000 loss: 0.0018 lr: 0.02\n", "iteration: 69000 loss: 0.0017 lr: 0.02\n", "iteration: 70000 loss: 0.0018 lr: 0.02\n", "iteration: 71000 loss: 0.0016 lr: 0.02\n", "iteration: 72000 loss: 0.0016 lr: 0.02\n", "iteration: 73000 loss: 0.0017 lr: 0.02\n", "iteration: 74000 loss: 0.0017 lr: 0.02\n", "iteration: 75000 loss: 0.0016 lr: 0.02\n", "iteration: 76000 loss: 0.0017 lr: 0.02\n", "iteration: 77000 loss: 0.0016 lr: 0.02\n", "iteration: 78000 loss: 0.0015 lr: 0.02\n", "iteration: 79000 loss: 0.0016 lr: 0.02\n", "iteration: 80000 loss: 0.0016 lr: 0.02\n", "iteration: 81000 loss: 0.0016 lr: 0.02\n", "iteration: 82000 loss: 0.0016 lr: 0.02\n", "iteration: 83000 loss: 0.0016 lr: 0.02\n", "iteration: 84000 loss: 0.0016 lr: 0.02\n", "iteration: 85000 loss: 0.0016 lr: 0.02\n", "iteration: 86000 loss: 0.0016 lr: 0.02\n", "iteration: 87000 loss: 0.0015 lr: 0.02\n", "iteration: 88000 loss: 0.0016 lr: 0.02\n", "iteration: 89000 loss: 0.0016 lr: 0.02\n", "iteration: 90000 loss: 0.0016 lr: 0.02\n", "iteration: 91000 loss: 0.0016 lr: 0.02\n", "iteration: 92000 loss: 0.0015 lr: 0.02\n", "iteration: 93000 loss: 0.0015 lr: 0.02\n", "iteration: 94000 loss: 0.0015 lr: 0.02\n", "iteration: 95000 loss: 0.0015 lr: 0.02\n", "iteration: 96000 loss: 0.0015 lr: 0.02\n", "iteration: 97000 loss: 0.0014 lr: 0.02\n", "iteration: 98000 loss: 0.0015 lr: 0.02\n", "iteration: 99000 loss: 0.0014 lr: 0.02\n", "iteration: 100000 loss: 0.0015 lr: 0.02\n", "iteration: 101000 loss: 0.0015 lr: 0.02\n", "iteration: 102000 loss: 0.0014 lr: 0.02\n", "iteration: 103000 loss: 0.0014 lr: 0.02\n", "iteration: 104000 loss: 0.0014 lr: 0.02\n", "iteration: 105000 loss: 0.0014 lr: 0.02\n", "iteration: 106000 loss: 0.0014 lr: 0.02\n", "iteration: 107000 loss: 0.0014 lr: 0.02\n", "iteration: 108000 loss: 0.0013 lr: 0.02\n", "iteration: 109000 loss: 0.0014 lr: 0.02\n", "iteration: 110000 loss: 0.0014 lr: 0.02\n", "iteration: 111000 loss: 0.0014 lr: 0.02\n", "iteration: 112000 loss: 0.0013 lr: 0.02\n", "iteration: 113000 loss: 0.0014 lr: 0.02\n", "iteration: 114000 loss: 0.0013 lr: 0.02\n", "iteration: 115000 loss: 0.0014 lr: 0.02\n", "iteration: 116000 loss: 0.0013 lr: 0.02\n", "iteration: 117000 loss: 0.0013 lr: 0.02\n", "iteration: 118000 loss: 0.0013 lr: 0.02\n", "iteration: 119000 loss: 0.0013 lr: 0.02\n", "iteration: 120000 loss: 0.0013 lr: 0.02\n", "iteration: 121000 loss: 0.0013 lr: 0.02\n", "iteration: 122000 loss: 0.0014 lr: 0.02\n", "iteration: 123000 loss: 0.0013 lr: 0.02\n", "iteration: 124000 loss: 0.0013 lr: 0.02\n", "iteration: 125000 loss: 0.0013 lr: 0.02\n", "iteration: 126000 loss: 0.0013 lr: 0.02\n", "iteration: 127000 loss: 0.0013 lr: 0.02\n", "iteration: 128000 loss: 0.0012 lr: 0.02\n", "iteration: 129000 loss: 0.0013 lr: 0.02\n", "iteration: 130000 loss: 0.0013 lr: 0.02\n", "iteration: 131000 loss: 0.0013 lr: 0.02\n", "iteration: 132000 loss: 0.0013 lr: 0.02\n", "iteration: 133000 loss: 0.0013 lr: 0.02\n", "iteration: 134000 loss: 0.0013 lr: 0.02\n", "iteration: 135000 loss: 0.0012 lr: 0.02\n", "iteration: 136000 loss: 0.0014 lr: 0.02\n", "iteration: 137000 loss: 0.0012 lr: 0.02\n", "iteration: 138000 loss: 0.0012 lr: 0.02\n", "iteration: 139000 loss: 0.0012 lr: 0.02\n", "iteration: 140000 loss: 0.0012 lr: 0.02\n", "iteration: 141000 loss: 0.0012 lr: 0.02\n", "iteration: 142000 loss: 0.0012 lr: 0.02\n", "iteration: 143000 loss: 0.0012 lr: 0.02\n", "iteration: 144000 loss: 0.0012 lr: 0.02\n", "iteration: 145000 loss: 0.0012 lr: 0.02\n", "iteration: 146000 loss: 0.0012 lr: 0.02\n", "iteration: 147000 loss: 0.0012 lr: 0.02\n", "iteration: 148000 loss: 0.0012 lr: 0.02\n", "iteration: 149000 loss: 0.0012 lr: 0.02\n", "iteration: 150000 loss: 0.0013 lr: 0.02\n", "iteration: 151000 loss: 0.0011 lr: 0.02\n", "iteration: 152000 loss: 0.0011 lr: 0.02\n", "iteration: 153000 loss: 0.0012 lr: 0.02\n", "iteration: 154000 loss: 0.0012 lr: 0.02\n", "iteration: 155000 loss: 0.0012 lr: 0.02\n", "iteration: 156000 loss: 0.0012 lr: 0.02\n", "iteration: 157000 loss: 0.0012 lr: 0.02\n", "iteration: 158000 loss: 0.0012 lr: 0.02\n", "iteration: 159000 loss: 0.0012 lr: 0.02\n", "iteration: 160000 loss: 0.0012 lr: 0.02\n", "iteration: 161000 loss: 0.0011 lr: 0.02\n", "iteration: 162000 loss: 0.0011 lr: 0.02\n", "iteration: 163000 loss: 0.0011 lr: 0.02\n", "iteration: 164000 loss: 0.0012 lr: 0.02\n", "iteration: 165000 loss: 0.0012 lr: 0.02\n", "iteration: 166000 loss: 0.0012 lr: 0.02\n", "iteration: 167000 loss: 0.0012 lr: 0.02\n", "iteration: 168000 loss: 0.0012 lr: 0.02\n", "iteration: 169000 loss: 0.0012 lr: 0.02\n", "iteration: 170000 loss: 0.0012 lr: 0.02\n", "iteration: 171000 loss: 0.0011 lr: 0.02\n", "iteration: 172000 loss: 0.0011 lr: 0.02\n", "iteration: 173000 loss: 0.0012 lr: 0.02\n", "iteration: 174000 loss: 0.0011 lr: 0.02\n", "iteration: 175000 loss: 0.0011 lr: 0.02\n", "iteration: 176000 loss: 0.0011 lr: 0.02\n", "iteration: 177000 loss: 0.0011 lr: 0.02\n", "iteration: 178000 loss: 0.0011 lr: 0.02\n", "iteration: 179000 loss: 0.0011 lr: 0.02\n", "iteration: 180000 loss: 0.0011 lr: 0.02\n", "iteration: 181000 loss: 0.0012 lr: 0.02\n", "iteration: 182000 loss: 0.0011 lr: 0.02\n", "iteration: 183000 loss: 0.0010 lr: 0.02\n", "iteration: 184000 loss: 0.0011 lr: 0.02\n", "iteration: 185000 loss: 0.0011 lr: 0.02\n", "iteration: 186000 loss: 0.0011 lr: 0.02\n", "iteration: 187000 loss: 0.0011 lr: 0.02\n", "iteration: 188000 loss: 0.0010 lr: 0.02\n", "iteration: 189000 loss: 0.0011 lr: 0.02\n", "iteration: 190000 loss: 0.0011 lr: 0.02\n", "iteration: 191000 loss: 0.0011 lr: 0.02\n", "iteration: 192000 loss: 0.0011 lr: 0.02\n", "iteration: 193000 loss: 0.0010 lr: 0.02\n", "iteration: 194000 loss: 0.0011 lr: 0.02\n", "iteration: 195000 loss: 0.0011 lr: 0.02\n", "iteration: 196000 loss: 0.0011 lr: 0.02\n", "iteration: 197000 loss: 0.0011 lr: 0.02\n", "iteration: 198000 loss: 0.0011 lr: 0.02\n", "iteration: 199000 loss: 0.0010 lr: 0.02\n", "iteration: 200000 loss: 0.0011 lr: 0.02\n", "iteration: 201000 loss: 0.0011 lr: 0.02\n", "iteration: 202000 loss: 0.0011 lr: 0.02\n", "iteration: 203000 loss: 0.0012 lr: 0.02\n", "iteration: 204000 loss: 0.0011 lr: 0.02\n", "iteration: 205000 loss: 0.0010 lr: 0.02\n", "iteration: 206000 loss: 0.0011 lr: 0.02\n", "iteration: 207000 loss: 0.0010 lr: 0.02\n", "iteration: 208000 loss: 0.0011 lr: 0.02\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 209000 loss: 0.0011 lr: 0.02\n", "iteration: 210000 loss: 0.0011 lr: 0.02\n", "iteration: 211000 loss: 0.0011 lr: 0.02\n", "iteration: 212000 loss: 0.0011 lr: 0.02\n", "iteration: 213000 loss: 0.0011 lr: 0.02\n", "iteration: 214000 loss: 0.0011 lr: 0.02\n", "iteration: 215000 loss: 0.0010 lr: 0.02\n", "iteration: 216000 loss: 0.0011 lr: 0.02\n", "iteration: 217000 loss: 0.0010 lr: 0.02\n", "iteration: 218000 loss: 0.0011 lr: 0.02\n", "iteration: 219000 loss: 0.0011 lr: 0.02\n", "iteration: 220000 loss: 0.0010 lr: 0.02\n", "iteration: 221000 loss: 0.0010 lr: 0.02\n", "iteration: 222000 loss: 0.0010 lr: 0.02\n", "iteration: 223000 loss: 0.0010 lr: 0.02\n", "iteration: 224000 loss: 0.0010 lr: 0.02\n", "iteration: 225000 loss: 0.0010 lr: 0.02\n", "iteration: 226000 loss: 0.0011 lr: 0.02\n", "iteration: 227000 loss: 0.0011 lr: 0.02\n", "iteration: 228000 loss: 0.0011 lr: 0.02\n", "iteration: 229000 loss: 0.0010 lr: 0.02\n", "iteration: 230000 loss: 0.0010 lr: 0.02\n", "iteration: 231000 loss: 0.0010 lr: 0.02\n", "iteration: 232000 loss: 0.0010 lr: 0.02\n", "iteration: 233000 loss: 0.0010 lr: 0.02\n", "iteration: 234000 loss: 0.0010 lr: 0.02\n", "iteration: 235000 loss: 0.0010 lr: 0.02\n", "iteration: 236000 loss: 0.0010 lr: 0.02\n", "iteration: 237000 loss: 0.0010 lr: 0.02\n", "iteration: 238000 loss: 0.0010 lr: 0.02\n", "iteration: 239000 loss: 0.0009 lr: 0.02\n", "iteration: 240000 loss: 0.0010 lr: 0.02\n", "iteration: 241000 loss: 0.0010 lr: 0.02\n", "iteration: 242000 loss: 0.0010 lr: 0.02\n", "iteration: 243000 loss: 0.0010 lr: 0.02\n", "iteration: 244000 loss: 0.0010 lr: 0.02\n", "iteration: 245000 loss: 0.0010 lr: 0.02\n", "iteration: 246000 loss: 0.0010 lr: 0.02\n", "iteration: 247000 loss: 0.0010 lr: 0.02\n", "iteration: 248000 loss: 0.0010 lr: 0.02\n", "iteration: 249000 loss: 0.0010 lr: 0.02\n", "iteration: 250000 loss: 0.0010 lr: 0.02\n", "iteration: 251000 loss: 0.0010 lr: 0.02\n", "iteration: 252000 loss: 0.0010 lr: 0.02\n", "iteration: 253000 loss: 0.0010 lr: 0.02\n", "iteration: 254000 loss: 0.0010 lr: 0.02\n", "iteration: 255000 loss: 0.0009 lr: 0.02\n", "iteration: 256000 loss: 0.0010 lr: 0.02\n", "iteration: 257000 loss: 0.0010 lr: 0.02\n", "iteration: 258000 loss: 0.0009 lr: 0.02\n", "iteration: 259000 loss: 0.0010 lr: 0.02\n", "iteration: 260000 loss: 0.0009 lr: 0.02\n", "iteration: 261000 loss: 0.0010 lr: 0.02\n", "iteration: 262000 loss: 0.0010 lr: 0.02\n", "iteration: 263000 loss: 0.0010 lr: 0.02\n", "iteration: 264000 loss: 0.0010 lr: 0.02\n", "iteration: 265000 loss: 0.0009 lr: 0.02\n", "iteration: 266000 loss: 0.0009 lr: 0.02\n", "iteration: 267000 loss: 0.0009 lr: 0.02\n", "iteration: 268000 loss: 0.0009 lr: 0.02\n", "iteration: 269000 loss: 0.0010 lr: 0.02\n", "iteration: 270000 loss: 0.0009 lr: 0.02\n", "iteration: 271000 loss: 0.0010 lr: 0.02\n", "iteration: 272000 loss: 0.0009 lr: 0.02\n", "iteration: 273000 loss: 0.0010 lr: 0.02\n", "iteration: 274000 loss: 0.0009 lr: 0.02\n", "iteration: 275000 loss: 0.0009 lr: 0.02\n", "iteration: 276000 loss: 0.0010 lr: 0.02\n", "iteration: 277000 loss: 0.0009 lr: 0.02\n", "iteration: 278000 loss: 0.0010 lr: 0.02\n", "iteration: 279000 loss: 0.0009 lr: 0.02\n", "iteration: 280000 loss: 0.0009 lr: 0.02\n", "iteration: 281000 loss: 0.0009 lr: 0.02\n", "iteration: 282000 loss: 0.0010 lr: 0.02\n", "iteration: 283000 loss: 0.0010 lr: 0.02\n", "iteration: 284000 loss: 0.0009 lr: 0.02\n", "iteration: 285000 loss: 0.0011 lr: 0.02\n", "iteration: 286000 loss: 0.0009 lr: 0.02\n", "iteration: 287000 loss: 0.0010 lr: 0.02\n", "iteration: 288000 loss: 0.0009 lr: 0.02\n", "iteration: 289000 loss: 0.0009 lr: 0.02\n", "iteration: 290000 loss: 0.0009 lr: 0.02\n", "iteration: 291000 loss: 0.0009 lr: 0.02\n", "iteration: 292000 loss: 0.0009 lr: 0.02\n", "iteration: 293000 loss: 0.0009 lr: 0.02\n", "iteration: 294000 loss: 0.0009 lr: 0.02\n", "iteration: 295000 loss: 0.0009 lr: 0.02\n", "iteration: 296000 loss: 0.0009 lr: 0.02\n", "iteration: 297000 loss: 0.0010 lr: 0.02\n", "iteration: 298000 loss: 0.0009 lr: 0.02\n", "iteration: 299000 loss: 0.0009 lr: 0.02\n", "iteration: 300000 loss: 0.0009 lr: 0.02\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "WARNING:tensorflow:From /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/training/saver.py:966: remove_checkpoint (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use standard file APIs to delete files with this prefix.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 301000 loss: 0.0009 lr: 0.02\n", "iteration: 302000 loss: 0.0009 lr: 0.02\n", "iteration: 303000 loss: 0.0009 lr: 0.02\n", "iteration: 304000 loss: 0.0009 lr: 0.02\n", "iteration: 305000 loss: 0.0009 lr: 0.02\n", "iteration: 306000 loss: 0.0009 lr: 0.02\n", "iteration: 307000 loss: 0.0009 lr: 0.02\n", "iteration: 308000 loss: 0.0009 lr: 0.02\n", "iteration: 309000 loss: 0.0009 lr: 0.02\n", "iteration: 310000 loss: 0.0009 lr: 0.02\n", "iteration: 311000 loss: 0.0009 lr: 0.02\n", "iteration: 312000 loss: 0.0009 lr: 0.02\n", "iteration: 313000 loss: 0.0009 lr: 0.02\n", "iteration: 314000 loss: 0.0009 lr: 0.02\n", "iteration: 315000 loss: 0.0009 lr: 0.02\n", "iteration: 316000 loss: 0.0009 lr: 0.02\n", "iteration: 317000 loss: 0.0009 lr: 0.02\n", "iteration: 318000 loss: 0.0008 lr: 0.02\n", "iteration: 319000 loss: 0.0009 lr: 0.02\n", "iteration: 320000 loss: 0.0009 lr: 0.02\n", "iteration: 321000 loss: 0.0009 lr: 0.02\n", "iteration: 322000 loss: 0.0009 lr: 0.02\n", "iteration: 323000 loss: 0.0009 lr: 0.02\n", "iteration: 324000 loss: 0.0009 lr: 0.02\n", "iteration: 325000 loss: 0.0009 lr: 0.02\n", "iteration: 326000 loss: 0.0009 lr: 0.02\n", "iteration: 327000 loss: 0.0009 lr: 0.02\n", "iteration: 328000 loss: 0.0009 lr: 0.02\n", "iteration: 329000 loss: 0.0008 lr: 0.02\n", "iteration: 330000 loss: 0.0008 lr: 0.02\n", "iteration: 331000 loss: 0.0009 lr: 0.02\n", "iteration: 332000 loss: 0.0008 lr: 0.02\n", "iteration: 333000 loss: 0.0009 lr: 0.02\n", "iteration: 334000 loss: 0.0009 lr: 0.02\n", "iteration: 335000 loss: 0.0008 lr: 0.02\n", "iteration: 336000 loss: 0.0008 lr: 0.02\n", "iteration: 337000 loss: 0.0009 lr: 0.02\n", "iteration: 338000 loss: 0.0009 lr: 0.02\n", "iteration: 339000 loss: 0.0009 lr: 0.02\n", "iteration: 340000 loss: 0.0009 lr: 0.02\n", "iteration: 341000 loss: 0.0008 lr: 0.02\n", "iteration: 342000 loss: 0.0009 lr: 0.02\n", "iteration: 343000 loss: 0.0009 lr: 0.02\n", "iteration: 344000 loss: 0.0009 lr: 0.02\n", "iteration: 345000 loss: 0.0010 lr: 0.02\n", "iteration: 346000 loss: 0.0009 lr: 0.02\n", "iteration: 347000 loss: 0.0009 lr: 0.02\n", "iteration: 348000 loss: 0.0009 lr: 0.02\n", "iteration: 349000 loss: 0.0009 lr: 0.02\n", "iteration: 350000 loss: 0.0009 lr: 0.02\n", "iteration: 351000 loss: 0.0008 lr: 0.02\n", "iteration: 352000 loss: 0.0009 lr: 0.02\n", "iteration: 353000 loss: 0.0009 lr: 0.02\n", "iteration: 354000 loss: 0.0009 lr: 0.02\n", "iteration: 355000 loss: 0.0009 lr: 0.02\n", "iteration: 356000 loss: 0.0009 lr: 0.02\n", "iteration: 357000 loss: 0.0009 lr: 0.02\n", "iteration: 358000 loss: 0.0008 lr: 0.02\n", "iteration: 359000 loss: 0.0009 lr: 0.02\n", "iteration: 360000 loss: 0.0008 lr: 0.02\n", "iteration: 361000 loss: 0.0008 lr: 0.02\n", "iteration: 362000 loss: 0.0008 lr: 0.02\n", "iteration: 363000 loss: 0.0008 lr: 0.02\n", "iteration: 364000 loss: 0.0008 lr: 0.02\n", "iteration: 365000 loss: 0.0008 lr: 0.02\n", "iteration: 366000 loss: 0.0008 lr: 0.02\n", "iteration: 367000 loss: 0.0008 lr: 0.02\n", "iteration: 368000 loss: 0.0009 lr: 0.02\n", "iteration: 369000 loss: 0.0009 lr: 0.02\n", "iteration: 370000 loss: 0.0008 lr: 0.02\n", "iteration: 371000 loss: 0.0009 lr: 0.02\n", "iteration: 372000 loss: 0.0008 lr: 0.02\n", "iteration: 373000 loss: 0.0009 lr: 0.02\n", "iteration: 374000 loss: 0.0010 lr: 0.02\n", "iteration: 375000 loss: 0.0008 lr: 0.02\n", "iteration: 376000 loss: 0.0008 lr: 0.02\n", "iteration: 377000 loss: 0.0009 lr: 0.02\n", "iteration: 378000 loss: 0.0009 lr: 0.02\n", "iteration: 379000 loss: 0.0008 lr: 0.02\n", "iteration: 380000 loss: 0.0009 lr: 0.02\n", "iteration: 381000 loss: 0.0008 lr: 0.02\n", "iteration: 382000 loss: 0.0008 lr: 0.02\n", "iteration: 383000 loss: 0.0008 lr: 0.02\n", "iteration: 384000 loss: 0.0008 lr: 0.02\n", "iteration: 385000 loss: 0.0009 lr: 0.02\n", "iteration: 386000 loss: 0.0008 lr: 0.02\n", "iteration: 387000 loss: 0.0008 lr: 0.02\n", "iteration: 388000 loss: 0.0009 lr: 0.02\n", "iteration: 389000 loss: 0.0008 lr: 0.02\n", "iteration: 390000 loss: 0.0008 lr: 0.02\n", "iteration: 391000 loss: 0.0009 lr: 0.02\n", "iteration: 392000 loss: 0.0008 lr: 0.02\n", "iteration: 393000 loss: 0.0008 lr: 0.02\n", "iteration: 394000 loss: 0.0008 lr: 0.02\n", "iteration: 395000 loss: 0.0008 lr: 0.02\n", "iteration: 396000 loss: 0.0009 lr: 0.02\n", "iteration: 397000 loss: 0.0008 lr: 0.02\n", "iteration: 398000 loss: 0.0008 lr: 0.02\n", "iteration: 399000 loss: 0.0008 lr: 0.02\n", "iteration: 400000 loss: 0.0008 lr: 0.02\n", "iteration: 401000 loss: 0.0008 lr: 0.02\n", "iteration: 402000 loss: 0.0008 lr: 0.02\n", "iteration: 403000 loss: 0.0009 lr: 0.02\n", "iteration: 404000 loss: 0.0008 lr: 0.02\n", "iteration: 405000 loss: 0.0009 lr: 0.02\n", "iteration: 406000 loss: 0.0009 lr: 0.02\n", "iteration: 407000 loss: 0.0008 lr: 0.02\n", "iteration: 408000 loss: 0.0008 lr: 0.02\n", "iteration: 409000 loss: 0.0008 lr: 0.02\n", "iteration: 410000 loss: 0.0008 lr: 0.02\n", "iteration: 411000 loss: 0.0008 lr: 0.02\n", "iteration: 412000 loss: 0.0008 lr: 0.02\n", "iteration: 413000 loss: 0.0008 lr: 0.02\n", "iteration: 414000 loss: 0.0009 lr: 0.02\n", "iteration: 415000 loss: 0.0008 lr: 0.02\n", "iteration: 416000 loss: 0.0008 lr: 0.02\n", "iteration: 417000 loss: 0.0009 lr: 0.02\n", "iteration: 418000 loss: 0.0009 lr: 0.02\n", "iteration: 419000 loss: 0.0009 lr: 0.02\n", "iteration: 420000 loss: 0.0008 lr: 0.02\n", "iteration: 421000 loss: 0.0008 lr: 0.02\n", "iteration: 422000 loss: 0.0008 lr: 0.02\n", "iteration: 423000 loss: 0.0008 lr: 0.02\n", "iteration: 424000 loss: 0.0009 lr: 0.02\n", "iteration: 425000 loss: 0.0009 lr: 0.02\n", "iteration: 426000 loss: 0.0008 lr: 0.02\n", "iteration: 427000 loss: 0.0008 lr: 0.02\n", "iteration: 428000 loss: 0.0008 lr: 0.02\n", "iteration: 429000 loss: 0.0008 lr: 0.02\n", "iteration: 430000 loss: 0.0008 lr: 0.02\n", "iteration: 431000 loss: 0.0007 lr: 0.002\n", "iteration: 432000 loss: 0.0007 lr: 0.002\n", "iteration: 433000 loss: 0.0006 lr: 0.002\n", "iteration: 434000 loss: 0.0006 lr: 0.002\n", "iteration: 435000 loss: 0.0006 lr: 0.002\n", "iteration: 436000 loss: 0.0006 lr: 0.002\n", "iteration: 437000 loss: 0.0006 lr: 0.002\n", "iteration: 438000 loss: 0.0006 lr: 0.002\n", "iteration: 439000 loss: 0.0006 lr: 0.002\n", "iteration: 440000 loss: 0.0006 lr: 0.002\n", "iteration: 441000 loss: 0.0006 lr: 0.002\n", "iteration: 442000 loss: 0.0006 lr: 0.002\n", "iteration: 443000 loss: 0.0006 lr: 0.002\n", "iteration: 444000 loss: 0.0006 lr: 0.002\n", "iteration: 445000 loss: 0.0006 lr: 0.002\n", "iteration: 446000 loss: 0.0006 lr: 0.002\n", "iteration: 447000 loss: 0.0006 lr: 0.002\n", "iteration: 448000 loss: 0.0006 lr: 0.002\n", "iteration: 449000 loss: 0.0006 lr: 0.002\n", "iteration: 450000 loss: 0.0006 lr: 0.002\n", "iteration: 451000 loss: 0.0006 lr: 0.002\n", "iteration: 452000 loss: 0.0006 lr: 0.002\n", "iteration: 453000 loss: 0.0006 lr: 0.002\n", "iteration: 454000 loss: 0.0006 lr: 0.002\n", "iteration: 455000 loss: 0.0006 lr: 0.002\n", "iteration: 456000 loss: 0.0006 lr: 0.002\n", "iteration: 457000 loss: 0.0006 lr: 0.002\n", "iteration: 458000 loss: 0.0006 lr: 0.002\n", "iteration: 459000 loss: 0.0006 lr: 0.002\n", "iteration: 460000 loss: 0.0006 lr: 0.002\n", "iteration: 461000 loss: 0.0006 lr: 0.002\n", "iteration: 462000 loss: 0.0006 lr: 0.002\n", "iteration: 463000 loss: 0.0006 lr: 0.002\n", "iteration: 464000 loss: 0.0006 lr: 0.002\n", "iteration: 465000 loss: 0.0006 lr: 0.002\n", "iteration: 466000 loss: 0.0006 lr: 0.002\n", "iteration: 467000 loss: 0.0006 lr: 0.002\n", "iteration: 468000 loss: 0.0006 lr: 0.002\n", "iteration: 469000 loss: 0.0006 lr: 0.002\n", "iteration: 470000 loss: 0.0006 lr: 0.002\n", "iteration: 471000 loss: 0.0006 lr: 0.002\n", "iteration: 472000 loss: 0.0006 lr: 0.002\n", "iteration: 473000 loss: 0.0006 lr: 0.002\n", "iteration: 474000 loss: 0.0005 lr: 0.002\n", "iteration: 475000 loss: 0.0006 lr: 0.002\n", "iteration: 476000 loss: 0.0006 lr: 0.002\n", "iteration: 477000 loss: 0.0006 lr: 0.002\n", "iteration: 478000 loss: 0.0006 lr: 0.002\n", "iteration: 479000 loss: 0.0006 lr: 0.002\n", "iteration: 480000 loss: 0.0006 lr: 0.002\n", "iteration: 481000 loss: 0.0006 lr: 0.002\n", "iteration: 482000 loss: 0.0006 lr: 0.002\n", "iteration: 483000 loss: 0.0006 lr: 0.002\n", "iteration: 484000 loss: 0.0006 lr: 0.002\n", "iteration: 485000 loss: 0.0006 lr: 0.002\n", "iteration: 486000 loss: 0.0006 lr: 0.002\n", "iteration: 487000 loss: 0.0005 lr: 0.002\n", "iteration: 488000 loss: 0.0005 lr: 0.002\n", "iteration: 489000 loss: 0.0005 lr: 0.002\n", "iteration: 490000 loss: 0.0006 lr: 0.002\n", "iteration: 491000 loss: 0.0006 lr: 0.002\n", "iteration: 492000 loss: 0.0006 lr: 0.002\n", "iteration: 493000 loss: 0.0005 lr: 0.002\n", "iteration: 494000 loss: 0.0006 lr: 0.002\n", "iteration: 495000 loss: 0.0006 lr: 0.002\n", "iteration: 496000 loss: 0.0006 lr: 0.002\n", "iteration: 497000 loss: 0.0006 lr: 0.002\n", "iteration: 498000 loss: 0.0006 lr: 0.002\n", "iteration: 499000 loss: 0.0005 lr: 0.002\n", "iteration: 500000 loss: 0.0005 lr: 0.002\n", "iteration: 501000 loss: 0.0005 lr: 0.002\n", "iteration: 502000 loss: 0.0005 lr: 0.002\n", "iteration: 503000 loss: 0.0005 lr: 0.002\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 504000 loss: 0.0005 lr: 0.002\n", "iteration: 505000 loss: 0.0006 lr: 0.002\n", "iteration: 506000 loss: 0.0006 lr: 0.002\n", "iteration: 507000 loss: 0.0006 lr: 0.002\n", "iteration: 508000 loss: 0.0005 lr: 0.002\n", "iteration: 509000 loss: 0.0005 lr: 0.002\n", "iteration: 510000 loss: 0.0006 lr: 0.002\n", "iteration: 511000 loss: 0.0006 lr: 0.002\n", "iteration: 512000 loss: 0.0005 lr: 0.002\n", "iteration: 513000 loss: 0.0005 lr: 0.002\n", "iteration: 514000 loss: 0.0006 lr: 0.002\n", "iteration: 515000 loss: 0.0005 lr: 0.002\n", "iteration: 516000 loss: 0.0006 lr: 0.002\n", "iteration: 517000 loss: 0.0005 lr: 0.002\n", "iteration: 518000 loss: 0.0006 lr: 0.002\n", "iteration: 519000 loss: 0.0005 lr: 0.002\n", "iteration: 520000 loss: 0.0005 lr: 0.002\n", "iteration: 521000 loss: 0.0006 lr: 0.002\n", "iteration: 522000 loss: 0.0005 lr: 0.002\n", "iteration: 523000 loss: 0.0005 lr: 0.002\n", "iteration: 524000 loss: 0.0005 lr: 0.002\n", "iteration: 525000 loss: 0.0005 lr: 0.002\n", "iteration: 526000 loss: 0.0006 lr: 0.002\n", "iteration: 527000 loss: 0.0005 lr: 0.002\n", "iteration: 528000 loss: 0.0005 lr: 0.002\n", "iteration: 529000 loss: 0.0005 lr: 0.002\n", "iteration: 530000 loss: 0.0005 lr: 0.002\n", "iteration: 531000 loss: 0.0006 lr: 0.002\n", "iteration: 532000 loss: 0.0005 lr: 0.002\n", "iteration: 533000 loss: 0.0005 lr: 0.002\n", "iteration: 534000 loss: 0.0005 lr: 0.002\n", "iteration: 535000 loss: 0.0005 lr: 0.002\n", "iteration: 536000 loss: 0.0005 lr: 0.002\n", "iteration: 537000 loss: 0.0006 lr: 0.002\n", "iteration: 538000 loss: 0.0006 lr: 0.002\n", "iteration: 539000 loss: 0.0005 lr: 0.002\n", "iteration: 540000 loss: 0.0005 lr: 0.002\n", "iteration: 541000 loss: 0.0006 lr: 0.002\n", "iteration: 542000 loss: 0.0005 lr: 0.002\n", "iteration: 543000 loss: 0.0005 lr: 0.002\n", "iteration: 544000 loss: 0.0005 lr: 0.002\n", "iteration: 545000 loss: 0.0006 lr: 0.002\n", "iteration: 546000 loss: 0.0005 lr: 0.002\n", "iteration: 547000 loss: 0.0005 lr: 0.002\n", "iteration: 548000 loss: 0.0005 lr: 0.002\n", "iteration: 549000 loss: 0.0006 lr: 0.002\n", "iteration: 550000 loss: 0.0005 lr: 0.002\n", "iteration: 551000 loss: 0.0005 lr: 0.002\n", "iteration: 552000 loss: 0.0005 lr: 0.002\n", "iteration: 553000 loss: 0.0005 lr: 0.002\n", "iteration: 554000 loss: 0.0006 lr: 0.002\n", "iteration: 555000 loss: 0.0005 lr: 0.002\n", "iteration: 556000 loss: 0.0005 lr: 0.002\n", "iteration: 557000 loss: 0.0005 lr: 0.002\n", "iteration: 558000 loss: 0.0005 lr: 0.002\n", "iteration: 559000 loss: 0.0005 lr: 0.002\n", "iteration: 560000 loss: 0.0005 lr: 0.002\n", "iteration: 561000 loss: 0.0005 lr: 0.002\n", "iteration: 562000 loss: 0.0005 lr: 0.002\n", "iteration: 563000 loss: 0.0005 lr: 0.002\n", "iteration: 564000 loss: 0.0005 lr: 0.002\n", "iteration: 565000 loss: 0.0005 lr: 0.002\n", "iteration: 566000 loss: 0.0005 lr: 0.002\n", "iteration: 567000 loss: 0.0005 lr: 0.002\n", "iteration: 568000 loss: 0.0005 lr: 0.002\n", "iteration: 569000 loss: 0.0005 lr: 0.002\n", "iteration: 570000 loss: 0.0005 lr: 0.002\n", "iteration: 571000 loss: 0.0006 lr: 0.002\n", "iteration: 572000 loss: 0.0005 lr: 0.002\n", "iteration: 573000 loss: 0.0005 lr: 0.002\n", "iteration: 574000 loss: 0.0005 lr: 0.002\n", "iteration: 575000 loss: 0.0005 lr: 0.002\n", "iteration: 576000 loss: 0.0005 lr: 0.002\n", "iteration: 577000 loss: 0.0005 lr: 0.002\n", "iteration: 578000 loss: 0.0005 lr: 0.002\n", "iteration: 579000 loss: 0.0005 lr: 0.002\n", "iteration: 580000 loss: 0.0005 lr: 0.002\n", "iteration: 581000 loss: 0.0005 lr: 0.002\n", "iteration: 582000 loss: 0.0005 lr: 0.002\n", "iteration: 583000 loss: 0.0005 lr: 0.002\n", "iteration: 584000 loss: 0.0005 lr: 0.002\n", "iteration: 585000 loss: 0.0005 lr: 0.002\n", "iteration: 586000 loss: 0.0005 lr: 0.002\n", "iteration: 587000 loss: 0.0005 lr: 0.002\n", "iteration: 588000 loss: 0.0005 lr: 0.002\n", "iteration: 589000 loss: 0.0005 lr: 0.002\n", "iteration: 590000 loss: 0.0005 lr: 0.002\n", "iteration: 591000 loss: 0.0005 lr: 0.002\n", "iteration: 592000 loss: 0.0005 lr: 0.002\n", "iteration: 593000 loss: 0.0005 lr: 0.002\n", "iteration: 594000 loss: 0.0005 lr: 0.002\n", "iteration: 595000 loss: 0.0005 lr: 0.002\n", "iteration: 596000 loss: 0.0005 lr: 0.002\n", "iteration: 597000 loss: 0.0005 lr: 0.002\n", "iteration: 598000 loss: 0.0005 lr: 0.002\n", "iteration: 599000 loss: 0.0005 lr: 0.002\n", "iteration: 600000 loss: 0.0006 lr: 0.002\n", "iteration: 601000 loss: 0.0005 lr: 0.002\n", "iteration: 602000 loss: 0.0005 lr: 0.002\n", "iteration: 603000 loss: 0.0005 lr: 0.002\n", "iteration: 604000 loss: 0.0005 lr: 0.002\n", "iteration: 605000 loss: 0.0005 lr: 0.002\n", "iteration: 606000 loss: 0.0005 lr: 0.002\n", "iteration: 607000 loss: 0.0005 lr: 0.002\n", "iteration: 608000 loss: 0.0005 lr: 0.002\n", "iteration: 609000 loss: 0.0005 lr: 0.002\n", "iteration: 610000 loss: 0.0006 lr: 0.002\n", "iteration: 611000 loss: 0.0005 lr: 0.002\n", "iteration: 612000 loss: 0.0005 lr: 0.002\n", "iteration: 613000 loss: 0.0005 lr: 0.002\n", "iteration: 614000 loss: 0.0005 lr: 0.002\n", "iteration: 615000 loss: 0.0005 lr: 0.002\n", "iteration: 616000 loss: 0.0005 lr: 0.002\n", "iteration: 617000 loss: 0.0005 lr: 0.002\n", "iteration: 618000 loss: 0.0005 lr: 0.002\n", "iteration: 619000 loss: 0.0005 lr: 0.002\n", "iteration: 620000 loss: 0.0006 lr: 0.002\n", "iteration: 621000 loss: 0.0005 lr: 0.002\n", "iteration: 622000 loss: 0.0005 lr: 0.002\n", "iteration: 623000 loss: 0.0005 lr: 0.002\n", "iteration: 624000 loss: 0.0005 lr: 0.002\n", "iteration: 625000 loss: 0.0005 lr: 0.002\n", "iteration: 626000 loss: 0.0005 lr: 0.002\n", "iteration: 627000 loss: 0.0005 lr: 0.002\n", "iteration: 628000 loss: 0.0005 lr: 0.002\n", "iteration: 629000 loss: 0.0005 lr: 0.002\n", "iteration: 630000 loss: 0.0005 lr: 0.002\n", "iteration: 631000 loss: 0.0005 lr: 0.002\n", "iteration: 632000 loss: 0.0005 lr: 0.002\n", "iteration: 633000 loss: 0.0005 lr: 0.002\n", "iteration: 634000 loss: 0.0005 lr: 0.002\n", "iteration: 635000 loss: 0.0005 lr: 0.002\n", "iteration: 636000 loss: 0.0005 lr: 0.002\n", "iteration: 637000 loss: 0.0005 lr: 0.002\n", "iteration: 638000 loss: 0.0005 lr: 0.002\n", "iteration: 639000 loss: 0.0005 lr: 0.002\n", "iteration: 640000 loss: 0.0005 lr: 0.002\n", "iteration: 641000 loss: 0.0005 lr: 0.002\n", "iteration: 642000 loss: 0.0005 lr: 0.002\n", "iteration: 643000 loss: 0.0005 lr: 0.002\n", "iteration: 644000 loss: 0.0005 lr: 0.002\n", "iteration: 645000 loss: 0.0005 lr: 0.002\n", "iteration: 646000 loss: 0.0005 lr: 0.002\n", "iteration: 647000 loss: 0.0005 lr: 0.002\n", "iteration: 648000 loss: 0.0005 lr: 0.002\n", "iteration: 649000 loss: 0.0005 lr: 0.002\n", "iteration: 650000 loss: 0.0005 lr: 0.002\n", "iteration: 651000 loss: 0.0005 lr: 0.002\n", "iteration: 652000 loss: 0.0005 lr: 0.002\n", "iteration: 653000 loss: 0.0005 lr: 0.002\n", "iteration: 654000 loss: 0.0005 lr: 0.002\n", "iteration: 655000 loss: 0.0005 lr: 0.002\n", "iteration: 656000 loss: 0.0005 lr: 0.002\n", "iteration: 657000 loss: 0.0005 lr: 0.002\n", "iteration: 658000 loss: 0.0005 lr: 0.002\n", "iteration: 659000 loss: 0.0005 lr: 0.002\n", "iteration: 660000 loss: 0.0005 lr: 0.002\n", "iteration: 661000 loss: 0.0005 lr: 0.002\n", "iteration: 662000 loss: 0.0005 lr: 0.002\n", "iteration: 663000 loss: 0.0005 lr: 0.002\n", "iteration: 664000 loss: 0.0005 lr: 0.002\n", "iteration: 665000 loss: 0.0005 lr: 0.002\n", "iteration: 666000 loss: 0.0005 lr: 0.002\n", "iteration: 667000 loss: 0.0005 lr: 0.002\n", "iteration: 668000 loss: 0.0005 lr: 0.002\n", "iteration: 669000 loss: 0.0005 lr: 0.002\n", "iteration: 670000 loss: 0.0005 lr: 0.002\n", "iteration: 671000 loss: 0.0005 lr: 0.002\n", "iteration: 672000 loss: 0.0005 lr: 0.002\n", "iteration: 673000 loss: 0.0005 lr: 0.002\n", "iteration: 674000 loss: 0.0005 lr: 0.002\n", "iteration: 675000 loss: 0.0005 lr: 0.002\n", "iteration: 676000 loss: 0.0005 lr: 0.002\n", "iteration: 677000 loss: 0.0005 lr: 0.002\n", "iteration: 678000 loss: 0.0005 lr: 0.002\n", "iteration: 679000 loss: 0.0005 lr: 0.002\n", "iteration: 680000 loss: 0.0005 lr: 0.002\n", "iteration: 681000 loss: 0.0005 lr: 0.002\n", "iteration: 682000 loss: 0.0005 lr: 0.002\n", "iteration: 683000 loss: 0.0005 lr: 0.002\n", "iteration: 684000 loss: 0.0005 lr: 0.002\n", "iteration: 685000 loss: 0.0005 lr: 0.002\n", "iteration: 686000 loss: 0.0005 lr: 0.002\n", "iteration: 687000 loss: 0.0005 lr: 0.002\n", "iteration: 688000 loss: 0.0005 lr: 0.002\n", "iteration: 689000 loss: 0.0005 lr: 0.002\n", "iteration: 690000 loss: 0.0005 lr: 0.002\n", "iteration: 691000 loss: 0.0005 lr: 0.002\n", "iteration: 692000 loss: 0.0005 lr: 0.002\n", "iteration: 693000 loss: 0.0005 lr: 0.002\n", "iteration: 694000 loss: 0.0005 lr: 0.002\n", "iteration: 695000 loss: 0.0005 lr: 0.002\n", "iteration: 696000 loss: 0.0005 lr: 0.002\n", "iteration: 697000 loss: 0.0005 lr: 0.002\n", "iteration: 698000 loss: 0.0005 lr: 0.002\n", "iteration: 699000 loss: 0.0005 lr: 0.002\n", "iteration: 700000 loss: 0.0005 lr: 0.002\n", "iteration: 701000 loss: 0.0005 lr: 0.002\n", "iteration: 702000 loss: 0.0005 lr: 0.002\n", "iteration: 703000 loss: 0.0005 lr: 0.002\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 704000 loss: 0.0005 lr: 0.002\n", "iteration: 705000 loss: 0.0005 lr: 0.002\n", "iteration: 706000 loss: 0.0005 lr: 0.002\n", "iteration: 707000 loss: 0.0005 lr: 0.002\n", "iteration: 708000 loss: 0.0005 lr: 0.002\n", "iteration: 709000 loss: 0.0005 lr: 0.002\n", "iteration: 710000 loss: 0.0005 lr: 0.002\n", "iteration: 711000 loss: 0.0005 lr: 0.002\n", "iteration: 712000 loss: 0.0005 lr: 0.002\n", "iteration: 713000 loss: 0.0005 lr: 0.002\n", "iteration: 714000 loss: 0.0005 lr: 0.002\n", "iteration: 715000 loss: 0.0005 lr: 0.002\n", "iteration: 716000 loss: 0.0005 lr: 0.002\n", "iteration: 717000 loss: 0.0005 lr: 0.002\n", "iteration: 718000 loss: 0.0005 lr: 0.002\n", "iteration: 719000 loss: 0.0005 lr: 0.002\n", "iteration: 720000 loss: 0.0005 lr: 0.002\n", "iteration: 721000 loss: 0.0005 lr: 0.002\n", "iteration: 722000 loss: 0.0005 lr: 0.002\n", "iteration: 723000 loss: 0.0005 lr: 0.002\n", "iteration: 724000 loss: 0.0005 lr: 0.002\n", "iteration: 725000 loss: 0.0005 lr: 0.002\n", "iteration: 726000 loss: 0.0005 lr: 0.002\n", "iteration: 727000 loss: 0.0005 lr: 0.002\n", "iteration: 728000 loss: 0.0005 lr: 0.002\n", "iteration: 729000 loss: 0.0005 lr: 0.002\n", "iteration: 730000 loss: 0.0005 lr: 0.002\n", "iteration: 731000 loss: 0.0005 lr: 0.001\n", "iteration: 732000 loss: 0.0005 lr: 0.001\n", "iteration: 733000 loss: 0.0005 lr: 0.001\n", "iteration: 734000 loss: 0.0005 lr: 0.001\n", "iteration: 735000 loss: 0.0005 lr: 0.001\n", "iteration: 736000 loss: 0.0005 lr: 0.001\n", "iteration: 737000 loss: 0.0005 lr: 0.001\n", "iteration: 738000 loss: 0.0005 lr: 0.001\n", "iteration: 739000 loss: 0.0005 lr: 0.001\n", "iteration: 740000 loss: 0.0005 lr: 0.001\n", "iteration: 741000 loss: 0.0004 lr: 0.001\n", "iteration: 742000 loss: 0.0005 lr: 0.001\n", "iteration: 743000 loss: 0.0005 lr: 0.001\n", "iteration: 744000 loss: 0.0005 lr: 0.001\n", "iteration: 745000 loss: 0.0005 lr: 0.001\n", "iteration: 746000 loss: 0.0005 lr: 0.001\n", "iteration: 747000 loss: 0.0005 lr: 0.001\n", "iteration: 748000 loss: 0.0005 lr: 0.001\n", "iteration: 749000 loss: 0.0005 lr: 0.001\n", "iteration: 750000 loss: 0.0005 lr: 0.001\n", "iteration: 751000 loss: 0.0005 lr: 0.001\n", "iteration: 752000 loss: 0.0005 lr: 0.001\n", "iteration: 753000 loss: 0.0005 lr: 0.001\n", "iteration: 754000 loss: 0.0005 lr: 0.001\n", "iteration: 755000 loss: 0.0005 lr: 0.001\n", "iteration: 756000 loss: 0.0005 lr: 0.001\n", "iteration: 757000 loss: 0.0005 lr: 0.001\n", "iteration: 758000 loss: 0.0005 lr: 0.001\n", "iteration: 759000 loss: 0.0005 lr: 0.001\n", "iteration: 760000 loss: 0.0004 lr: 0.001\n", "iteration: 761000 loss: 0.0005 lr: 0.001\n", "iteration: 762000 loss: 0.0005 lr: 0.001\n", "iteration: 763000 loss: 0.0005 lr: 0.001\n", "iteration: 764000 loss: 0.0005 lr: 0.001\n", "iteration: 765000 loss: 0.0005 lr: 0.001\n", "iteration: 766000 loss: 0.0005 lr: 0.001\n", "iteration: 767000 loss: 0.0005 lr: 0.001\n", "iteration: 768000 loss: 0.0005 lr: 0.001\n", "iteration: 769000 loss: 0.0005 lr: 0.001\n", "iteration: 770000 loss: 0.0005 lr: 0.001\n", "iteration: 771000 loss: 0.0005 lr: 0.001\n", "iteration: 772000 loss: 0.0005 lr: 0.001\n", "iteration: 773000 loss: 0.0005 lr: 0.001\n", "iteration: 774000 loss: 0.0005 lr: 0.001\n", "iteration: 775000 loss: 0.0005 lr: 0.001\n", "iteration: 776000 loss: 0.0005 lr: 0.001\n", "iteration: 777000 loss: 0.0005 lr: 0.001\n", "iteration: 778000 loss: 0.0005 lr: 0.001\n", "iteration: 779000 loss: 0.0004 lr: 0.001\n", "iteration: 780000 loss: 0.0004 lr: 0.001\n", "iteration: 781000 loss: 0.0005 lr: 0.001\n", "iteration: 782000 loss: 0.0005 lr: 0.001\n", "iteration: 783000 loss: 0.0005 lr: 0.001\n", "iteration: 784000 loss: 0.0005 lr: 0.001\n", "iteration: 785000 loss: 0.0005 lr: 0.001\n", "iteration: 786000 loss: 0.0005 lr: 0.001\n", "iteration: 787000 loss: 0.0005 lr: 0.001\n", "iteration: 788000 loss: 0.0005 lr: 0.001\n", "iteration: 789000 loss: 0.0005 lr: 0.001\n", "iteration: 790000 loss: 0.0005 lr: 0.001\n", "iteration: 791000 loss: 0.0005 lr: 0.001\n", "iteration: 792000 loss: 0.0005 lr: 0.001\n", "iteration: 793000 loss: 0.0005 lr: 0.001\n", "iteration: 794000 loss: 0.0004 lr: 0.001\n", "iteration: 795000 loss: 0.0005 lr: 0.001\n", "iteration: 796000 loss: 0.0005 lr: 0.001\n", "iteration: 797000 loss: 0.0005 lr: 0.001\n", "iteration: 798000 loss: 0.0005 lr: 0.001\n", "iteration: 799000 loss: 0.0005 lr: 0.001\n", "iteration: 800000 loss: 0.0005 lr: 0.001\n", "iteration: 801000 loss: 0.0005 lr: 0.001\n", "iteration: 802000 loss: 0.0004 lr: 0.001\n", "iteration: 803000 loss: 0.0005 lr: 0.001\n", "iteration: 804000 loss: 0.0005 lr: 0.001\n", "iteration: 805000 loss: 0.0004 lr: 0.001\n", "iteration: 806000 loss: 0.0005 lr: 0.001\n", "iteration: 807000 loss: 0.0004 lr: 0.001\n", "iteration: 808000 loss: 0.0005 lr: 0.001\n", "iteration: 809000 loss: 0.0005 lr: 0.001\n", "iteration: 810000 loss: 0.0005 lr: 0.001\n", "iteration: 811000 loss: 0.0005 lr: 0.001\n", "iteration: 812000 loss: 0.0005 lr: 0.001\n", "iteration: 813000 loss: 0.0005 lr: 0.001\n", "iteration: 814000 loss: 0.0005 lr: 0.001\n", "iteration: 815000 loss: 0.0005 lr: 0.001\n", "iteration: 816000 loss: 0.0005 lr: 0.001\n", "iteration: 817000 loss: 0.0005 lr: 0.001\n", "iteration: 818000 loss: 0.0005 lr: 0.001\n", "iteration: 819000 loss: 0.0005 lr: 0.001\n", "iteration: 820000 loss: 0.0005 lr: 0.001\n", "iteration: 821000 loss: 0.0005 lr: 0.001\n", "iteration: 822000 loss: 0.0004 lr: 0.001\n", "iteration: 823000 loss: 0.0004 lr: 0.001\n", "iteration: 824000 loss: 0.0005 lr: 0.001\n", "iteration: 825000 loss: 0.0005 lr: 0.001\n", "iteration: 826000 loss: 0.0005 lr: 0.001\n", "iteration: 827000 loss: 0.0005 lr: 0.001\n", "iteration: 828000 loss: 0.0005 lr: 0.001\n", "iteration: 829000 loss: 0.0005 lr: 0.001\n", "iteration: 830000 loss: 0.0005 lr: 0.001\n", "iteration: 831000 loss: 0.0005 lr: 0.001\n", "iteration: 832000 loss: 0.0005 lr: 0.001\n", "iteration: 833000 loss: 0.0005 lr: 0.001\n", "iteration: 834000 loss: 0.0004 lr: 0.001\n", "iteration: 835000 loss: 0.0005 lr: 0.001\n", "iteration: 836000 loss: 0.0004 lr: 0.001\n", "iteration: 837000 loss: 0.0004 lr: 0.001\n", "iteration: 838000 loss: 0.0004 lr: 0.001\n", "iteration: 839000 loss: 0.0005 lr: 0.001\n", "iteration: 840000 loss: 0.0005 lr: 0.001\n", "iteration: 841000 loss: 0.0005 lr: 0.001\n", "iteration: 842000 loss: 0.0005 lr: 0.001\n", "iteration: 843000 loss: 0.0005 lr: 0.001\n", "iteration: 844000 loss: 0.0005 lr: 0.001\n", "iteration: 845000 loss: 0.0005 lr: 0.001\n", "iteration: 846000 loss: 0.0005 lr: 0.001\n", "iteration: 847000 loss: 0.0004 lr: 0.001\n", "iteration: 848000 loss: 0.0005 lr: 0.001\n", "iteration: 849000 loss: 0.0005 lr: 0.001\n", "iteration: 850000 loss: 0.0005 lr: 0.001\n", "iteration: 851000 loss: 0.0005 lr: 0.001\n", "iteration: 852000 loss: 0.0004 lr: 0.001\n", "iteration: 853000 loss: 0.0005 lr: 0.001\n", "iteration: 854000 loss: 0.0005 lr: 0.001\n", "iteration: 855000 loss: 0.0005 lr: 0.001\n", "iteration: 856000 loss: 0.0004 lr: 0.001\n", "iteration: 857000 loss: 0.0004 lr: 0.001\n", "iteration: 858000 loss: 0.0005 lr: 0.001\n", "iteration: 859000 loss: 0.0005 lr: 0.001\n", "iteration: 860000 loss: 0.0005 lr: 0.001\n", "iteration: 861000 loss: 0.0005 lr: 0.001\n", "iteration: 862000 loss: 0.0005 lr: 0.001\n", "iteration: 863000 loss: 0.0005 lr: 0.001\n", "iteration: 864000 loss: 0.0005 lr: 0.001\n", "iteration: 865000 loss: 0.0005 lr: 0.001\n", "iteration: 866000 loss: 0.0005 lr: 0.001\n", "iteration: 867000 loss: 0.0005 lr: 0.001\n", "iteration: 868000 loss: 0.0005 lr: 0.001\n", "iteration: 869000 loss: 0.0005 lr: 0.001\n", "iteration: 870000 loss: 0.0005 lr: 0.001\n", "iteration: 871000 loss: 0.0005 lr: 0.001\n", "iteration: 872000 loss: 0.0005 lr: 0.001\n", "iteration: 873000 loss: 0.0005 lr: 0.001\n", "iteration: 874000 loss: 0.0005 lr: 0.001\n", "iteration: 875000 loss: 0.0005 lr: 0.001\n", "iteration: 876000 loss: 0.0005 lr: 0.001\n", "iteration: 877000 loss: 0.0005 lr: 0.001\n", "iteration: 878000 loss: 0.0005 lr: 0.001\n", "iteration: 879000 loss: 0.0005 lr: 0.001\n", "iteration: 880000 loss: 0.0004 lr: 0.001\n", "iteration: 881000 loss: 0.0005 lr: 0.001\n", "iteration: 882000 loss: 0.0005 lr: 0.001\n", "iteration: 883000 loss: 0.0005 lr: 0.001\n", "iteration: 884000 loss: 0.0004 lr: 0.001\n", "iteration: 885000 loss: 0.0005 lr: 0.001\n", "iteration: 886000 loss: 0.0005 lr: 0.001\n", "iteration: 887000 loss: 0.0004 lr: 0.001\n", "iteration: 888000 loss: 0.0004 lr: 0.001\n", "iteration: 889000 loss: 0.0005 lr: 0.001\n", "iteration: 890000 loss: 0.0005 lr: 0.001\n", "iteration: 891000 loss: 0.0004 lr: 0.001\n", "iteration: 892000 loss: 0.0005 lr: 0.001\n", "iteration: 893000 loss: 0.0004 lr: 0.001\n", "iteration: 894000 loss: 0.0005 lr: 0.001\n", "iteration: 895000 loss: 0.0005 lr: 0.001\n", "iteration: 896000 loss: 0.0005 lr: 0.001\n", "iteration: 897000 loss: 0.0004 lr: 0.001\n", "iteration: 898000 loss: 0.0005 lr: 0.001\n", "iteration: 899000 loss: 0.0005 lr: 0.001\n", "iteration: 900000 loss: 0.0004 lr: 0.001\n", "iteration: 901000 loss: 0.0005 lr: 0.001\n", "iteration: 902000 loss: 0.0005 lr: 0.001\n", "iteration: 903000 loss: 0.0005 lr: 0.001\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "iteration: 904000 loss: 0.0004 lr: 0.001\n", "iteration: 905000 loss: 0.0005 lr: 0.001\n", "iteration: 906000 loss: 0.0004 lr: 0.001\n", "iteration: 907000 loss: 0.0005 lr: 0.001\n", "iteration: 908000 loss: 0.0005 lr: 0.001\n", "iteration: 909000 loss: 0.0004 lr: 0.001\n", "iteration: 910000 loss: 0.0004 lr: 0.001\n", "iteration: 911000 loss: 0.0005 lr: 0.001\n", "iteration: 912000 loss: 0.0004 lr: 0.001\n", "iteration: 913000 loss: 0.0004 lr: 0.001\n", "iteration: 914000 loss: 0.0005 lr: 0.001\n", "iteration: 915000 loss: 0.0005 lr: 0.001\n", "iteration: 916000 loss: 0.0004 lr: 0.001\n", "iteration: 917000 loss: 0.0004 lr: 0.001\n", "iteration: 918000 loss: 0.0004 lr: 0.001\n", "iteration: 919000 loss: 0.0005 lr: 0.001\n", "iteration: 920000 loss: 0.0005 lr: 0.001\n", "iteration: 921000 loss: 0.0005 lr: 0.001\n", "iteration: 922000 loss: 0.0004 lr: 0.001\n", "iteration: 923000 loss: 0.0005 lr: 0.001\n", "iteration: 924000 loss: 0.0005 lr: 0.001\n", "iteration: 925000 loss: 0.0005 lr: 0.001\n", "iteration: 926000 loss: 0.0005 lr: 0.001\n", "iteration: 927000 loss: 0.0005 lr: 0.001\n", "iteration: 928000 loss: 0.0005 lr: 0.001\n", "iteration: 929000 loss: 0.0004 lr: 0.001\n", "iteration: 930000 loss: 0.0005 lr: 0.001\n", "iteration: 931000 loss: 0.0004 lr: 0.001\n", "iteration: 932000 loss: 0.0005 lr: 0.001\n", "iteration: 933000 loss: 0.0004 lr: 0.001\n", "iteration: 934000 loss: 0.0004 lr: 0.001\n", "iteration: 935000 loss: 0.0005 lr: 0.001\n", "iteration: 936000 loss: 0.0005 lr: 0.001\n", "iteration: 937000 loss: 0.0004 lr: 0.001\n", "iteration: 938000 loss: 0.0005 lr: 0.001\n", "iteration: 939000 loss: 0.0005 lr: 0.001\n", "iteration: 940000 loss: 0.0005 lr: 0.001\n", "iteration: 941000 loss: 0.0004 lr: 0.001\n", "iteration: 942000 loss: 0.0004 lr: 0.001\n", "iteration: 943000 loss: 0.0005 lr: 0.001\n", "iteration: 944000 loss: 0.0005 lr: 0.001\n", "iteration: 945000 loss: 0.0005 lr: 0.001\n", "iteration: 946000 loss: 0.0005 lr: 0.001\n", "iteration: 947000 loss: 0.0005 lr: 0.001\n", "iteration: 948000 loss: 0.0004 lr: 0.001\n", "iteration: 949000 loss: 0.0005 lr: 0.001\n", "iteration: 950000 loss: 0.0004 lr: 0.001\n", "iteration: 951000 loss: 0.0004 lr: 0.001\n", "iteration: 952000 loss: 0.0005 lr: 0.001\n", "iteration: 953000 loss: 0.0005 lr: 0.001\n", "iteration: 954000 loss: 0.0004 lr: 0.001\n", "iteration: 955000 loss: 0.0005 lr: 0.001\n", "iteration: 956000 loss: 0.0004 lr: 0.001\n", "iteration: 957000 loss: 0.0004 lr: 0.001\n", "iteration: 958000 loss: 0.0004 lr: 0.001\n", "iteration: 959000 loss: 0.0005 lr: 0.001\n", "iteration: 960000 loss: 0.0005 lr: 0.001\n", "iteration: 961000 loss: 0.0004 lr: 0.001\n", "iteration: 962000 loss: 0.0004 lr: 0.001\n", "iteration: 963000 loss: 0.0004 lr: 0.001\n", "iteration: 964000 loss: 0.0005 lr: 0.001\n", "iteration: 965000 loss: 0.0004 lr: 0.001\n", "iteration: 966000 loss: 0.0005 lr: 0.001\n", "iteration: 967000 loss: 0.0005 lr: 0.001\n", "iteration: 968000 loss: 0.0005 lr: 0.001\n", "iteration: 969000 loss: 0.0004 lr: 0.001\n", "iteration: 970000 loss: 0.0005 lr: 0.001\n", "iteration: 971000 loss: 0.0005 lr: 0.001\n", "iteration: 972000 loss: 0.0004 lr: 0.001\n", "iteration: 973000 loss: 0.0004 lr: 0.001\n", "iteration: 974000 loss: 0.0004 lr: 0.001\n", "iteration: 975000 loss: 0.0005 lr: 0.001\n", "iteration: 976000 loss: 0.0004 lr: 0.001\n", "iteration: 977000 loss: 0.0005 lr: 0.001\n", "iteration: 978000 loss: 0.0005 lr: 0.001\n", "iteration: 979000 loss: 0.0005 lr: 0.001\n", "iteration: 980000 loss: 0.0004 lr: 0.001\n", "iteration: 981000 loss: 0.0004 lr: 0.001\n", "iteration: 982000 loss: 0.0005 lr: 0.001\n", "iteration: 983000 loss: 0.0005 lr: 0.001\n", "iteration: 984000 loss: 0.0004 lr: 0.001\n", "iteration: 985000 loss: 0.0005 lr: 0.001\n", "iteration: 986000 loss: 0.0004 lr: 0.001\n", "iteration: 987000 loss: 0.0005 lr: 0.001\n", "iteration: 988000 loss: 0.0004 lr: 0.001\n", "iteration: 989000 loss: 0.0005 lr: 0.001\n", "iteration: 990000 loss: 0.0005 lr: 0.001\n", "iteration: 991000 loss: 0.0005 lr: 0.001\n", "iteration: 992000 loss: 0.0005 lr: 0.001\n", "iteration: 993000 loss: 0.0005 lr: 0.001\n", "iteration: 994000 loss: 0.0004 lr: 0.001\n", "iteration: 995000 loss: 0.0004 lr: 0.001\n", "iteration: 996000 loss: 0.0004 lr: 0.001\n", "iteration: 997000 loss: 0.0005 lr: 0.001\n", "iteration: 998000 loss: 0.0005 lr: 0.001\n", "iteration: 999000 loss: 0.0005 lr: 0.001\n", "iteration: 1000000 loss: 0.0005 lr: 0.001\n", "iteration: 1001000 loss: 0.0004 lr: 0.001\n", "iteration: 1002000 loss: 0.0004 lr: 0.001\n", "iteration: 1003000 loss: 0.0005 lr: 0.001\n", "iteration: 1004000 loss: 0.0004 lr: 0.001\n", "iteration: 1005000 loss: 0.0004 lr: 0.001\n", "iteration: 1006000 loss: 0.0004 lr: 0.001\n", "iteration: 1007000 loss: 0.0004 lr: 0.001\n", "iteration: 1008000 loss: 0.0004 lr: 0.001\n", "iteration: 1009000 loss: 0.0004 lr: 0.001\n", "iteration: 1010000 loss: 0.0004 lr: 0.001\n", "iteration: 1011000 loss: 0.0004 lr: 0.001\n", "iteration: 1012000 loss: 0.0005 lr: 0.001\n", "iteration: 1013000 loss: 0.0005 lr: 0.001\n", "iteration: 1014000 loss: 0.0005 lr: 0.001\n", "iteration: 1015000 loss: 0.0004 lr: 0.001\n", "iteration: 1016000 loss: 0.0004 lr: 0.001\n", "iteration: 1017000 loss: 0.0005 lr: 0.001\n", "iteration: 1018000 loss: 0.0004 lr: 0.001\n", "iteration: 1019000 loss: 0.0004 lr: 0.001\n", "iteration: 1020000 loss: 0.0005 lr: 0.001\n", "iteration: 1021000 loss: 0.0005 lr: 0.001\n", "iteration: 1022000 loss: 0.0005 lr: 0.001\n", "iteration: 1023000 loss: 0.0005 lr: 0.001\n", "iteration: 1024000 loss: 0.0004 lr: 0.001\n", "iteration: 1025000 loss: 0.0005 lr: 0.001\n", "iteration: 1026000 loss: 0.0004 lr: 0.001\n", "iteration: 1027000 loss: 0.0005 lr: 0.001\n", "iteration: 1028000 loss: 0.0004 lr: 0.001\n", "iteration: 1029000 loss: 0.0005 lr: 0.001\n", "iteration: 1030000 loss: 0.0004 lr: 0.001\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "The network is now trained and ready to evaluate. Use the function 'evaluate_network' to evaluate the network.\n", "--> training took 11.5 hours\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Exception in thread Thread-5:\n", "Traceback (most recent call last):\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1334, in _do_call\n", " return fn(*args)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1319, in _run_fn\n", " options, feed_dict, fetch_list, target_list, run_metadata)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1407, in _call_tf_sessionrun\n", " run_metadata)\n", "tensorflow.python.framework.errors_impl.CancelledError: Enqueue operation was cancelled\n", "\t [[{{node fifo_queue_enqueue}}]]\n", "\n", "During handling of the above exception, another exception occurred:\n", "\n", "Traceback (most recent call last):\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/threading.py\", line 926, in _bootstrap_inner\n", " self.run()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/threading.py\", line 870, in run\n", " self._target(*self._args, **self._kwargs)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py\", line 81, in load_and_enqueue\n", " sess.run(enqueue_op, feed_dict=food)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 929, in run\n", " run_metadata_ptr)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1152, in _run\n", " feed_dict_tensor, options, run_metadata)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1328, in _do_run\n", " run_metadata)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/client/session.py\", line 1348, in _do_call\n", " raise type(e)(node_def, op, message)\n", "tensorflow.python.framework.errors_impl.CancelledError: Enqueue operation was cancelled\n", "\t [[node fifo_queue_enqueue (defined at /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py:67) ]]\n", "\n", "Caused by op 'fifo_queue_enqueue', defined at:\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/runpy.py\", line 193, in _run_module_as_main\n", " \"__main__\", mod_spec)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/runpy.py\", line 85, in _run_code\n", " exec(code, run_globals)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel_launcher.py\", line 16, in \n", " app.launch_new_instance()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/traitlets/config/application.py\", line 664, in launch_instance\n", " app.start()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelapp.py\", line 612, in start\n", " self.io_loop.start()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/platform/asyncio.py\", line 149, in start\n", " self.asyncio_loop.run_forever()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/asyncio/base_events.py\", line 541, in run_forever\n", " self._run_once()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/asyncio/base_events.py\", line 1786, in _run_once\n", " handle._run()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/asyncio/events.py\", line 88, in _run\n", " self._context.run(self._callback, *self._args)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/ioloop.py\", line 690, in \n", " lambda f: self._run_callback(functools.partial(callback, future))\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/ioloop.py\", line 743, in _run_callback\n", " ret = callback()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 787, in inner\n", " self.run()\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 748, in run\n", " yielded = self.gen.send(value)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 365, in process_one\n", " yield gen.maybe_future(dispatch(*args))\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 268, in dispatch_shell\n", " yield gen.maybe_future(handler(stream, idents, msg))\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/kernelbase.py\", line 545, in execute_request\n", " user_expressions, allow_stdin,\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tornado/gen.py\", line 209, in wrapper\n", " yielded = next(result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/ipkernel.py\", line 306, in do_execute\n", " res = shell.run_cell(code, store_history=store_history, silent=silent)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/ipykernel/zmqshell.py\", line 536, in run_cell\n", " return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2877, in run_cell\n", " raw_cell, store_history, silent, shell_futures)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 2922, in _run_cell\n", " return runner(coro)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/async_helpers.py\", line 68, in _pseudo_sync_runner\n", " coro.send(None)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3146, in run_cell_async\n", " interactivity=interactivity, compiler=compiler, result=result)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3337, in run_ast_nodes\n", " if (await self.run_code(code, result, async_=asy)):\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/IPython/core/interactiveshell.py\", line 3417, in run_code\n", " exec(code_obj, self.user_global_ns, self.user_ns)\n", " File \"\", line 2, in \n", " dlc.train_network(cfg)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/training.py\", line 132, in train_network\n", " train(str(poseconfigfile),displayiters,saveiters,maxiters,max_to_keep=max_snapshots_to_keep,keepdeconvweights=keepdeconvweights,allow_growth=allow_growth) #pass on path and file name for pose_cfg.yaml!\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py\", line 118, in train\n", " batch, enqueue_op, placeholders = setup_preloading(batch_spec)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py\", line 67, in setup_preloading\n", " enqueue_op = q.enqueue(placeholders_list)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/ops/data_flow_ops.py\", line 345, in enqueue\n", " self._queue_ref, vals, name=scope)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/ops/gen_data_flow_ops.py\", line 4158, in queue_enqueue_v2\n", " timeout_ms=timeout_ms, name=name)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py\", line 788, in _apply_op_helper\n", " op_def=op_def)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/util/deprecation.py\", line 507, in new_func\n", " return func(*args, **kwargs)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\", line 3300, in create_op\n", " op_def=op_def)\n", " File \"/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\", line 1801, in __init__\n", " self._traceback = tf_stack.extract_stack()\n", "\n", "CancelledError (see above for traceback): Enqueue operation was cancelled\n", "\t [[node fifo_queue_enqueue (defined at /home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/train.py:67) ]]\n", "\n", "\n" ] } ], "source": [ "start = now()\n", "dlc.train_network(cfg)\n", "end = now()\n", "print(f\"--> training took {(end - start)/3600:.1f} hours\")" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [], "source": [ "annotation = Path(\"../labeled/S005_2020-09-05/iteration-1\")\n", "if not dest.exists():\n", " annotation.mkdir(parents=True)" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Config:\n", "{'all_joints': [[0], [1], [2]],\n", " 'all_joints_names': ['Tip1', 'Tip2', 'Tip3'],\n", " 'batch_size': 1,\n", " 'bottomheight': 400,\n", " 'crop': True,\n", " 'crop_pad': 0,\n", " 'cropratio': 0.4,\n", " 'dataset': 'training-datasets/iteration-1/UnaugmentedDataSet_S005-tipsSep5/S005-tips_Keisuke95shuffle1.mat',\n", " 'dataset_type': 'default',\n", " 'deconvolutionstride': 2,\n", " 'deterministic': False,\n", " 'display_iters': 1000,\n", " 'fg_fraction': 0.25,\n", " 'global_scale': 0.8,\n", " 'init_weights': '/home/mouse/anaconda3/envs/dlc2.1/lib/python3.7/site-packages/deeplabcut/pose_estimation_tensorflow/models/pretrained/resnet_v1_50.ckpt',\n", " 'intermediate_supervision': False,\n", " 'intermediate_supervision_layer': 12,\n", " 'leftwidth': 400,\n", " 'location_refinement': True,\n", " 'locref_huber_loss': True,\n", " 'locref_loss_weight': 0.05,\n", " 'locref_stdev': 7.2801,\n", " 'log_dir': 'log',\n", " 'max_input_size': 1500,\n", " 'mean_pixel': [123.68, 116.779, 103.939],\n", " 'metadataset': 'training-datasets/iteration-1/UnaugmentedDataSet_S005-tipsSep5/Documentation_data-S005-tips_95shuffle1.pickle',\n", " 'min_input_size': 64,\n", " 'minsize': 100,\n", " 'mirror': False,\n", " 'multi_step': [[0.005, 10000],\n", " [0.02, 430000],\n", " [0.002, 730000],\n", " [0.001, 1030000]],\n", " 'net_type': 'resnet_50',\n", " 'num_joints': 3,\n", " 'optimizer': 'sgd',\n", " 'output_stride': 16,\n", " 'pos_dist_thresh': 17,\n", " 'project_path': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05',\n", " 'regularize': False,\n", " 'rightwidth': 400,\n", " 'save_iters': 50000,\n", " 'scale_jitter_lo': 0.5,\n", " 'scale_jitter_up': 1.25,\n", " 'scoremap_dir': 'test',\n", " 'shuffle': True,\n", " 'snapshot_prefix': '/home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-1/S005-tipsSep5-trainset95shuffle1/test/snapshot',\n", " 'stride': 8.0,\n", " 'topheight': 400,\n", " 'weigh_negatives': False,\n", " 'weigh_only_present_joints': False,\n", " 'weigh_part_predictions': False,\n", " 'weight_decay': 0.0001}\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Using snapshot-1030000 for model /home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-1/S005-tipsSep5-trainset95shuffle1\n", "Initializing ResNet\n", "INFO:tensorflow:Restoring parameters from /home/mouse/ks/Projects/whiskers-realtime/dlc_projects/S005-tips-Keisuke-2020-09-05/dlc-models/iteration-1/S005-tipsSep5-trainset95shuffle1/train/snapshot-1030000\n", "Starting to analyze: ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", " 0%| | 0/3781 [00:00iteration-1\n", "Loading ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz and data.\n", "reader=NumpyPicker, writer=SkVideoWriter\n", "opening source: ../videos/animal-train_2020-09-05/S005-19_2020-09-05_run124520.npz\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/3781 [00:00