Browse Source

Initial commit

Hio-Been Han 3 years ago
parent
commit
865a462b6e

+ 1 - 0
data/video_original/sample_01.mp4

@@ -0,0 +1 @@
+/annex/objects/MD5-s50695609--ea5237a4b4aaec6cb78d9d02cbf50613

+ 1 - 0
data/video_original/sample_02.mp4

@@ -0,0 +1 @@
+/annex/objects/MD5-s22656224--49ca39d2346ef34c4027c106516a8dd3

+ 1 - 0
data/video_original/sample_03.mp4

@@ -0,0 +1 @@
+/annex/objects/MD5-s29063596--cb1001f43d181167b52a77a4aa2db3b2

+ 1 - 0
data/video_original/sample_04.mp4

@@ -0,0 +1 @@
+/annex/objects/MD5-s14911434--e03316126fa4707add5de030a7d9ea9f

+ 126 - 0
step1_parse_video.m

@@ -0,0 +1,126 @@
+parse_video();
+
+%% 
+function parse_video(resize_shape, crop_flag)
+% 
+% Parsing video (*.mp4/*.avi) file(s) into image (*.jpg) files
+% 
+% Written by Hio-Been Han, hiobeen.han@kaist.ac.kr
+%
+% 2020-09-30
+%
+if nargin < 1, resize_shape = [448 448]; end
+if nargin < 2, crop_flag = false; end
+
+%% (1) Get file info & set save directory
+[file_list, vid_directory] = uigetfile('*.mp4','MultiSelect','on');
+if ischar(file_list),  file_list = {file_list}; end
+img_directory = [ 'data/frames/'];
+if ~isdir(img_directory), mkdir(img_directory); end
+convert_resolution = sprintf( '%dx%d', resize_shape(1),resize_shape(2));
+save_format = '.jpg';
+
+%% (2) Parse video into JPEG files
+for fileIdx = 1:length(file_list)
+    
+    % Set save directory
+    fname = file_list{fileIdx};
+    uniqname = fname(1:end-4);
+    fileformat = fname(end-3:end);
+    vidname = [vid_directory uniqname];
+    save_directory = [img_directory uniqname '/'];
+    if ~isdir(save_directory), mkdir(save_directory); end
+    
+    % Read frames
+    v = VideoReader( [ vidname fileformat ]);
+    nFrames = v.Duration * v.FrameRate;
+    for frameIdx = 1:nFrames
+        if v.hasFrame
+            frame = v.readFrame();
+            
+            % Set ROI
+            if frameIdx == 1
+                if crop_flag
+                    mask = image_ROI_selection(frame, ['Select ROI to save' ...
+                        ' [Occlusion => Right Click]']);
+                else
+                    mask = logical(ones([size(frame,1), size(frame,2)]));
+                end
+                idx_x = find( nanmean(mask,2) );
+                idx_y = find( nanmean(mask,1) );
+                original_resolution = sprintf( '%dx%d', size(frame,1), size(frame,2));
+            end
+            
+            % Save frame
+%             frame_valid = rgb2gray( frame(idx_x,idx_y,:) );
+            frame_valid = rgb2gray( frame(:,60:end-60,:) );
+            imgname_write = [num2str(sprintf('frame-%06d',frameIdx))...
+                '-' uniqname '-from' original_resolution '-to' convert_resolution save_format];
+            imwrite( imresize( frame_valid, resize_shape ), [ save_directory imgname_write ] )
+        end
+    end
+end
+end
+
+%% Subfunc
+function mask = image_ROI_selection(I, textInput)
+if nargin == 1, textInput = ''; end
+figure(1), set(gcf, 'Color', [1 1 1]);
+colormap(gray);
+imshow(uint8(I), 'DisplayRange', [], 'InitialMagnification',600/max(size(I))*100 );
+title( textInput )
+
+%%-- Display instructions to user
+%title('click points to make an initial contour, right click to close contour and finish');
+disp('click points to make an initial contour,');
+disp('right click to close contour and finish');
+
+%%-- begin getting points
+[y1 x1 b] = ginput(1);
+xi = x1;
+yi = y1;
+if(b ~= 1) return; end
+
+[ny nx c] = size(I);
+mask = zeros(ny,nx);
+
+while(1)
+    x2 = x1;  y2 = y1;
+    [y1 x1 b] = ginput(1);
+    if(b ~= 1),  x1 = xi;  y1 = yi;   end
+    lx = x2-x1;  ly = y2-y1;
+    len = ceil((lx^2+ly^2)^(1/2))+1;
+    x = round(x1:(lx)/(len-1):x2);    y = round(y1:(ly)/(len-1):y2); %make another one for y
+    if(length(x) == 0),  x = round(x1) * ones(1,len); end
+    if(length(y) == 0)
+        y = round(y1) * ones(1,len);
+    end
+    
+    this_index2 = sub2ind(size(mask),x, y);
+    if size(this_index2) == [1 1]
+        title('Try Again!', 'FontSize', 18)
+        sprintf Try_Again!
+    end
+    
+    try    mask(this_index2) = 1; end
+    idx = find(mask==1);
+    backup_mask = mask;
+    %%-- draw the users line in the image (color or grayscale)
+    if(c-1)
+        Ir = I(:,:,1); Ig = I(:,:,2); Ib = I(:,:,3);
+        Ir(idx) = 0;
+        Ig(idx) = 255;
+        Ib(idx) = 0;
+        I(:,:,1) = Ir; I(:,:,2) = Ig; I(:,:,3) = Ib;
+    else
+        I(idx) = 255;
+    end
+    imshow(uint8(I),'DisplayRange', [], 'InitialMagnification', 600/max(size(I))*100);
+    title( textInput )
+    if(b ~= 1) break; end
+end
+
+mask = bwfill(mask, 'holes');
+% mask = ~mask;
+end
+

+ 168 - 0
step2_gen_trainset.m

@@ -0,0 +1,168 @@
+
+
+%% (2) Randomly-select partial dataset
+% Getting full frame list
+frame_directory = 'data/frames/';
+video_names = dir( [frame_directory 'sample*' ]);
+full_frame_names = {};
+counter = 1;
+for videoIdx = 1:length( video_names )
+    frame_names = dir( [frame_directory video_names(videoIdx).name '/*.jpg']);
+    for frameIdx = 1:length( frame_names )
+        full_frame_names{ counter, 1 } = [...
+            frame_names(frameIdx).folder...
+            '/' frame_names(frameIdx).name];
+        counter=counter+1;
+    end
+end
+
+% Random select & copy
+trainset_directory = 'data/trainset/';
+if ~isdir(trainset_directory), mkdir(trainset_directory); end
+n_all_image = length(full_frame_names);
+n_train_image = 101;
+rng(abs(010-6207-8179)) % Fix random seed
+target_frame_number = randi( n_all_image, [1 n_train_image] );
+for frameIdx = 1:n_train_image
+    copyfile( full_frame_names{target_frame_number(frameIdx)}, trainset_directory );
+end
+
+%% (3) Manual segmentation
+mask_directory = trainset_directory;
+img_list = dir( [trainset_directory 'frame*.jpg'] );
+close all;
+for frameIdx = 1:length(img_list)
+    img_fname = img_list(frameIdx).name;
+    mask_fname = ['mask' img_list(frameIdx).name(6:end-4) '.jpg'];
+    frame = imread( [trainset_directory img_fname] );
+    
+    % Visualization
+%     open frame; figure(1); clf; imagesc( frame ); drawnow; colormap gray;
+    
+    % Check existence
+    if exist([trainset_directory mask_fname])
+        disp(['Already exist::: ' trainset_directory mask_fname]);
+    else
+        figure(2); clf;
+        gcf_pos = [50 50 1500 600];
+        set(gcf, 'Position', gcf_pos, 'Color', [1 1 1]);
+        
+        % Segmentation
+        subplot(1,2,1);
+        try % if exist
+            mask = image_ROI_selection(frame, [ 'Select area' ...
+                '.. imgIdx=' num2str(sprintf('%04d',frameIdx))]);
+        catch % in case of no mouse
+            mask  = zeros(size(frame));
+        end
+        xlabel(img_fname)
+        set(gca, 'FontSize', 15, 'Box', 'off', 'LineWidth', 2);
+        
+        % Show result
+        subplot(1,2,2);
+        imagesc( mask ); colormap gray;
+        title('Result mask (binary)');
+        drawnow;
+        imwrite( mask, [mask_directory mask_fname]);
+    end
+end
+%open mask
+zip( 'data/trainset.zip', 'data/trainset/')
+zip( 'data/frames.zip','data/frames/' )
+
+%% Subfunctions
+
+
+%% Subfunction
+function mask = image_ROI_selection(I, textInput)
+if nargin == 1, textInput = ''; end
+colormap(gray);
+imshow(uint8(I), 'DisplayRange', [], 'InitialMagnification',600/max(size(I))*100 );
+title( textInput )
+
+%%-- Display instructions to user
+%title('click points to make an initial contour, right click to close contour and finish');
+disp('click points to make an initial contour,');
+disp('right click to close contour and finish');
+
+%%-- begin getting points
+[y1 x1 b] = ginput(1);
+xi = x1;
+yi = y1;
+if(b ~= 1) return; end
+
+[ny nx c] = size(I);
+mask = zeros(ny,nx);
+
+while(1)
+    x2 = x1;
+    y2 = y1;
+    [y1 x1 b] = ginput(1);
+    
+    if(b ~= 1)
+        x1 = xi;
+        y1 = yi;
+    end
+    
+    %%--figure out the length of x & y component
+    lx = x2-x1;
+    ly = y2-y1;
+    
+    %%--figure out line length = pythagorian length + fudge factor
+    len = ceil((lx^2+ly^2)^(1/2))+1;
+    
+    %%--make a linearly spaced vector (some values repeated)
+    x = round(x1:(lx)/(len-1):x2);
+    y = round(y1:(ly)/(len-1):y2); %make another one for y
+    
+    %%--if it was a constant level the lines above mess up make a constant line
+    if(length(x) == 0)
+        x = round(x1) * ones(1,len);
+    end
+    if(length(y) == 0)
+        y = round(y1) * ones(1,len);
+    end
+    
+    this_index2 = sub2ind(size(mask),x, y);
+    if size(this_index2) == [1 1]
+        title('Try Again!', 'FontSize', 18)
+        sprintf Try_Again!
+    end
+    
+    try  mask(this_index2) = 1; end
+    idx = find(mask==1);
+    backup_mask = mask;
+    %%-- draw the users line in the image (color or grayscale)
+    if(c-1)
+        Ir = I(:,:,1); Ig = I(:,:,2); Ib = I(:,:,3);
+        Ir(idx) = 0;
+        Ig(idx) = 255;
+        Ib(idx) = 0;
+        I(:,:,1) = Ir; I(:,:,2) = Ig; I(:,:,3) = Ib;
+    else
+        I(idx) = 255;
+    end
+    imshow(uint8(I),'DisplayRange', [], 'InitialMagnification', 600/max(size(I))*100);
+    title( textInput )
+    if(b ~= 1) break; end
+end
+mask = bwfill(mask, 'holes');
+% mask = ~mask;
+end
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

File diff suppressed because it is too large
+ 5702 - 0
step3_CNN_model_handling_(use Google Colab).ipynb


+ 114 - 0
step4_extract_image_statistics.m

@@ -0,0 +1,114 @@
+% Directory setting
+frame_directory = 'data/frames/';
+mask_directory = 'data/predicts/';
+video_names = dir( [frame_directory 'sample*'] );
+
+% Enhancing figure visibility
+set(gcf, 'Position', [0 0 800 800], 'Color', [1 1 1] );
+
+% Figure to video 
+output_video_option = 1;
+if output_video_option
+    output_video_directory = 'data/output/';
+    if ~isdir(output_video_directory), mkdir(output_video_directory); end
+end
+
+% Main loop
+for videoIdx = 1:length( video_names )
+    frame_names = dir( [frame_directory video_names(videoIdx).name '/*.jpg']);
+    mask_names = dir( [mask_directory video_names(videoIdx).name '/*.jpg']);
+    
+    %% For one video
+    xy = nan([2,length( frame_names )]);
+    bb = nan([4,length( frame_names )]);
+    for frameIdx = 1:length( frame_names )
+        
+        %% Single frame loading
+        frame_name = [ frame_names(frameIdx).folder ...
+            '/' frame_names(frameIdx).name];
+        mask_name = [ mask_names(frameIdx).folder ...
+            '/' mask_names(frameIdx).name];
+        
+        frame = double(imread( frame_name ))/255; % Scaling to 0-1
+        mask = double(imread( mask_name ))/255; % Scaling to 0-1
+        
+        % Manual artifact rejection
+        mask(:,1:30) = 0;
+
+        % Visualization
+        
+        % Original frame
+        sp1 = subplot(2,2,1); hold off
+        imagesc( frame );
+        set(gca, 'Box', 'off', 'LineWidth', 2, 'XTick', [], 'YTick', [], 'FontSize', 15);
+        title( sprintf( '(1) Frame #%04d', frameIdx ));
+        colormap(sp1, 'gray');
+        
+        % Mask 
+        sp2 = subplot(2,2,2); hold off
+        imagesc( mask' );
+        set(gca, 'Box', 'off', 'LineWidth', 2, 'XTick', [], 'YTick', [], 'FontSize', 15);
+        title( sprintf( '(2) Mask #%04d', frameIdx ));
+        colormap(sp2, 'gray');
+        
+        % Overlay
+        overlay = repmat( frame, [1 1 3] );
+        overlay(:,:,1) = 0.5*mask';
+        
+        sp3 = subplot(2,2,3); hold off
+        imagesc( overlay );
+        set(gca, 'Box', 'off', 'LineWidth', 2, 'XTick', [], 'YTick', [], 'FontSize', 15);
+        title( sprintf( '(3) Overlay #%04d', frameIdx ));
+        
+        %% Extracting image statistics using regionprops
+        mask_binary = mask > probability_threshold;
+        stats = regionprops( mask_binary', 'BoundingBox', 'Area', 'Centroid', 'Orientation' );
+        
+        probability_threshold = 0.5; % Probability, anonymous unit (0-1)
+        size_threshold = 100; % Pixel
+        drawing_regionprops = false;
+        
+        big_enough = find( [stats.Area]>size_threshold );
+        if ~isempty( big_enough )
+            stat = stats(big_enough);
+            xy(:,frameIdx) = stat.Centroid;
+            bb(:,frameIdx) = stat.BoundingBox;
+        else % if failed
+            xy(:,frameIdx) = xy(:,frameIdx-1);
+            bb(:,frameIdx) = bb(:,frameIdx-1);
+        end
+            
+        sp4 = subplot(2,2,4); hold off
+        imagesc( frame );
+        set(gca, 'Box', 'off', 'LineWidth', 2, 'XTick', [], 'YTick', [], 'FontSize', 15);
+        title( sprintf( '(4) Tracking result #%04d', frameIdx ));
+        colormap(sp4,'gray');
+        hold on;
+        % Centroid
+        plot( xy(1,frameIdx), xy(2,frameIdx), 'g*', 'MarkerSize', 20);
+        % BoundingBox
+        rectangle('Position', bb(:,frameIdx),'EdgeColor','g','LineWidth',2 )
+        % Text annotation
+        text_contents = sprintf( 'X: [%.1f], Y: [%.1f]',  xy(:,frameIdx));
+        text( xy(1,frameIdx),xy(2,frameIdx) - 50, text_contents, 'Color', 'g',...
+            'HorizontalAlignment', 'center' );
+        
+        %% Etc.
+        drawnow;
+        
+        % Writing output video
+        if output_video_option
+            if frameIdx==1
+                try vid_obj.close(); clear vid_obj; end
+                vid_obj = VideoWriter( [output_video_directory ...
+                    video_names(videoIdx).name '.mp4'], 'MPEG-4');
+                vid_obj.open();
+            end
+            capture_frame = getframe(gcf);
+            vid_obj.writeVideo(capture_frame.cdata);
+            if frameIdx==length( frame_names )
+                vid_obj.close();
+            end
+        end
+    end
+end