123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284 |
- import matplotlib.pyplot as plt
- from sklearn.datasets import make_classification
- from sklearn.model_selection import train_test_split
- from sklearn.ensemble import RandomForestClassifier
- from sklearn.inspection import permutation_importance
- from sklearn.preprocessing import StandardScaler
- from sklearn.metrics import accuracy_score
- import numpy as np
- import pandas as pd
- from master_funcs import *
- from imblearn.over_sampling import SMOTE
- from imblearn.under_sampling import RandomUnderSampler
- from imblearn.pipeline import Pipeline
- from imblearn.over_sampling import SMOTE
- import dill
- mr_gw_1=pd.read_csv("/mnt/DLC/DLC/3D_videos/manual_raters/Manual_GridWalk_Analyse_Jan.csv", index_col=0, delimiter=',', header=None)
- mr_gw_2=pd.read_csv("/mnt/DLC/DLC/3D_videos/manual_raters/Manual_GridWalk_Analyse_Nicole.csv", index_col=0, delimiter=',', header=None)
- mr_gw_3=pd.read_csv("/mnt/DLC/DLC/3D_videos/manual_raters/Manual_GridWalk_Analyse_Jule.csv", index_col=0, delimiter=',', header=None)
- patients=['PB_T2_3_1', 'PB_T2_3_2', 'PB_T2_3_3', 'PB_T2_4_2', 'PB_T2_5_1', 'PB_T2_6_1', 'PB_T2_6_2', 'PB_T3_23_1', 'PB_T3_23_2', 'PB_T3_23_3', 'PB_T3_24_1', 'PB_T3_24_2', 'PB_T3_24_3']
- for z in patients:
- path1='/mnt/DLC/DLC/3D_videos/GW_2D/'+z+'_P3_GW_camera-1DLC_resnet50_GridWalk_camera-1Jun15shuffle1_250000.csv'
- path2='/mnt/DLC/DLC/3D_videos/GW_2D/'+z+'_P3_GW_camera-2DLC_resnet50_GridWalk_camera-2Jun15shuffle1_250000.csv'
- data1=prep_dlc(path1, 0.9, 1080)
- data2=prep_dlc(path2, 0.9, 1080)
- vel1, acc1=vel_acc(data1, '2D')
- vel2, acc2=vel_acc(data2, '2D')
- names=[i[0] for i in data1.columns[::3]]
- likeli1=np.array(np.zeros(shape=(len(data1,))))
- likeli2=np.array(np.zeros(shape=(len(data2,))))
- for j in names:
- data1[j, 'x']=data1[j, 'x'].interpolate(method='polynomial', order=1)
- data1[j, 'y']=data1[j, 'y'].interpolate(method='polynomial', order=1)
- likeli1+=data1[j, 'likelihood']
- data1=data1.drop([(j, 'likelihood')], axis=1)
- data2[j, 'x']=data2[j, 'x'].interpolate(method='polynomial', order=1)
- data2[j, 'y']=data2[j, 'y'].interpolate(method='polynomial', order=1)
- likeli2+=data2[j, 'likelihood']
- data2=data2.drop([(j, 'likelihood')], axis=1)
- df_vel1=pd.DataFrame(vel1)
- df_vel1.columns=names
- df_vel1=pd.concat([df_vel1], keys=['velocity'], axis=1, names=['coords'])
- df_vel1.columns.names=['coords', 'bodyparts']
- df_vel2=pd.DataFrame(vel2)
- df_vel2.columns=names
- df_vel2=pd.concat([df_vel2], keys=['velocity'], axis=1, names=['coords'])
- df_vel2.columns.names=['coords', 'bodyparts']
- df_acc1=pd.DataFrame(acc1)
- df_acc1.columns=names
- df_acc1=pd.concat([df_acc1], keys=['accocity'], axis=1, names=['coords'])
- df_acc1.columns.names=['coords', 'bodyparts']
- df_acc2=pd.DataFrame(acc2)
- df_acc2.columns=names
- df_acc2=pd.concat([df_acc2], keys=['accocity'], axis=1, names=['coords'])
- df_acc2.columns.names=['coords', 'bodyparts']
-
-
- nnames=[]
- angle1=angle_all_2D(data1, names[0], names[1], names[2])
- nnames.append(names[0]+'_'+names[1]+'_'+names[2])
- sum=0
- for i in range(1, len(names)):
- for j in range(i+1, len(names)):
- for k in range(i+2, len(names)):
- if i!=j!=k:
- angle1=np.c_[angle1,angle_all_2D(data1, names[i], names[j], names[k])]
- nnames.append(names[i]+'_'+names[j]+'_'+names[k])
- sum+=1
- print(sum, ' of 2870 angles')
- df_angle1=pd.DataFrame(angle1)
- df_angle1.columns=nnames
- df_angle1=pd.concat([df_angle1], keys=['angle'], axis=1, names=['coords'])
- df_angle1.columns.names=['coords', 'bodyparts']
- angle_vel1=np.diff(df_angle1[df_angle1.columns[0]])
- for i in df_angle1.columns[1:]:
- angle_vel1=np.c_[angle_vel1, np.diff(df_angle1[i])]
- angle_vel1=np.r_[angle_vel1, [[0]*angle_vel1.shape[1]]]
- df_angle_vel1=pd.DataFrame(angle_vel1)
- df_angle_vel1.columns=nnames
- df_angle_vel1=pd.concat([df_angle_vel1], keys=['angle_velocity'], axis=1, names=['coords'])
- df_angle_vel1.columns.names=['coords', 'bodyparts']
- angle_acc1=np.diff(df_angle_vel1[df_angle_vel1.columns[0]])
- for i in df_angle_vel1.columns[1:]:
- angle_acc1=np.c_[angle_acc1, np.diff(df_angle_vel1[i])]
- angle_acc1=np.r_[angle_acc1, [[0]*angle_acc1.shape[1], [0]*angle_acc1.shape[1]]]
- df_angle_acc1=pd.DataFrame(angle_acc1)
- df_angle_acc1.columns=nnames
- df_angle_acc1=pd.concat([df_angle_acc1], keys=['angle_acceleration'], axis=1, names=['coords'])
- df_angle_acc1.columns.names=['coords', 'bodyparts']
- nnames=[]
- dis1=distance_all_2D(data1, names[0], names[1])
- nnames.append(names[0]+'_'+names[1])
- sum=0
- for i in range(1, len(names)):
- for j in range(i+1, len(names)):
- dis1=np.c_[dis1, distance_all_2D(data1, names[i], names[j])]
- nnames.append(names[i]+'_'+names[j])
- sum+=1
- print(sum, 'of 231')
- df_dis1=pd.DataFrame(dis1)
- df_dis1.columns=nnames
- df_dis1=pd.concat([df_dis1], keys=['distance'], axis=1, names=['coords'])
- df_dis1.columns.names=['coords', 'bodyparts']
- dis_vel1=np.diff(df_dis1[df_dis1.columns[0]])
- for i in df_dis1.columns[1:]:
- dis_vel1=np.c_[dis_vel1, np.diff(df_dis1[i])]
- dis_vel1=np.r_[dis_vel1, [[0]*dis_vel1.shape[1]]]
- df_dis_vel1=pd.DataFrame(dis_vel1)
- df_dis_vel1.columns=nnames
- df_dis_vel1=pd.concat([df_dis_vel1], keys=['distance_velocity'], axis=1, names=['coords'])
- df_dis_vel1.columns.names=['coords', 'bodyparts']
- dis_acc1=np.diff(df_dis_vel1[df_dis_vel1.columns[0]])
- for i in df_dis_vel1.columns[1:]:
- dis_acc1=np.c_[dis_acc1, np.diff(df_dis_vel1[i])]
- dis_acc1=np.r_[dis_acc1, [[0]*dis_acc1.shape[1], [0]*dis_acc1.shape[1]]]
- df_dis_acc1=pd.DataFrame(dis_acc1)
- df_dis_acc1.columns=nnames
- df_dis_acc1=pd.concat([df_dis_acc1], keys=['distance_acceleration'], axis=1, names=['coords'])
- df_dis_acc1.columns.names=['coords', 'bodyparts']
- df_vel_acc1=pd.concat((df_vel1, df_acc1, df_angle1, df_angle_vel1, df_angle_acc1, df_dis1, df_dis_vel1, df_dis_acc1), axis=1)
- names=[]
- for i in range(0, len(df_vel_acc1.columns)):
- names.append(df_vel_acc1.columns[i][0]+'_'+df_vel_acc1.columns[i][1])
- df_vel_acc1.columns=names
- df_vel_acc1=df_vel_acc1.interpolate(method='polynomial', order=1)
- df_vel_acc1=df_vel_acc1.fillna(0)
-
- del df_vel1
- del df_acc1
- del acc1
- del vel1
- del angle1
- del df_angle1
- del angle_vel1
- del df_angle_vel1
- del angle_acc1
- del df_angle_acc1
- del dis1
- del df_dis1
- del dis_vel1
- del df_dis_vel1
- del dis_acc1
- del df_dis_acc1
-
- names=[i[0] for i in data2.columns[::2]]
- nnames=[]
- angle2=angle_all_2D(data2, names[0], names[1], names[2])
- nnames.append(names[0]+'_'+names[1]+'_'+names[2])
- sum=0
- for i in range(1, len(names)):
- for j in range(i+1, len(names)):
- for k in range(i+2, len(names)):
- if i!=j!=k:
- angle2=np.c_[angle2,angle_all_2D(data2, names[i], names[j], names[k])]
- nnames.append(names[i]+'_'+names[j]+'_'+names[k])
- sum+=1
- print(sum, ' of 2870 angles')
- df_angle2=pd.DataFrame(angle2)
- df_angle2.columns=nnames
- df_angle2=pd.concat([df_angle2], keys=['angle'], axis=1, names=['coords'])
- df_angle2.columns.names=['coords', 'bodyparts']
-
- angle_vel2=np.diff(df_angle2[df_angle2.columns[0]])
- for i in df_angle2.columns[1:]:
- angle_vel2=np.c_[angle_vel2, np.diff(df_angle2[i])]
- angle_vel2=np.r_[angle_vel2, [[0]*angle_vel2.shape[1]]]
- df_angle_vel2=pd.DataFrame(angle_vel2)
- df_angle_vel2.columns=nnames
- df_angle_vel2=pd.concat([df_angle_vel2], keys=['angle_velocity'], axis=1, names=['coords'])
- df_angle_vel2.columns.names=['coords', 'bodyparts']
-
- angle_acc2=np.diff(df_angle_vel2[df_angle_vel2.columns[0]])
- for i in df_angle_vel2.columns[1:]:
- angle_acc2=np.c_[angle_acc2, np.diff(df_angle_vel2[i])]
- angle_acc2=np.r_[angle_acc2, [[0]*angle_acc2.shape[1], [0]*angle_acc2.shape[1]]]
- df_angle_acc2=pd.DataFrame(angle_acc2)
- df_angle_acc2.columns=nnames
- df_angle_acc2=pd.concat([df_angle_acc2], keys=['angle_acceleration'], axis=1, names=['coords'])
- df_angle_acc2.columns.names=['coords', 'bodyparts']
- nnames=[]
- dis2=distance_all_2D(data2, names[0], names[1])
- nnames.append(names[0]+'_'+names[1])
- sum=0
- for i in range(1, len(names)):
- for j in range(i+1, len(names)):
- dis2=np.c_[dis2, distance_all_2D(data2, names[i], names[j])]
- nnames.append(names[i]+'_'+names[j])
- sum+=1
- print(sum, 'of 231')
- df_dis2=pd.DataFrame(dis2)
- df_dis2.columns=nnames
- df_dis2=pd.concat([df_dis2], keys=['distance'], axis=1, names=['coords'])
- df_dis2.columns.names=['coords', 'bodyparts']
-
- dis_vel2=np.diff(df_dis2[df_dis2.columns[0]])
- for i in df_dis2.columns[1:]:
- dis_vel2=np.c_[dis_vel2, np.diff(df_dis2[i])]
- dis_vel2=np.r_[dis_vel2, [[0]*dis_vel2.shape[1]]]
- df_dis_vel2=pd.DataFrame(dis_vel2)
- df_dis_vel2.columns=nnames
- df_dis_vel2=pd.concat([df_dis_vel2], keys=['distance_velocity'], axis=1, names=['coords'])
- df_dis_vel2.columns.names=['coords', 'bodyparts']
- dis_acc2=np.diff(df_dis_vel2[df_dis_vel2.columns[0]])
- for i in df_dis_vel2.columns[1:]:
- dis_acc2=np.c_[dis_acc2, np.diff(df_dis_vel2[i])]
- dis_acc2=np.r_[dis_acc2, [[0]*dis_acc2.shape[1], [0]*dis_acc2.shape[1]]]
- df_dis_acc2=pd.DataFrame(dis_acc2)
- df_dis_acc2.columns=nnames
- df_dis_acc2=pd.concat([df_dis_acc2], keys=['distance_acceleration'], axis=1, names=['coords'])
- df_dis_acc2.columns.names=['coords', 'bodyparts']
- df_vel_acc2=pd.concat((df_vel2, df_acc2, df_angle2, df_angle_vel2, df_angle_acc2, df_dis2, df_dis_vel2, df_dis_acc2), axis=1)
- names=[]
- for i in range(0, len(df_vel_acc2.columns)):
- names.append(df_vel_acc2.columns[i][0]+'_'+df_vel_acc2.columns[i][1])
- df_vel_acc2.columns=names
- df_vel_acc2=df_vel_acc2.interpolate(method='polynomial', order=1)
- df_vel_acc2=df_vel_acc2.fillna(0)
-
- del df_vel2
- del df_acc2
- del acc2
- del vel2
- del angle2
- del df_angle2
- del angle_vel2
- del df_angle_vel2
- del angle_acc2
- del df_angle_acc2
- del dis2
- del df_dis2
- del dis_vel2
- del df_dis_vel2
- del dis_acc2
- del df_dis_acc2
- if likeli1[0]>likeli2[0]:
- df_big=df_vel_acc1.iloc[[0]]
- else:
- df_big=df_vel_acc2.iloc[[0]]
- for i in range(1, len(likeli1)):
- if (i<=df_vel_acc1.shape[0]-1) and (i<=df_vel_acc2.shape[0]-1):
- if likeli1[i]>likeli2[i]:
- df_big=np.r_[df_big, df_vel_acc1.iloc[[i]]]
- elif likeli2[i]>likeli1[i]:
- df_big=np.r_[df_big, df_vel_acc2.iloc[[i]]]
- else:
- df_big=np.r_[df_big, df_vel_acc1.iloc[[i]]]
- df_big=pd.DataFrame(df_big)
- df_big.columns=df_vel_acc1.columns
- vid1=[]
- for j in [mr_gw_1, mr_gw_2, mr_gw_3]:
- df=j.fillna(0)
- for i in df.loc[z, :]:
- if i>0:
- vid1.append(i)
- y=np.zeros(shape=(len(df_big),))
- for i in vid1:
- y[int(i)]=1
- if np.count_nonzero(y)>=6:
- over = SMOTE(sampling_strategy=0.1)
- under = RandomUnderSampler(sampling_strategy=0.2)
- steps = [('o', over), ('u', under)]
- pipeline = Pipeline(steps=steps)
- X, y = pipeline.fit_resample(df_big, y)
-
- acc_score = 0
- for i in range(100):
- if (i+1) % 10 == 0:
- print("iterations: {}/100".format(i+1))
- X_train, X_test, y_train, y_test = train_test_split(X, y)
- sc = StandardScaler()
- X_train = sc.fit_transform(X_train)
- X_test = sc.transform(X_test)
- rfc = RandomForestClassifier(bootstrap=True).fit(X_train, y_train)
- pred_rfc = rfc.predict(X_test)
- if accuracy_score(y_test, pred_rfc) > acc_score:
- best_model = rfc
- acc_score = accuracy_score(y_test, pred_rfc)
- result = permutation_importance(best_model, X_test, y_test, n_repeats=10, random_state=42)
-
- dill.dump_session('GW_2D_'+z+'_data.pkl')
- print(z+' ready!')
|