RatCAVE_Round2_Backup/Code/CliffAnalysis/scripts/cleaning.py~

import os
from os import path
from glob import glob
import numpy as np
import pandas as pd
import h5py
import datashader as ds
import datashader.transfer_functions as tf
# from plots import plot_occupancy_projections

HDF_FORMAT = 'fixed'


def strip_sessions(pos):
	"""Remove the beginning and end of each session's tracking data, using its position XYZ info."""
	# Get Only the data after entry into the arena for each session.
	def get_first_index(pos):
	    indices = np.where(
	        (pos.Y < .2)   # Rat is low.
	        & (pos.Z.abs() < .2)  # Rat is inside the Arena.
	    )
	    return indices[0][0] if len(indices[0]) > 0 else 0

	def get_last_index(pos):
	    indices = np.where(
	        (.20 < np.abs(pos.X))  # Rat is off board
	        & (np.abs(pos.X) < .45)  # Rat is still close to board, though.
	        & (pos.Y > .06)  # Rat is above the floor, not sitting on it.
	        & (pos.Y < .20)  # Rat is below the board.
	        & (np.sign(pos.X) == np.sign(pos.X.diff()))  # Rat is moving away from the board.
	        & (np.abs(pos.X.diff() / pos.Time.diff()) > .08)  # Rat is moving away from the board at speed.
	        & ((pos.Y.diff() / pos.Time.diff()) < -.20)
	    )
	    
	    return indices[0][0] if len(indices[0]) > 0 else 0

	get_window = lambda df: df.iloc[(get_first_index(df) + 300):get_last_index(df) + 28 ]
	pos = pos.groupby('Session_id').apply(get_window)
	pos.reset_index('Session_id', inplace=True, drop=True)
	return pos


def filter_sessions(pos):
	"""Remove Bad Sessions"""
	pos = pos.groupby('Session_id').filter(lambda df: (np.abs(df.X) > .38).any() == False )
	pos = pos.groupby('Session_id').filter(lambda df: (np.abs(df.Y) > .49).any() == False )
	return pos.copy()


def add_relative_time(pos):
	# Get Relative Time in Session
	pos = pos.copy()
	normalize = lambda t: (t - t.min()) / (t.max() - t.min())
	pos['TimeNormed'] = pos.groupby('Session_id').Time.apply(normalize)	
	return pos


def smooth_trajectories(df, **rolling_kwargs):
	"""Transform Data (Velocity and Acceleration)"""
	pos_s = df.copy()
	g = pos_s.groupby('Session_id')
	for dim in 'XYZ':
	    pos_s[dim] = g[dim].rolling(**rolling_kwargs).mean().values
	    pos_s['Vel' + dim] = g.apply(lambda df: df[dim].diff() / df.Time.diff()).values
	    pos_s['Acc' + dim] = g.apply(lambda df: df['Vel' + dim].diff() / df.Time.diff()).values
	return pos_s



def get_jump_trajectory(df, search_size=100, win_size=40):
	def get_end_of_fall(df):
	    end = np.where(df.VelY == np.min(df.VelY.iloc[-search_size:]))[0][0]
	    return df.iloc[(end - win_size):end]
	falls = df.copy().groupby('Session_id').apply(get_end_of_fall)
	falls.reset_index('Session_id', drop=True, inplace=True)
	return falls


def get_jump_point(df, session_filter='VelY < -.3'):
	"""Get last frame of falling: The minimum VelY"""
	jumps = df.groupby('Session_id').apply(lambda df: df[df.VelY == df.VelY.min()])
	jumps.reset_index('Session_id', inplace=True, drop=True)
	jumps = jumps.query(session_filter)
	return jumps.copy()


if __name__ == '__main__':

	# Load Data
	fname = '../data/VRCliff_AllData.h5'
	if not path.exists(fname):
	    os.system("python merge_sessions.py")
	pos = pd.read_hdf(fname, '/Position').dropna()
	pos.reset_index(inplace=True, drop=True)  # Because original file put frame as index.

	sessions = pd.read_hdf(fname, '/Session')
	sessions.set_index('id', inplace=True)

	# Correct for different Board Height
	sessions['BoardCorrection'] = sessions.BoardHeight.apply(lambda x: -.01 if x > .15 else 0)
	boardcorr = pd.merge(pos[['Session_id']], sessions[['BoardCorrection']], left_on='Session_id', right_index=True,)
	pos.Y.update(pos.Y + boardcorr.BoardCorrection)


    # Process, Clean, Filter Data
	pos = strip_sessions(pos)
	pos = filter_sessions(pos)
	pos = add_relative_time(pos)
	pos_s = smooth_trajectories(pos, window=15, center=True, min_periods=9, win_type='hamming')
	falls = get_jump_trajectory(pos_s, search_size=100, win_size=40)
	jumps = get_jump_point(falls, session_filter='VelY < -.3')


    
    # # Save Cleaned Data to a new file.
    # savename = '../data/VRCliff_AllData_cleaned.h5'
    # pos.to_hdf(savename, '/Position', format='table')
    # sessions.to_hdf(savename, '/Session', format='table')

# ###########




# Correct_for Left Cliff Condition (onlyh in RealCliff=Left condition)
# dfsub = sessions.query('CliffType == "Real" & CliffSide == "L"')
# swap_lr = lambda x: {'L': 'R', 'R': 'L', np.nan: np.nan}[x]
# sessions.loc[dfsub.index, 'JumpSide'] = dfsub.JumpSide.apply(swap_lr)
# todo: Move Left Cliff correction to earlier in the process.

# Evaluate Whether jump was on correct side or not
# sessions['JumpRight'] = ((sessions.JumpSide == 'R') - 0.5) * 2
# sessions['Correct'] = sessions.JumpSide != sessions.CliffSide

# #########





# falls.to_hdf(fname, '/Falls', format=HDF_FORMAT)



# jumps.to_hdf(fname, '/Jumps', format=HDF_FORMAT)