123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102 |
- import os
- import pandas as pd
- from scipy.stats import spearmanr, shapiro, pearsonr
- import numpy as np
- from statsmodels.stats.multitest import multipletests
- # Get the directory where the code file is located
- code_dir = os.path.dirname(os.path.abspath(__file__))
- # Get the parent directory of the code directory
- parent_dir = os.path.dirname(code_dir)
- # Step 4: Save the resulting dataframe to a CSV file
- input_file_path = os.path.join(parent_dir, 'output', "Final_Quantitative_output", 'Quantitative_results_from_dwi_processing_merged_with_behavior_data.csv')
- input_anova_file = os.path.join(parent_dir, 'output', "Final_Quantitative_output","mixed_model_analysis",'mixed_model_results.csv')
- df_annova = pd.read_csv(input_anova_file)
- df_annova2 = df_annova[df_annova["p_value"]<=0.05]
- df_temp = pd.read_csv(input_file_path)
- # Filter df based on condition
- df_f1 = df_temp[df_temp["dialation_amount"] == 2]
- # Merge df_f1 and df_annova2 on common columns "Qtype" and "mask_name"
- merged_df = pd.merge(df_f1, df_annova2[['Qtype', 'mask_name', 'p_value']], on=['Qtype', 'mask_name'], how='left')
- # Display the merged DataFrame
- print(merged_df)
- #%%
- # Empty list to store the results
- results = []
- df = merged_df[(merged_df["p_value"]<=0.05) & (merged_df["merged_timepoint"]<40)]
- df.dropna(subset=['p_value'], inplace=True)
- # Iterate over unique values of 'Group', 'merged_timepoint', 'Qtype', and 'mask_name'
- for ss in df["Group"].unique():
- for time_point in df["merged_timepoint"].unique():
- for qq in df["Qtype"].unique():
- for mm in df["mask_name"].unique():
- # Filter the DataFrame for the current combination of 'Group', 'merged_timepoint', 'Qtype', and 'mask_name'
- df_f2 = df[(df["Group"] == ss) & (df["merged_timepoint"] == time_point) & (df["Qtype"] == qq) & (df["mask_name"] == mm)]
- # Remove rows with NaN values in 'Value' or 'averageScore' columns
- df_f2 = df_f2.dropna(subset=['Value', 'averageScore'])
- if not df_f2.empty:
-
- shapiro_statValue, shapiro_pvalueQValue = shapiro(df_f2["Value"])
- shapiro_statScore, shapiro_pvalueBehavior = shapiro(df_f2["averageScore"])
-
- if shapiro_pvalueQValue < 0.05 or shapiro_pvalueBehavior < 0.05:
- correlation_coefficient, p_value = pearsonr(df_f2["Value"], df_f2["averageScore"])
- else:
- correlation_coefficient, p_value = pearsonr(df_f2["Value"], df_f2["averageScore"])
-
- # Store the results in a dictionary
- result = {'Group': ss, 'merged_timepoint': time_point, 'Qtype': qq, 'mask_name': mm,
- 'Pval': p_value, 'R': correlation_coefficient,
- 'shapiro-wilk_pvalue_qtype': shapiro_pvalueQValue,'shapiro-wilk_pvalue_behavior': shapiro_pvalueBehavior}
- # Append the dictionary to the results list
- results.append(result)
- else:
- print(
- f"No valid data found for Group: {ss}, merged_timepoint: {time_point}, Qtype: {qq}, and mask_name: {mm}. Skipping.")
- # Create a DataFrame from the results list
- correlation_results_df = pd.DataFrame(results)
- unique_groups = df["Group"].unique()
- unique_masks = df["mask_name"].unique()
- unique_tp = df["merged_timepoint"].unique()
- for tt in unique_tp:
- for mask in unique_masks:
- for group in unique_groups:
- time_mask = correlation_results_df['merged_timepoint'] == tt
- mask_mask = correlation_results_df['mask_name'] == mask
- group_mask = correlation_results_df['Group'] == group
- combined_mask = time_mask & mask_mask & group_mask
-
- p_values = correlation_results_df[combined_mask]['Pval']
- rejected, p_values_corrected, _, _ = multipletests(p_values, method='fdr_bh')
- # Assign the corrected p-values to the DataFrame
- correlation_results_df.loc[combined_mask, 'Pval_corrected'] = p_values_corrected
- # Define the output file path
- output_file_path = os.path.join(parent_dir, 'output', "Correlation_with_behavior", 'correlation_dti_with_behavior.csv')
- # Save the correlation results DataFrame to a CSV file
- correlation_results_df.to_csv(output_file_path, index=False)
- print("Correlation results with corrected p-values saved successfully to 'correlation_results.csv' in the output folder.")
|