Final State of dataset

outputs/code_for_figures/AnatallDataMergew.py

@@ -0,0 +1,49 @@
+import os
+import glob
+import pandas as pd
+
+# Step 1: Define the starting path and file pattern
+start_path =  r"C:\Users\aswen\Desktop\Code\Validation3"
+file_pattern = '*anat*.csv'
+
+# Step 2: Find all matching CSV files in the specified directory and its subdirectories
+csv_files = glob.glob(os.path.join(start_path, '*','*', file_pattern), recursive=True)
+
+# Step 3: Initialize an empty DataFrame to store the extracted data
+combined_df = pd.DataFrame()
+combined_df1 = pd.DataFrame()
+combined_df2 = pd.DataFrame()
+
+
+# Step 4: Loop through the CSV files and extract the specified columns
+for csv_file in csv_files:
+    try:
+        df = pd.read_csv(csv_file)
+        selected_columns = ["FileAddress", "corresponding_img","Goasting", "SNR Chang", "SNR Normal"]
+        df = df[selected_columns]
+        df["dataset"] = csv_file.split(os.sep)[-3]
+        # Concatenate the current DataFrame with the combined DataFrame
+        combined_df1 = pd.concat([combined_df1, df], ignore_index=True)
+    except Exception as e:
+        print(f"Error reading {csv_file}: {e}")
+
+for csv_file in csv_files:
+    try:
+        df = pd.read_csv(csv_file)
+        selected_columns = ["FileAddress", "img name","Goasting", "SNR Chang", "SNR Normal"]
+        df = df[selected_columns]
+        df["dataset"] = csv_file.split(os.sep)[-3]
+        # Concatenate the current DataFrame with the combined DataFrame
+        combined_df2 = pd.concat([combined_df2, df], ignore_index=True)
+    except Exception as e:
+        print(f"Error reading {csv_file}: {e}")
+        
+combined_df2=combined_df2.rename({"img name": "corresponding_img"}, axis=1)
+combined_df = pd.concat([combined_df1, combined_df2], ignore_index=True)
+
+# Step 5: Print the combined DataFrame
+print(combined_df)
+
+# Optionally, you can save the combined DataFrame to a CSV file
+p = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input"
+combined_df.to_csv(os.path.join(p,'combined_data_anat.csv'), index=False)

outputs/code_for_figures/CSVplot.py

@@ -0,0 +1,271 @@
+
+
+#%% SNR Chang vs Standard plot for RT and Cryp
+
+p_address = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_Standard\94_m_As_allslice\calculated_features\caculated_features_structural.csv"
+p_address2= r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_Standard\94c_m_As_allslices\calculated_features\caculated_features_structural.csv"
+#p_save = r"\\10.209.5.114\Publications\2023_Kalantari_AIDAqc\outputs\QC_Final\validation\ChangVSStandard"
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+
+cm = 1/2.54  # centimeters in inches
+# Set the color palette to 'Set2'
+
+plt.figure(figsize=(18*cm/3,7.30*cm),dpi=300)
+# Load the CSV data into a pandas DataFrame
+data = pd.read_csv(p_address)
+data2 = pd.read_csv(p_address2)
+
+
+
+chang = data["SNR Chang"].to_frame().rename(columns={"SNR Chang": "SNR"})
+normal = data["SNR Normal"].to_frame().rename(columns={"SNR Normal": "SNR"})
+chang["type"] = "chang_RT"
+normal["type"] = "normal_RT"
+
+
+
+chang2 = data2["SNR Chang"].to_frame().rename(columns={"SNR Chang": "SNR"})
+normal2 = data2["SNR Normal"].to_frame().rename(columns={"SNR Normal": "SNR"})
+chang2["type"] = "chang_Cryo"
+normal2["type"] = "normal_Cryo"
+
+
+
+
+Data_merged = data[["SNR Chang","SNR Normal"]]
+Data_merged.rename(columns={"SNR Normal":"SNR-standard"})
+#Data_merged[["SNR Chang","SNR Normal"]] = Data_merged[["SNR Chang","SNR Normal"]].apply(lambda x: np.power((x/20),10))
+
+Data_merged2 = data2[["SNR Chang","SNR Normal"]]
+Data_merged2.rename(columns={"SNR Normal":"SNR-standard"})
+#Data_merged2[["SNR Chang","SNR Normal"]] = Data_merged2[["SNR Chang","SNR Normal"]].apply(lambda x: np.power((x/20),10))
+
+
+SNR = pd.concat([chang, normal,chang2,normal2])
+#SNR["SNR"] = SNR["SNR"].apply(lambda x: np.power(x/20,10))
+
+tips = SNR
+x = "type"
+y = "SNR"
+sns.set_style('ticks')
+# Create circular marker style for left side
+circle_marker = {"marker": "o", "markerfacecolor": "black", "markersize": 3}
+
+
+#tips = sns.load_dataset("tips")
+plt.rcParams['font.family'] = 'Times New Roman'
+plt.rcParams['font.size'] = 8
+plt.title("(c) Chang vs Standard SNR",weight='bold')
+ax=sns.barplot(x="type", y="SNR", data=tips, capsize=0 ,palette="pastel",errorbar="se",ci=None)
+#sns.swarmplot(x="type", y="SNR", data=tips, color="0", alpha=.5)
+for i in range(len(Data_merged)):
+    plt.plot(["chang_RT", "normal_RT"], Data_merged.iloc[i], color="black", **circle_marker,linewidth=0.5)
+
+for i in range(len(Data_merged2)):
+    plt.plot(["chang_Cryo", "normal_Cryo"], Data_merged2.iloc[i], color="black", **circle_marker,linewidth=0.5)
+
+ax.spines['top'].set_linewidth(0)  # Top border
+ax.spines['right'].set_linewidth(0)  # Right border
+ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax.spines['left'].set_linewidth(0.5)  # Left border
+# Set the font to Times New Roman and font size to 8 points
+#ax.set_ylim(bottom=0, top=45)
+ax.set_xticklabels(ax.get_xticklabels(), rotation=20,fontsize=8)
+# Rest of your code...
+ax.set_xlabel('')
+ax.tick_params(axis='both', which='both', width=0.5,color='gray',length=2)
+plt.show()
+
+
+#%% SNR Chang vs Standard plot for RT and Cryp Version 2
+
+p_address = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_Standard\94_m_As\calculated_features\caculated_features_anatomical.csv"
+p_address2= r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_Standard\94c_m_As\calculated_features\caculated_features_anatomical.csv"
+#p_save = r"\\10.209.5.114\Publications\2023_Kalantari_AIDAqc\outputs\QC_Final\validation\ChangVSStandard"
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import numpy as np
+
+cm = 1/2.54  # centimeters in inches
+# Set the color palette to 'Set2'
+
+plt.figure(figsize=(18*cm/3,7.30*cm),dpi=300)
+# Load the CSV data into a pandas DataFrame
+data = pd.read_csv(p_address)
+data2 = pd.read_csv(p_address2)
+
+
+
+chang = data["SNR Chang"].to_frame().rename(columns={"SNR Chang": "SNR"})
+normal = data["SNR Normal"].to_frame().rename(columns={"SNR Normal": "SNR"})
+chang["type"] = "chang_rt"
+normal["type"] = "normal_rt"
+
+
+
+chang2 = data2["SNR Chang"].to_frame().rename(columns={"SNR Chang": "SNR"})
+normal2 = data2["SNR Normal"].to_frame().rename(columns={"SNR Normal": "SNR"})
+chang2["type"] = "chang_cryo"
+normal2["type"] = "normal_cryo"
+
+
+
+
+SNR = pd.concat([chang, normal,chang2,normal2])
+SNR["SNR"] = SNR["SNR"].apply(lambda x: np.power(10,x/20))
+
+
+SNR = SNR.sort_values('type')
+
+
+
+
+#Data_of_selected_feature = Data_of_selected_feature.sort_values("Dataset",ascending=False)
+# creating boxplots
+#plt.figure(figsize=(6*cm,6*cm*np.e),dpi=300)
+sns.set_style('ticks')
+sns.set(font='Times New Roman', font_scale=0.9,style=None)  # Set font to Times New Roman and font size to 9
+palette = 'Set2'
+ax = sns.violinplot(x="type", y="SNR", data=SNR, hue="type", dodge=False,
+                    palette=palette,
+                    scale="width", inner=None,linewidth=1)
+patches = ax.patches
+#legend_colors = [patch.get_facecolor() for patch in patches[:]]
+
+xlim = ax.get_xlim()
+ylim = ax.get_ylim()
+for violin in ax.collections:
+    bbox = violin.get_paths()[0].get_extents()
+    x0, y0, width, height = bbox.bounds
+    violin.set_clip_path(plt.Rectangle((x0, y0), width / 2, height, transform=ax.transData))
+
+sns.boxplot(x="type", y="SNR", data=SNR, saturation=1, showfliers=False,
+            width=0.3, boxprops={'zorder': 3, 'facecolor': 'none'}, ax=ax, linewidth=1)
+old_len_collections = len(ax.collections)
+sns.stripplot(x="type", y="SNR", data=SNR, size=1.1, hue="type", palette=palette, dodge=False, ax=ax)
+for dots in ax.collections[old_len_collections:]:
+    dots.set_offsets(dots.get_offsets() + np.array([0.12, 0]))
+ax.set_xlim(xlim)
+ax.set_ylim(ylim)
+ax.legend_.remove()
+ax
+ax.set_xlabel('')
+ax.set_xticklabels(ax.get_xticklabels(), rotation=20,fontsize=8)
+#ax.set_ylim(bottom=15, top=40)
+
+#%% Plot all chang vs normal
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import scipy.stats as stats
+import numpy as np
+import os
+
+
+cm = 1/2.54  # centimeters in inches
+# Specify the path to your Excel file
+excel_file_path = r"C:\Users\arefk\OneDrive\Desktop\Projects\2023_Kalantari_AIDAqc\outputs\files_4figs\combined_data_anat.csv"
+plt.figure(figsize=(10*cm,10*cm),dpi=300)
+# Read the data into a pandas DataFrame
+df = pd.read_csv(excel_file_path)
+
+# Drop rows with NaN values in the specified columns
+#df = df.dropna(subset=['SNR-Chang (dB)', 'SNR-Standard (dB)'])
+
+df['SNR Chang'] = df['SNR Chang'].apply(lambda x: np.power(10,x/20))
+df['SNR Normal'] = df['SNR Normal'].apply(lambda x: np.power(10,x/20))
+df['names'] = df['FileAddress'].apply(lambda x:x.split("mri")[1].split(os.path.sep)[1])
+
+
+
+plt.rcParams['font.family'] = 'Times New Roman'
+plt.rcParams['font.size'] = 8
+plt.title("All anatomical data",weight='bold', fontsize=10)
+
+# Calculate the correlation and p-value between 'SNR-Chang' and 'SNR-Standard'
+#correlation, p_value = stats.pearsonr(df['SNR-Chang (dB)'], df['SNR-Standard (dB)'])
+correlation, p_value = stats.spearmanr(df['SNR Chang'], df['SNR Normal'], nan_policy='omit',alternative='two-sided')
+
+# Set seaborn style
+sns.set_style('whitegrid')
+
+
+# Create a scatter plot
+#ax=sns.scatterplot(data=df, x='SNR-Chang', y='SNR-Standard',palette="gray_r",s=7)
+ax=sns.scatterplot(data=df, x='SNR Chang', y='SNR Normal',hue="names",palette="Spectral_r",s=7)
+ax.set_title("All anatomical data", weight='bold', fontsize=11)
+
+
+
+
+# Set title and labels including the correlation and p-value
+#plt.title(f'Correlation: {correlation:.8f}, P-value: {p_value:.8f}')
+plt.xlabel('SNR-Chang')
+plt.ylabel('SNR-Standard')
+
+ax.set_xlim(0,200)
+ax.set_ylim(0,140)
+
+ax.spines['top'].set_linewidth(0)  # Top border
+ax.spines['right'].set_linewidth(0)  # Right border
+ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax.spines['left'].set_linewidth(0.5)  # Left border
+# Show the plot
+# Move the legend outside the plot to the right side
+legend =plt.legend(title="Dataset", loc='center left', bbox_to_anchor=(1, 0.5), fontsize=5,handlelength=0.5)
+legend.get_title().set_fontfamily('Times New Roman')
+for text in legend.get_texts():
+    text.set_fontfamily('Times New Roman')
+
+#ax.legend_.remove()
+plt.show()
+
+#%% Plot all chang vs normal and corrolate 0 to 50 and 50 to end
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import scipy.stats as stats
+import numpy as np
+
+
+cm = 1/2.54  # centimeters in inches
+# Specify the path to your Excel file
+excel_file_path = r"Y:\2023_Kalantari_AIDAqc\outputs\QC_Final\validation\Chang&normal_all.xlsx"
+plt.figure(figsize=(8*cm,8*cm),dpi=300)
+# Read the data into a pandas DataFrame
+df = pd.read_excel(excel_file_path, engine='openpyxl')
+
+# Drop rows with NaN values in the specified columns
+#df = df.dropna(subset=['SNR-Chang (dB)', 'SNR-Standard (dB)'])
+
+df['SNR-Chang'] = df['SNR-Chang'].apply(lambda x: np.power(10,x/20))
+df['SNR-Standard'] = df['SNR-Standard'].apply(lambda x: np.power(10,x/20))
+
+df = df.sort_values('sort')
+
+
+filtered_df = df[df['Dataset'].isin(['94_m_We', '94_m_Va'])]
+
+plt.rcParams['font.family'] = 'Times New Roman'
+plt.rcParams['font.size'] = 8
+plt.title("All anatomical data",weight='bold')
+
+# Create two dataframes, one for values smaller than 50 and another for values 50 or greater
+df_smaller_than_50 = df[df['SNR-Chang'] < 60 & df['SNR-Standard'] < 75]
+
+# Calculate the correlation and p-value for the group where 'SNR-Chang' < 50
+correlation_smaller_than_50, p_value_smaller_than_50 = stats.spearmanr(df_smaller_than_50['SNR-Chang'], df_smaller_than_50['SNR-Standard'], nan_policy='omit', alternative='two-sided')
+print("Correlation:", correlation_smaller_than_50)
+print("P-value:", p_value_smaller_than_50)

outputs/code_for_figures/ChangvsSNR.py

@@ -0,0 +1,78 @@
+#%% Plot all chang vs normal
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import scipy.stats as stats
+import numpy as np
+import os
+import glob
+import shutil
+import matplotlib.pyplot as plt
+
+cm = 1/2.54  # centimeters in inches
+# Specify the path to your Excel file
+img_path = r"C:\Users\aswen\Desktop\Code\Validation\*\manual_slice_inspection"
+new_path = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures"
+excel_file_path = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input\combined_data_anat.csv"
+plt.figure(figsize=(10*cm,10*cm),dpi=300)
+# Read the data into a pandas DataFrame
+df = pd.read_csv(excel_file_path)
+
+# Drop rows with NaN values in the specified columns
+#df = df.dropna(subset=['SNR-Chang (dB)', 'SNR-Standard (dB)'])
+
+df['SNR Chang'] = df['SNR Chang'].apply(lambda x: np.power(10,x/20))
+df['SNR Normal'] = df['SNR Normal'].apply(lambda x: np.power(10,x/20))
+#f['FileAddress2'] = df['FileAddress'].apply(lambda x:x.split(os.path.sep)[0])
+df_filterd = df[df["SNR Normal"] < 19]
+df_filterd= df_filterd[df_filterd["SNR Chang"]>90]
+df_filterd['names'] = df_filterd['FileAddress'].apply(lambda x:x.split("mri")[1].split(os.path.sep)[1])
+
+
+df_filterd.to_csv(r"C:\Users\arefk\OneDrive\Desktop\Projects\2023_Kalantari_AIDAqc\outputs\files_4figs\combined_data_anat_changVSstandard_Check.csv")
+P = []
+for i in df_filterd["corresponding_img"]:
+    print(i)
+    
+    old_add = glob.glob(os.path.join(img_path,i),recursive=True)
+    new_add = os.path.join(new_path,i)
+    #shutil.copyfile(old_add[0],new_add)
+    
+
+plt.rcParams['font.family'] = 'Times New Roman'
+plt.rcParams['font.size'] = 8
+plt.title("All anatomical data",weight='bold', fontsize=10)
+
+# Calculate the correlation and p-value between 'SNR-Chang' and 'SNR-Standard'
+
+# Set seaborn style
+sns.set_style('whitegrid')
+
+ax=sns.scatterplot(data=df_filterd, x='SNR Chang', y='SNR Normal',hue="names",palette="Spectral_r",s=7)
+ax.set_title("All anatomical data", weight='bold', fontsize=11)
+
+
+
+
+# Set title and labels including the correlation and p-value
+#plt.title(f'Correlation: {correlation:.8f}, P-value: {p_value:.8f}')
+plt.xlabel('SNR-Chang')
+plt.ylabel('SNR-Standard')
+
+ax.set_xlim(0,200)
+ax.set_ylim(0,140)
+
+ax.spines['top'].set_linewidth(0)  # Top border
+ax.spines['right'].set_linewidth(0)  # Right border
+ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax.spines['left'].set_linewidth(0.5)  # Left border
+# Show the plot
+# Move the legend outside the plot to the right side
+legend =plt.legend(title="Dataset", loc='center left', bbox_to_anchor=(1, 0.5), fontsize=5,handlelength=0.5)
+legend.get_title().set_fontfamily('Times New Roman')
+for text in legend.get_texts():
+    text.set_fontfamily('Times New Roman')
+
+#ax.legend_.remove()
+plt.show()

outputs/code_for_figures/ChangvsStandardSNR.py

@@ -0,0 +1,94 @@
+#%% Plot all chang vs normal
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import scipy.stats as stats
+import numpy as np
+import os
+
+sns.set_style('ticks')
+cm = 1/2.54  # centimeters in inches
+# Specify the path to your Excel file
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+excel_file_path = os.path.join(script_dir, '..', 'input', 'combined_data_anat.csv')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+
+plt.figure(figsize=(10*cm,10*cm),dpi=300)
+# Read the data into a pandas DataFrame
+df = pd.read_csv(excel_file_path)
+
+# Drop rows with NaN values in the specified columns
+#df = df.dropna(subset=['SNR-Chang (dB)', 'SNR-Standard (dB)'])
+# Custom sorting key function
+def extract_number(dataset):
+    # Extract numeric part from the 'dataset' column
+    # Assuming the numeric part is always at the beginning of the string
+    # If it's not, you might need a more sophisticated method
+    return int(''.join(filter(str.isdigit, dataset)))
+
+df['sorting_key'] = df['dataset']#.apply(extract_number)
+df['SNR Chang'] = df['SNR Chang']#.apply(lambda x: np.power(10,x/20))
+df['SNR Normal'] = df['SNR Normal']#.apply(lambda x: np.power(10,x/20))
+df = df.sort_values(by=["sorting_key"],ascending=True)
+
+
+plt.rcParams['font.family'] = 'Times New Roman'
+plt.rcParams['font.size'] = 8
+#plt.title("All anatomical data",weight='bold', fontsize=10)
+
+# Calculate the correlation and p-value between 'SNR-Chang' and 'SNR-Standard'
+#correlation, p_value = stats.pearsonr(df['SNR-Chang (dB)'], df['SNR-Standard (dB)'])
+# Set seaborn style
+Wanted_datasets = ["117_m_Ra","94_m_Va","7_m_Se","7_rab_Mu","7_h_He"]
+
+# Filter DataFrame based on Wanted_datasets
+filtered_df = df[df['dataset'].isin(Wanted_datasets)]
+
+#
+
+# Your DataFrame and plot
+ax = sns.scatterplot(data=filtered_df, x='SNR Chang', y='SNR Normal', hue="dataset", palette= "Set1", s=20)
+
+#ax.set_title("All anatomical data", weight='bold', fontsize=11)
+
+# Set title and labels including the correlation and p-value
+plt.xlabel('SNR-Chang (db)', fontname='Times New Roman', fontsize=8)
+plt.ylabel('SNR-Standard (db)', fontname='Times New Roman', fontsize=8)
+
+# Calculate the correlation and p-value between 'SNR-Chang' and 'SNR-Standard'
+correlation, p_value = stats.spearmanr(df['SNR Chang'], df['SNR Normal'], nan_policy='omit', alternative='two-sided')
+
+
+# Sort the dataframe by 'SNR Chang' in descending order
+df_sorted = df.sort_values(by='SNR Chang', ascending=False)
+
+# Exclude the top 3 highest values
+top3_indices = df_sorted.head(1).index
+df_filtered = df.drop(top3_indices)
+
+
+ax.spines['top'].set_linewidth(0.5)  # Top border
+ax.spines['right'].set_linewidth(0.5)  # Right border
+ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax.spines['left'].set_linewidth(0.5)  # Left border
+ax.set_xlim(15,50)
+ax.set_ylim(10,50)
+
+# Move the legend outside the plot to the right side
+legend = plt.legend(title="Dataset", loc='center left', bbox_to_anchor=(1, 0.5), fontsize=8, handlelength=0.5)
+legend.get_title().set_fontfamily('Times New Roman')
+for text in legend.get_texts():
+    text.set_fontfamily('Times New Roman')
+
+fig_path_png = os.path.join(out_path, 'StandardVSchangtogether.png')
+fig_path_svg = os.path.join(out_path, 'StandardVSchangtogether.svg')
+
+plt.savefig(fig_path_png, format='png', bbox_inches='tight')
+plt.savefig(fig_path_svg, format='svg', bbox_inches='tight')
+
+# Show the plot
+plt.show()
+

outputs/code_for_figures/ChangvsStandardSNR_Seperated.py

@@ -0,0 +1,97 @@
+import seaborn as sns
+import matplotlib.pyplot as plt
+import pandas as pd
+import scipy.stats as stats
+import numpy as np
+import os
+
+cm = 1/2.54  # centimeters in inches
+# Specify the path to your Excel file
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+excel_file_path = os.path.join(script_dir, '..', 'input', 'combined_data_anat.csv')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+# Read the data into a pandas DataFrame
+df = pd.read_csv(excel_file_path)
+
+# Set seaborn style
+sns.set_style('ticks')
+
+# Create a list to store correlation and p-value for each dataset
+correlation_list = []
+p_value_list = []
+
+plt.figure(figsize=(10*cm,10*cm),dpi=300)
+# Get unique values in the 'dataset' column
+def extract_number(dataset):
+    # Extract numeric part from the 'dataset' column
+    # Assuming the numeric part is always at the beginning of the string
+    # If it's not, you might need a more sophisticated method
+    return int(''.join(filter(str.isdigit, dataset)))
+
+df['sorting_key'] = df['dataset'].apply(extract_number)
+df['SNR Chang'] = df['SNR Chang']#.apply(lambda x: np.power(10,x/20))
+df['SNR Normal'] = df['SNR Normal']#.apply(lambda x: np.power(10,x/20))
+df = df.sort_values(by=["sorting_key"],ascending=True)
+datasets = df['dataset'].unique()
+SS = int(np.ceil(np.sqrt(len(datasets))))
+# Create subplots based on the number of datasets
+fig, axes = plt.subplots(SS, SS, figsize=(18*cm, 18*cm), dpi=300, constrained_layout=True)
+
+# Flatten the axes array to iterate over it
+axes = axes.flatten()
+
+for i, dataset in enumerate(datasets):
+    # Filter the dataframe for the current dataset
+    df_subset = df[df['dataset'] == dataset]
+
+    # Calculate the correlation and p-value for the current dataset
+    correlation, p_value = stats.spearmanr(df_subset['SNR Chang'], df_subset['SNR Normal'], nan_policy='omit', alternative='two-sided')
+    
+    # Append the correlation and p-value to the lists
+    correlation_list.append(correlation)
+    p_value_list.append(p_value)
+
+    # Create a scatter plot for the current dataset
+    ax = sns.scatterplot(data=df_subset, x='SNR Chang', y='SNR Normal', s=7, ax=axes[i],color="red")
+    ax.set_title(f"{dataset}", weight='bold', fontsize=8, fontname='Times New Roman')
+    
+
+    # Set title and labels including the correlation and p-value
+    ax.set_xlabel('SNR-Chang (db)', fontname='Times New Roman',fontsize=8)
+    ax.set_ylabel('SNR-Standard (db)', fontname='Times New Roman',fontsize=8)
+    for tick in ax.get_xticklabels():
+        tick.set_fontname("Times New Roman")
+        tick.set_fontsize(8)
+    for tick in ax.get_yticklabels():
+        tick.set_fontname("Times New Roman")
+        tick.set_fontsize(8)        
+    
+    
+    # Set xlim and ylim
+    #ax.set_xlim(20.978242760551243, 88.420371212099)
+    #ax.set_ylim(3.251536979292914, 43.47414376123412)
+
+    # Remove borders
+    ax.spines['top'].set_linewidth(0.5)
+    ax.spines['right'].set_linewidth(0.5)
+    ax.spines['bottom'].set_linewidth(0.5)
+    ax.spines['left'].set_linewidth(0.5)
+    
+
+# Show the plot
+fig_path_png = os.path.join(out_path, 'StandardVSchang_all.png')
+fig_path_svg = os.path.join(out_path, 'StandardVSchangall.svg')
+
+plt.savefig(fig_path_png, format='png', bbox_inches='tight')
+plt.savefig(fig_path_svg, format='svg', bbox_inches='tight')
+
+# Show the plot
+plt.show()
+
+# Print the correlation and p-value for each dataset
+for i, dataset in enumerate(datasets):
+    print(f"{dataset} - Correlation: {correlation_list[i]}, p-value: {p_value_list[i]}")
+
+

outputs/code_for_figures/CompareVotes.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep  1 17:57:59 2023
+
+@author: arefks
+"""
+
+import glob
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+
+#%function
+
+def calculate_and_display_fleiss_kappa(data):
+    kappa, _ = fleiss_kappa(data)
+    print("Fleiss' Kappa:", kappa)
+    return kappa
+
+#%% Expert
+
+Path_votings_chang = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_Chang\*\votings.csv"
+
+All_csv_votings = glob.glob(Path_votings_chang)
+
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        data_dict[ff] = df[df["Voting outliers (from 5)"]>0] 
+            
+    return data_dict
+
+All_values = read_csv_files(All_csv_votings)
+
+img_values = []
+
+for df_name, df in All_values.items():
+    if "corresponding_img" in df.columns:
+        img_values.extend(df["corresponding_img"].tolist())
+
+chang_df = pd.DataFrame({"corresponding_img": img_values})
+###
+###
+
+Path_votings_standard = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_standard\*\votings.csv"
+
+All_csv_votings = glob.glob(Path_votings_standard)
+
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        data_dict[ff] = df[df["Voting outliers (from 5)"]>0] 
+            
+    return data_dict
+
+All_values = read_csv_files(All_csv_votings)
+
+img_values = []
+
+for df_name, df in All_values.items():
+    if "corresponding_img" in df.columns:
+        img_values.extend(df["corresponding_img"].tolist())
+
+standard_df = pd.DataFrame({"corresponding_img": img_values})

outputs/code_for_figures/ConfusionBoxplot.py

@@ -0,0 +1,228 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep  1 17:57:59 2023
+
+@author: arefks
+"""
+
+import glob
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+
+#%function
+text=[]
+def calculate_and_display_fleiss_kappa(data):
+    kappa, _ = fleiss_kappa(data)
+    print("Fleiss' Kappa:", kappa)
+    return kappa
+
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        data_dict[ff] = df[df["Voting outliers (from 5)"]>0] 
+            
+    return data_dict
+#%% Expert
+
+Path_votings = r"C:\Users\aswen\Desktop\validation\*\votings.csv"
+#P = r"C:\Users\arefk\OneDrive\Desktop\Projects\validation\7_m_Br\votings.csv"
+
+
+All_csv_votings = glob.glob(Path_votings)
+
+for aa in All_csv_votings:
+    
+    temp = glob.glob(aa)
+    All_values = read_csv_files(temp)  
+    print(temp)
+    
+    img_values = []
+    
+    for df_name, df in All_values.items():
+        if "corresponding_img" in df.columns:
+            img_values.extend(df["corresponding_img"].tolist())
+    
+    new_df = pd.DataFrame({"corresponding_img": img_values})
+    
+    MU_strings = ["beginner","experienced","expert"]
+    Sequence_type = ["anatomical","functional","structural"]
+    cc=0
+    cm = 1/2.54  # centimeters in inches
+    
+    plt.figure(figsize=(18*cm, 9*cm))
+    fig, ax = plt.subplots(3,3, dpi=300,figsize=(12*cm, 12*cm))#,sharex="row",sharey="row")
+    #fig.suptitle('Confusion matrix',fontname='Times New Roman')
+            
+    Matric_sequneces = []
+    for ss,S in enumerate(Sequence_type):
+        for mu,MU in enumerate(MU_strings):
+            
+            #Path = r"C:\Users\arefk\OneDrive\Desktop\Projects\validation\94_m_As\manual_slice_inspection"
+            Path = os.path.join(os.path.dirname(temp[0]), "manual_slice_inspection")
+            
+            
+            p_afs = os.path.join(Path,S+"*.png")
+            
+            afs_all = glob.glob(p_afs,recursive=True)
+            afs_all = [os.path.basename(path) for path in afs_all]
+            if not afs_all:
+                print("fuck that " + MU + "and " + S)
+                continue
+            
+            
+            count_afs_all = len(afs_all)
+            
+    
+            #Path_gt = r"C:\Users\arefk\OneDrive\Desktop\Projects\validation\94_m_As\validation_" + MU
+            Path_gt = os.path.join(os.path.dirname(temp[0]),("validation_" + MU))
+            pgt_afs = os.path.join(Path_gt, S+"*.png")
+            
+            afsgt_all = glob.glob(pgt_afs,recursive=True)
+            
+            afsgt_all = [os.path.basename(path) for path in afsgt_all]
+            
+            countgt_afs_bad = len(afsgt_all)
+            
+            countgt_afs_good = count_afs_all - countgt_afs_bad
+            
+            
+            
+            
+            
+            # Separate the new_df DataFrame based on specific prefixes
+            afsqc_all = new_df[new_df['corresponding_img'].str.startswith(S)]['corresponding_img'].tolist()
+            
+                     
+            countqc_afs_bad = len(afsqc_all)
+            countqc_afs_good = count_afs_all - countqc_afs_bad
+            
+            
+            afs_intersect_qc_gt = set(afsgt_all) & set(afsqc_all)
+    # =============================================================================
+    #         afs_TN = len(afs_intersect_qc_gt)
+    #         afs_FN = countqc_afs_bad - afs_TN
+    #         afs_FP = countgt_afs_bad - afs_TN
+    #         afs_TP = countgt_afs_good - afs_FN
+    # =============================================================================
+            afs_TP = len(afs_intersect_qc_gt)
+            afs_FN = countgt_afs_bad - afs_TP
+            afs_FP = countqc_afs_bad - afs_TP
+            afs_TN = countgt_afs_good - afs_FP
+            
+    # =============================================================================
+            afs_percent_TP = (afs_TP / countgt_afs_bad)
+            afs_percent_FN = (1 - afs_percent_TP)
+            afs_percent_FP = (afs_FP /countgt_afs_good)
+            afs_percent_TN = (1 - afs_percent_FP)
+    # =============================================================================
+            
+            
+            # Calculate precision
+            precision = afs_TP / (afs_TP + afs_FP)
+
+            # Calculate recall
+            recall = afs_TP / (afs_TP + afs_FN)
+
+            
+            # Calculate F1 score
+            f1_score = 2 * (precision * recall) / (precision + recall)
+            
+            # Print the F1 score
+            print("F1 Score:", f1_score)
+    
+            
+            confusion_matrix = [[afs_percent_TP, afs_percent_FN],
+                                 [afs_percent_FP, afs_percent_TN]]
+                
+            
+            # Create a heatmap using Seaborn
+            sns.set(font_scale=0.8)  # Adjust the font size
+            heatmap = sns.heatmap(confusion_matrix, annot=True, fmt='.2%', cmap='Greys',
+                                  annot_kws={"fontname": "Times New Roman"},
+                                  xticklabels=False, yticklabels=False, cbar=False,ax=ax[ss,mu])
+            ax[ss, mu].set_xlabel('AIDAqc', fontname='Times New Roman')
+            ax[ss, mu].set_ylabel(MU, fontname='Times New Roman')
+            ax[ss, mu].set_title(S.capitalize()+'\n F1-score: %.2f' % f1_score + "", fontname='Times New Roman', weight="bold")
+            ax[ss, mu].set_xticks([0.5, 1.5])
+            ax[ss, mu].set_xticklabels(['good', 'bad'], fontname='Times New Roman')
+            ax[ss, mu].set_yticks([0.5, 1.5])
+            ax[ss, mu].set_yticklabels(['good', 'bad'], fontname='Times New Roman',rotation=90)
+            
+            
+            
+            Vec = [0 if item in afsgt_all else 1 for item in afs_all]
+            if not Vec:
+                print("fuck that precsion and recall " + MU + "and " + S)
+                continue
+            
+            Matric_sequneces.append(Vec)
+            
+    
+        Vec_qc = [0 if item in afsqc_all else 1 for item in afs_all]   
+        Matric_sequneces.append(Vec_qc)   
+            
+            
+            
+            
+            
+            
+    # Show the plot
+    #plt.subplots_adjust(left=0.00, right=1, top=1, bottom=0.00, wspace=0.3, hspace=1.8)
+    plt.tight_layout()
+    plt.show()
+    
+    
+    
+    try:
+        Stacked_anat = np.stack(Matric_sequneces[0:4],axis=0).transpose()
+        Stacked_func = np.stack(Matric_sequneces[4:8],axis=0).transpose()
+        Stacked_struct = np.stack(Matric_sequneces[8:12],axis=0).transpose()
+        
+        Stacked_anat_bool = Stacked_anat.astype(bool)
+        Stacked_func_bool = Stacked_func.astype(bool)
+        Stacked_struct_bool = Stacked_struct.astype(bool)
+    except ValueError:
+        print("fuck that precsion and recall " + MU + "and " + S)
+        continue
+    
+    # Define column labels
+    column_labels = ["beginner", "experienced", "expert", "aidaqc"]
+    
+    # Create pandas DataFrames
+    try:
+        df_anat = pd.DataFrame(Stacked_anat_bool, columns=column_labels)
+        df_func = pd.DataFrame(Stacked_func_bool, columns=column_labels)
+        df_struct = pd.DataFrame(Stacked_struct_bool, columns=column_labels)
+    except ValueError:
+        print("fuck that precsion and recall " + MU + "and " + S)
+        continue
+    
+        
+    from sklearn.metrics import cohen_kappa_score
+    
+    # Create a list of DataFrames
+    dataframes = [df_anat, df_func, df_struct]
+    
+    # Loop through the DataFrames
+    for i, df in enumerate(dataframes):
+        print(f"DataFrame {i + 1}:")
+        for col1 in df.columns:
+            for col2 in df.columns:
+                if col1 != col2:
+                    kappa_score = cohen_kappa_score(df[col1], df[col2])
+                    print(f"Cohen's Kappa Score for {Sequence_type[i]} and between {col1} and {col2}: {kappa_score:.4f}")
+                    text.append(f"Cohen's Kappa Score for {Sequence_type[i]} and {aa} between {col1} and {col2}: {kappa_score:.4f}")
+        print()
+    
+    

outputs/code_for_figures/ConfusionMatrix.py

@@ -0,0 +1,161 @@
+import pandas as pd
+from sklearn.metrics import confusion_matrix, precision_recall_fscore_support,ConfusionMatrixDisplay
+import os
+from sklearn.metrics import precision_recall_curve
+import matplotlib.pyplot as plt
+# Load the CSV files
+
+
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+
+
+
+human_voters_df = pd.read_csv(os.path.join(file_path,"combined_Human_Voters_from_votings3_Final.csv"))
+votings_df = pd.read_csv(os.path.join(file_path,"combined_votings3.csv"))
+C = 0
+
+# Create an empty DataFrame to store results
+result_df = pd.DataFrame(columns=[
+    'dataset_name', 'sequence_name', 'Thresold_ML_Voters', 'Thresold_Human_Voters',
+    'TP', 'TN', 'FP', 'FN',"SUM" , 'Precision','Accuracy',
+    'Prevalence', 'Sensitivity-Recall', 'Specificity', 'F1 Score',
+    'Informedness', 'Markedness', 'Diagnostic Odds Ratio', 'Phi Coefficient',
+    'Fowlkes-Mallows Index', 'Negative Predictive Value', 'Miss Rate',
+    'Fall-out', 'False Discovery Rate', 'False Omission Rate', 'Positive Likelihood Ratio',
+    'Negative Likelihood Ratio', 'Prevalence Threshold', 'Threat Score'
+])
+
+# Iterate over unique datasets
+for dataset_name in human_voters_df['dataset_name'].unique():
+    # Filter data for the current dataset
+    human_data = human_voters_df[human_voters_df['dataset_name'] == dataset_name]
+    voting_data = votings_df[votings_df['dataset'] == dataset_name]
+
+    # Iterate over unique sequence names
+    for sequence_name in human_data['SequenceType'].unique():
+
+        # Filter data for the current sequence
+        human_sequence_data = human_data[human_data['SequenceType'] == sequence_name]
+        voting_sequence_data = voting_data[voting_data['SequenceType'] == sequence_name]
+        voting_sequence_data["complete_image_path"] = voting_sequence_data.apply(
+            lambda row: os.path.join(row['corresponding_img_Path'], row['corresponding_img']), axis=1)
+
+        # Create a list to store rows for missing paths
+        missing_rows = []
+        missing_paths = set(human_sequence_data['Path']) - set(voting_sequence_data['complete_image_path'])
+        for missing_path in missing_paths:
+            # Get relevant information from human_voters_df based on the missing path
+            relevant_info = human_voters_df[human_voters_df['Path'] == missing_path].iloc[0]
+
+            # Create a new row with relevant information
+            new_row = {'Pathes': relevant_info['Path'],
+                       'corresponding_img': None,
+                       'One_class_SVM': False,
+                       'IsolationForest': False,
+                       'LocalOutlierFactor': False,
+                       ' EllipticEnvelope': False,
+                       'statistical_method': False,
+                       'Voting outliers (from 5)': None,
+                       'dataset': relevant_info['dataset_name'],
+                       'SequenceType': relevant_info['SequenceType'],
+                       'complete_image_path': None
+                       }
+
+            # Append the new row to the list
+            missing_rows.append(new_row)
+
+        # Concatenate the list of new rows to the voting DataFrame
+        voting_sequence_data_complemented = pd.concat([voting_sequence_data, pd.DataFrame(missing_rows)],
+                                                      ignore_index=True)
+
+        # Iterate over thresholds for Thresold_ML_Voters
+        for Thresold_ML_Voters in range(1, 6):
+            # Iterate over thresholds for Thresold_Human_Voters
+            for Thresold_Human_Voters in range(1, 6):
+                voting_sequence_data_complemented["Predictor"] = voting_sequence_data_complemented.iloc[:, 2:7].sum(
+                    axis=1) >= Thresold_ML_Voters
+                human_sequence_data["Predictor"] = human_sequence_data.iloc[:, [1] + list(range(4, 7))].sum(
+                    axis=1) >= Thresold_Human_Voters
+
+                # Extract labels from human voters
+                human_labels = human_sequence_data[['Predictor']].values.flatten()
+
+                # Extract labels from voting methods
+                voting_labels = voting_sequence_data_complemented[['Predictor']].values.flatten()
+
+                # Calculate confusion matrix
+                try:
+                    tn, fp, fn, tp = confusion_matrix(human_labels, voting_labels).ravel()
+                    a = (tn, fp, fn, tp)
+                except ValueError:
+                    print("exception for: " + sequence_name + " + " + dataset_name)
+                    C = 1 + C
+                    continue
+
+                # Calculate additional metrics
+                prevalence = (tp + fn) / (tp + tn + fp + fn)
+                accuracy = (tp + tn) / (tp + tn + fp + fn)
+                sensitivity = recall = tp / (tp + fn)
+                specificity = tn / (tn + fp)
+                f1_score = 2 * tp / (2 * tp + fp + fn)
+                informedness = sensitivity + specificity - 1
+                markedness = precision = tp / (tp + fp)
+                npv = tn / (tn + fn)
+                miss_rate = fn / (tp + fn)
+                fall_out = fp / (tn + fp)
+                false_discovery_rate = fp / (tp + fp)
+                false_omission_rate = fn / (tn + fn)
+                positive_likelihood_ratio = sensitivity / (1 - specificity) if (1 - specificity) != 0 else float('inf')
+                negative_likelihood_ratio = (1 - sensitivity) / specificity if specificity != 0 else float('inf')
+                prevalence_threshold = (tp + fp) / (tp + tn + fp + fn)
+                threat_score = 2 * tp / (2 * tp + fp + fn)
+                # Calculate Phi Coefficient
+                phi_coefficient = (tp * tn - fp * fn) / ((tp + fp) * (tp + fn) * (tn + fp) * (tn + fn))**0.5 if (tp + fp) * (tp + fn) * (tn + fp) * (tn + fn) != 0 else 0
+                
+                # Calculate Fowlkes-Mallows Index
+                fowlkes_mallows_index = (precision * recall)**0.5
+
+                # Append the results to the result DataFrame
+                result_df = pd.concat([result_df, pd.DataFrame({
+                    'dataset_name': [dataset_name],
+                    'sequence_name': [sequence_name],
+                    'Thresold_ML_Voters': [Thresold_ML_Voters],
+                    'Thresold_Human_Voters': [Thresold_Human_Voters],
+                    'TP': [tp],
+                    'TN': [tn],
+                    'FP': [fp],
+                    'FN': [fn],
+                    "SUM":[tp+tn+fp+fn],
+                    'Precision': [precision],
+                    'Accuracy': [accuracy],
+                    'Prevalence': [prevalence],
+                    'Sensitivity-Recall': [sensitivity],
+                    'Specificity': [specificity],
+                    'F1 Score': [f1_score],
+                    'Informedness': [informedness],
+                    'Markedness': [markedness],
+                    'Diagnostic Odds Ratio': [positive_likelihood_ratio],
+                    'Phi Coefficient': [phi_coefficient],
+                    'Fowlkes-Mallows Index': [fowlkes_mallows_index],
+                    'Negative Predictive Value': [npv],
+                    'Miss Rate': [miss_rate],
+                    'Fall-out': [fall_out],
+                    'False Discovery Rate': [false_discovery_rate],
+                    'False Omission Rate': [false_omission_rate],
+                    'Positive Likelihood Ratio': [positive_likelihood_ratio],
+                    'Negative Likelihood Ratio': [negative_likelihood_ratio],
+                    'Prevalence Threshold': [prevalence_threshold],
+                    'Threat Score': [threat_score]
+                })], ignore_index=True)
+
+# Print the count of exceptions
+print(C)
+
+outpath = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input"
+# Save the result DataFrame to a CSV file
+#result_df.to_csv(os.path.join(outpath, 'Confusion_matrix_metrics.csv'), index=False)

outputs/code_for_figures/ConfusionMatrix_getter.py

@@ -0,0 +1,187 @@
+
+import glob
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+
+def calculate_confusion_matrix(path, Sequence, Voting_threshold):
+    # Step 1: Accept an initial path as input and look for "voting.csv" files
+    Errors = []
+    initial_path = path  # Replace with the actual path
+    voting_csv_files = []
+    for root, dirs, files in os.walk(initial_path):
+        for file in files:
+            if file.endswith("votings.csv"):
+                voting_csv_files.append(os.path.join(root, file))
+    
+    # Step 2: Store all "voting.csv" files into a single Pandas DataFrame
+    dataframes = []
+    for file in voting_csv_files:
+        df = pd.read_csv(file)
+        dataframes.append(df)
+    aidaqc_voting = pd.concat(dataframes, ignore_index=True)
+    
+    # Step 3: Filter rows based on the "majority vote" column
+    MM = aidaqc_voting["Voting outliers (from 5)"].max()
+    threshold_qc_adaptive = MM -1
+    
+    threshold = Voting_threshold
+    aidaqc_voting_thresolded = aidaqc_voting[aidaqc_voting["Voting outliers (from 5)"] >= threshold]
+    S = Sequence
+    # Step 4: List all PNG images in the "manual_slice_inspection" folder
+    manual_slice_inspection_path = os.path.join(initial_path, "manual_slice_inspection")
+    manual_slice_images = [file for file in os.listdir(manual_slice_inspection_path) if file.endswith(".png")]
+    
+    # Create a DataFrame for "manual_slice_inspection"
+    data = {"manual_slice_inspection": manual_slice_images}
+    manual_voting = pd.DataFrame(data)
+    
+    # Step 5: Create columns for "validation_" folders
+    validation_folders = [os.path.join(initial_path, folder) for folder in os.listdir(initial_path) if folder.startswith("validation_") and "Markus" not in folder]
+    
+    for folder in validation_folders:
+        folder_name = os.path.basename(folder)
+        folder_images = [file for file in os.listdir(folder) if file.endswith(".png")]
+        
+        # Create a column with 1 if the image is in the folder, 0 otherwise
+        manual_voting[folder_name] = manual_voting["manual_slice_inspection"].apply(lambda x: 1 if x in folder_images else 0)
+    
+    # Now you have "VoteTemp" and "manual_voting" dataframes as described in steps 3 and 5 with the corrected code.
+    manual_voting_afs = manual_voting[manual_voting['manual_slice_inspection'].str.startswith(S)] 
+    
+    # Create a new DataFrame containing only the relevant columns
+    columns_to_include = [col for col in manual_voting_afs.columns if col != "manual_slice_inspection" and col != "MajorityVoteMR"]
+    ratings_data = manual_voting_afs[columns_to_include]
+
+    try:
+        # Convert the ratings data to a format expected by fleiss_kappa
+        ratings_matrix,d = aggregate_raters(ratings_data.values, n_cat=None) 
+    
+        # Calculate Fleiss' Kappa
+        kappa = fleiss_kappa(ratings_matrix,method='fleiss')
+    except ValueError:
+        print("\n "+"value Error, no kappa could be calculated for:")
+        print(path + Sequence)
+        Errors.append(path + Sequence)
+        kappa = np.nan
+
+       
+    manual_voting['MajorityVoteMR'] = manual_voting.iloc[:, 1:].sum(axis=1)
+    manual_voting_thresholded = manual_voting[manual_voting["MajorityVoteMR"] >= threshold]
+    
+    #Separate the new_df DataFrame based on specific prefixes
+    manual_voting_thresholded_afs = manual_voting_thresholded[manual_voting_thresholded['manual_slice_inspection'].str.startswith(S)]
+    aidaqc_voting_thresolded_afs = aidaqc_voting_thresolded[aidaqc_voting_thresolded['corresponding_img'].str.startswith(S)] 
+    # Define a custom function to remove the '_number.png' suffix
+    
+    def remove_suffix(path):
+        parts = path.split('_')
+        return '_'.join(parts[:-1]) + '.png'
+    
+    manual_voting_thresholded_afs['manual_slice_inspection'] = manual_voting_thresholded_afs['manual_slice_inspection'].apply(remove_suffix)
+    aidaqc_voting_thresolded_afs['corresponding_img'] = aidaqc_voting_thresolded_afs['corresponding_img'].apply(remove_suffix)
+    
+    
+    
+    temp_afs_all = manual_voting[manual_voting['manual_slice_inspection'].str.startswith(S)]
+    temp_afs_all["manual_slice_inspection"] = temp_afs_all["manual_slice_inspection"].apply(remove_suffix)
+    count_afs_all = len(temp_afs_all)
+    countqc_afs_bad = len(aidaqc_voting_thresolded_afs)
+    countqc_afs_good = count_afs_all - countqc_afs_bad
+    countgt_afs_bad = len(manual_voting_thresholded_afs)
+    countgt_afs_good = len(temp_afs_all[temp_afs_all["MajorityVoteMR"] < threshold])
+    
+    
+    afs_intersect_qc_gt_TP = set(aidaqc_voting_thresolded_afs['corresponding_img']) & set(manual_voting_thresholded_afs['manual_slice_inspection'])
+    afs_intersect_qc_gt_FN = set(manual_voting_thresholded_afs['manual_slice_inspection']) & set(set(temp_afs_all['manual_slice_inspection'])-set(aidaqc_voting_thresolded_afs['corresponding_img']))
+    afs_intersect_qc_gt_FP = set(set(temp_afs_all['manual_slice_inspection'])-set(manual_voting_thresholded_afs['manual_slice_inspection'])) & set(aidaqc_voting_thresolded_afs['corresponding_img'])
+    afs_intersect_qc_gt_TN = set(set(temp_afs_all['manual_slice_inspection'])-set(manual_voting_thresholded_afs['manual_slice_inspection'])) &  set(set(temp_afs_all['manual_slice_inspection'])-set(aidaqc_voting_thresolded_afs['corresponding_img']))
+    
+    
+    
+    afs_TP = len(afs_intersect_qc_gt_TP)
+    afs_FN = len(afs_intersect_qc_gt_FN)
+    afs_FP = len(afs_intersect_qc_gt_FP)
+    afs_TN = len(afs_intersect_qc_gt_TN)
+    
+    if countgt_afs_bad == 0:
+        print("\n "+"countgt_afs_bad was zero for:")
+        print(path + Sequence)
+        afs_percent_TP = 0
+        Errors.append(path + Sequence)
+    else:
+        afs_percent_TP = (afs_TP / countgt_afs_bad)
+    
+    afs_percent_FN = (1 - afs_percent_TP)
+    
+    if countgt_afs_good == 0:
+        afs_percent_FP = 0
+        print("\n "+"countgt_afs_good was zero for:")
+        print(path + Sequence)
+        Errors.append(path + Sequence)
+    else:
+        afs_percent_FP = (afs_FP /countgt_afs_good)
+    
+    afs_percent_TN = (1 - afs_percent_FP)
+    
+    
+    if afs_TP + afs_FP == 0:
+        print("\n "+"afs_TP + afs_FP was zero for:")
+        print(path + Sequence)
+        precision = 0
+        Errors.append(path + Sequence)
+    else:
+        precision = afs_TP / (afs_TP + afs_FP)
+
+    # Calculate recall
+    if afs_TP + afs_FN == 0:
+        print("\n "+"afs_TP + afs_FN was zero for:")
+        print(path + Sequence)
+        recall = 0
+        Errors.append(path + Sequence)
+    else:
+        recall = afs_TP / (afs_TP + afs_FN)
+
+    # Calculate F1 score
+    if precision + recall == 0:
+        print("\n "+"precision + recall was zero for:")
+        print(path + Sequence)
+        f1_score = 0
+        Errors.append(path + Sequence)
+    else:
+        f1_score = 2 * (precision * recall) / (precision + recall)    
+    
+    if np.isnan(kappa) and f1_score == 0:
+        f1_score=np.nan
+        recall = np.nan
+    
+    confusion_matrix = [[afs_percent_TP, afs_percent_FN],
+                         [afs_percent_FP, afs_percent_TN]]
+    return confusion_matrix, f1_score, kappa, np.unique(Errors)
+    
+    
+
+    
+    
+    
+    
+
+
+
+
+
+
+
+
+
+
+

outputs/code_for_figures/Confusionplot.py

@@ -0,0 +1,212 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep  1 17:57:59 2023
+
+@author: arefks
+"""
+
+import glob
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+
+#%function
+
+def calculate_and_display_fleiss_kappa(data):
+    kappa, _ = fleiss_kappa(data)
+    print("Fleiss' Kappa:", kappa)
+    return kappa
+
+#%% Expert
+
+Path_votings = r"Z:\2023_Kalantari_AIDAqc\outputs\QC_Final\validation\*\votings.csv"
+
+All_csv_votings = glob.glob(Path_votings)
+
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        #data_dict[ff] = df[df["Voting outliers (from 5)"]] 
+        data_dict[ff] = df
+            
+    return data_dict
+
+All_values = read_csv_files(All_csv_votings)
+
+img_values = []
+
+for df_name, df in All_values.items():
+    if "corresponding_img" in df.columns:
+        img_values.extend(df["corresponding_img"].tolist())
+
+new_df = pd.DataFrame({"corresponding_img": img_values})
+
+MU_strings = ["adam","joanes"]
+
+Sequence_type = ["anatomical"]
+cc=0
+cm = 1/2.54  # centimeters in inches
+
+plt.figure(figsize=(18*cm, 9*cm))
+fig, ax = plt.subplots(1,2, dpi=300,figsize=(18*cm, 9*cm))#,sharex="row",sharey="row")
+#fig.suptitle('Confusion matrix',fontname='Times New Roman')
+        
+Matric_sequneces = []
+for ss,S in enumerate(Sequence_type):
+    for mu,MU in enumerate(MU_strings):
+        
+        Path = r"Z:\2023_Kalantari_AIDAqc\outputs\QC_Final\validation\*\votings.csv"
+        
+        p_afs = os.path.join(os.path.dirname(Path),S+"*.png")
+        
+        afs_all = glob.glob(p_afs,recursive=True)
+        afs_all = [os.path.basename(path) for path in afs_all]
+        
+        
+        count_afs_all = len(afs_all)
+        
+
+        Path_gt = r"Z:\2023_Kalantari_AIDAqc\outputs\QC_Final\validation\*\validation_" + MU
+        
+        pgt_afs = os.path.join(Path_gt, S+"*.png")
+        
+        afsgt_all = glob.glob(pgt_afs,recursive=True)
+        
+        afsgt_all = [os.path.basename(path) for path in afsgt_all]
+        
+        countgt_afs_bad = len(afsgt_all)
+        
+        countgt_afs_good = count_afs_all - countgt_afs_bad
+        
+        
+        
+        
+        
+        # Separate the new_df DataFrame based on specific prefixes
+        afsqc_all = new_df[new_df['corresponding_img'].str.startswith(S)]['corresponding_img'].tolist()
+        
+        countqc_afs_bad = len(afsqc_all)
+        countqc_afs_good = count_afs_all - countqc_afs_bad
+        
+        
+        afs_intersect_qc_gt = set(afsgt_all) & set(afsqc_all)
+# =============================================================================
+#         afs_TN = len(afs_intersect_qc_gt)
+#         afs_FN = countqc_afs_bad - afs_TN
+#         afs_FP = countgt_afs_bad - afs_TN
+#         afs_TP = countgt_afs_good - afs_FN
+# =============================================================================
+        afs_TP = len(afs_intersect_qc_gt)
+        afs_FN = countgt_afs_bad - afs_TP
+        afs_FP = countqc_afs_bad - afs_TP
+        afs_TN = countgt_afs_good - afs_FP
+        
+# =============================================================================
+        afs_percent_TP = (afs_TP / countgt_afs_bad)
+        afs_percent_FN = (1 - afs_percent_TP)
+        afs_percent_FP = (afs_FP /countgt_afs_good)
+        afs_percent_TN = (1 - afs_percent_FP)
+# =============================================================================
+        
+        
+        # Calculate precision
+        precision = afs_TP / (afs_TP + afs_FP)
+        #print("precision"+str(precision))
+        # Calculate recall
+        recall = afs_TP / (afs_TP + afs_FN)
+        #print("recall:"+str(recall))
+        
+        # Calculate F1 score
+        f1_score = 2 * (precision * recall) / (precision + recall)
+        
+        # Print the F1 score
+        #print("F1 Score:", f1_score)
+
+        
+        confusion_matrix = [[afs_percent_TP, afs_percent_FN],
+                             [afs_percent_FP, afs_percent_TN]]
+# =============================================================================
+#         Per = afs_TP + afs_FN+afs_FP +afs_TN
+#         confusion_matrix = [[afs_TP/Per, afs_FN/Per],
+#                     [afs_FP/Per, afs_TN/Per]]
+# 
+# =============================================================================
+        
+        
+        # Create a heatmap using Seaborn
+        sns.set(font_scale=0.8)  # Adjust the font size
+        heatmap = sns.heatmap(confusion_matrix, annot=True, fmt='.2%', cmap='Greys',
+                              annot_kws={"fontname": "Times New Roman"},
+                              xticklabels=False, yticklabels=False, cbar=False,ax=ax[mu,ss])
+        ax[mu, ss].set_xlabel('AIDAqc', fontname='Times New Roman')
+        ax[mu, ss].set_ylabel("Test", fontname='Times New Roman')
+        ax[mu, ss].set_title(S.capitalize()+'\n F1-score: %.2f' % f1_score + "", fontname='Times New Roman', weight="bold")
+        ax[mu, ss].set_xticks([0.5, 1.5])
+        ax[mu, ss].set_xticklabels(['bad', 'good'], fontname='Times New Roman')
+        ax[mu, ss].set_yticks([0.5, 1.5])
+        ax[mu, ss].set_yticklabels(['bad', 'good'], fontname='Times New Roman',rotation=90)
+        
+        
+        
+        Vec = [0 if item in afsgt_all else 1 for item in afs_all]
+        Matric_sequneces.append(Vec)
+        
+
+    Vec_qc = [0 if item in afsqc_all else 1 for item in afs_all]   
+    Matric_sequneces.append(Vec_qc)   
+        
+        
+        
+        
+        
+        
+# Show the plot
+#plt.subplots_adjust(left=0.00, right=1, top=1, bottom=0.00, wspace=0.3, hspace=1.8)
+plt.tight_layout()
+plt.show()
+
+
+
+
+Stacked_anat = np.stack(Matric_sequneces[0:4],axis=0).transpose()
+Stacked_func = np.stack(Matric_sequneces[4:8],axis=0).transpose()
+Stacked_struct = np.stack(Matric_sequneces[8:12],axis=0).transpose()
+
+Stacked_anat_bool = Stacked_anat.astype(bool)
+Stacked_func_bool = Stacked_func.astype(bool)
+Stacked_struct_bool = Stacked_struct.astype(bool)
+
+# Define column labels
+column_labels = ["experienced", "expert", "aidaqc"]
+
+# Create pandas DataFrames
+df_anat = pd.DataFrame(Stacked_anat_bool, columns=column_labels)
+df_func = pd.DataFrame(Stacked_func_bool, columns=column_labels)
+df_struct = pd.DataFrame(Stacked_struct_bool, columns=column_labels)
+
+from sklearn.metrics import cohen_kappa_score
+
+# Create a list of DataFrames
+dataframes = [df_anat, df_func, df_struct]
+
+# Loop through the DataFrames
+for i, df in enumerate(dataframes):
+    print(f"DataFrame {i + 1}:")
+    for col1 in df.columns:
+        for col2 in df.columns:
+            if col1 != col2:
+                kappa_score = cohen_kappa_score(df[col1], df[col2])
+                print(f"Cohen's Kappa Score between {col1} and {col2}: {kappa_score:.4f}")
+    print()
+
+

outputs/code_for_figures/Confusionplot_new.py

@@ -0,0 +1,217 @@
+
+import glob
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+from ConfusionMatrix_getter import *
+
+#path = r"C:\Users\aswen\Desktop\Code\Validation\*/"
+path = r"C:\Users\aswen\Desktop\Code\Validation_Chang\*"
+List_folders = [item for item in glob.glob(path) if os.path.isdir(item)]
+Result = []
+
+Sequence = ["anatomical","structural","functional"]
+Voting_threshold = [1,2,3,4,5]
+ErrorAll = []
+for v in Voting_threshold:
+    for L in List_folders:
+        for S in Sequence:
+            
+            confusion_matrix,f1_score,kappa,Errors = calculate_confusion_matrix(L, S, v)
+            Result.append({"Dataset":os.path.basename(os.path.abspath(L)),"Thresold":v,"Sequence":S,"Kappa":kappa,"F1_score":f1_score})
+            if any(Errors):
+                ErrorAll.append(Errors[0])
+            # Create a heatmap using Seaborn
+            cm = 1/2.54  # centimeters in inches
+            
+            plt.figure(figsize=(18*cm, 9*cm))
+            fig, ax = plt.subplots(1,1, dpi=300,figsize=(18*cm, 9*cm))#,sharex="row",sharey="row")
+            #fig.suptitle('Confusion matrix',fontname='Times New Roman')
+                    
+            
+            sns.set(font_scale=0.8)  # Adjust the font size
+            heatmap = sns.heatmap(confusion_matrix, annot=True, fmt='.2%', cmap='Greys',
+                                  annot_kws={"fontname": "Times New Roman"},
+                                  xticklabels=False, yticklabels=False, cbar=False)
+            ax.set_xlabel('AIDAqc', fontname='Times New Roman')
+            ax.set_ylabel("Manual Rater", fontname='Times New Roman')
+            ax.set_title(S.capitalize()+" & "+ os.path.basename(os.path.abspath(L))  +'\n F1-score: %.2f' % f1_score + '  |  Kappa: %.2f' % kappa , fontname='Times New Roman', weight="bold")
+            ax.set_xticks([0.5, 1.5])
+            ax.set_xticklabels(['bad', 'good'], fontname='Times New Roman')
+            ax.set_yticks([0.5, 1.5])
+            ax.set_yticklabels(['bad', 'good'], fontname='Times New Roman')
+            plt.show()
+ResultD = pd.DataFrame(Result)  
+ResultD.to_csv(r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input\Kappa_and_F1Score_resultsTemp.csv")
+#%% All together
+
+import glob
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters, aggregate_raters
+#from ConfusionMatrix_getter import *
+
+path =  r"C:\Users\aswen\Desktop\Code\Validation\94_r_We"
+
+Sequence = "anatomical"
+Voting_threshold = 1
+# Step 1: Accept an initial path as input and look for "voting.csv" files
+initial_path = path  # Replace with the actual path
+voting_csv_files = []
+for root, dirs, files in os.walk(initial_path):
+    for file in files:
+        if file.endswith("votings.csv"):
+            voting_csv_files.append(os.path.join(root, file))
+
+# Step 2: Store all "voting.csv" files into a single Pandas DataFrame
+dataframes = []
+for file in voting_csv_files:
+    df = pd.read_csv(file)
+    dataframes.append(df)
+aidaqc_voting = pd.concat(dataframes, ignore_index=True)
+
+# Step 3: Filter rows based on the "majority vote" column
+threshold = Voting_threshold
+aidaqc_voting_thresolded = aidaqc_voting[aidaqc_voting["Voting outliers (from 5)"] >= threshold]
+
+# Step 4: List all PNG images in the "manual_slice_inspection" folder
+manual_slice_inspection_path = os.path.join(initial_path, "manual_slice_inspection")
+manual_slice_images = [file for file in os.listdir(manual_slice_inspection_path) if file.endswith(".png")]
+
+# Create a DataFrame for "manual_slice_inspection"
+data = {"manual_slice_inspection": manual_slice_images}
+manual_voting = pd.DataFrame(data)
+S = Sequence
+# Step 5: Create columns for "validation_" folders
+validation_folders = [os.path.join(initial_path, folder) for folder in os.listdir(initial_path) if folder.startswith("validation_") and "Markus" not in folder]
+
+for folder in validation_folders:
+    folder_name = os.path.basename(folder)
+    folder_images = [file for file in os.listdir(folder) if file.endswith(".png")]
+    
+    # Create a column with 1 if the image is in the folder, 0 otherwise
+    manual_voting[folder_name] = manual_voting["manual_slice_inspection"].apply(lambda x: 1 if x in folder_images else 0)
+
+# Now you have "VoteTemp" and "manual_voting" dataframes as described in steps 3 and 5 with the corrected code.
+manual_voting['MajorityVoteMR'] = manual_voting.iloc[:, 1:].sum(axis=1)
+manual_voting_afs = manual_voting[manual_voting['manual_slice_inspection'].str.startswith(S)]
+
+
+
+# Create a new DataFrame containing only the relevant columns
+columns_to_include = [col for col in manual_voting_afs.columns if col != "manual_slice_inspection" and col != "MajorityVoteMR"]
+ratings_data = manual_voting_afs[columns_to_include]
+
+# Convert the ratings data to a format expected by fleiss_kappa
+ratings_matrix,d = aggregate_raters(ratings_data.values, n_cat=None) 
+
+# Calculate Fleiss' Kappa
+kappa = fleiss_kappa(ratings_matrix,method='fleiss')
+
+
+
+manual_voting_thresholded = manual_voting[manual_voting["MajorityVoteMR"] >= threshold]
+
+#Separate the new_df DataFrame based on specific prefixes
+manual_voting_thresholded_afs = manual_voting_thresholded[manual_voting_thresholded['manual_slice_inspection'].str.startswith(S)]
+aidaqc_voting_thresolded_afs = aidaqc_voting_thresolded[aidaqc_voting_thresolded['corresponding_img'].str.startswith(S)] 
+
+
+temp_afs_all = manual_voting[manual_voting['manual_slice_inspection'].str.startswith(S)]
+count_afs_all = len(temp_afs_all)
+countqc_afs_bad = len(aidaqc_voting_thresolded_afs)
+countqc_afs_good = count_afs_all - countqc_afs_bad
+countgt_afs_bad = len(manual_voting_thresholded_afs)
+countgt_afs_good = count_afs_all - countgt_afs_bad
+
+
+afs_intersect_qc_gt_TP = set(aidaqc_voting_thresolded_afs['corresponding_img']) & set(manual_voting_thresholded_afs['manual_slice_inspection'])
+afs_intersect_qc_gt_FN = set(manual_voting_thresholded_afs['manual_slice_inspection']) & set(set(temp_afs_all['manual_slice_inspection'])-set(aidaqc_voting_thresolded_afs['corresponding_img']))
+afs_intersect_qc_gt_FP = set(set(temp_afs_all['manual_slice_inspection'])-set(manual_voting_thresholded_afs['manual_slice_inspection'])) & set(aidaqc_voting_thresolded_afs['corresponding_img'])
+afs_intersect_qc_gt_TN = set(set(temp_afs_all['manual_slice_inspection'])-set(manual_voting_thresholded_afs['manual_slice_inspection'])) &  set(set(temp_afs_all['manual_slice_inspection'])-set(aidaqc_voting_thresolded_afs['corresponding_img']))
+
+
+
+afs_TP = len(afs_intersect_qc_gt_TP)
+afs_FN = len(afs_intersect_qc_gt_FN)
+afs_FP = len(afs_intersect_qc_gt_FP)
+afs_TN = len(afs_intersect_qc_gt_TN)
+
+
+afs_percent_TP = (afs_TP / countgt_afs_bad)
+afs_percent_FN = (1 - afs_percent_TP)
+afs_percent_FP = (afs_FP /countgt_afs_good)
+afs_percent_TN = (1 - afs_percent_FP)
+
+
+# Calculate precision
+precision = afs_TP / (afs_TP + afs_FP)
+#print("precision"+str(precision))
+# Calculate recall
+recall = afs_TP / (afs_TP + afs_FN)
+#print("recall:"+str(recall))
+
+# Calculate F1 score
+f1_score = 2 * (precision * recall) / (precision + recall)
+
+# Print the F1 score
+#print("F1 Score:", f1_score)
+
+
+confusion_matrix = [[afs_percent_TP, afs_percent_FN],
+                     [afs_percent_FP, afs_percent_TN]]
+confusion_matrix, f1_score
+
+    
+# Create a heatmap using Seaborn
+cm = 1/2.54  # centimeters in inches
+
+plt.figure(figsize=(18*cm, 9*cm))
+fig, ax = plt.subplots(1,1, dpi=300,figsize=(18*cm, 9*cm))#,sharex="row",sharey="row")
+#fig.suptitle('Confusion matrix',fontname='Times New Roman')
+        
+
+sns.set(font_scale=0.8)  # Adjust the font size
+heatmap = sns.heatmap(confusion_matrix, annot=True, fmt='.2%', cmap='Greys',
+                      annot_kws={"fontname": "Times New Roman"},
+                      xticklabels=False, yticklabels=False, cbar=False)
+ax.set_xlabel('AIDAqc', fontname='Times New Roman')
+ax.set_ylabel("Manual Rater", fontname='Times New Roman')
+ax.set_title(Sequence.capitalize()+'\n F1-score: %.2f' % f1_score + "", fontname='Times New Roman', weight="bold")
+ax.set_xticks([0.5, 1.5])
+ax.set_xticklabels(['bad', 'good'], fontname='Times New Roman')
+ax.set_yticks([0.5, 1.5])
+ax.set_yticklabels(['bad', 'good'], fontname='Times New Roman')
+
+
+    
+    
+
+
+
+
+
+
+
+
+
+
+

outputs/code_for_figures/CorroloationDTISNR_SNR.py

@@ -0,0 +1,109 @@
+import pandas as pd
+import os
+
+# Step 1: Read three CSV files
+
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+func_df = pd.read_csv(os.path.join(file_path, 'combined_data_func.csv'))
+struct_df = pd.read_csv(os.path.join(file_path, 'combined_data_struct.csv'))
+anat_df = pd.read_csv(os.path.join(file_path, 'combined_data_anat.csv'))
+
+# Step 4: Sort all dataframes based on the FileAddress column
+func_df.sort_values(by='FileAddress', inplace=True)
+struct_df.sort_values(by='FileAddress', inplace=True)
+anat_df.sort_values(by='FileAddress', inplace=True)
+
+# Step 5: Process the FileAddress column
+def process_file_address(file_address):
+    elements = file_address.split('\\')  # Use '\\' to split on backslash
+    return '\\'.join(elements[:-1])  # Use '\\' to join elements with backslash
+
+struct_df['FileAddress'] = struct_df['FileAddress'].apply(process_file_address)
+anat_df['FileAddress'] = anat_df['FileAddress'].apply(process_file_address)
+
+# Step 6: Create a new dataframe
+common_file_addresses = set(anat_df['FileAddress']).intersection(set(struct_df['FileAddress']))
+result_data = []
+
+for file_address in common_file_addresses:
+    anat_rows = anat_df[anat_df['FileAddress'] == file_address]
+    struct_rows = struct_df[struct_df['FileAddress'] == file_address]
+    
+    # Calculate the average of 'SNR Chang' and 'tSNR (Averaged Brain ROI)' values, ignoring NaNs
+    avg_snr_chang = anat_rows['SNR Normal'].mean()
+    avg_tsnr_avg_brain_roi = struct_rows['SNR Normal'].mean()
+    
+    result_data.append({
+        'Common FileAddress': file_address,
+        'Average SNR Chang': avg_snr_chang,
+        'Average tSNR (Averaged Brain ROI)': avg_tsnr_avg_brain_roi
+    })
+
+# Create the result DataFrame
+result_df = pd.DataFrame(result_data)
+
+# Print the result
+print(result_df)
+import pandas as pd
+import os
+import seaborn as sns
+import matplotlib.pyplot as plt
+from scipy.stats import pearsonr
+from scipy.stats import spearmanr
+
+
+# Calculate the Spearman correlation coefficient and p-value
+corr, p_value = spearmanr(result_df['Average SNR Chang'], result_df['Average tSNR (Averaged Brain ROI)'])
+
+# Define the centimeters to inches conversion
+cm = 1/2.54  # centimeters in inches
+sns.set_style("ticks")
+# Set the font style to Times New Roman
+plt.rcParams['font.family'] = 'Times New Roman'
+
+# Create the plot with a specified figure size
+h = 5 * cm
+width = 9.5 * cm
+aspect = width / h
+fig = sns.lmplot(x='Average SNR Chang', y='Average tSNR (Averaged Brain ROI)',
+                data=result_df, palette='Set1', ci=100,
+                scatter_kws={'s': 8, 'color': '#4C72B0', 'edgecolor': 'w', 'linewidths': .3},
+                line_kws={'lw': 2, 'color': '#4682b4'}, aspect=aspect, height=h)
+
+plt.rcParams['figure.dpi'] = 300
+
+plt.xlabel('Anatomical SNR-standard (dB)', fontsize=8)
+plt.ylabel('Diffusion SNR-standard (dB)', fontsize=8)
+
+ax = plt.gca()
+# Set font size for tick labels
+ax.tick_params(axis='both', which='both', labelsize=8)
+# Explicitly set the visibility of top and right spines
+ax.spines['top'].set_visible(True)
+ax.spines['right'].set_visible(True)
+# Customize the border linewidth
+ax.spines['top'].set_linewidth(0.5)     # Top border
+ax.spines['right'].set_linewidth(0.5)   # Right border
+ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax.spines['left'].set_linewidth(0.5)   # Left border
+ax.locator_params(axis='y', nbins=8)
+ax.locator_params(axis='x', nbins=16)
+
+# Add horizontal lines from yticks
+
+ax.tick_params(axis='both', which='both', width=0.5, color='gray', length=2)
+#ax.set_title("(b) SNR structural vs anatomical", weight='bold', fontsize=10)
+
+# Save the figure as PNG and SVG
+fig_path_png = os.path.join(out_path, 'AnatVSDiffAllDataCorr.png')
+fig_path_svg = os.path.join(out_path, 'AnatVSDiffAllDataCorr.svg')
+
+fig.savefig(fig_path_png, format='png', bbox_inches='tight')
+fig.savefig(fig_path_svg, format='svg', bbox_inches='tight')
+
+plt.show()

outputs/code_for_figures/CorroloationTSNR_SNR.py

@@ -0,0 +1,119 @@
+import pandas as pd
+import os
+
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+func_df = pd.read_csv(os.path.join(file_path, 'combined_data_func.csv'))
+struct_df = pd.read_csv(os.path.join(file_path, 'combined_data_struct.csv'))
+anat_df = pd.read_csv(os.path.join(file_path, 'combined_data_anat.csv'))
+
+
+# Step 4: Sort all dataframes based on the FileAddress column
+func_df.sort_values(by='FileAddress', inplace=True)
+struct_df.sort_values(by='FileAddress', inplace=True)
+anat_df.sort_values(by='FileAddress', inplace=True)
+
+# Step 5: Process the FileAddress column
+def process_file_address(file_address):
+    elements = file_address.split('\\')  # Use '\\' to split on backslash
+    return '\\'.join(elements[:-1])  # Use '\\' to join elements with backslash
+
+func_df['FileAddress'] = func_df['FileAddress'].apply(process_file_address)
+anat_df['FileAddress'] = anat_df['FileAddress'].apply(process_file_address)
+
+# Step 6: Create a new dataframe
+common_file_addresses = set(anat_df['FileAddress']).intersection(set(func_df['FileAddress']))
+result_data = []
+
+for file_address in common_file_addresses:
+    anat_rows = anat_df[anat_df['FileAddress'] == file_address]
+    func_rows = func_df[func_df['FileAddress'] == file_address]
+    
+    # Calculate the average of 'SNR Chang' and 'tSNR (Averaged Brain ROI)' values, ignoring NaNs
+    avg_snr_chang = anat_rows['SNR Normal'].mean() #Here you have to set what kind of SNR you want to plot
+    avg_tsnr_avg_brain_roi = func_rows['tSNR (Averaged Brain ROI)'].mean()
+    
+    result_data.append({
+        'Common FileAddress': file_address,
+        'Average SNR Chang': avg_snr_chang,
+        'Average tSNR (Averaged Brain ROI)': avg_tsnr_avg_brain_roi
+    })
+
+# Create the result DataFrame
+result_df = pd.DataFrame(result_data)
+
+# Print the result
+print(result_df)
+
+
+import pandas as pd
+import os
+import seaborn as sns
+import matplotlib.pyplot as plt
+from scipy.stats import pearsonr
+from scipy.stats import spearmanr
+
+# Calculate the correlation coefficient and p-value
+corr, p_value = pearsonr(result_df['Average SNR Chang'], result_df['Average tSNR (Averaged Brain ROI)'])
+# corr, p_value = spearmanr(result_df['Average SNR Chang'], result_df['Average tSNR (Averaged Brain ROI)'])
+sns.set_style('ticks')
+# Set the font style to Times New Roman
+plt.rcParams['font.family'] = 'Times New Roman'
+
+# Define the centimeters to inches conversion
+cm = 1/2.54  # centimeters in inches
+
+# Create the plot
+
+#plt.figure(figsize=(4.5*cm, 5*cm), dpi=100)
+
+# Create the plot with a specified figure size
+h = 5 * cm
+width = 9.5 * cm
+aspect = width / h
+fig =sns.lmplot(x='Average SNR Chang', y='Average tSNR (Averaged Brain ROI)',
+                data=result_df, palette='Set2',height=h,aspect = aspect,ci=100,scatter_kws={'s': 8,'color': '#4C72B0','edgecolor':'w','linewidths':.3},line_kws={'lw': 2, 'color': '#4682b4'})
+plt.rcParams['figure.dpi'] = 300
+
+plt.xlabel('Anatomical SNR-standard (dB)', fontsize=8)
+plt.ylabel('Functional tSNR (dB)', fontsize=8)
+
+# Set font size for tick labels
+plt.xticks(fontsize=8)
+plt.yticks(fontsize=8)
+
+# Add correlation coefficient and p-value to the plot
+# plt.text(0.1, 0.9, f'r: {corr:.2f}', transform=plt.gca().transAxes, fontsize=8)
+# plt.text(0.1, 0.85, f'p-value: {p_value:}', transform=plt.gca().transAxes, fontsize=8)
+ax = plt.gca()
+# Remove right and upper bordersax = plt.gca()
+# Set font size for tick labels
+ax.tick_params(axis='both', which='both', labelsize=8)
+# Explicitly set the visibility of top and right spines
+ax.spines['top'].set_visible(True)
+ax.spines['right'].set_visible(True)
+# Customize the border linewidth
+ax.spines['top'].set_linewidth(0.5)     # Top border
+ax.spines['right'].set_linewidth(0.5)   # Right border
+ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax.spines['left'].set_linewidth(0.5)   # Left border
+ax.locator_params(axis='y', nbins=8)
+ax.locator_params(axis='x', nbins=16)
+
+# Add horizontal lines from yticks
+
+ax.tick_params(axis='both', which='both', width=0.5, color='gray', length=2)
+#ax.set_title("(b) SNR structural vs anatomical", weight='bold', fontsize=10)
+
+# Save the figure as PNG and SVG
+fig_path_png = os.path.join(out_path, 'AnatVSFuncAllDataCorr.png')
+fig_path_svg = os.path.join(out_path, 'AnatVSFuncAllDataCorr.svg')
+
+#fig.savefig(fig_path_png, format='png', bbox_inches='tight')
+#fig.savefig(fig_path_svg, format='svg', bbox_inches='tight')
+
+plt.show()

outputs/code_for_figures/CreatBadFolderqc.py

@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Sep  1 17:57:59 2023
+
+@author: arefks
+"""
+
+import glob
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+from shutil import copyfile
+#%function
+
+def calculate_and_display_fleiss_kappa(data):
+    kappa, _ = fleiss_kappa(data)
+    print("Fleiss' Kappa:", kappa)
+    return kappa
+
+#%% Expert
+
+Path_votings = r"Z:\2023_Kalantari_AIDAqc\outputs\QC_Final\test\*\votings.csv"
+
+All_csv_votings = glob.glob(Path_votings)
+
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        data_dict[ff] = df[df["Voting outliers (from 5)"]>0] 
+            
+    return data_dict
+
+All_values = read_csv_files(All_csv_votings)
+
+img_values = []
+
+for df_name, df in All_values.items():
+    if "corresponding_img" in df.columns:
+        img_values.extend(df["corresponding_img"].tolist())
+
+new_df = pd.DataFrame({"corresponding_img": img_values})
+
+
+PP =  r"Z:\2023_Kalantari_AIDAqc\outputs\QC_Final\test"
+    
+for tt in new_df["corresponding_img"]: 
+    
+    String = os.path.join(PP,"**",tt)
+    PathIm = glob.glob(String,recursive=True)
+    temp = os.path.dirname(PathIm[0])
+    Path_study = os.path.dirname(temp)
+    Path_new_folder = os.path.join(Path_study,"votedImages")
+    PathIm_new = os.path.join(Path_new_folder,tt)
+    if not os.path.exists(Path_new_folder):
+        os.mkdir(Path_new_folder)
+        
+    copyfile(PathIm[0],PathIm_new)
+    print(String)
+
+
+
+
+

outputs/code_for_figures/FuncAllDataMerge.py

@@ -0,0 +1,51 @@
+import os
+import glob
+import pandas as pd
+
+# Step 1: Define the starting path and file pattern
+start_path =  r"C:\Users\aswen\Desktop\Code\Validation3"
+file_pattern = '*func*.csv'
+
+# Step 2: Find all matching CSV files in the specified directory and its subdirectories
+csv_files = glob.glob(os.path.join(start_path, '**', file_pattern), recursive=True)
+
+# Step 3: Initialize an empty DataFrame to store the extracted data
+combined_df = pd.DataFrame()
+
+# Step 4: Loop through the CSV files and extract the specified columns
+for csv_file in csv_files:
+    try:
+        df = pd.read_csv(csv_file)
+        selected_columns = ["FileAddress", "tSNR (Averaged Brain ROI)", "Displacement factor (std of Mutual information)", "Goasting"]
+        df = df[selected_columns]
+        df["dataset"] = csv_file.split(os.sep)[-3]
+        # Concatenate the current DataFrame with the combined DataFrame
+        combined_df = pd.concat([combined_df, df], ignore_index=True)
+    except Exception as e:
+        print(f"Error reading {csv_file}: {e}")
+
+# Step 5: Calculate the mean, standard deviation, minimum, and maximum for tSNR and Displacement factor
+mean_tsnr = combined_df["tSNR (Averaged Brain ROI)"].mean()
+std_tsnr = combined_df["tSNR (Averaged Brain ROI)"].std()
+min_tsnr = combined_df["tSNR (Averaged Brain ROI)"].min()
+max_tsnr = combined_df["tSNR (Averaged Brain ROI)"].max()
+
+mean_displacement_factor = combined_df["Displacement factor (std of Mutual information)"].mean()
+std_displacement_factor = combined_df["Displacement factor (std of Mutual information)"].std()
+min_displacement_factor = combined_df["Displacement factor (std of Mutual information)"].min()
+max_displacement_factor = combined_df["Displacement factor (std of Mutual information)"].max()
+
+# Print the results
+print(f"Mean tSNR: {mean_tsnr}")
+print(f"Standard Deviation tSNR: {std_tsnr}")
+print(f"Minimum tSNR: {min_tsnr}")
+print(f"Maximum tSNR: {max_tsnr}")
+
+print(f"Mean Displacement Factor: {mean_displacement_factor}")
+print(f"Standard Deviation Displacement Factor: {std_displacement_factor}")
+print(f"Minimum Displacement Factor: {min_displacement_factor}")
+print(f"Maximum Displacement Factor: {max_displacement_factor}")
+
+# Optionally, you can save the combined DataFrame to a CSV file
+p = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input"
+combined_df.to_csv(os.path.join(p,'combined_data_func.csv'), index=False)

outputs/code_for_figures/GhostingPlots.py

@@ -0,0 +1,85 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os
+
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input', 'Ghostplot.csv')
+out_path = os.path.join(script_dir,'..','figures')
+df = pd.read_csv(file_path, header=None, names=['Shift (Voxel)', 'Severe motion', 'No motion'], skiprows=1)
+
+# Set the seaborn style and palette
+sns.set(style='ticks', palette='Set1')
+
+# Set font properties
+font_properties = {'family': 'Times New Roman', 'size': 8}
+font_properties2 = {'family': 'Times New Roman', 'size': 8}
+
+cm = 1/2.54  # centimeters in inches
+
+# Create the first plot (No ghost)
+fig1, ax1 = plt.subplots(figsize=(5.2*cm, 4.91*cm), dpi=300)
+ax1.plot(df['Shift (Voxel)'], df['No motion'], label='No ghost', linewidth=1, color='red')
+ax1.set_xlabel('Shift (Voxel)', **font_properties)
+ax1.set_ylabel('Mutual Information (a.u)', **font_properties)
+ax1.tick_params(axis='both', which='both', width=0.5, color='gray', length=2)
+ax1.locator_params(axis='x', nbins=8)  # Set the number of ticks for the x-axis
+ax1.locator_params(axis='y', nbins=8)  # Set the number of ticks for the y-axis
+
+for tick in ax1.get_xticklabels():
+    tick.set_fontname('Times New Roman')
+    tick.set_fontsize(8)
+for tick in ax1.get_yticklabels():
+    tick.set_fontname('Times New Roman')
+    tick.set_fontsize(8)
+
+ax1.spines['top'].set_linewidth(0.5)     # Top border
+ax1.spines['right'].set_linewidth(0.5)   # Right border
+ax1.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax1.spines['left'].set_linewidth(0.5)   # Left border
+ax1.xaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0)
+ax1.xaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0)
+ax1.yaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0)
+ax1.yaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0)        
+plt.tight_layout()
+# Create the second plot (Ghost)
+fig2, ax2 = plt.subplots(figsize=(5.2*cm, 4.91*cm), dpi=300)
+ax2.plot(df['Shift (Voxel)'], df['Severe motion'], label='Ghost', linewidth=1, color='blue')
+ax2.set_xlabel('Shift (Voxel)', **font_properties)
+ax2.set_ylabel('Mutual Information (a.u)', **font_properties)
+ax2.tick_params(axis='both', which='both', width=0.5, color='gray', length=2)
+ax2.locator_params(axis='x', nbins=8)  # Set the number of ticks for the x-axis
+ax2.locator_params(axis='y', nbins=8)  # Set the number of ticks for the y-axis
+
+for tick in ax2.get_xticklabels():
+    tick.set_fontname('Times New Roman')
+    tick.set_fontsize(8)
+for tick in ax2.get_yticklabels():
+    tick.set_fontname('Times New Roman')
+    tick.set_fontsize(8)
+
+ax2.spines['top'].set_linewidth(0.5)     # Top border
+ax2.spines['right'].set_linewidth(0.5)   # Right border
+ax2.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax2.spines['left'].set_linewidth(0.5)   # Left border
+
+ax2.xaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0)
+ax2.xaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0)
+ax2.yaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0)
+ax2.yaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0)
+plt.tight_layout()
+# Save figures as SVG and PNG with 300 dpi
+fig1_path = os.path.join(out_path, 'GhostingPlotHighMotion.svg')
+fig2_path = os.path.join(out_path, 'GhostingPlotLowMotion.svg')
+fig3_path = os.path.join(out_path, 'GhostingPlotHighMotion.png')
+fig4_path = os.path.join(out_path, 'GhostingPlotLowMotion.png')
+
+fig1.savefig(fig1_path, format='svg')
+fig2.savefig(fig2_path, format='svg')
+
+
+fig1.savefig(fig3_path, format='png')
+fig2.savefig(fig4_path, format='png')
+
+# Show the plots
+plt.show()

outputs/code_for_figures/HeatMaps_Accuracy.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Nov 23 02:42:50 2023
+
+@author: arefk
+"""
+
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import matplotlib.ticker as ticker
+import os
+# Load your Excel file into a Pandas DataFrame
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+csv_file = os.path.join(file_path,"Confusion_matrix_metrics.csv")
+
+df = pd.read_csv(csv_file)
+
+# Set the user-defined thresholds to get desired values
+Thresold_Human_Voters = 4  # Replace with the user-set threshold value
+Thresold_ML_Voters = 1
+
+# Filter the DataFrame based on the user-set threshold
+filtered_df = df[(df['Thresold_Human_Voters'] == Thresold_Human_Voters) &
+                 (df['Thresold_ML_Voters'] == Thresold_ML_Voters) &
+                (df['TP']+df['FN']  > 0)]
+                 
+
+filtered_df.loc[filtered_df["sequence_name"] == "anat","sequence_name"] = "Anatomical"
+filtered_df.loc[filtered_df["sequence_name"] == "diff","sequence_name"] = "Diffusion"
+filtered_df.loc[filtered_df["sequence_name"] == "func","sequence_name"] = "Functional"
+
+cm = 1/2.54  # centimeters in inches
+# Create two subplots for Kappa and F1 scores heatmaps
+fig, axes = plt.subplots(1, 1, figsize=(20*cm, 4*cm), dpi=300)
+#sns.set_style('whitegrid')
+# Specify the font properties
+font_properties = fm.FontProperties(family='Times New Roman', size=8)
+font_properties2 = fm.FontProperties(family='Times New Roman', size=10)
+Title = ["(a) Fleiss kappa score: inter rater reliability "," (b) F1_score: raters vs AIDAqc"]
+for i, metric in enumerate(['Accuracy']):
+    ax = axes
+    pivot_df = filtered_df.pivot(index='sequence_name', columns='dataset_name', values=metric)
+    pivot_df['mean'] = pivot_df.mean(axis=1)
+    pivot_df['std'] = pivot_df.std(axis=1)
+    t=Title[i]
+    sns.heatmap(pivot_df, annot=True, fmt=".1f", cmap="YlGnBu", cbar=True, ax=ax)
+
+    # Set font properties for labels, titles, and annotations
+    ax.set_xticklabels(ax.get_xticklabels(), fontproperties=font_properties)
+    ax.set_yticklabels(ax.get_yticklabels(), fontproperties=font_properties)
+    ax.set_xlabel('Datasets', fontproperties=font_properties)
+    ax.set_ylabel('Sequences', fontproperties=font_properties)
+    #ax.set_title(f'{t} ', fontsize=10, fontproperties=font_properties2, fontweight='bold')
+    ax.set(xlabel=None)
+    # Set the color bar legend font size and font properties
+    cbar = ax.collections[0].colorbar
+    cbar.ax.tick_params(labelsize=8)
+    cbar.ax.set_yticklabels(cbar.ax.get_yticklabels(), fontproperties=font_properties)
+    
+    # Customize the color bar ticks (increase the number of ticks)
+    cbar.ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=5))
+    
+    a = []
+    for text in ax.texts:
+        a.append(float(text.get_text()))
+        
+    
+
+    for text in ax.texts:
+        text.set_font("Times New Roman")
+        text.set_size(10)
+                
+# Show the plots
+plt.tight_layout()
+
+output_path = out_path
+output_filename = "HeatMap_Accurracy_ManualRater4_AIDAqc1"
+
+# Save as SVG
+fig.savefig(f"{output_path}/{output_filename}.svg", format="svg")
+
+# Save as PNG
+fig.savefig(f"{output_path}/{output_filename}.png", format="png")
+
+plt.show()
+#%%
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import matplotlib.ticker as ticker
+import os
+# Load your Excel file into a Pandas DataFrame
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+csv_file = os.path.join(file_path,"Confusion_matrix_metrics.csv")
+
+df = pd.read_csv(csv_file)
+
+# Set the user-defined thresholds to get desired values
+
+for Thresold_Human_Voters in df["Thresold_Human_Voters"].unique():
+    for Thresold_ML_Voters in df["Thresold_ML_Voters"].unique():
+        # Filter the DataFrame based on the user-set threshold
+        filtered_df = df[(df['Thresold_Human_Voters'] == Thresold_Human_Voters) &
+                         (df['Thresold_ML_Voters'] == Thresold_ML_Voters) &
+                        (df['TP']+df['FN']  > 0)]
+                         
+        
+        filtered_df.loc[filtered_df["sequence_name"] == "anat","sequence_name"] = "Anatomical"
+        filtered_df.loc[filtered_df["sequence_name"] == "diff","sequence_name"] = "Diffusion"
+        filtered_df.loc[filtered_df["sequence_name"] == "func","sequence_name"] = "Functional"
+        
+        cm = 1/2.54  # centimeters in inches
+        # Create two subplots for Kappa and F1 scores heatmaps
+        fig, axes = plt.subplots(1, 1, figsize=(20*cm, 5*cm), dpi=300)
+        sns.set_style('darkgrid')
+        # Specify the font properties
+        font_properties = fm.FontProperties(family='Times New Roman', size=8)
+        font_properties2 = fm.FontProperties(family='Times New Roman', size=10)
+        Title = ["(a) Fleiss kappa score: inter rater reliability "," (b) F1_score: raters vs AIDAqc"]
+        for i, metric in enumerate(['Accuracy']):
+            ax = axes
+            pivot_df = filtered_df.pivot(index='sequence_name', columns='dataset_name', values=metric)
+            pivot_df['mean'] = pivot_df.mean(axis=1)
+            pivot_df['std'] = pivot_df.std(axis=1)
+            t = "Human: " + str(Thresold_Human_Voters) + "|| AIDAqc:" +str(Thresold_ML_Voters) 
+            sns.heatmap(pivot_df, annot=True, fmt=".2f", cmap="YlGnBu", cbar=True, ax=ax)
+        
+            # Set font properties for labels, titles, and annotations
+            ax.set_xticklabels(ax.get_xticklabels(), fontproperties=font_properties)
+            ax.set_yticklabels(ax.get_yticklabels(), fontproperties=font_properties)
+            ax.set_xlabel('Datasets', fontproperties=font_properties)
+            ax.set_ylabel('Sequences', fontproperties=font_properties)
+            ax.set_title(f'{t} ', fontsize=10, fontproperties=font_properties2, fontweight='bold')
+            ax.set(xlabel=None)
+            # Set the color bar legend font size and font properties
+            cbar = ax.collections[0].colorbar
+            cbar.ax.tick_params(labelsize=8)
+            cbar.ax.set_yticklabels(cbar.ax.get_yticklabels(), fontproperties=font_properties)
+            
+            # Customize the color bar ticks (increase the number of ticks)
+            cbar.ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=5))
+            
+            a = []
+            for text in ax.texts:
+                a.append(float(text.get_text()))
+                
+            
+        
+            for text in ax.texts:
+                text.set_font("Times New Roman")
+                text.set_size(8)
+                
+                
+        # Show the plots
+        plt.tight_layout()
+        
+        plt.show()

outputs/code_for_figures/HeatMaps_Sensitivity.py

@@ -0,0 +1,163 @@
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import matplotlib.ticker as ticker
+import os
+# Load your Excel file into a Pandas DataFrame
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+csv_file = os.path.join(file_path,"Confusion_matrix_metrics.csv")
+
+df = pd.read_csv(csv_file)
+
+# Set the user-defined thresholds to get desired values
+Thresold_Human_Voters = 4  # Replace with the user-set threshold value
+Thresold_ML_Voters = 1
+
+# Filter the DataFrame based on the user-set threshold
+filtered_df = df[(df['Thresold_Human_Voters'] == Thresold_Human_Voters) &
+                 (df['Thresold_ML_Voters'] == Thresold_ML_Voters) &
+                (df['TP']+df['FN']  > 0)]
+                 
+
+filtered_df.loc[filtered_df["sequence_name"] == "anat","sequence_name"] = "Anatomical"
+filtered_df.loc[filtered_df["sequence_name"] == "diff","sequence_name"] = "Diffusion"
+filtered_df.loc[filtered_df["sequence_name"] == "func","sequence_name"] = "Functional"
+
+cm = 1/2.54  # centimeters in inches
+# Create two subplots for Kappa and F1 scores heatmaps
+fig, axes = plt.subplots(1, 1, figsize=(20*cm, 4*cm), dpi=300)
+#sns.set_style('whitegrid')
+# Specify the font properties
+font_properties = fm.FontProperties(family='Times New Roman', size=8)
+font_properties2 = fm.FontProperties(family='Times New Roman', size=10)
+Title = ["(a) Fleiss kappa score: inter rater reliability "," (b) F1_score: raters vs AIDAqc"]
+for i, metric in enumerate(['Sensitivity-Recall']):
+    ax = axes
+    pivot_df = filtered_df.pivot(index='sequence_name', columns='dataset_name', values=metric)
+    pivot_df['mean'] = pivot_df.mean(axis=1)
+    pivot_df['std'] = pivot_df.std(axis=1)
+    t=Title[i]
+    sns.heatmap(pivot_df, annot=True, fmt=".1f", cmap="YlGnBu", cbar=True, ax=ax)
+
+    # Set font properties for labels, titles, and annotations
+    ax.set_xticklabels(ax.get_xticklabels(), fontproperties=font_properties)
+    ax.set_yticklabels(ax.get_yticklabels(), fontproperties=font_properties)
+    ax.set_xlabel('Datasets', fontproperties=font_properties)
+    ax.set_ylabel('Sequences', fontproperties=font_properties)
+    #ax.set_title(f'{t} ', fontsize=10, fontproperties=font_properties2, fontweight='bold')
+    ax.set(xlabel=None)
+    # Set the color bar legend font size and font properties
+    cbar = ax.collections[0].colorbar
+    cbar.ax.tick_params(labelsize=8)
+    cbar.ax.set_yticklabels(cbar.ax.get_yticklabels(), fontproperties=font_properties)
+    
+    # Customize the color bar ticks (increase the number of ticks)
+    cbar.ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=5))
+    
+    a = []
+    for text in ax.texts:
+        a.append(float(text.get_text()))
+        
+    
+
+    for text in ax.texts:
+        text.set_font("Times New Roman")
+        text.set_size(8)
+                
+# Show the plots
+plt.tight_layout()
+
+output_path = out_path
+output_filename = "HeatMap_Recall_ManualRater4_AIDAqc1"
+
+# Save as SVG
+fig.savefig(f"{output_path}/{output_filename}.svg", format="svg")
+
+# Save as PNG
+fig.savefig(f"{output_path}/{output_filename}.png", format="png")
+
+plt.show()
+#%%
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import matplotlib.ticker as ticker
+import os
+# Load your Excel file into a Pandas DataFrame
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+csv_file = os.path.join(file_path,"Confusion_matrix_metrics.csv")
+
+df = pd.read_csv(csv_file)
+
+# Set the user-defined thresholds to get desired values
+
+for Thresold_Human_Voters in df["Thresold_Human_Voters"].unique():
+    for Thresold_ML_Voters in df["Thresold_ML_Voters"].unique():
+        # Filter the DataFrame based on the user-set threshold
+        filtered_df = df[(df['Thresold_Human_Voters'] == Thresold_Human_Voters) &
+                         (df['Thresold_ML_Voters'] == Thresold_ML_Voters) &
+                        (df['TP']+df['FN']  > 0)]
+                         
+        
+        filtered_df.loc[filtered_df["sequence_name"] == "anat","sequence_name"] = "Anatomical"
+        filtered_df.loc[filtered_df["sequence_name"] == "diff","sequence_name"] = "Diffusion"
+        filtered_df.loc[filtered_df["sequence_name"] == "func","sequence_name"] = "Functional"
+        
+        cm = 1/2.54  # centimeters in inches
+        # Create two subplots for Kappa and F1 scores heatmaps
+        fig, axes = plt.subplots(1, 1, figsize=(20*cm, 5*cm), dpi=300)
+        sns.set_style('darkgrid')
+        # Specify the font properties
+        font_properties = fm.FontProperties(family='Times New Roman', size=8)
+        font_properties2 = fm.FontProperties(family='Times New Roman', size=10)
+        Title = ["(a) Fleiss kappa score: inter rater reliability "," (b) F1_score: raters vs AIDAqc"]
+        for i, metric in enumerate(['Sensitivity-Recall']):
+            ax = axes
+            pivot_df = filtered_df.pivot(index='sequence_name', columns='dataset_name', values=metric)
+            pivot_df['mean'] = pivot_df.mean(axis=1)
+            pivot_df['std'] = pivot_df.std(axis=1)
+            t = "Human: " + str(Thresold_Human_Voters) + "|| AIDAqc:" +str(Thresold_ML_Voters) 
+            sns.heatmap(pivot_df, annot=True, fmt=".2f", cmap="YlGnBu", cbar=True, ax=ax)
+        
+            # Set font properties for labels, titles, and annotations
+            ax.set_xticklabels(ax.get_xticklabels(), fontproperties=font_properties)
+            ax.set_yticklabels(ax.get_yticklabels(), fontproperties=font_properties)
+            ax.set_xlabel('Datasets', fontproperties=font_properties)
+            ax.set_ylabel('Sequences', fontproperties=font_properties)
+            ax.set_title(f'{t} ', fontsize=10, fontproperties=font_properties2, fontweight='bold')
+            ax.set(xlabel=None)
+            # Set the color bar legend font size and font properties
+            cbar = ax.collections[0].colorbar
+            cbar.ax.tick_params(labelsize=8)
+            cbar.ax.set_yticklabels(cbar.ax.get_yticklabels(), fontproperties=font_properties)
+            
+            # Customize the color bar ticks (increase the number of ticks)
+            cbar.ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=5))
+            
+            a = []
+            for text in ax.texts:
+                a.append(float(text.get_text()))
+                
+            
+        
+            for text in ax.texts:
+                text.set_font("Times New Roman")
+                text.set_size(8)
+                
+                
+        # Show the plots
+        plt.tight_layout()
+        
+        plt.show()

outputs/code_for_figures/HeatMaps_Version2.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Wed Nov 22 17:50:28 2023
+
+@author: arefk
+"""
+
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import os
+
+# Load your Excel file into a Pandas DataFrame
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+csv_file = os.path.join(file_path, "Confusion_matrix_metrics.csv")
+
+df = pd.read_csv(csv_file)
+
+# Set the user-defined thresholds to get desired values
+Threshold_Human_Voters = 3  # Replace with the user-set threshold value
+Threshold_ML_Voters = 1
+
+# Filter the DataFrame based on the user-set threshold
+filtered_df = df[(df['Thresold_Human_Voters'] == Threshold_Human_Voters) &
+                 (df['Thresold_ML_Voters'] == Threshold_ML_Voters) &
+                 (df['TP'] > 0)]
+
+# Specify the columns you want to include in the heatmap
+columns_to_include = ['TP', 'TN', 'FP', 'FN', 'Sensitivity-Recall', 'Specificity', 'Precision', 'Accuracy', 'F1 Score']
+
+cm = 1 / 2.54  # centimeters in inches
+
+# Create a subplot for the heatmap
+fig, ax = plt.subplots(figsize=(20 * cm, 10 * cm), dpi=300)
+
+# Specify the font properties
+font_properties = fm.FontProperties(family='Times New Roman', size=10)
+
+# Pivot the DataFrame to have "sequence_name" as columns and "dataset_name" as index
+pivot_df = filtered_df.pivot(index='sequence_name', columns='dataset_name')
+
+# Filter the pivot DataFrame to include only the specified columns
+pivot_df = pivot_df[columns_to_include]
+
+# Calculate the average for each "dataset_name"
+avg_df = pivot_df.mean(level=0, axis=1)
+
+# Plot the heatmap without color coding
+sns.heatmap(avg_df, annot=True, fmt=".1f", cmap="Blues", cbar=False, ax=ax)
+
+# Set font properties for labels, titles, and annotations
+ax.set_xticklabels(ax.get_xticklabels(), fontproperties=font_properties)
+ax.set_yticklabels(ax.get_yticklabels(), fontproperties=font_properties)
+ax.set_xlabel('Sequences', fontproperties=font_properties)
+ax.set_ylabel('Datasets', fontproperties=font_properties)
+
+# Set the font properties for the cell annotations
+for text in ax.texts:
+    text.set_font("Times New Roman")
+    text.set_size(8)
+
+# Show the plot
+plt.tight_layout()
+plt.show()

outputs/code_for_figures/HighVSLowMotion.py

@@ -0,0 +1,73 @@
+import os
+import nibabel as nib
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+from matplotlib.colors import Normalize
+from matplotlib.gridspec import GridSpec
+
+# Define the directory path where the NIfTI and PNG files are located
+path = r"C:\Users\aswen\Desktop\Code\2023_Kalantari_AIDAqc\outputs\files_4figs"  # Replace with your actual directory path
+
+# File names for NIfTI images (assuming they represent motion data)
+motion_best_filename = "MotionBest.nii.gz"
+motion_worst_filename = "MotionWorst.nii.gz"
+
+# Load the NIfTI files
+motion_best = nib.load(os.path.join(path, motion_best_filename))
+motion_worst = nib.load(os.path.join(path, motion_worst_filename))
+
+# Get the NIfTI data as NumPy arrays
+motion_best_data = motion_best.get_fdata()
+motion_worst_data = motion_worst.get_fdata()
+
+# Rotate the data arrays 90 degrees counterclockwise
+motion_best_data = np.rot90(motion_best_data, k=-1)
+motion_worst_data = np.rot90(motion_worst_data, k=1)
+
+# Average over the 4th dimension and select the middle slice
+motion_best_data = np.mean(motion_best_data, axis=3)
+motion_worst_data = np.mean(motion_worst_data, axis=3)
+slice_index1 = motion_best_data.shape[2] // 2
+slice_index2 = motion_worst_data.shape[2] // 2
+
+motion_best_data = motion_best_data[:, :, slice_index1]
+motion_best_data = np.rot90(np.rot90(motion_best_data, k=1), k=1)
+
+motion_worst_data = motion_worst_data[:, :, slice_index2]
+
+# Normalize the data to the range of 0 to 1
+norm = Normalize(vmin=motion_best_data.min(), vmax=motion_best_data.max())
+motion_best_data = norm(motion_best_data)
+motion_worst_data = norm(motion_worst_data)
+
+# Create a figure with GridSpec
+cm = 1/2.54  # centimeters in inches
+fig = plt.figure(figsize=(9*cm, 6.68*cm), dpi=300)
+gs = GridSpec(1, 2, width_ratios=[1, 1])
+
+# Create the axes for the "Motion Data (Best)" NIfTI image
+ax0 = plt.subplot(gs[0])
+im0 = ax0.imshow(motion_best_data, cmap='viridis', origin='lower', vmin=0, vmax=1)
+ax0.set_title("rsfMRI low motion", fontsize=10, fontweight='bold', fontname='Times New Roman')
+ax0.set_xticks([])
+ax0.set_yticks([])
+
+# Create the axes for the "Motion Data (Worst)" NIfTI image
+ax1 = plt.subplot(gs[1])
+im1 = ax1.imshow(motion_worst_data, cmap='viridis', origin='lower', vmin=0, vmax=1)
+ax1.set_title("rsfMRI high motion", fontsize=10, fontweight='bold', fontname='Times New Roman')
+ax1.set_xticks([])
+ax1.set_yticks([])
+
+# Add a single colorbar for the NIfTI images with more ticks
+divider0 = make_axes_locatable(ax1)
+cax0 = divider0.append_axes("right", size="5%", pad=0.05)
+cbar0 = plt.colorbar(im0, cax=cax0, ticks=np.linspace(0, 1, 6))  # Adjust the number of ticks (6 in this case)
+cbar0.ax.tick_params(labelsize=8)
+
+# Adjust the layout for better visibility
+plt.tight_layout()
+
+# Display the figure
+plt.show()

outputs/code_for_figures/MergeAllManualRaters.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Nov 17 11:34:11 2023
+
+@author: arefks
+"""
+import os
+import glob
+import pandas as pd
+from PIL import Image
+import numpy as np
+import concurrent.futures
+
+# Step 1: Define the starting path and file pattern
+start_path = r"C:\Users\aswen\Desktop\Code\Validation3"
+file_pattern_all_images = os.path.join(start_path, "*", "manual_slice_inspection", "*.png")
+manual_slice_inspection_image_files = glob.glob(file_pattern_all_images, recursive=True)
+
+file_pattern_all_raters = os.path.join(start_path, "*", "validation*", "*.png")
+validators_image_files = glob.glob(file_pattern_all_raters, recursive=True)
+
+# Step 2: Create DataFrames
+column_names = ["Path", "dataset_name", "validator_name"]
+
+manual_slice_inspection_df = pd.DataFrame(
+    [
+        [file_path, path_elements[-3] if len(path_elements) >= 3 else None, path_elements[-2] if len(path_elements) >= 2 else None]
+        for file_path in manual_slice_inspection_image_files
+        for path_elements in [file_path.split(os.sep)]
+    ],
+    columns=column_names,
+)
+
+validators_df = pd.DataFrame(
+    [
+        [file_path, path_elements[-3] if len(path_elements) >= 3 else None, path_elements[-2] if len(path_elements) >= 2 else None]
+        for file_path in validators_image_files
+        for path_elements in [file_path.split(os.sep)]
+    ],
+    columns=column_names,
+)
+
+# Get unique dataset names and validators
+unique_datasets = manual_slice_inspection_df["dataset_name"].unique()
+
+
+def process_dataset(dataset_name):
+    ma_subset = manual_slice_inspection_df[manual_slice_inspection_df["dataset_name"] == dataset_name]
+    va_subset = validators_df[validators_df["dataset_name"] == dataset_name]
+
+    results = []
+
+    for index, ma_row in ma_subset.iterrows():
+        ma_path = ma_row["Path"]
+        ma_image = Image.open(ma_path).convert('RGB')
+        ma_image_array = np.array(ma_image)
+
+        result_row = {"Path": ma_path}
+        is_same_all = []
+        for _, va_row in va_subset.iterrows():
+            va_path = va_row["Path"]
+            va_image = Image.open(va_path).convert('RGB')
+            va_image_array = np.array(va_image)
+
+            diff_image = abs(ma_image_array - va_image_array)
+            threshold = 1e-6  # You can adjust this threshold based on your needs
+            is_same = np.sum(diff_image) < threshold
+            is_same_all.append(is_same)
+
+            result_row[va_row["validator_name"]] = is_same
+            result_row["dataset_name"] = dataset_name
+            result_row["SequenceType"] = ma_path.split(os.sep)[-1].split("_")[0]
+
+        validator_names = va_subset[is_same_all].validator_name
+        for v in validator_names:
+            result_row[v] = True
+        results.append(result_row)
+        #print(C)
+    
+    print("finished: " + dataset_name)
+    return results
+
+# Get the number of available CPUs
+num_cpus = os.cpu_count()
+
+# Create a ThreadPoolExecutor
+with concurrent.futures.ThreadPoolExecutor(max_workers=num_cpus) as executor:
+    # Process datasets in parallel
+    all_results = list(executor.map(process_dataset, unique_datasets))
+
+# Concatenate the results into the final DataFrame
+result_df = pd.concat([pd.DataFrame(results) for results in all_results], ignore_index=True)
+
+# Fill NaN values with False
+result_df = result_df.fillna(False)
+
+# Optionally, you can save the combined DataFrame to a CSV file
+output_path = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input"
+result_df.to_csv(os.path.join(output_path, "combined_Human_Voters_from_votings3_Final.csv"), index=False)

outputs/code_for_figures/MergeAllVoitings.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Nov 17 11:34:11 2023
+
+@author: arefks
+"""
+
+import os
+import glob
+import pandas as pd
+
+# Step 1: Define the starting path and file pattern
+start_path =  r"C:\Users\aswen\Desktop\Code\Validation3"
+file_pattern = '*votin*.csv'
+
+# Step 2: Find all matching CSV files in the specified directory and its subdirectories
+csv_files = glob.glob(os.path.join(start_path, '*', file_pattern), recursive=True)
+
+# Step 3: Initialize an empty DataFrame to store the extracted data
+combined_df = pd.DataFrame()
+combined_df1 = pd.DataFrame()
+
+
+# Step 4: Loop through the CSV files and extract the specified columns
+for csv_file in csv_files:
+    try:
+        df = pd.read_csv(csv_file)
+        selected_columns = ["Pathes",	"corresponding_img"
+                            ,	"One_class_SVM",
+                            "IsolationForest",	"LocalOutlierFactor",
+                            " EllipticEnvelope",	"statistical_method",
+                            "Voting outliers (from 5)"]
+        df = df[selected_columns]
+        df["dataset"] = csv_file.split(os.sep)[-2]
+        df["corresponding_img_Path"] = os.path.join(os.path.dirname(csv_file),"manual_slice_inspection")
+        # Concatenate the current DataFrame with the combined DataFrame
+        combined_df1 = pd.concat([combined_df1, df], ignore_index=True)
+    except Exception as e:
+        print(f"Error reading {csv_file}: {e}")
+
+
+
+combined_df = combined_df1
+combined_df["SequenceType"] = combined_df["corresponding_img"].str.split("_").str[0]
+# Step 5: Print the combined DataFrame
+print(combined_df)
+
+# Optionally, you can save the combined DataFrame to a CSV file
+p = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input"
+combined_df.to_csv(os.path.join(p,'combined_votings3.csv'), index=False)

outputs/code_for_figures/MutualInfoPlot.py

@@ -0,0 +1,62 @@
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import os
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input', 'MUplot.csv')
+out_path = os.path.join(script_dir, '..', 'figures')
+df = pd.read_csv(file_path, header=None, names=['Shift (Voxel)', 'Severe motion', 'No motion'], skiprows=1)
+
+# Set the seaborn style and palette
+sns.set(style='ticks', palette='Set1')
+
+# Set font properties
+font_properties = {'family': 'Times New Roman', 'size': 8}
+font_properties2 = {'family': 'Times New Roman', 'size': 6}
+
+cm = 1/2.54  # centimeters in inches
+
+# Create the plot
+fig, ax = plt.subplots(figsize=(7.01*cm, 3.21*cm), dpi=300)
+ax.plot(df['Shift (Voxel)'], df['Severe motion'], label='Severe motion', linewidth=1)  # Adjust the line width
+ax.plot(df['Shift (Voxel)'], df['No motion'], label='No motion', linewidth=1, color='blue')  # Adjust the line width
+
+# Set axis labels
+ax.set_xlabel('Shift (Voxel)', **font_properties)
+ax.set_ylabel('Mutual information (a.u)', **font_properties)
+
+# Set axis ticks font and number of ticks
+ax.tick_params(axis='both', which='both', width=0.5, color='gray', length=2)
+ax.locator_params(axis='x', nbins=8)  # Set the number of ticks for the x-axis
+ax.locator_params(axis='y', nbins=8)  # Set the number of ticks for the y-axis
+
+for tick in ax.get_xticklabels():
+    tick.set_fontname('Times New Roman')
+    tick.set_fontsize(8)
+for tick in ax.get_yticklabels():
+    tick.set_fontname('Times New Roman')
+    tick.set_fontsize(8)
+
+# Set legend font and remove the legend border
+legend = ax.legend(prop=font_properties2, frameon=False)
+
+# Customize the border linewidth
+ax.spines['top'].set_linewidth(0.5)     # Top border
+ax.spines['right'].set_linewidth(0.5)   # Right border
+ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+ax.spines['left'].set_linewidth(0.5)   # Left border
+
+# Adjust layout to include labels
+plt.subplots_adjust(left=0.15, right=0.95, top=1.1, bottom=0.25)
+
+# Save figures as PNG and SVG with 300 dpi
+fig_path_png = os.path.join(out_path, 'MutualInformation.png')
+fig_path_svg = os.path.join(out_path, 'MutualInformation.svg')
+
+fig.savefig(fig_path_png, format='png', bbox_inches='tight')
+fig.savefig(fig_path_svg, format='svg', bbox_inches='tight')
+
+# Show the plot
+plt.show()

outputs/code_for_figures/PieCharts.py

@@ -0,0 +1,67 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+
+# Set font to Times New Roman
+plt.rcParams['font.family'] = 'Times New Roman'
+
+script_dir = os.path.dirname(__file__)
+p_address = os.path.join(script_dir, '..', 'input', 'AIDAqc_Testdaten_PieChart.csv')
+p_save = os.path.join(script_dir, '..', 'figures')
+
+# Load the CSV data into a pandas DataFrame
+data = pd.read_csv(p_address)
+
+# Clean up the 'Scanner' column to extract the field strength (including decimals)
+def extract_field_strength(scanner):
+    match = re.search(r'(\d+(\.\d+)?)T', scanner)
+    if match:
+        return f"{match.group(1)}T"
+    else:
+        return None
+
+data['Scanner'] = data['Scanner'].apply(extract_field_strength)
+cm = 1/2.54  # centimeters in inches
+
+# Set up the figure and axes using subplots
+fig, axes = plt.subplots(1, 4, figsize=(22*cm, 10*cm), dpi=300)
+
+rrr = 1.1
+# Species pie chart
+species_counts = data['Species'].value_counts()
+axes[0].pie(species_counts, labels=species_counts.index, autopct='%1.1f%%', startangle=180, pctdistance=0.75,radius=rrr)
+#axes[0].set_title('(a) Species', weight='bold', fontsize=10)
+
+# Field strength pie chart
+scanner_counts = data['Scanner'].value_counts()
+axes[1].pie(scanner_counts, labels=scanner_counts.index, autopct='%1.1f%%', startangle=180, pctdistance=0.70,radius=rrr)
+#axes[1].set_title('(b) Field strength', weight='bold', fontsize=10)
+
+# Sequence type pie chart
+sequences_data = data['Sequences'].str.split(', ', expand=True)
+sequences_melted = sequences_data.melt(value_name='Sequence').dropna()['Sequence']
+sequence_counts = sequences_melted.value_counts()
+axes[2].pie(sequence_counts, labels=sequence_counts.index, autopct='%1.1f%%', startangle=180, pctdistance=0.65,radius=rrr)
+#axes[2].set_title('(c) Sequence type', weight='bold', fontsize=10)
+
+# Data format pie chart
+format_counts = data['Data format'].value_counts()
+axes[3].pie(format_counts, labels=format_counts.index, autopct='%1.1f%%', startangle=180,radius=rrr)
+#axes[3].set_title('(d) Data format', weight='bold', fontsize=10)
+
+# Turn off axes for all subplots
+for ax in axes:
+    ax.axis('off')
+
+# Adjust layout and save the figure
+#plt.tight_layout()
+plt.savefig(os.path.join(p_save , "All_Pie_Charts" + ".svg"), format='svg', bbox_inches='tight')
+plt.savefig(os.path.join(p_save , "All_Pie_Charts" + ".png"), format='png', bbox_inches='tight')
+
+plt.show()

outputs/code_for_figures/ROCcurve.py

@@ -0,0 +1,101 @@
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import os
+import numpy as np
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+# Load the data 
+result_df = pd.read_csv(os.path.join(file_path, 'Confusion_matrix_metrics.csv'))
+cm = 1/2.54
+
+# Calculate Actual_Label
+
+# Specify the font size for the plot
+sns.set_style('ticks')
+sns.set(font='Times New Roman', style=None)  # Set font to Times New Roman and font size to 9
+palette = 'Set1'
+
+subset_df = result_df[(result_df['TP']+result_df['FN'] > 0)]
+
+# Create a Seaborn line plot
+g = sns.relplot(
+    data=subset_df, kind="line",
+    x="Thresold_ML_Voters", y="Sensitivity-Recall", hue="sequence_name",
+    dashes=False, markers=True, ci=30,
+    height=5*cm, aspect=2 # Adjust the aspect to achieve the desired width of 9 cm
+)
+
+# Access the individual axes
+axes = g.axes.flatten()
+axes[0].set_xlabel("AIDAqc Voting Threshold", fontsize=8)
+axes[0].set_ylabel("Sensitivity", fontsize=8)
+
+# Loop through each axis and customize spines and tick parameters
+for ax in axes:
+    ax.tick_params(axis='both', which='both', labelsize=8)
+    ax.spines['top'].set_visible(True)
+    ax.spines['right'].set_visible(True)
+    ax.spines['bottom'].set_visible(True)
+    ax.spines['left'].set_visible(True)
+    ax.spines['top'].set_linewidth(0.5)
+    ax.spines['right'].set_linewidth(0.5)
+    ax.spines['bottom'].set_linewidth(0.5)
+    ax.spines['left'].set_linewidth(0.5)
+g._legend.set_title("")
+g._legend.set_fontsize(8)
+# Save the figure as SVG and PNG
+output_path = out_path
+output_filename = "Sensitivity_AIDAqc_Voting_Threshold"
+
+# Save as SVG
+g.savefig(f"{output_path}/{output_filename}.svg", format="svg")
+
+# Save as PNG
+g.savefig(f"{output_path}/{output_filename}.png", format="png")
+
+plt.show()
+#%%
+# Create a Seaborn line plot
+g = sns.relplot(
+    data=subset_df, kind="line",
+    x="Thresold_Human_Voters", y="Sensitivity-Recall", hue="sequence_name",
+    dashes=False, markers=True, ci=30,
+    height=5*cm, aspect=2 # Adjust the aspect to achieve the desired width of 9 cm
+)
+
+# Access the individual axes
+# Access the individual axes
+axes = g.axes.flatten()
+axes[0].set_xlabel("Manual-rater Voting Threshold", fontsize=8)
+axes[0].set_ylabel("Sensitivity", fontsize=8)
+
+# Loop through each axis and customize spines and tick parameters
+for ax in axes:
+    ax.tick_params(axis='both', which='both', labelsize=8)
+    ax.spines['top'].set_visible(True)
+    ax.spines['right'].set_visible(True)
+    ax.spines['bottom'].set_visible(True)
+    ax.spines['left'].set_visible(True)
+    ax.spines['top'].set_linewidth(0.5)
+    ax.spines['right'].set_linewidth(0.5)
+    ax.spines['bottom'].set_linewidth(0.5)
+    ax.spines['left'].set_linewidth(0.5)
+g._legend.set_title("")
+g._legend.set_fontsize(8)  # Set legend label size to 8 points
+
+# Save the figure as SVG and PNG
+output_path = out_path
+output_filename = "Sensitivity_ManualRater_Voting_Threshold"
+
+# Save as SVG
+g.savefig(f"{output_path}/{output_filename}.svg", format="svg")
+
+# Save as PNG
+g.savefig(f"{output_path}/{output_filename}.png", format="png")
+
+plt.show()

outputs/code_for_figures/ResoPlot.py

@@ -0,0 +1,160 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+
+#% Function to read CSV files and store them in a dictionary
+
+cm = 1/2.54  # centimeters in inches
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        # Extract the data type from the file name (assuming the file name contains "anatomical", "structural", or "functional")
+        data_type = "anatomical" if "anatomical" in file else ("structural" if "structural" in file else "functional")
+        data_dict[ff] = {data_type:df} 
+            
+    return data_dict
+
+
+# Function for statistical comparison and plotting
+def compare_and_plot(data, column_name, group_column):
+    sns.set_palette("Set2")  # Use colors for color-blind people
+    plt.figure(figsize=(9*cm, 6*cm))
+    sns.boxplot(x=group_column, y=column_name, data=data)
+    plt.xlabel(group_column)
+    plt.ylabel(column_name)
+    plt.title(f"Statistical Comparison of {column_name} by {group_column}")
+    plt.tight_layout()
+    #plt.savefig(f"{column_name}_by_{group_column}_boxplot.png")
+    plt.show()
+
+#% List of CSV files for each data type
+Path = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_Plot_VoxelsSize"
+
+anatomical_files = glob.glob(os.path.join(Path,"**/*caculated_features_anatomical.csv"), recursive=True)
+structural_files = glob.glob(os.path.join(Path,"**/*caculated_features_structural.csv"), recursive=True)
+functional_files = glob.glob(os.path.join(Path,"**/*caculated_features_functional.csv"), recursive=True)
+
+All_files = [anatomical_files,structural_files,functional_files]
+
+# Read the CSV files and store them in dictionaries
+anatomical_data = read_csv_files(anatomical_files)
+structural_data = read_csv_files(structural_files)
+functional_data = read_csv_files(functional_files)
+All_Data = [anatomical_data]
+
+All_type = ["anatomical"]
+#% data statistisc figure 7
+
+#features_to_compare = ["SNR Chang", "SNR Normal", "tSNR (Averaged Brain ROI)", "Displacement factor (std of Mutual information)"]
+#features_to_compare = ["SNR Chang"]
+features_to_compare = ["voxel size [mm³]"]
+
+for dd,data in enumerate(All_Data):
+    for feature in features_to_compare:
+        cc = 0
+        temp = pd.DataFrame()
+        Data_of_selected_feature = pd.DataFrame()
+        temp_data = pd.DataFrame()
+
+        for key in data:
+            try:
+                #temp_data[feature] = data[key][All_type[dd]][feature]
+                temp_data["SpatRx"] = data[key][All_type[dd]]["SpatRx"]
+                temp_data["SpatRy"] = data[key][All_type[dd]]["SpatRy"]
+                temp_data["Slicethick"] = data[key][All_type[dd]]["Slicethick"]
+                
+            except KeyError:
+                continue
+            temp_data["Dataset"] = All_files[dd][cc].split(os.sep)[-3]
+            cc = cc +1
+            Data_of_selected_feature = pd.concat([Data_of_selected_feature, temp_data], ignore_index=True)
+            #Data_of_selected_feature2 = pd.concat([Data_of_selected_feature2, temp_data], ignore_index=True)
+            
+            
+            
+        if not Data_of_selected_feature.empty:
+            Data_of_selected_feature['sort'] = Data_of_selected_feature['Dataset'].str.extract('(\d+)', expand=True).astype(int)
+            Data_of_selected_feature = Data_of_selected_feature.sort_values('Dataset')
+            
+            if feature == "SNR Normal":
+                Data_of_selected_feature.rename(columns={"SNR Normal": "SNR-Standard (dB)"}, inplace=True)
+                feature = "SNR-Standard (dB)"
+            if feature == "SNR Chang":
+                Data_of_selected_feature.rename(columns={"SNR Chang": "SNR-Chang (dB)"}, inplace=True)
+                feature = "SNR-Chang (dB)"    
+            elif feature == "tSNR (Averaged Brain ROI)":
+                Data_of_selected_feature.rename(columns={"tSNR (Averaged Brain ROI)": "tSNR (dB)"}, inplace=True)
+                feature = "tSNR (dB)"
+            elif feature == "Displacement factor (std of Mutual information)":
+                Data_of_selected_feature.rename(columns={"Displacement factor (std of Mutual information)": "Motion severity (A.U)"}, inplace=True)
+                feature = "Motion severity (A.U)"
+            
+            
+            Data_of_selected_feature["voxel size [mm³]"] = Data_of_selected_feature["SpatRx"]*Data_of_selected_feature["SpatRy"]*Data_of_selected_feature["Slicethick"]
+            #Data_of_selected_feature = Data_of_selected_feature.sort_values("Dataset",ascending=False)
+            # creating boxplots
+            plt.figure(figsize=(5*cm,10*cm),dpi=300)
+            sns.set_style('ticks')
+            sns.set(font='Times New Roman', font_scale=0.8,style=None)  # Set font to Times New Roman and font size to 9
+            palette = 'Set2'
+            ax = sns.violinplot(x="Dataset", y=feature, data=Data_of_selected_feature, hue="Dataset", dodge=False,
+                                palette=palette,
+                                scale="width", inner=None,linewidth=1)
+            patches = ax.patches
+            #legend_colors = [patch.get_facecolor() for patch in patches[:]]
+
+            
+            ax.legend_.remove()
+            ax
+            ax.set_xticklabels(ax.get_xticklabels(), rotation=45,fontsize=8)
+            #ax.set_ylabel(ax.get_ylabels(),fontsize=8)
+# =============================================================================
+#             for label, color in zip(ax.get_xticklabels(), legend_colors):
+#                 label.set_color(color)
+# =============================================================================
+            ax.set_xlabel('')
+            ax.set_title(All_type[dd].capitalize(),weight='bold',fontsize=10)
+            y_label = ax.set_ylabel(ax.get_ylabel(),fontsize=8)
+
+# =============================================================================
+            #ax.xaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            #ax.xaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)
+# 
+            ax.yaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            ax.yaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)        
+# =============================================================================
+            ax.spines['top'].set_linewidth(0)  # Top border
+            ax.spines['right'].set_linewidth(0)  # Right border
+            ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+            ax.spines['left'].set_linewidth(0.5)  # Left border
+            ax.tick_params(axis='both', which='both', width=0.5,color='gray',length=2)
+            plt.xticks(ha='right')
+            #plt.savefig(os.path.join(Path,feature+"_"+All_type[dd]+".svg"), format='svg', bbox_inches='tight',transparent=False)
+            plt.show()
+                        
+            import pandas as pd
+            import scikit_posthocs as sp
+             
+            # reading CSV file
+            #dataset= pd.read_csv(r"C:\Users\arefk\OneDrive\Desktop\Projects\iris.csv")
+             
+            # data which contains sepal width of the three species
+            data = [Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_Rei"]["voxel size [mm³]"],
+                    Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_We"]["voxel size [mm³]"],
+                    Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94c_m_As"]["voxel size [mm³]"]]
+             
+            # using the posthoc_dunn() function
+            
+            p_values= sp.posthoc_dunn(data, p_adjust = 'holm')
+             
+            print(p_values)
+            
+            

outputs/code_for_figures/ResoPlotAll.py

@@ -0,0 +1,153 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+
+#% Function to read CSV files and store them in a dictionary
+
+cm = 1/2.54  # centimeters in inches
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        # Extract the data type from the file name (assuming the file name contains "anatomical", "structural", or "functional")
+        data_type = "anatomical" if "anatomical" in file else ("structural" if "structural" in file else "functional")
+        data_dict[ff] = {data_type:df} 
+            
+    return data_dict
+
+
+# Function for statistical comparison and plotting
+def compare_and_plot(data, column_name, group_column):
+    sns.set_palette("Set2")  # Use colors for color-blind people
+    plt.figure(figsize=(9*cm, 6*cm))
+    sns.boxplot(x=group_column, y=column_name, data=data)
+    plt.xlabel(group_column)
+    plt.ylabel(column_name)
+    plt.title(f"Statistical Comparison of {column_name} by {group_column}")
+    plt.tight_layout()
+    #plt.savefig(f"{column_name}_by_{group_column}_boxplot.png")
+    plt.show()
+
+#% List of CSV files for each data type
+Path = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_Plot_VoxelSizesAll"
+
+anatomical_files = glob.glob(os.path.join(Path,"*/*/*caculated_features_anatomical.csv"), recursive=True)
+structural_files = glob.glob(os.path.join(Path,"*/*/*caculated_features_structural.csv"), recursive=True)
+functional_files = glob.glob(os.path.join(Path,"*/*/*caculated_features_functional.csv"), recursive=True)
+
+All_files = [anatomical_files,structural_files,functional_files]
+
+# Read the CSV files and store them in dictionaries
+anatomical_data = read_csv_files(anatomical_files)
+structural_data = read_csv_files(structural_files)
+functional_data = read_csv_files(functional_files)
+All_Data = [anatomical_data,structural_data,functional_data]
+
+All_type = ["anatomical","structural","functional"]
+#% data statistisc figure 7
+
+#features_to_compare = ["SNR Chang", "SNR Normal", "tSNR (Averaged Brain ROI)", "Displacement factor (std of Mutual information)"]
+
+features_to_compare = ["SpatRx", "SpatRy", "Slicethick"]
+
+
+Data_of_selected_feature2 = pd.DataFrame()
+for dd,data in enumerate(All_Data):
+    for feature in features_to_compare:
+        cc = 0
+        temp = pd.DataFrame()
+        Data_of_selected_feature = pd.DataFrame()
+        temp_data = pd.DataFrame()
+
+        for key in data:
+            try:
+                temp_data[feature] = data[key][All_type[dd]][feature]
+            except KeyError:
+                continue
+            temp_data["Dataset"] = All_files[dd][cc].split(os.sep)[-3]
+            cc = cc +1
+            Data_of_selected_feature = pd.concat([Data_of_selected_feature, temp_data], ignore_index=True)
+            #Data_of_selected_feature2 = pd.concat([Data_of_selected_feature2, temp_data], ignore_index=True)
+            
+            
+            
+        if not Data_of_selected_feature.empty:
+            Data_of_selected_feature['sort'] = Data_of_selected_feature['Dataset'].str.extract('(\d+)', expand=True).astype(int)
+            Data_of_selected_feature = Data_of_selected_feature.sort_values('sort')
+            
+            if feature == "SNR Normal":
+                Data_of_selected_feature.rename(columns={"SNR Normal": "SNR-Standard (dB)"}, inplace=True)
+                feature = "SNR-Standard (dB)"
+            if feature == "SNR Chang":
+                Data_of_selected_feature.rename(columns={"SNR Chang": "SNR-Chang (dB)"}, inplace=True)
+                feature = "SNR-Chang (dB)"    
+            elif feature == "tSNR (Averaged Brain ROI)":
+                Data_of_selected_feature.rename(columns={"tSNR (Averaged Brain ROI)": "tSNR (dB)"}, inplace=True)
+                feature = "tSNR (dB)"
+            elif feature == "Displacement factor (std of Mutual information)":
+                Data_of_selected_feature.rename(columns={"Displacement factor (std of Mutual information)": "Motion severity (A.U)"}, inplace=True)
+                feature = "Motion severity (A.U)"
+            
+            
+            #Data_of_selected_feature2["Vol"] = Data_of_selected_feature2["SpatRx"]*Data_of_selected_feature2["SpatRy"]*Data_of_selected_feature2["Slicethick"]
+            
+            #Data_of_selected_feature = Data_of_selected_feature.sort_values("Dataset",ascending=False)
+            # creating boxplots
+            plt.figure(figsize=(6*cm,3.527*cm),dpi=300)
+            sns.set_style('ticks')
+            sns.set(font='Times New Roman', font_scale=0.9,style=None)  # Set font to Times New Roman and font size to 9
+            palette = 'Set2'
+            ax = sns.violinplot(x="Dataset", y=feature, data=Data_of_selected_feature, hue="Dataset", dodge=False,
+                                palette=palette,
+                                scale="width", inner=None,linewidth=1)
+            patches = ax.patches
+            #legend_colors = [patch.get_facecolor() for patch in patches[:]]
+
+            xlim = ax.get_xlim()
+            ylim = ax.get_ylim()
+            for violin in ax.collections:
+                bbox = violin.get_paths()[0].get_extents()
+                x0, y0, width, height = bbox.bounds
+                violin.set_clip_path(plt.Rectangle((x0, y0), width / 2, height, transform=ax.transData))
+            
+            sns.boxplot(x="Dataset", y=feature, data=Data_of_selected_feature, saturation=1, showfliers=False,
+                        width=0.3, boxprops={'zorder': 3, 'facecolor': 'none'}, ax=ax, linewidth=1)
+            old_len_collections = len(ax.collections)
+            sns.stripplot(x="Dataset", y=feature, data=Data_of_selected_feature,size=1.1, hue="Dataset", palette=palette, dodge=False, ax=ax)
+            for dots in ax.collections[old_len_collections:]:
+                dots.set_offsets(dots.get_offsets() + np.array([0.12, 0]))
+            ax.set_xlim(xlim)
+            ax.set_ylim(ylim)
+            ax.legend_.remove()
+            ax
+            ax.set_xticklabels(ax.get_xticklabels(), rotation=45,fontsize=8)
+# =============================================================================
+#             for label, color in zip(ax.get_xticklabels(), legend_colors):
+#                 label.set_color(color)
+# =============================================================================
+            ax.set_xlabel('')
+            ax.set_title(All_type[dd].capitalize(),weight='bold',fontsize=9)
+            y_label = ax.set_ylabel(ax.get_ylabel(), fontweight='bold',fontsize=9)
+
+# =============================================================================
+            ax.xaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            ax.xaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)
+# 
+            ax.yaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            ax.yaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)        
+# =============================================================================
+            ax.spines['top'].set_linewidth(0.5)  # Top border
+            ax.spines['right'].set_linewidth(0.5)  # Right border
+            ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+            ax.spines['left'].set_linewidth(0.5)  # Left border
+            ax.tick_params(axis='both', which='both', width=0.5,color='gray',length=2)
+            plt.xticks(ha='right')
+            #plt.savefig(os.path.join(Path,feature+"_"+All_type[dd]+".svg"), format='svg', bbox_inches='tight',transparent=False)
+            plt.show()

outputs/code_for_figures/SameFSdifferentVS.py

@@ -0,0 +1,199 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+
+#% Function to read CSV files and store them in a dictionary
+
+cm = 1/2.54  # centimeters in inches
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        # Extract the data type from the file name (assuming the file name contains "anatomical", "structural", or "functional")
+        data_type = "anatomical" if "anatomical" in file else ("structural" if "structural" in file else "functional")
+        data_dict[ff] = {data_type:df} 
+            
+    return data_dict
+
+
+# Function for statistical comparison and plotting
+def compare_and_plot(data, column_name, group_column):
+    sns.set_palette("Set2")  # Use colors for color-blind people
+    plt.figure(figsize=(9*cm, 6*cm))
+    sns.boxplot(x=group_column, y=column_name, data=data)
+    plt.xlabel(group_column)
+    plt.ylabel(column_name)
+    plt.title(f"Statistical Comparison of {column_name} by {group_column}")
+    plt.tight_layout()
+    #plt.savefig(f"{column_name}_by_{group_column}_boxplot.png")
+    plt.show()
+
+#% List of CSV files for each data type
+Path = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_SameFSdiffVS" #QC_Plot_VoxelsSize
+
+anatomical_files = glob.glob(os.path.join(Path,"**/*caculated_features_anatomical.csv"), recursive=True)
+structural_files = glob.glob(os.path.join(Path,"**/*caculated_features_structural.csv"), recursive=True)
+functional_files = glob.glob(os.path.join(Path,"**/*caculated_features_functional.csv"), recursive=True)
+
+All_files = [anatomical_files,structural_files,functional_files]
+
+# Read the CSV files and store them in dictionaries
+anatomical_data = read_csv_files(anatomical_files)
+structural_data = read_csv_files(structural_files)
+functional_data = read_csv_files(functional_files)
+All_Data = [anatomical_data]
+
+All_type = ["anatomical"]
+#% data statistisc figure 7
+
+#features_to_compare = ["SNR Chang", "SNR Normal", "tSNR (Averaged Brain ROI)", "Displacement factor (std of Mutual information)"]
+features_to_compare = ["SNR Chang"]
+#features_to_compare = ["Vol"]
+
+for dd,data in enumerate(All_Data):
+    for feature in features_to_compare:
+        cc = 0
+        temp = pd.DataFrame()
+        Data_of_selected_feature = pd.DataFrame()
+        temp_data = pd.DataFrame()
+
+        for key in data:
+            try:
+                temp_data[feature] = data[key][All_type[dd]][feature]
+                temp_data["SpatRx"] = data[key][All_type[dd]]["SpatRx"]
+                temp_data["SpatRy"] = data[key][All_type[dd]]["SpatRy"]
+                temp_data["Slicethick"] = data[key][All_type[dd]]["Slicethick"]
+                
+            except KeyError:
+                continue
+            temp_data["Dataset"] = All_files[dd][cc].split(os.sep)[-3]
+            cc = cc +1
+            Data_of_selected_feature = pd.concat([Data_of_selected_feature, temp_data], ignore_index=True)
+            #Data_of_selected_feature2 = pd.concat([Data_of_selected_feature2, temp_data], ignore_index=True)
+            
+            
+            
+        if not Data_of_selected_feature.empty:
+            Data_of_selected_feature['sort'] = Data_of_selected_feature['Dataset'].str.extract('(\d+)', expand=True).astype(int)
+            Data_of_selected_feature = Data_of_selected_feature.sort_values('Dataset')
+            
+            if feature == "SNR Normal":
+                Data_of_selected_feature.rename(columns={"SNR Normal": "SNR-Standard (dB)"}, inplace=True)
+                feature = "SNR-Standard (dB)"
+            if feature == "SNR Chang":
+                Data_of_selected_feature.rename(columns={"SNR Chang": "SNR-Chang (dB)"}, inplace=True)
+                feature = "SNR-Chang (dB)"    
+            elif feature == "tSNR (Averaged Brain ROI)":
+                Data_of_selected_feature.rename(columns={"tSNR (Averaged Brain ROI)": "tSNR (dB)"}, inplace=True)
+                feature = "tSNR (dB)"
+            elif feature == "Displacement factor (std of Mutual information)":
+                Data_of_selected_feature.rename(columns={"Displacement factor (std of Mutual information)": "Motion severity (A.U)"}, inplace=True)
+                feature = "Motion severity (A.U)"
+            
+            
+            Data_of_selected_feature["Vol"] = Data_of_selected_feature["SpatRx"]*Data_of_selected_feature["SpatRy"]*Data_of_selected_feature["Slicethick"]
+            #Data_of_selected_feature = Data_of_selected_feature.sort_values("Dataset",ascending=False)
+            # creating boxplots
+            plt.figure(figsize=(4.5*cm,5*cm),dpi=300)
+            sns.set_style('ticks')
+            sns.set(font='Times New Roman', font_scale=0.8,style=None)  # Set font to Times New Roman and font size to 9
+            palette = 'pastel'
+            ax = sns.violinplot(x="Dataset", y=feature, data=Data_of_selected_feature, hue="Dataset", dodge=False,
+                                palette=palette,
+                                scale="width", inner=None,linewidth=1)
+            patches = ax.patches
+            #legend_colors = [patch.get_facecolor() for patch in patches[:]]
+
+            xlim = ax.get_xlim()
+            ylim = ax.get_ylim()
+            for violin in ax.collections:
+                bbox = violin.get_paths()[0].get_extents()
+                x0, y0, width, height = bbox.bounds
+                violin.set_clip_path(plt.Rectangle((x0, y0), width / 2, height, transform=ax.transData))
+            
+            sns.boxplot(x="Dataset", y=feature, data=Data_of_selected_feature, saturation=1, showfliers=False,
+                        width=0.3, boxprops={'zorder': 3, 'facecolor': 'none'}, ax=ax, linewidth=1)
+            old_len_collections = len(ax.collections)
+            sns.stripplot(x="Dataset", y=feature, data=Data_of_selected_feature,size=1.1, hue="Dataset", palette=palette, dodge=False, ax=ax)
+            for dots in ax.collections[old_len_collections:]:
+                dots.set_offsets(dots.get_offsets() + np.array([0.12, 0]))
+            ax.set_xlim(xlim)
+            ax.set_ylim(ylim)
+            ax.legend_.remove()
+            ax
+            ax.set_xticklabels(ax.get_xticklabels(), rotation=45,fontsize=8)
+            #ax.set_ylabel(ax.get_ylabels(),fontsize=8)
+# =============================================================================
+#             for label, color in zip(ax.get_xticklabels(), legend_colors):
+#                 label.set_color(color)
+# =============================================================================
+            ax.set_xlabel('')
+            ax.set_title("(b) Voxel size effect",weight='bold',fontsize=10)
+            y_label = ax.set_ylabel(ax.get_ylabel(),fontsize=8)
+
+# =============================================================================
+            ax.xaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            ax.xaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)
+# 
+            ax.yaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            ax.yaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)        
+# =============================================================================
+            ax.spines['top'].set_linewidth(0)  # Top border
+            ax.spines['right'].set_linewidth(0)  # Right border
+            ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+            ax.spines['left'].set_linewidth(0.5)  # Left border
+            ax.tick_params(axis='both', which='both', width=0.5,color='gray',length=2)
+            plt.xticks(ha='right')
+            #plt.savefig(os.path.join(Path,feature+"_"+All_type[dd]+".svg"), format='svg', bbox_inches='tight',transparent=False)
+            plt.show()
+                        
+            import pandas as pd
+            import scikit_posthocs as sp
+            from scipy.stats import kruskal
+
+            # reading CSV file
+            #dataset= pd.read_csv(r"C:\Users\arefk\OneDrive\Desktop\Projects\iris.csv")
+             
+            # data which contains sepal width of the three species
+            # data = [Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_Rei"]['Vol'],
+            #        Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_We"]['Vol'],
+            #        Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94c_m_As"]['Vol']]
+            
+            data = [Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_As"][feature],
+                    Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_Ce"][feature],
+                    Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_Je"][feature]]
+            
+            
+            
+                         
+            # Perform Kruskal-Wallis test
+            kruskal_statistic, p_value_kruskal = kruskal(*data,nan_policy="omit")
+            print(kruskal_statistic)
+            print(p_value_kruskal)
+            # using the posthoc_dunn() function
+            p_values= sp.posthoc_dunn(data, p_adjust = 'holm')
+             
+            print(p_values)
+                        
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            

outputs/code_for_figures/SameVSdifferentFS.py

@@ -0,0 +1,199 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+
+#% Function to read CSV files and store them in a dictionary
+
+cm = 1/2.54  # centimeters in inches
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        # Extract the data type from the file name (assuming the file name contains "anatomical", "structural", or "functional")
+        data_type = "anatomical" if "anatomical" in file else ("structural" if "structural" in file else "functional")
+        data_dict[ff] = {data_type:df} 
+            
+    return data_dict
+
+
+# Function for statistical comparison and plotting
+def compare_and_plot(data, column_name, group_column):
+    sns.set_palette("Set2")  # Use colors for color-blind people
+    plt.figure(figsize=(9*cm, 6*cm))
+    sns.boxplot(x=group_column, y=column_name, data=data)
+    plt.xlabel(group_column)
+    plt.ylabel(column_name)
+    plt.title(f"Statistical Comparison of {column_name} by {group_column}")
+    plt.tight_layout()
+    #plt.savefig(f"{column_name}_by_{group_column}_boxplot.png")
+    plt.show()
+
+#% List of CSV files for each data type
+Path = r"C:\Users\aswen\Documents\Data\2023_Kalantari_AIDAqc\outputs\validation\QC_SameVSdiffFS" #QC_Plot_VoxelsSize
+
+anatomical_files = glob.glob(os.path.join(Path,"**/*caculated_features_anatomical.csv"), recursive=True)
+structural_files = glob.glob(os.path.join(Path,"**/*caculated_features_structural.csv"), recursive=True)
+functional_files = glob.glob(os.path.join(Path,"**/*caculated_features_functional.csv"), recursive=True)
+
+All_files = [anatomical_files,structural_files,functional_files]
+
+# Read the CSV files and store them in dictionaries
+anatomical_data = read_csv_files(anatomical_files)
+structural_data = read_csv_files(structural_files)
+functional_data = read_csv_files(functional_files)
+All_Data = [anatomical_data]
+
+All_type = ["anatomical"]
+#% data statistisc figure 7
+
+#features_to_compare = ["SNR Chang", "SNR Normal", "tSNR (Averaged Brain ROI)", "Displacement factor (std of Mutual information)"]
+features_to_compare = ["SNR Chang"]
+#features_to_compare = ["Vol"]
+
+for dd,data in enumerate(All_Data):
+    for feature in features_to_compare:
+        cc = 0
+        temp = pd.DataFrame()
+        Data_of_selected_feature = pd.DataFrame()
+        temp_data = pd.DataFrame()
+
+        for key in data:
+            try:
+                temp_data[feature] = data[key][All_type[dd]][feature]
+                temp_data["SpatRx"] = data[key][All_type[dd]]["SpatRx"]
+                temp_data["SpatRy"] = data[key][All_type[dd]]["SpatRy"]
+                temp_data["Slicethick"] = data[key][All_type[dd]]["Slicethick"]
+                
+            except KeyError:
+                continue
+            temp_data["Dataset"] = All_files[dd][cc].split(os.sep)[-3]
+            cc = cc +1
+            Data_of_selected_feature = pd.concat([Data_of_selected_feature, temp_data], ignore_index=True)
+            #Data_of_selected_feature2 = pd.concat([Data_of_selected_feature2, temp_data], ignore_index=True)
+            
+            
+            
+        if not Data_of_selected_feature.empty:
+            Data_of_selected_feature['sort'] = Data_of_selected_feature['Dataset'].str.extract('(\d+)', expand=True).astype(int)
+            Data_of_selected_feature = Data_of_selected_feature.sort_values('sort')
+            
+            if feature == "SNR Normal":
+                Data_of_selected_feature.rename(columns={"SNR Normal": "SNR-Standard (dB)"}, inplace=True)
+                feature = "SNR-Standard (dB)"
+            if feature == "SNR Chang":
+                Data_of_selected_feature.rename(columns={"SNR Chang": "SNR-Chang (dB)"}, inplace=True)
+                feature = "SNR-Chang (dB)"    
+            elif feature == "tSNR (Averaged Brain ROI)":
+                Data_of_selected_feature.rename(columns={"tSNR (Averaged Brain ROI)": "tSNR (dB)"}, inplace=True)
+                feature = "tSNR (dB)"
+            elif feature == "Displacement factor (std of Mutual information)":
+                Data_of_selected_feature.rename(columns={"Displacement factor (std of Mutual information)": "Motion severity (A.U)"}, inplace=True)
+                feature = "Motion severity (A.U)"
+            
+            
+            Data_of_selected_feature["Vol"] = Data_of_selected_feature["SpatRx"]*Data_of_selected_feature["SpatRy"]*Data_of_selected_feature["Slicethick"]
+            #Data_of_selected_feature = Data_of_selected_feature.sort_values("Dataset",ascending=False)
+            # creating boxplots
+            plt.figure(figsize=(4.5*cm,5*cm),dpi=300)
+            sns.set_style('ticks')
+            sns.set(font='Times New Roman', font_scale=0.8,style=None)  # Set font to Times New Roman and font size to 9
+            palette = 'Set1'
+            ax = sns.violinplot(x="Dataset", y=feature, data=Data_of_selected_feature, hue="Dataset", dodge=False,
+                                palette=palette,
+                                scale="width", inner=None,linewidth=1)
+            patches = ax.patches
+            #legend_colors = [patch.get_facecolor() for patch in patches[:]]
+
+            xlim = ax.get_xlim()
+            ylim = ax.get_ylim()
+            for violin in ax.collections:
+                bbox = violin.get_paths()[0].get_extents()
+                x0, y0, width, height = bbox.bounds
+                violin.set_clip_path(plt.Rectangle((x0, y0), width / 2, height, transform=ax.transData))
+            
+            sns.boxplot(x="Dataset", y=feature, data=Data_of_selected_feature, saturation=1, showfliers=False,
+                        width=0.3, boxprops={'zorder': 3, 'facecolor': 'none'}, ax=ax, linewidth=1)
+            old_len_collections = len(ax.collections)
+            sns.stripplot(x="Dataset", y=feature, data=Data_of_selected_feature,size=1.1, hue="Dataset", palette=palette, dodge=False, ax=ax)
+            for dots in ax.collections[old_len_collections:]:
+                dots.set_offsets(dots.get_offsets() + np.array([0.12, 0]))
+            ax.set_xlim(xlim)
+            ax.set_ylim(15,50)
+            ax.legend_.remove()
+            ax
+            ax.set_xticklabels(ax.get_xticklabels(), rotation=45,fontsize=8)
+            #ax.set_ylabel(ax.get_ylabels(),fontsize=8)
+# =============================================================================
+#             for label, color in zip(ax.get_xticklabels(), legend_colors):
+#                 label.set_color(color)
+# =============================================================================
+            ax.set_xlabel('')
+            ax.set_title("(a) Field strength effect",weight='bold',fontsize=10)
+            y_label = ax.set_ylabel(ax.get_ylabel(),fontsize=8)
+
+# =============================================================================
+            ax.xaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            ax.xaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)
+# 
+            ax.yaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0.5)
+            ax.yaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0.5)        
+# =============================================================================
+            ax.spines['top'].set_linewidth(0)  # Top border
+            ax.spines['right'].set_linewidth(0)  # Right border
+            ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+            ax.spines['left'].set_linewidth(0.5)  # Left border
+            ax.tick_params(axis='both', which='both', width=0.5,color='gray',length=2)
+            plt.xticks(ha='right')
+            #plt.savefig(os.path.join(Path,feature+"_"+All_type[dd]+".svg"), format='svg', bbox_inches='tight',transparent=False)
+            plt.show()
+                        
+            import pandas as pd
+            import scikit_posthocs as sp
+            from scipy.stats import kruskal
+
+            # reading CSV file
+            #dataset= pd.read_csv(r"C:\Users\arefk\OneDrive\Desktop\Projects\iris.csv")
+             
+            # data which contains sepal width of the three species
+            # data = [Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_Rei"]['Vol'],
+            #        Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_We"]['Vol'],
+            #        Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94c_m_As"]['Vol']]
+            
+            data = [Data_of_selected_feature[Data_of_selected_feature['Dataset']=="7_rab_Mu"][feature],
+                    Data_of_selected_feature[Data_of_selected_feature['Dataset']=="94_m_We"][feature],
+                    Data_of_selected_feature[Data_of_selected_feature['Dataset']=="117_m_Vr"][feature]]
+            
+            
+            
+                         
+            # Perform Kruskal-Wallis test
+            kruskal_statistic, p_value_kruskal = kruskal(*data,nan_policy="omit")
+            print(kruskal_statistic)
+            print(p_value_kruskal)
+            # using the posthoc_dunn() function
+            p_values= sp.posthoc_dunn(data, p_adjust = 'holm')
+             
+            print(p_values)
+                        
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            
+            

outputs/code_for_figures/StructuralAllDataMerge.py

@@ -0,0 +1,47 @@
+import os
+import glob
+import pandas as pd
+
+# Step 1: Define the starting path and file pattern
+# Step 1: Define the starting path and file pattern
+start_path =  r"C:\Users\aswen\Desktop\Code\Validation3"
+file_pattern = '*diff*.csv'
+# Step 2: Find all matching CSV files in the specified directory and its subdirectories
+csv_files = glob.glob(os.path.join(start_path, '**', file_pattern), recursive=True)
+
+# Step 3: Initialize an empty DataFrame to store the extracted data
+combined_df = pd.DataFrame()
+
+# Step 4: Loop through the CSV files and extract the specified columns
+for csv_file in csv_files:
+    try:
+        df = pd.read_csv(csv_file)
+        selected_columns = ["FileAddress", "SNR Chang", "SNR Normal", "Displacement factor (std of Mutual information)", "Goasting"]
+        df = df[selected_columns]
+        df["dataset"] = csv_file.split(os.sep)[-3]
+        # Concatenate the current DataFrame with the combined DataFrame
+        combined_df = pd.concat([combined_df, df], ignore_index=True)
+    except Exception as e:
+        print(f"Error reading {csv_file}: {e}")
+
+# Step 5: Calculate the mean and standard deviation for SNR Chang, SNR Normal, and Displacement factor
+mean_snr_chang = combined_df["SNR Chang"].mean()
+std_snr_chang = combined_df["SNR Chang"].std()
+
+mean_snr_normal = combined_df["SNR Normal"].mean()
+std_snr_normal = combined_df["SNR Normal"].std()
+
+mean_displacement_factor = combined_df["Displacement factor (std of Mutual information)"].mean()
+std_displacement_factor = combined_df["Displacement factor (std of Mutual information)"].std()
+
+# Print the results
+print(f"Mean SNR Chang: {mean_snr_chang}")
+print(f"Standard Deviation SNR Chang: {std_snr_chang}")
+print(f"Mean SNR Normal: {mean_snr_normal}")
+print(f"Standard Deviation SNR Normal: {std_snr_normal}")
+print(f"Mean Displacement Factor: {mean_displacement_factor}")
+print(f"Standard Deviation Displacement Factor: {std_displacement_factor}")
+
+# Optionally, you can save the combined DataFrame to a CSV file
+p = r"C:\Users\aswen\Desktop\Code\AIDAqc_Figures\input"
+combined_df.to_csv(os.path.join(p,'combined_data_diff.csv'), index=False)

outputs/code_for_figures/ViolinPlots.py

@@ -0,0 +1,171 @@
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+import seaborn as sns
+from scipy.stats import ttest_ind, f_oneway
+import glob
+import os
+from scipy.stats import ttest_ind
+from statsmodels.stats.multitest import multipletests
+import re
+
+#% Function to read CSV files and store them in a dictionary
+script_dir = os.path.dirname(__file__)
+out_path = os.path.join(script_dir, '..', 'figures')
+cm = 1/2.54  # centimeters in inches
+def read_csv_files(files):
+    data_dict = {}
+    for ff,file in enumerate(files):
+        df = pd.read_csv(file)    
+        # Extract the data type from the file name (assuming the file name contains "anatomical", "structural", or "functional")
+        data_type = "anatomical" if "anatomical" in file else ("structural" if "structural" in file else "functional")
+        data_dict[ff] = {data_type:df} 
+            
+    return data_dict
+
+
+# Function for statistical comparison and plotting
+def compare_and_plot(data, column_name, group_column):
+    sns.set_palette("Set2")  # Use colors for color-blind people
+    plt.figure(figsize=(9*cm, 6*cm))
+    sns.boxplot(x=group_column, y=column_name, data=data)
+    plt.xlabel(group_column)
+    plt.ylabel(column_name)
+    plt.title(f"Statistical Comparison of {column_name} by {group_column}")
+    plt.tight_layout()
+    #plt.savefig(f"{column_name}_by_{group_column}_boxplot.png")
+    plt.show()
+
+#% List of CSV files for each data type
+Path = r"C:\Users\aswen\Desktop\Code\Validation2"
+
+anatomical_files = glob.glob(os.path.join(Path,"*/*/*caculated_features_anatomical.csv"), recursive=True)
+structural_files = glob.glob(os.path.join(Path,"*/*/*caculated_features_structural.csv"), recursive=True)
+functional_files = glob.glob(os.path.join(Path,"*/*/*caculated_features_functional.csv"), recursive=True)
+
+All_files = [anatomical_files,structural_files,functional_files]
+
+# Read the CSV files and store them in dictionaries
+anatomical_data = read_csv_files(anatomical_files)
+structural_data = read_csv_files(structural_files)
+functional_data = read_csv_files(functional_files)
+All_Data = [anatomical_data,structural_data,functional_data]
+
+All_type = ["anatomical","structural","functional"]
+#% data statistisc figure 7
+BINS = [8,8,8]
+features_to_compare = ["SNR Chang", "SNR Normal", "tSNR (Averaged Brain ROI)", "Displacement factor (std of Mutual information)"]
+
+#features_to_compare = ["SpatRx", "SpatRy", "Slicethick"]
+
+
+Data_of_selected_feature2 = pd.DataFrame()
+for dd,data in enumerate(All_Data):
+    for feature in features_to_compare:
+        cc = 0
+        temp = pd.DataFrame()
+        Data_of_selected_feature = pd.DataFrame()
+        temp_data = pd.DataFrame()
+
+        for key in data:
+            try:
+                temp_data[feature] = data[key][All_type[dd]][feature]
+            except KeyError:
+                continue
+            temp_data["Dataset"] = All_files[dd][cc].split(os.sep)[-3]
+            cc = cc +1
+            Data_of_selected_feature = pd.concat([Data_of_selected_feature, temp_data], ignore_index=True)
+            #Data_of_selected_feature2 = pd.concat([Data_of_selected_feature2, temp_data], ignore_index=True)
+            
+            
+            
+        if not Data_of_selected_feature.empty:
+            Data_of_selected_feature['sort'] = Data_of_selected_feature['Dataset'].str.extract('(\d+)', expand=True).astype(int)
+            Data_of_selected_feature = Data_of_selected_feature.sort_values('sort')
+            
+            if feature == "SNR Normal":
+                Data_of_selected_feature.rename(columns={"SNR Normal": "SNR-Standard (dB)"}, inplace=True)
+                feature = "SNR-Standard (dB)"
+            if feature == "SNR Chang":
+                Data_of_selected_feature.rename(columns={"SNR Chang": "SNR-Chang (dB)"}, inplace=True)
+                feature = "SNR-Chang (dB)"    
+            elif feature == "tSNR (Averaged Brain ROI)":
+                Data_of_selected_feature.rename(columns={"tSNR (Averaged Brain ROI)": "tSNR (dB)"}, inplace=True)
+                feature = "tSNR (dB)"
+            elif feature == "Displacement factor (std of Mutual information)":
+                Data_of_selected_feature.rename(columns={"Displacement factor (std of Mutual information)": "Motion severity (a.u)"}, inplace=True)
+                BINS[dd] = 10
+                feature = "Motion severity (a.u)"
+            
+            
+            #Data_of_selected_feature2["Vol"] = Data_of_selected_feature2["SpatRx"]*Data_of_selected_feature2["SpatRy"]*Data_of_selected_feature2["Slicethick"]
+            
+            #Data_of_selected_feature = Data_of_selected_feature.sort_values("Dataset",ascending=False)
+            # creating boxplots
+            if All_type[dd] == "anatomical":
+                plt.figure(figsize=(21.3*cm,3.527*cm),dpi=600)
+            else:
+                plt.figure(figsize=(9.70*cm,3.527*cm),dpi=600)
+                
+            sns.set_style('ticks')
+            sns.set(font='Times New Roman',style=None)  # Set font to Times New Roman and font size to 9
+            palette = 'Set2'
+            ax = sns.violinplot(x="Dataset", y=feature, data=Data_of_selected_feature, hue="Dataset", dodge=False,
+                                palette=palette,
+                                scale="width", inner=None,linewidth=1)
+            patches = ax.patches
+            #legend_colors = [patch.get_facecolor() for patch in patches[:]]
+
+            xlim = ax.get_xlim()
+            ylim = ax.get_ylim()
+            for violin in ax.collections:
+                bbox = violin.get_paths()[0].get_extents()
+                x0, y0, width, height = bbox.bounds
+                violin.set_clip_path(plt.Rectangle((x0, y0), width / 2, height, transform=ax.transData))
+            
+            sns.boxplot(x="Dataset", y=feature, data=Data_of_selected_feature, saturation=1, showfliers=False,
+                        width=0.3, boxprops={'zorder': 3, 'facecolor': 'none'}, ax=ax, linewidth=1)
+            old_len_collections = len(ax.collections)
+            sns.stripplot(x="Dataset", y=feature, data=Data_of_selected_feature,size=1.1, hue="Dataset", palette=palette, dodge=False, ax=ax)
+            for dots in ax.collections[old_len_collections:]:
+                dots.set_offsets(dots.get_offsets() + np.array([0.12, 0]))
+            ax.set_xlim(xlim)
+            ax.set_ylim(ylim)
+            ax.legend_.remove()
+            ax.locator_params(axis='y', nbins=BINS[dd])  # Set the number of ticks for the y-axis
+            
+            ax
+            ax.set_xticklabels(ax.get_xticklabels(), rotation=45,fontsize=8)
+            ax.set_yticklabels(ax.get_yticklabels(),fontsize=8)
+            
+            
+            #ax.set_yticks(np.linspace(ax.get_ylim()[0], ax.get_ylim()[1], BINS[dd]))
+
+
+# =============================================================================
+#             for label, color in zip(ax.get_xticklabels(), legend_colors):
+#                 label.set_color(color)
+# =============================================================================
+            ax.set_xlabel('')
+            ax.set_title(All_type[dd].capitalize(),weight='bold',fontsize=10)
+            y_label = ax.set_ylabel(ax.get_ylabel(),fontsize=8)
+
+# =============================================================================
+            ax.xaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0)
+            ax.xaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0)
+# 
+            ax.yaxis.grid(True, linestyle='-', which='major', color='gray', linewidth=0)
+            ax.yaxis.grid(True, linestyle='--', which='minor', color='gray', linewidth=0)        
+# =============================================================================
+            ax.spines['top'].set_linewidth(0.5)  # Top border
+            ax.spines['right'].set_linewidth(0.5)  # Right border
+            ax.spines['bottom'].set_linewidth(0.5)  # Bottom border
+            ax.spines['left'].set_linewidth(0.5)  # Left border
+                        # Set axis ticks font and number of ticks
+            ax.tick_params(axis='both', which='both', width=0.5, color='gray', length=2)
+            
+            ax.tick_params(axis='both', which='both', width=0.5,color='gray',length=2)
+            plt.xticks(ha='right')
+            plt.savefig(os.path.join(out_path,feature+"_"+All_type[dd]+".svg"), format='svg', bbox_inches='tight',transparent=False)
+            plt.savefig(os.path.join(out_path,feature+"_"+All_type[dd]+".png"), format='png', bbox_inches='tight',transparent=False)
+            plt.show()

outputs/code_for_figures/fleissKappa_HumanRaters.py

@@ -0,0 +1,164 @@
+import pandas as pd
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+import os
+# Read the CSV file
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+csv_path = os.path.join(file_path,"combined_Human_Voters_from_votings3_Final.csv")
+df = pd.read_csv(csv_path)
+
+# Extract relevant columns
+path_column = "Path"
+validation_columns = ["validation_adam", "validation_giovanna", "validation_joanes", "validation_pboehmsturm", "validation_Susanne"]
+dataset_column = "dataset_name"
+
+
+
+# Create a DataFrame for ratings
+data = {
+    'Path': df[path_column],
+    'Dataset': df[dataset_column],
+    'SequenceType': df['SequenceType'],
+    'Methods': df[validation_columns].apply(tuple, axis=1)
+}
+ratings_df = pd.DataFrame(data)
+
+# Define sequence types
+sequence_types = ["anat", "func", "diff"]
+
+results_per_sequence_type = {}
+
+for seq_type in sequence_types:
+    seq_type_ratings = ratings_df[ratings_df['SequenceType']==seq_type]
+    unique_datasets = seq_type_ratings['Dataset'].unique()
+
+    results_per_dataset = {}
+
+    for dataset in unique_datasets:
+        dataset_ratings = seq_type_ratings[seq_type_ratings['Dataset'] == dataset]
+        try:
+            # Convert the ratings data to a format expected by fleiss_kappa
+            ratings_matrix, _ = aggregate_raters(list(dataset_ratings['Methods'].values), n_cat=None)
+
+            # Calculate Fleiss' Kappa
+            kappa = fleiss_kappa(ratings_matrix, method='fleiss')
+            results_per_dataset[dataset] = kappa
+        except ValueError:
+            print(f"\nValue Error: Fleiss' Kappa could not be calculated for dataset {dataset} and sequence type {seq_type}.")
+
+    results_per_sequence_type[seq_type] = results_per_dataset
+
+# Save results to CSV file
+output_csv_path = os.path.join(file_path,"fleiss_kappa_results_Human_Voters.csv")
+output_data = []
+
+for seq_type, datasets in results_per_sequence_type.items():
+    for dataset, kappa in datasets.items():
+        output_data.append({'SequenceType': seq_type, 'Dataset': dataset, 'FleissKappa': kappa})
+
+output_df = pd.DataFrame(output_data)
+output_df.to_csv(output_csv_path, index=False)
+
+print(f"\nResults saved to: {output_csv_path}")
+
+
+#%% Create Heat map of it
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import matplotlib.ticker as ticker
+import os
+# Load your Excel file into a Pandas DataFrame
+filtered_df = output_df
+filtered_df.loc[filtered_df["SequenceType"] == "anat","SequenceType"] = "Anatomical"
+filtered_df.loc[filtered_df["SequenceType"] == "diff","SequenceType"] = "Diffusion"
+filtered_df.loc[filtered_df["SequenceType"] == "func","SequenceType"] = "Functional"
+
+sns.set_style("darkgrid")
+cm = 1/2.54  # centimeters in inches
+# Create two subplots for Kappa and F1 scores heatmaps
+fig, axes = plt.subplots(1, 1, figsize=(20*cm, 4*cm), dpi=300)
+#sns.set_style('whitegrid')
+# Specify the font properties
+font_properties = fm.FontProperties(family='Times New Roman', size=8)
+font_properties2 = fm.FontProperties(family='Times New Roman', size=10)
+Title = ["(a) Fleiss kappa score: inter rater reliability "," (b) F1_score: raters vs AIDAqc"]
+for i, metric in enumerate(['FleissKappa']):
+    ax = axes
+    pivot_df = filtered_df.pivot(index='SequenceType', columns='Dataset', values=metric)
+    pivot_df['mean'] = pivot_df.mean(axis=1)
+    pivot_df['std'] = pivot_df.std(axis=1)
+    t=Title[i]
+    sns.heatmap(pivot_df, annot=True, fmt=".1f", cmap="YlGnBu", cbar=True, ax=ax)
+
+    # Set font properties for labels, titles, and annotations
+    ax.set_xticklabels(ax.get_xticklabels(), fontproperties=font_properties)
+    ax.set_yticklabels(ax.get_yticklabels(), fontproperties=font_properties)
+    ax.set_xlabel('Datasets', fontproperties=font_properties)
+    ax.set_ylabel('Sequences', fontproperties=font_properties)
+    #ax.set_title(f'{t} ', fontsize=10, fontproperties=font_properties2, fontweight='bold')
+    ax.set(xlabel=None)
+    # Set the color bar legend font size and font properties
+    cbar = ax.collections[0].colorbar
+    cbar.ax.tick_params(labelsize=8)
+    cbar.ax.set_yticklabels(cbar.ax.get_yticklabels(), fontproperties=font_properties)
+    
+    # Customize the color bar ticks (increase the number of ticks)
+    cbar.ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=5))
+    
+    a = []
+    for text in ax.texts:
+        a.append(float(text.get_text()))
+        
+    
+
+    for text in ax.texts:
+        text.set_font("Times New Roman")
+        text.set_size(8)
+        
+        
+# Show the plots
+plt.tight_layout()
+
+
+# Save the figure as PNG and SVG
+fig_path_png = os.path.join(out_path, 'FleissKappaManualRatersInterraterVariability.png')
+fig_path_svg = os.path.join(out_path, 'FleissKappaManualRatersInterraterVariability.svg')
+
+fig.savefig(fig_path_png, format='png', bbox_inches='tight')
+fig.savefig(fig_path_svg, format='svg', bbox_inches='tight')
+plt.show()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+

outputs/code_for_figures/fleissKappa_MLraters.py

@@ -0,0 +1,140 @@
+import pandas as pd
+from statsmodels.stats.inter_rater import fleiss_kappa
+from statsmodels.stats.inter_rater import aggregate_raters
+import os
+import numpy
+
+# Read data from the CSV file
+script_dir = os.path.dirname(__file__)
+file_path = os.path.join(script_dir, '..', 'input')
+out_path = os.path.join(script_dir, '..', 'figures')
+
+# Read the CSV file
+df = pd.read_csv(os.path.join(file_path,"combined_votings3.csv"))
+
+# Extract relevant columns
+sequence_name_column = "SequenceType"
+dataset_column = "dataset"
+corresponding_img_column = "corresponding_img"
+method_columns = ["One_class_SVM", "IsolationForest", "LocalOutlierFactor", " EllipticEnvelope", "statistical_method"]
+
+sequence_name = df[sequence_name_column]
+dataset_names = df[dataset_column]
+corresponding_imgs = df[corresponding_img_column]
+methods_data = df[list(method_columns)]
+
+# Create a DataFrame for ratings
+data = {
+    sequence_name_column: sequence_name,
+    'Dataset': dataset_names,
+    'CorrespondingImg': corresponding_imgs,
+    'Methods': methods_data.apply(tuple, axis=1)
+}
+ratings_df = pd.DataFrame(data)
+
+# Define sequence types
+sequence_types = ["anat", "func", "diff"]
+
+results_per_sequence_type = {}
+
+for seq_type in sequence_types:
+    seq_type_ratings = ratings_df[ratings_df['CorrespondingImg'].str.startswith(seq_type)]
+    unique_datasets = seq_type_ratings['Dataset'].unique()
+
+    results_per_dataset = {}
+
+    for dataset in unique_datasets:
+        dataset_ratings = seq_type_ratings[seq_type_ratings['Dataset'] == dataset]
+        try:
+            # Convert the ratings data to a format expected by fleiss_kappa
+            ratings_matrix, _ = aggregate_raters(list(dataset_ratings['Methods'].values), n_cat=None)
+
+            # Calculate Fleiss' Kappa
+            kappa = fleiss_kappa(ratings_matrix, method='fleiss')
+            results_per_dataset[dataset] = kappa
+        except ValueError:
+            print(f"\nValue Error: Fleiss' Kappa could not be calculated for dataset {dataset} and sequence type {seq_type}.")
+
+    results_per_sequence_type[seq_type] = results_per_dataset
+
+# Save results to CSV file
+output_csv_path = os.path.join(file_path, "fleiss_kappa_results_ML_Voters.csv")
+output_data = []
+
+for seq_type, datasets in results_per_sequence_type.items():
+    for dataset, kappa in datasets.items():
+        output_data.append({'SequenceType': seq_type, 'Dataset': dataset, 'FleissKappa': kappa})
+
+output_df = pd.DataFrame(output_data)
+output_df.to_csv(output_csv_path, index=False)
+
+print(f"\nResults saved to: {output_csv_path}")
+#%% Create Heat map of it
+import pandas as pd
+import seaborn as sns
+import matplotlib.pyplot as plt
+import matplotlib.font_manager as fm
+import matplotlib.ticker as ticker
+import os
+# Load your Excel file into a Pandas DataFrame
+filtered_df = output_df
+filtered_df.loc[filtered_df["SequenceType"] == "anat","SequenceType"] = "Anatomical"
+filtered_df.loc[filtered_df["SequenceType"] == "diff","SequenceType"] = "Diffusion"
+filtered_df.loc[filtered_df["SequenceType"] == "func","SequenceType"] = "Functional"
+
+sns.set_style("darkgrid")
+cm = 1/2.54  # centimeters in inches
+# Create two subplots for Kappa and F1 scores heatmaps
+fig, axes = plt.subplots(1, 1, figsize=(20*cm, 5*cm), dpi=300)
+#sns.set_style('whitegrid')
+# Specify the font properties
+font_properties = fm.FontProperties(family='Times New Roman', size=8)
+font_properties2 = fm.FontProperties(family='Times New Roman', size=10)
+Title = ["(a) Fleiss kappa score: inter rater reliability "," (b) F1_score: raters vs AIDAqc"]
+for i, metric in enumerate(['FleissKappa']):
+    ax = axes
+    pivot_df = filtered_df.pivot(index='SequenceType', columns='Dataset', values=metric)
+    pivot_df['mean'] = pivot_df.mean(axis=1)
+    pivot_df['std'] = pivot_df.std(axis=1)
+    t=Title[i]
+    sns.heatmap(pivot_df, annot=True, fmt=".1f", cmap="YlGnBu", cbar=True, ax=ax)
+
+    # Set font properties for labels, titles, and annotations
+    ax.set_xticklabels(ax.get_xticklabels(), fontproperties=font_properties)
+    ax.set_yticklabels(ax.get_yticklabels(), fontproperties=font_properties)
+    ax.set_xlabel('Datasets', fontproperties=font_properties)
+    ax.set_ylabel('Sequences', fontproperties=font_properties)
+    #ax.set_title(f'{t} ', fontsize=10, fontproperties=font_properties2, fontweight='bold')
+    ax.set(xlabel=None)
+    # Set the color bar legend font size and font properties
+    cbar = ax.collections[0].colorbar
+    cbar.ax.tick_params(labelsize=8)
+    cbar.ax.set_yticklabels(cbar.ax.get_yticklabels(), fontproperties=font_properties)
+    
+    # Customize the color bar ticks (increase the number of ticks)
+    cbar.ax.yaxis.set_major_locator(ticker.MaxNLocator(nbins=5))
+    
+    a = []
+    for text in ax.texts:
+        a.append(float(text.get_text()))
+        
+    
+
+    for text in ax.texts:
+        text.set_font("Times New Roman")
+        text.set_size(8)
+        
+        
+# Show the plots
+plt.tight_layout()
+
+
+# Save the figure as PNG and SVG
+fig_path_png = os.path.join(out_path, 'FleissKappa_AIDAqc_Raters_InterraterVariability.png')
+fig_path_svg = os.path.join(out_path, 'FleissKappa_AIDAqc_Raters_InterraterVariability.svg')
+
+fig.savefig(fig_path_png, format='png', bbox_inches='tight')
+fig.savefig(fig_path_svg, format='svg', bbox_inches='tight')
+plt.show()
+
+

outputs/code_for_figures/tSNRWorstVSBest.py

@@ -0,0 +1,80 @@
+import os
+import nibabel as nib
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+
+# Define the directory path where the NIfTI and PNG files are located
+path = r"C:\Users\aswen\Desktop\Code\2023_Kalantari_AIDAqc\outputs\files_4figs"  # Replace with your actual directory path
+
+# File names for NIfTI images
+tsnr_best_filename = "TSNRmap_best.nii.gz"
+tsnr_worst_filename = "TSNRmap_worst.nii.gz"
+
+# Load the NIfTI files
+tsnr_best = nib.load(os.path.join(path, tsnr_best_filename))
+tsnr_worst = nib.load(os.path.join(path, tsnr_worst_filename))
+
+# Get the NIfTI data as NumPy arrays
+tsnr_best_data = tsnr_best.get_fdata()
+tsnr_worst_data = tsnr_worst.get_fdata()
+
+# Rotate the data arrays 90 degrees counterclockwise
+tsnr_best_data = np.rot90(tsnr_best_data, k=-1)
+tsnr_worst_data = np.roll(tsnr_worst_data, shift=5, axis=1)
+tsnr_worst_data = np.rot90(tsnr_worst_data, k=1)
+
+# Determine a common color scale range for both rotated images
+vmin = min(tsnr_best_data.min(), tsnr_worst_data.min())
+vmax = max(tsnr_best_data.max(), tsnr_worst_data.max())
+
+# Load the PNG images
+tsnr_best_png = plt.imread(os.path.join(path, "TSNRbest.png"))
+tsnr_worst_png = plt.imread(os.path.join(path, "TSNRworst.png"))
+
+# Create a figure with a 2x2 grid
+cm = 1/2.54  # centimeters in inches
+fig, axes = plt.subplots(2, 2, figsize=(9*cm, 7*cm), dpi=300)
+
+# Define the width of the images
+image_width = 0.39
+
+# Create the axes for the "best" tSNR NIfTI image
+ax0 = axes[1, 0]
+im0 = ax0.imshow(tsnr_best_data[:, :, tsnr_best_data.shape[2] // 2], cmap='viridis', origin='lower', vmin=vmin, vmax=vmax)
+ax0.set_title("tSNR-map (best)", fontsize=10, fontweight='bold', fontname='Times New Roman')
+ax0.set_xticks([])
+ax0.set_yticks([])
+
+# Create the axes for the "worst" tSNR NIfTI image
+ax1 = axes[1, 1]
+im1 = ax1.imshow(tsnr_worst_data[:, :, tsnr_worst_data.shape[2] // 2], cmap='viridis', origin='lower', vmin=vmin, vmax=vmax)
+ax1.set_title("tSNR-map (worst)", fontsize=10, fontweight='bold', fontname='Times New Roman')
+ax1.set_xticks([])
+ax1.set_yticks([])
+
+# Create the axes for the "best" tSNR PNG image
+ax2 = axes[0, 0]
+ax2.imshow(tsnr_best_png)
+ax2.set_title("rsfMR image", fontsize=10, fontweight='bold', fontname='Times New Roman')
+ax2.set_xticks([])
+ax2.set_yticks([])
+
+# Create the axes for the "worst" tSNR PNG image
+ax3 = axes[0, 1]
+ax3.imshow(tsnr_worst_png)
+ax3.set_title("rsfMR image", fontsize=10, fontweight='bold', fontname='Times New Roman')
+ax3.set_xticks([])
+ax3.set_yticks([])
+
+# Add a single colorbar for the NIfTI images with more ticks
+divider = make_axes_locatable(ax1)
+cax = divider.append_axes("right", size="5%", pad=0.05)
+cbar = plt.colorbar(im1, cax=cax)  # Adjust the number of ticks (11 in this case)
+cbar.ax.tick_params(labelsize=8)
+
+# Adjust the layout for better visibility
+plt.tight_layout()
+
+# Display the figure
+plt.show()