2023_Kalantari_AIDAqc_old/code/RunAllDatasets/datasets_all.py

import os
import pandas as pd
from concurrent.futures import ThreadPoolExecutor

# Function to execute the ParsingData.py script
def execute_parsing_data(input_value, output_value, type_value):
    command = f"python ParsingData.py -i {input_value} -o {output_value} -f {type_value}"
    os.system(command)

# Load the dataset from the XLSX file using openpyxl engine
dataset = pd.read_excel(r"C:\Users\aswen\Desktop\Code\2023_Kalantari_AIDAqc\outputs\files_4figs\datasetrun.xlsx", engine="openpyxl")

# Determine the number of available processors
num_processors = os.cpu_count() or 1  # Use at least 1 if os.cpu_count() returns None

# Create a ThreadPoolExecutor with the determined number of threads
with ThreadPoolExecutor(max_workers=num_processors) as executor:
    # Iterate through the rows of the dataset and submit tasks to the executor
    for index, row in dataset.iterrows():
        input_value = row["input"]
        output_value = row["output"]
        type_value = row["type"]

        # Submit the task to the ThreadPoolExecutor
        executor.submit(execute_parsing_data, input_value, output_value, type_value)