add pipeline files

WORKFLOW/SECTIONS/base/02_experimental_phase/check_pipeline.ipynb

@@ -0,0 +1,100 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# 実験パイプラインのチェック\n",
+    "\n",
+    "これは実験パイプライン構成をチェックするためのノートブックです。  "
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 1. パイプライン定義ファイルのチェック\n",
+    "\n",
+    "- このステップでは、パイプライン定義ファイルとパッケージ群の情報に相違がないか確認します。\n",
+    "- 以下のセルを実行し、エラーが起きないことを確認してください。\n",
+    "- エラーが起きた場合は、[pipeline.json](../../../../experiments/pipeline.json)を修正＆エラーが起きなくなるまで再実行を繰り返してください。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import json\n",
+    "import glob\n",
+    "\n",
+    "# 作業ディレクトリに移動\n",
+    "%cd ~/experiments/\n",
+    "\n",
+    "# 実験パイプライン定義ファイルの読み込み\n",
+    "pipeline = json.load(open(\"pipeline.json\"))\n",
+    "print(\"実験パイプライン：\" + str(pipeline))\n",
+    "\n",
+    "# 実験パッケージ群の読み込み\n",
+    "files = glob.glob(\"./*/\")\n",
+    "\n",
+    "experiments = []\n",
+    "for file in files:\n",
+    "    experiments += [os.path.basename(file[:-1])]\n",
+    "print(\"実験パッケージ　：\" + str(experiments))\n",
+    "\n",
+    "# 元のディレクトリに移動\n",
+    "%cd -\n",
+    "\n",
+    "# 過不足の確認\n",
+    "if set(pipeline) != set(experiments):\n",
+    "    raise(\"実験パイプラインと実験パッケージ間で相違がある\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## 2. 変更内容を書き戻す"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import papermill as pm\n",
+    "\n",
+    "pm.execute_notebook(\n",
+    "    '../util/datalad_save_push.ipynb',\n",
+    "    '-',\n",
+    "    parameters = dict(SAVE_MESSAGE = 'パイプライン構築', PATH = [未定])\n",
+    ")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

WORKFLOW/SECTIONS/base/02_experimental_phase/run_pipeline.ipynb

@@ -0,0 +1,119 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "79b4d068-3b7a-4717-aa03-8a077b34af9d",
+   "metadata": {},
+   "source": [
+    "# 実験パイプラインの実行\n",
+    "\n",
+    "これは実験パイプラインを実行するためのノートブックです。"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "20f2d8a2-3503-4702-b16c-6701eb3dc7ba",
+   "metadata": {},
+   "source": [
+    "## 1. 実験パイプライン情報の取得\n",
+    "\n",
+    "- このステップでは、実行する実験パイプラインの定義情報を取得します。\n",
+    "- 以下のセルを実行してください。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "208579b9-070c-449d-bc26-a152ff52e400",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "# 作業ディレクトリに移動\n",
+    "%cd ~/experiments/\n",
+    "\n",
+    "# 実験パイプライン定義ファイルの読み込み\n",
+    "pipeline = json.load(open(\"pipeline.json\"))\n",
+    "print(\"実験パイプライン：\" + str(pipeline))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "aafc65db-adf6-4ec5-931d-6bae110ec306",
+   "metadata": {},
+   "source": [
+    "## 2. 実験パイプラインの実行\n",
+    "\n",
+    "- このステップでは、定義情報に基づいて実験パイプラインを実行します。\n",
+    "- 以下のセルを実行してください。"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "39552e78-48cb-44cf-86d1-cf3d759f00a9",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import papermill\n",
+    "\n",
+    "outputs = []\n",
+    "\n",
+    "for experiment in pipeline:\n",
+    "    print(\"\\n============================================================ \" + experiment)\n",
+    "\n",
+    "    print(\"\\n# 出力ディレクトリの初期化\")\n",
+    "    cmd = \"rm -rf \" + experiment + \"/output_data/*\"\n",
+    "    print(\"$ \" + cmd)\n",
+    "    !$cmd\n",
+    "    \n",
+    "    print(\"\\n前実験の出力データを当実験の入力データとして複製\")\n",
+    "    for output in outputs:\n",
+    "        cmd = \"cp -rf \" + output + \"/output_data/* \" + experiment + \"/input_data/\" \n",
+    "        print(\"$ \" + cmd)\n",
+    "        !$cmd\n",
+    "\n",
+    "    print(\"\\n実験再現のための環境構築\")\n",
+    "    cmd = \"pip install -r \" + experiment + \"/requirements.txt\"\n",
+    "    print(\"$ \" + cmd)\n",
+    "    !$cmd\n",
+    "\n",
+    "    print(\"\\n実験ソースコードの実行\")\n",
+    "    cmd = experiment + \"/source/\"\n",
+    "    print(\"$ cd \" + cmd)\n",
+    "    %cd $cmd\n",
+    "\n",
+    "    papermill.execute_notebook(\"main.ipynb\", \"log.ipynb\")\n",
+    "    \n",
+    "    cmd = \"-\"\n",
+    "    print(\"$ cd \" + cmd)\n",
+    "    %cd $cmd\n",
+    "\n",
+    "    print(\"\\n次実験に備えて本実験の出力データを登録\")\n",
+    "    outputs += [experiment]"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}