{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# ようこそ\n", "\n", "maDMPへようこそ。ここでは、DMPに入力いただいた研究分野等情報に基づき、研究活動の支援、およびデータ管理品質向上のためにデータガバナンス機能が提供するワークフローを最適化します。\n", "\n", "![maDMP_to_workflow](https://raw.githubusercontent.com/NII-DG/maDMP-template/main/images/maDMP_to_workflow.jpg)\n", "\n", "このmaDMPには、データガバナンス機能で作成いただいたDMPの情報の中で、ワークフローの最適化に必要な情報が既に埋め込まれています。以下を順番に実行することで、システムがDMP情報を活用した最適化処理を行います。" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "deletable": false, "editable": false, "run_through_control": { "frozen": true }, "scrolled": true }, "outputs": [], "source": [ "# DMP情報\n", "field = 'basic'\n", "dataSize = '1GB'\n", "datasetStructure = 'RCOS_with_code'" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "deletable": false, "editable": false, "run_through_control": { "frozen": true }, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cloning into '/home/jovyan/WORKFLOW'...\n", "remote: Enumerating objects: 3964, done.\u001b[K\n", "remote: Counting objects: 100% (445/445), done.\u001b[K\n", "remote: Compressing objects: 100% (211/211), done.\u001b[K\n", "remote: Total 3964 (delta 238), reused 421 (delta 233), pack-reused 3519\u001b[K\n", "Receiving objects: 100% (3964/3964), 51.95 MiB | 9.39 MiB/s, done.\n", "Resolving deltas: 100% (1813/1813), done.\n" ] } ], "source": [ "# ワークフローテンプレート取得\n", "clone_url = 'https://github.com/NII-DG/workflow-template'+'.git'\n", "workflow_path = '/home/jovyan/WORKFLOW'\n", "!git clone $clone_url $workflow_path\n", "!rm -rf $workflow_path/.git" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "deletable": false, "editable": false, "run_through_control": { "frozen": true }, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "最適化処理が完了しました。次の手順へお進みください。\n" ] } ], "source": [ "# dmp.jsonに\"fields\"プロパティがある想定\n", "import os\n", "import glob\n", "import shutil\n", "\n", "path_flows = os.path.join('WORKFLOW', 'FLOW')\n", "\n", "templates = glob.glob(os.path.join(path_flows, '**'), recursive=True)\n", "\n", "# 選択外の分野のセクション群を削除\n", "for tmpl in templates:\n", " file = os.path.basename(tmpl)\n", " if not os.path.isdir(tmpl) and os.path.splitext(file)[1] == '.ipynb':\n", " if 'base_' not in file and field not in file:\n", " os.remove(tmpl)\n", " \n", "print('最適化処理が完了しました。次の手順へお進みください。')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": { "deletable": false, "editable": false, "run_through_control": { "frozen": true } }, "outputs": [], "source": [ "# モニタリング準備\n", "import os\n", "import json\n", "\n", "path_params = os.path.join('WORKFLOW', 'FLOW', 'param_files', 'monitoring_params.json')\n", "monitoring_dict = {\n", " 'dataSize': dataSize,\n", " 'datasetStructure': datasetStructure\n", " # モニタリング項目を追加する場合は以下に追記してください\n", " # e.g. 'dirStructure': 'withCode'\n", "}\n", "\n", "with open(path_params, 'w') as f:\n", " json.dump(monitoring_dict, f, indent=4)\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# ワークフロー全体の関連図と目次を生成する\n", "\n", "次に、最適化したワークフロー全体が俯瞰できる図を生成します。 \n", "以下を実行することで、システムがワークフロー関連図、および目次を作成します。\n", "\n", "◆◆◆開発メモ◆◆◆\n", "\n", "*.diagに分野固有のセクションはまだ反映されていない？" ] }, { "cell_type": "code", "execution_count": 5, "metadata": { "deletable": false, "editable": false, "run_through_control": { "frozen": true } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ワークフローの生成が完了しました。以下のURLをクリックすることで\n", "ワークフロー全体図を閲覧できるページに遷移します。\n", "https://jupyter.cs.rcos.nii.ac.jp//user/ivis_rino_mizuguchi@rdm.nii.ac.jp/ivis-mizuguchi-0606-test-rx9y76gn/notebooks/WORKFLOW/FLOW/base_FLOW.ipynb\n" ] } ], "source": [ "import os\n", "import nbformat\n", "\n", "flow_path = 'WORKFLOW/FLOW/base_FLOW.ipynb'\n", "rcos_binder_url = 'https://jupyter.cs.rcos.nii.ac.jp/'\n", "flow_path_notebook = 'notebooks/WORKFLOW/FLOW/base_FLOW.ipynb'\n", "# flow_path_lab = 'lab/tree/WORKFLOW/FLOW/base_FLOW.ipynb'\n", "\n", "nb = nbformat.read(flow_path, 4)\n", "\n", "title1 = \"\"\"## ワークフロー図を作成する\n", "\n", "以下のセルを実行してワークフロー図を作成できます。 \n", "すでに実行したワークフローを確認したい場合は再度実行してください。\"\"\"\n", "\n", "code1 = \"\"\"import os\n", "from IPython.display import SVG\n", "\n", "nb_utils_path = os.path.join(os.environ['HOME'], 'WORKFLOW/FLOW/util/scripts/nb_utils.py')\n", "\n", "basic_path = os.path.join(os.environ['HOME'], 'WORKFLOW/images')\n", "basic_path_diag = os.path.join(basic_path, 'notebooks.diag')\n", "basic_path_svg = os.path.join(basic_path, 'notebooks.svg')\n", "\n", "os.chdir(os.environ['HOME'])\n", "!python3 $nb_utils_path $basic_path_diag\n", "\n", "SVG(filename=basic_path_svg)\"\"\"\n", "\n", "title2 = \"\"\"ワークフロー図の左のノードはワークフローのフェーズとその流れを表しており、フェーズの右側にあるノードは個々のワークフローを表しています。 \n", "各ワークフローを実行するには、ワークフローの各ノードのリンクをクリックしてください。\"\"\"\n", "\n", "title3 = \"\"\"## データガバナンス機能へ戻る\n", "\n", "以下のセルを実行し、表示されるURLをクリックするとデータガバナンス機能へアクセスできます。\"\"\"\n", "\n", "code2 = \"\"\"!git config --get remote.origin.url\"\"\"\n", "\n", "\n", "nb[\"cells\"] += [\n", " nbformat.v4.new_markdown_cell(title1),\n", " nbformat.v4.new_code_cell(code1),\n", " nbformat.v4.new_markdown_cell(title2),\n", " nbformat.v4.new_markdown_cell(title3),\n", " nbformat.v4.new_code_cell(code2)\n", "]\n", "\n", "with open(flow_path, \"w\") as f:\n", " nbformat.write(nb, f)\n", "\n", "print('ワークフローの生成が完了しました。以下のURLをクリックすることで')\n", "print('ワークフロー全体図を閲覧できるページに遷移します。')\n", "print(rcos_binder_url + os.environ['JUPYTERHUB_SERVICE_PREFIX'] + flow_path_notebook)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.10" } }, "nbformat": 4, "nbformat_minor": 2 }