Skip to content

update with nvflare 1.1 and monai 0.7 #373

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wyli merged 3 commits into from
Sep 27, 2021
Merged

update with nvflare 1.1 and monai 0.7 #373

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
98647de
update with nvflare 1.1 and monai 0.7
yiheng-wang-nv Sep 27, 2021
7dddce5
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Sep 27, 2021
230af59
Merge branch 'master' into 365-update-nvflare-example
yiheng-wang-nv Sep 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 10 additions & 48 deletions federated_learning/nvflare/nvflare_example_docker/1-Startup.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,19 +37,18 @@
"data": {
"text/plain": [
"['project.yml',\n",
" 'prerpare_expr_files.sh',\n",
" 'researcher@nvidia.com.zip',\n",
" 'download_dataset.py',\n",
" 'authz_config.json',\n",
" 'org1-b.zip',\n",
" 'researcher@org2.com.zip',\n",
" 'admin@nvidia.com.zip',\n",
" 'org1-a.zip',\n",
" 'audit.pkl',\n",
" 'org3.zip',\n",
" 'server.zip',\n",
" 'researcher@org1.com.zip',\n",
" 'org2.zip',\n",
" 'it@org2.com.zip']"
" 'it@org2.com.zip',\n",
" 'prepare_expr_files.sh']"
]
},
"execution_count": 2,
Expand Down Expand Up @@ -83,14 +82,9 @@
"text": [
"demo_workspace created!\n",
"unzip: server finished.\n",
"unzip: admin@nvidia.com finished.\n",
"unzip: researcher@nvidia.com finished.\n",
"unzip: researcher@org1.com finished.\n",
"unzip: researcher@org2.com finished.\n",
"unzip: it@org2.com finished.\n",
"unzip: org1-a finished.\n",
"unzip: org1-b finished.\n",
"unzip: org2 finished.\n"
"unzip: admin@nvidia.com finished.\n"
]
}
],
Expand All @@ -102,39 +96,15 @@
" os.makedirs(workspace)\n",
" print(workspace, \" created!\")\n",
"\n",
"admin_name = \"admin@nvidia.com\"\n",
"client_name_1 = \"org1-a\"\n",
"client_name_2 = \"org1-b\"\n",
"server_name = \"server\"\n",
"used_file_names = [\"server\", \"org1-a\", \"org1-b\", \"admin@nvidia.com\"]\n",
"\n",
"# access audit file get passwords for unzipping packages\n",
"with open(os.path.join(startup_path, \"audit.pkl\"), 'rb') as handle:\n",
" audit_file = pickle.load(handle)\n",
"\n",
"proj_name = list(audit_file.keys())[0]\n",
"pw_key = \"zip_pw\"\n",
"server_folder_list = [\"server\"]\n",
"client_folder_list = [\"admin_clients\", \"fl_clients\"]\n",
"\n",
"folder_pwd_dict = {}\n",
"for obj in server_folder_list:\n",
" unzip_pw = audit_file[proj_name][obj][pw_key]\n",
" folder_pwd_dict[obj] = unzip_pw\n",
"\n",
"for obj in client_folder_list:\n",
" obj_sub_dict = audit_file[proj_name][obj]\n",
" for client in obj_sub_dict.keys():\n",
" unzip_pw = obj_sub_dict[client][pw_key]\n",
" folder_pwd_dict[client] = unzip_pw\n",
"\n",
"# unzip all folders into workspace\n",
"for name, pwd in folder_pwd_dict.items():\n",
"for name in used_file_names:\n",
" zip_file_path = os.path.join(startup_path, name + \".zip\")\n",
" dst_file_path = os.path.join(workspace, name)\n",
" if not os.path.exists(dst_file_path):\n",
" os.makedirs(dst_file_path)\n",
" with ZipFile(zip_file_path, 'r') as zip_ref:\n",
" zip_ref.extractall(path=dst_file_path, pwd=bytes(pwd, 'utf-8'))\n",
" zip_ref.extractall(path=dst_file_path)\n",
" # change permissions\n",
" if \".com\" in name:\n",
" sub_file_list = [\"docker.sh\", \"fl_admin.sh\"]\n",
Expand All @@ -153,15 +123,7 @@
{
"data": {
"text/plain": [
"['org1-b',\n",
" 'researcher@nvidia.com',\n",
" 'server',\n",
" 'admin@nvidia.com',\n",
" 'researcher@org2.com',\n",
" 'org1-a',\n",
" 'org2',\n",
" 'researcher@org1.com',\n",
" 'it@org2.com']"
"['org1-b', 'server', 'admin@nvidia.com', 'org1-a']"
]
},
"execution_count": 4,
Expand Down Expand Up @@ -197,7 +159,7 @@
"source": [
"config_folder = \"spleen_example\"\n",
"\n",
"transfer_path = os.path.join(workspace, admin_name, \"transfer/\")\n",
"transfer_path = os.path.join(workspace, \"admin@nvidia.com\", \"transfer/\")\n",
"if not os.path.exists(transfer_path):\n",
" os.makedirs(transfer_path)\n",
"shutil.copytree(config_folder, os.path.join(transfer_path, config_folder))\n",
Expand Down Expand Up @@ -233,7 +195,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand Down
18 changes: 6 additions & 12 deletions federated_learning/nvflare/nvflare_example_docker/2-Server.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@
"== PyTorch ==\n",
"=============\n",
"\n",
"NVIDIA Release 21.04 (build 22382700)\n",
"PyTorch Version 1.9.0a0+2ecb2c7\n",
"NVIDIA Release 21.08 (build 26011915)\n",
"PyTorch Version 1.10.0a0+3fd9dcf\n",
"\n",
"Container image Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.\n",
"\n",
Expand All @@ -89,15 +89,9 @@
"By pulling and using the container, you accept the terms and conditions of this license:\n",
"https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license\n",
"\n",
"WARNING: The NVIDIA Driver was not detected. GPU functionality will not be available.\n",
" Use 'nvidia-docker run' to start this container; see\n",
" https://github.com/NVIDIA/nvidia-docker/wiki/nvidia-docker .\n",
"\n",
"NOTE: MOFED driver for multi-node communication was not detected.\n",
" Multi-node communication performance may be reduced.\n",
"\n",
"\u001b]0;root@sys: /workspace\u0007root@sys:/workspace# ^C\n",
"\n",
"\u001b]0;root@sys: /workspace\u0007root@sys:/workspace# "
]
}
Expand All @@ -106,7 +100,7 @@
"server_name = \"server\"\n",
"workspace = \"demo_workspace\"\n",
"\n",
"server_startup_path = os.path.join(workspace, server_name, 'startup')\n",
"server_startup_path = os.path.join(workspace, server_name, \"startup\")\n",
"cmd = server_startup_path + \"/docker.sh\"\n",
"\n",
"\n",
Expand Down Expand Up @@ -137,8 +131,8 @@
"name": "stdout",
"output_type": "stream",
"text": [
"CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n",
"a4d31af29041 monai_nvflare:latest \"/usr/local/bin/nvid…\" 3 seconds ago Up 3 seconds flserver\n"
"CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n",
"08492335d003 monai_nvflare:latest \"/usr/local/bin/nvid…\" 2 seconds ago Up 1 second flserver\n"
]
}
],
Expand Down Expand Up @@ -216,7 +210,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand Down
34 changes: 15 additions & 19 deletions federated_learning/nvflare/nvflare_example_docker/3-Client.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 3,
"metadata": {
"pycharm": {
"metadata": false,
Expand All @@ -171,17 +171,17 @@
"name": "stdout",
"output_type": "stream",
"text": [
"running cmd demo_workspace/org1-a/startup/docker.sh\n",
"running cmd demo_workspace/org1-b/startup/docker.sh\n",
"running cmd demo_workspace/org1-a/startup/docker.sh\n",
"Starting docker with monai_nvflare:latest\n",
"Starting docker with monai_nvflare:latest\n",
"\n",
"=============\n",
"== PyTorch ==\n",
"=============\n",
"\n",
"NVIDIA Release 21.04 (build 22382700)\n",
"PyTorch Version 1.9.0a0+2ecb2c7\n",
"NVIDIA Release 21.08 (build 26011915)\n",
"PyTorch Version 1.10.0a0+3fd9dcf\n",
"\n",
"Container image Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.\n",
"\n",
Expand Down Expand Up @@ -210,8 +210,8 @@
"== PyTorch ==\n",
"=============\n",
"\n",
"NVIDIA Release 21.04 (build 22382700)\n",
"PyTorch Version 1.9.0a0+2ecb2c7\n",
"NVIDIA Release 21.08 (build 26011915)\n",
"PyTorch Version 1.10.0a0+3fd9dcf\n",
"\n",
"Container image Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.\n",
"\n",
Expand Down Expand Up @@ -239,14 +239,10 @@
"NOTE: MOFED driver for multi-node communication was not detected.\n",
" Multi-node communication performance may be reduced.\n",
"\n",
"\n",
"venn@sys:/workspace$ \n",
"NOTE: MOFED driver for multi-node communication was not detected.\n",
" Multi-node communication performance may be reduced.\n",
"\n",
"venn@sys:/workspace$ venn@sys:/workspace$ ^C\n",
"\n",
"venn@sys:/workspace$ ^C\n",
"\n",
"venn@sys:/workspace$ "
]
}
Expand Down Expand Up @@ -280,17 +276,17 @@
},
{
"cell_type": "code",
"execution_count": 5,
"execution_count": 4,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n",
"d87439228f1a monai_nvflare:latest \"/usr/local/bin/nvid…\" 4 seconds ago Up 3 seconds org1-b\n",
"fc4227a0abfd monai_nvflare:latest \"/usr/local/bin/nvid…\" 4 seconds ago Up 3 seconds org1-a\n",
"a4d31af29041 monai_nvflare:latest \"/usr/local/bin/nvid…\" About a minute ago Up About a minute flserver\n"
"CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES\n",
"96c8f9406303 monai_nvflare:latest \"/usr/local/bin/nvid…\" 2 seconds ago Up 1 second org1-a\n",
"6b5d9eefc699 monai_nvflare:latest \"/usr/local/bin/nvid…\" 2 seconds ago Up 1 second org1-b\n",
"08492335d003 monai_nvflare:latest \"/usr/local/bin/nvid…\" 40 seconds ago Up 39 seconds flserver\n"
]
}
],
Expand All @@ -317,7 +313,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 5,
"metadata": {},
"outputs": [
{
Expand All @@ -329,7 +325,7 @@
"<IPython.core.display.HTML object>"
]
},
"execution_count": 6,
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
Expand Down Expand Up @@ -390,7 +386,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
Expand Down
Loading