From bacfeab8f0b7dbbc0d3eb79c557966da33e64f68 Mon Sep 17 00:00:00 2001 From: Christian Kadner Date: Thu, 2 Dec 2021 12:59:21 -0800 Subject: [PATCH] Simplify Quickstart * Since we have /catalog/upload_from_url API now (#259), we no longer need to keep a copy of the catalog_upload.json file under quickstart * Since all assets get published and featured automatically with catalog upload (#256), there is no longer a need for the init_catalog.sh script either * Since the MLX API also creates the Pipelines table (3c10fda) we no longer need to mount the init_db.sql to the mysql service * Since the Docker Compose File v3+ spec no longer supports condition as an option of depends_on we no longer need the healthcheck on services, but instead have to use simple while-wait-loops in the commands of downstream services Related #256 Related #259 Related 3c10fda Related https://forums.docker.com/t/depends-on-version-3/29463 Signed-off-by: Christian Kadner --- quickstart/catalog_upload.json | 196 ------------------ quickstart/docker-compose.yaml | 57 +++-- quickstart/init_catalog.sh | 32 --- quickstart/init_db.sql | 43 ---- .../python/regenerate_catalog_upload_json.py | 2 +- 5 files changed, 29 insertions(+), 301 deletions(-) delete mode 100644 quickstart/catalog_upload.json delete mode 100755 quickstart/init_catalog.sh delete mode 100644 quickstart/init_db.sql diff --git a/quickstart/catalog_upload.json b/quickstart/catalog_upload.json deleted file mode 100644 index 7a5dc91f..00000000 --- a/quickstart/catalog_upload.json +++ /dev/null @@ -1,196 +0,0 @@ -{ - "components": [ - { - "name": "Create Secret - Kubernetes Cluster", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/component-samples/create-secret/component.yaml" - }, - { - "name": "Generate Dataset Metadata", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/component-samples/dax-to-dlf/component.yaml" - }, - { - "name": "Create Dataset Volume", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/component-samples/dlf/component.yaml" - }, - { - "name": "Echo Sample", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/component-samples/echo/component.yaml" - }, - { - "name": "Kubernetes Model Deploy", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/component-samples/kube-model-deployment/component.yaml" - }, - { - "name": "Create Model Config", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/component-samples/model-config/component.yaml" - }, - { - "name": "Model Fairness Check", - "url": "https://raw.githubusercontent.com/Trusted-AI/AIF360/master/mlops/kubeflow/bias_detector_pytorch/component.yaml" - }, - { - "name": "Adversarial Robustness Evaluation", - "url": "https://raw.githubusercontent.com/Trusted-AI/adversarial-robustness-toolbox/main/utils/mlops/kubeflow/robustness_evaluation_fgsm_pytorch/component.yaml" - } - ], - "datasets": [ - { - "name": "Project CodeNet", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/codenet/codenet.yaml" - }, - { - "name": "Project CodeNet - Language Classifier", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/codenet_langclass/codenet_langclass.yaml" - }, - { - "name": "Project CodeNet - MLM", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/codenet_mlm/codenet_mlm.yaml" - }, - { - "name": "Finance Proposition Bank", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/fpb/fpb.yaml" - }, - { - "name": "Groningen Meaning Bank", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/gmb/gmb.yaml" - }, - { - "name": "NOAA Weather Data", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/jfk/jfk.yaml" - }, - { - "name": "PubLayNet", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/publaynet/publaynet.yaml" - }, - { - "name": "PubTabNet", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/pubtabnet/pubtabnet.yaml" - }, - { - "name": "Thematic Clustering", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/thematic_clustering/thematic_clustering.yaml" - }, - { - "name": "TensorFlow Speech Commands", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/dataset-samples/tsc/tsc.yaml" - } - ], - "models": [ - { - "name": "CodeNet Language Classification", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/codenet-language-classification/codenet-language-classification.yaml" - }, - { - "name": "Human Pose Estimator", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-human-pose-estimator/max-human-pose-estimator.yaml" - }, - { - "name": "Image Caption Generator", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-image-caption-generator/max-image-caption-generator.yaml" - }, - { - "name": "Image Resolution Enhancer", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-image-resolution-enhancer/max-image-resolution-enhancer.yaml" - }, - { - "name": "Named Entity Tagger", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-named-entity-tagger/max-named-entity-tagger.yaml" - }, - { - "name": "Object Detector", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-object-detector/max-object-detector.yaml" - }, - { - "name": "Optical Character Recognition", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-ocr/max-ocr.yaml" - }, - { - "name": "Question Answering", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-question-answering/max-question-answering.yaml" - }, - { - "name": "Recommender System", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-recommender/max-recommender.yaml" - }, - { - "name": "Text Sentiment Classifier", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-text-sentiment-classifier/max-text-sentiment-classifier.yaml" - }, - { - "name": "Toxic Comment Classifier", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-toxic-comment-classifier/max-toxic-comment-classifier.yaml" - }, - { - "name": "Weather Forecaster", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/model-samples/max-weather-forecaster/max-weather-forecaster.yaml" - } - ], - "notebooks": [ - { - "name": "JFK Airport Analysis", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/JFK-airport.yaml" - }, - { - "name": "AIF360 Bias detection example", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/aif-bias.yaml" - }, - { - "name": "ART detector model", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/art-detector.yaml" - }, - { - "name": "ART poisoning attack", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/art-poison.yaml" - }, - { - "name": "Project CodeNet - Language Classification", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/codenet-lang.yaml" - }, - { - "name": "Project CodeNet - MLM", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/codenet-mlm.yaml" - }, - { - "name": "Qiskit Quantum Kernel Machine Learning", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/qiskit-ml.yaml" - }, - { - "name": "Qiskit Neural Network Classifier and Regressor", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/notebook-samples/qiskit-nncr.yaml" - } - ], - "pipelines": [ - { - "name": "Parallel Join", - "url": "https://github.com/kubeflow/kfp-tekton/blob/master/sdk/python/tests/compiler/testdata/parallel_join.yaml" - }, - { - "name": "Sequential Pipeline", - "url": "https://github.com/kubeflow/kfp-tekton/blob/master/sdk/python/tests/compiler/testdata/sequential.yaml" - }, - { - "name": "ResourceOp Basic", - "url": "https://github.com/kubeflow/kfp-tekton/blob/master/sdk/python/tests/compiler/testdata/resourceop_basic.yaml" - }, - { - "name": "Calculation Pipeline", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/pipeline-samples/calculation-pipeline.yaml" - }, - { - "name": "Katib Early Stopping Experiment", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/pipeline-samples/katib-pipeline.yaml" - }, - { - "name": "Nested Pipeline", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/pipeline-samples/nested-pipeline.yaml" - }, - { - "name": "Trusted AI Pipeline", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/pipeline-samples/trusted-ai-pipeline.yaml" - }, - { - "name": "Watson Machine Learning", - "url": "https://raw.githubusercontent.com/machine-learning-exchange/katalog/main/pipeline-samples/wml-pipeline.yaml" - } - ] -} diff --git a/quickstart/docker-compose.yaml b/quickstart/docker-compose.yaml index 6f5bf8a1..d697579f 100644 --- a/quickstart/docker-compose.yaml +++ b/quickstart/docker-compose.yaml @@ -1,6 +1,7 @@ # Copyright 2021 The MLX Contributors # # SPDX-License-Identifier: Apache-2.0 + version: '3.7' # starts 4 Docker containers: Minio, MySQL, MLX-API, MLX-UI @@ -20,20 +21,20 @@ services: MINIO_ROOT_USER: "minio" MINIO_ROOT_PASSWORD: "minio123" command: server /data --console-address ":9001" - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] - interval: 5s - timeout: 5s - retries: 10 miniosetup: image: minio/mc:RELEASE.2021-06-13T17-48-22Z depends_on: - minio: - condition: service_healthy + - minio entrypoint: ["/bin/sh","-c"] + environment: + HEALTH_API: "http://minio:9000/minio/health/live" command: - | + until curl -I -s "$${HEALTH_API}" | grep -q "200 OK"; do + echo "Waiting for Minio at $${HEALTH_API}" + sleep 1 + done /usr/bin/mc config host add miniohost http://minio:9000 minio minio123 /usr/bin/mc mb -p miniohost/mlpipeline /usr/bin/mc policy set download miniohost/mlpipeline @@ -49,16 +50,13 @@ services: ports: - "3306:3306" volumes: - - ./init_db.sql:/docker-entrypoint-initdb.d/init_db.sql - data-mysql:/var/lib/mysql - healthcheck: - test: [ "CMD", "mysqladmin" ,"ping", "-h", "localhost" ] - interval: 5s - timeout: 5s - retries: 10 mlx-api: image: mlexchange/mlx-api:nightly-main + depends_on: + - miniosetup + - mysql ports: - "8080:8080" environment: @@ -68,11 +66,6 @@ services: MYSQL_SERVICE_PORT: "3306" ML_PIPELINE_SERVICE_HOST: "UNAVAILABLE" ML_PIPELINE_SERVICE_PORT: "UNAVAILABLE" -# healthcheck: # disable healthcheck, it's filling up the logs -# test: [ "CMD", "wget", "-qO-", 'http://localhost:8080/apis/v1alpha1/health_check?check_database=true&check_object_store=true' ] -# interval: 5s -# timeout: 5s -# retries: 10 mlx-ui: image: mlexchange/mlx-ui:nightly-origin-main @@ -93,18 +86,24 @@ services: catalog: image: curlimages/curl depends_on: - miniosetup: - condition: service_completed_successfully - mysql: - condition: service_healthy - mlx-api: - condition: service_started # NOT service_healthy as that pollutes the logs - volumes: - - ./catalog_upload.json:/catalog_upload.json - - ./init_catalog.sh:/init_catalog.sh - entrypoint: ["/bin/sh", "-c", "/init_catalog.sh"] + - miniosetup + - mysql + - mlx-api + entrypoint: [ "/bin/sh", "-c" ] environment: - MLX_API_SERVER: "mlx-api:8080" + HEALTH_API: "http://mlx-api:8080/apis/v1alpha1/health_check?check_database=true&check_object_store=true" + UPLOAD_API: "http://mlx-api:8080/apis/v1alpha1/catalog/upload_from_url" + CATALOG_URL: "https://raw.githubusercontent.com/machine-learning-exchange/mlx/main/bootstrapper/catalog_upload.json" + command: + - | + until curl -I -s "$${HEALTH_API}" | grep -q "200 OK"; do + echo "Waiting for MLX API at $${HEALTH_API}" + sleep 3 + done + curl -X POST \ + -H "Content-Type: multipart/form-data" \ + -F url="$${CATALOG_URL}" \ + -s "$${UPLOAD_API}" | grep -iE "total_|error" dashboard: image: curlimages/curl diff --git a/quickstart/init_catalog.sh b/quickstart/init_catalog.sh deleted file mode 100755 index f8b479f7..00000000 --- a/quickstart/init_catalog.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -# Copyright 2021 The MLX Contributors -# -# SPDX-License-Identifier: Apache-2.0 - -MLX_API_SERVER=${MLX_API_SERVER:-"localhost:8080"} -MLX_API_URL="http://${MLX_API_SERVER}/apis/v1alpha1" - -# wait for the MLX API server, but more importantly the MySQL server to be ready -until curl -X GET -H 'Accept: application/json' -s "${MLX_API_URL}/health_check?check_database=true&check_object_store=true" | grep -q 'Healthy'; do - echo 'Waiting for MLX-API, Minio, MySQL ...' - sleep 1 -done - -# echo the MLX API server status, should be "Healthy" -curl -X GET -H 'Accept: application/json' -s "${MLX_API_URL}/health_check?check_database=true&check_object_store=true" - -# upload the pipeline asset catalog -curl -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d @catalog_upload.json -s "${MLX_API_URL}/catalog" | grep -iE "total_|error" - -# mark all the catalog assets as approved and featured -for asset_type in components datasets models notebooks pipelines; do - curl -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '["*"]' -s "${MLX_API_URL}/$asset_type/publish_approved" - curl -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' -d '["*"]' -s "${MLX_API_URL}/$asset_type/featured" -done - -# disable the Inference Services, since we don't have a KFP cluster -curl -X PUT -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"Inference Services": false}' -s "${MLX_API_URL}/settings" -o /dev/null --show-error - -# disable pipeline execution, since we don't have a KFP cluster -curl -X PUT -H 'Content-Type: application/json' -H 'Accept: application/json' -d '{"Execution enabled": false}' -s "${MLX_API_URL}/settings" -o /dev/null --show-error diff --git a/quickstart/init_db.sql b/quickstart/init_db.sql deleted file mode 100644 index a10fc985..00000000 --- a/quickstart/init_db.sql +++ /dev/null @@ -1,43 +0,0 @@ --- --------------------------------------------- --- Generated using MySQL dump --- Server version: 5.7.33 --- Database: mlpipeline --- --------------------------------------------- --- $ brew install mysql-client --- $ /usr/local/opt/mysql-client/bin/mysqldump mlpipeline --result-file=init_db.sql pipelines pipeline_versions --skip-add-drop-table --skip-disable-keys --skip-lock-tables --skip-add-locks --create-options --extended-insert --column-statistics=0 --user=root --host=169.45.121.104 --port=32259 --- --------------------------------------------- - -CREATE TABLE IF NOT EXISTS `pipelines` ( - `UUID` varchar(255) NOT NULL, - `CreatedAtInSec` bigint(20) NOT NULL, - `Name` varchar(255) NOT NULL, - `Description` longtext NOT NULL, - `Parameters` longtext NOT NULL, - `Status` varchar(255) NOT NULL, - `DefaultVersionId` varchar(255) DEFAULT NULL, - `Namespace` varchar(63) DEFAULT '', - PRIMARY KEY (`UUID`), - UNIQUE KEY `name_namespace_index` (`Name`,`Namespace`) -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - -CREATE TABLE IF NOT EXISTS `pipeline_versions` ( - `UUID` varchar(255) NOT NULL, - `CreatedAtInSec` bigint(20) NOT NULL, - `Name` varchar(255) NOT NULL, - `Parameters` longtext NOT NULL, - `PipelineId` varchar(255) NOT NULL, - `Status` varchar(255) NOT NULL, - `CodeSourceUrl` varchar(255) DEFAULT NULL, - PRIMARY KEY (`UUID`), - UNIQUE KEY `idx_pipelineid_name` (`Name`,`PipelineId`), - KEY `idx_pipeline_versions_CreatedAtInSec` (`CreatedAtInSec`), - KEY `idx_pipeline_versions_PipelineId` (`PipelineId`), - CONSTRAINT `pipeline_versions_PipelineId_pipelines_UUID_foreign` FOREIGN KEY (`PipelineId`) REFERENCES `pipelines` (`UUID`) ON DELETE CASCADE ON UPDATE CASCADE -) ENGINE=InnoDB DEFAULT CHARSET=latin1; - --- --- don't populate tables since we won't have the accompanying "template" YAML stored on Minio --- - --- INSERT INTO `pipelines` VALUES ('c616527f-1f3c-4f0c-aada-35fa2cc4feb3',1619547888,'[Demo] flip-coin','[source code](https://github.com/kubeflow/kfp-tekton/tree/master/samples/flip-coin) A conditional pipeline to flip coins based on a random number generator.','[]','READY','c616527f-1f3c-4f0c-aada-35fa2cc4feb3',''); --- INSERT INTO `pipeline_versions` VALUES ('c616527f-1f3c-4f0c-aada-35fa2cc4feb3',1619547888,'[Demo] flip-coin','[]','c616527f-1f3c-4f0c-aada-35fa2cc4feb3','READY',''); diff --git a/tools/python/regenerate_catalog_upload_json.py b/tools/python/regenerate_catalog_upload_json.py index 43483461..66e60b2f 100755 --- a/tools/python/regenerate_catalog_upload_json.py +++ b/tools/python/regenerate_catalog_upload_json.py @@ -29,7 +29,7 @@ catalog_upload_json_files = [ f"{project_dir}/bootstrapper/catalog_upload.json", - f"{project_dir}/quickstart/catalog_upload.json", + # f"{project_dir}/quickstart/catalog_upload.json", ]