From 59b73cf7c45504c3f4206ef74f6ffa652319529b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 13 Oct 2023 16:44:53 +0200 Subject: [PATCH 01/45] Add mkdocs-material minimal example --- docs/index.md | 17 +++++++++++++++++ mkdocs.yml | 5 +++++ pyproject.toml | 3 +++ 3 files changed, 25 insertions(+) create mode 100644 docs/index.md create mode 100644 mkdocs.yml diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..000ea345 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,17 @@ +# Welcome to MkDocs + +For full documentation visit [mkdocs.org](https://www.mkdocs.org). + +## Commands + +* `mkdocs new [dir-name]` - Create a new project. +* `mkdocs serve` - Start the live-reloading docs server. +* `mkdocs build` - Build the documentation site. +* `mkdocs -h` - Print help message and exit. + +## Project layout + + mkdocs.yml # The configuration file. + docs/ + index.md # The documentation homepage. + ... # Other markdown pages, images and other files. diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 00000000..57fe6d38 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,5 @@ +# yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json + +site_name: OpenML Server Documentation +theme: + name: material diff --git a/pyproject.toml b/pyproject.toml index fe7ecdba..a6000adc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,9 @@ dev = [ "pytest", "httpx", ] +docs = [ + "mkdocs-material" +] [project.urls] "Homepage" = "https://github.com/openml/server-api" From f0be89e992f0e553c890d172e46f9e7e7edc1297 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 13 Oct 2023 16:54:39 +0200 Subject: [PATCH 02/45] Replace minimal example with README and CONTRIBUTING --- CONTRIBUTING.md => docs/CONTRIBUTING.md | 0 README.md => docs/README.md | 0 docs/index.md | 17 ----------------- mkdocs.yml | 6 +++++- 4 files changed, 5 insertions(+), 18 deletions(-) rename CONTRIBUTING.md => docs/CONTRIBUTING.md (100%) rename README.md => docs/README.md (100%) delete mode 100644 docs/index.md diff --git a/CONTRIBUTING.md b/docs/CONTRIBUTING.md similarity index 100% rename from CONTRIBUTING.md rename to docs/CONTRIBUTING.md diff --git a/README.md b/docs/README.md similarity index 100% rename from README.md rename to docs/README.md diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 000ea345..00000000 --- a/docs/index.md +++ /dev/null @@ -1,17 +0,0 @@ -# Welcome to MkDocs - -For full documentation visit [mkdocs.org](https://www.mkdocs.org). - -## Commands - -* `mkdocs new [dir-name]` - Create a new project. -* `mkdocs serve` - Start the live-reloading docs server. -* `mkdocs build` - Build the documentation site. -* `mkdocs -h` - Print help message and exit. - -## Project layout - - mkdocs.yml # The configuration file. - docs/ - index.md # The documentation homepage. - ... # Other markdown pages, images and other files. diff --git a/mkdocs.yml b/mkdocs.yml index 57fe6d38..f5de7e40 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,5 +1,9 @@ # yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json -site_name: OpenML Server Documentation +site_name: OpenML Server theme: name: material + +nav: + - Getting Started: README.md + - Contributing: CONTRIBUTING.md From dd01c443b08a6ef64596d6116e83d877502f5e73 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 13 Oct 2023 17:28:47 +0200 Subject: [PATCH 03/45] Split up readme into github readme and documentation pages --- README.md | 17 +++++++++++++++ docs/migration.md | 40 ++++++++++++++++++++++++++++++++++ docs/{README.md => roadmap.md} | 38 ++------------------------------ mkdocs.yml | 1 + 4 files changed, 60 insertions(+), 36 deletions(-) create mode 100644 README.md create mode 100644 docs/migration.md rename docs/{README.md => roadmap.md} (52%) diff --git a/README.md b/README.md new file mode 100644 index 00000000..104c4021 --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +![Python 3.12](https://img.shields.io/badge/python-3.12-green?logo=python) + +# OpenML Server +This is the Python-based OpenML REST API server. +It's a rewrite of our [old backend](http://github.com/openml/openml) built with a +modern Python-based stack. + +> [!WARNING] +> This software is in early stages of development and not ready for production. + +If you simply want to access data stored on OpenML in a programmatic way, +please have a look at connector packages in +[Python](https://openml.github.io/openml-python/main/), +[Java](https://github.com/openml/openml-java), +or [R](http://openml.github.io/openml-r/). + +For information on getting started, please visit our [documentation](https://openml.github.io/server-api). diff --git a/docs/migration.md b/docs/migration.md new file mode 100644 index 00000000..7a8fc8fb --- /dev/null +++ b/docs/migration.md @@ -0,0 +1,40 @@ +# Migration to V1 +The first iteration of the new server has nearly identical responses to the old JSON +endpoints, but there are exceptions. + +## All Endpoints +The following changes affect all endpoints. + +### Error on Invalid Input +When providing input of invalid types (e.g., a non-integer dataset id) the HTTP header +and JSON content will be different. + +HTTP Header: +```diff +- 412 Precondition Failed ++ 422 Unprocessable Entity +``` + +JSON Content +```diff +- {"error":{"code":"100","message":"Function not valid"}} ++ {"detail":[{"loc":["query","_dataset_id"],"msg":"value is not a valid integer","type":"type_error.integer"}]} +``` + +### Other Errors +For any other error messages, the response is identical except that outer field will be `"detail"` instead of `"error"`: +```diff +- {"error":{"code":"112","message":"No access granted"}} ++ {"detail":{"code":"112","message":"No access granted"}} +``` + +## Datasets + + - Dataset format names are normalized to be all lower-case + (`"Sparse_ARFF"` -> `"sparse_arff"`). + - Non-`arff` datasets will not incorrectly have a `"parquet_ur"`: + https://github.com/openml/OpenML/issues/1189 + - If `"creator"` contains multiple comma-separated creators it is always returned + as a list, instead of it depending on the quotation used by the original uploader. + - For (some?) datasets that have multiple values in `"ignore_attribute"`, this field + is correctly populated instead of omitted. diff --git a/docs/README.md b/docs/roadmap.md similarity index 52% rename from docs/README.md rename to docs/roadmap.md index 8a9b6946..57e3d27e 100644 --- a/docs/README.md +++ b/docs/roadmap.md @@ -1,5 +1,5 @@ -# server -Python-based server prototype + +## Getting Started ## Development Roadmap First we will mimic current server functionality, relying on many implementation details @@ -26,37 +26,3 @@ new version of the JSON API which is more standardized, leverages typing, and so There is no planned sunset date for the old API. This will depend on the progress with the new API as well as the usage numbers of the old API. - -## Change Notes -The first iteration of the new server has nearly identical responses to the old JSON -endpoints, but there are exceptions: - - - Providing input of invalid types (e.g., a non-integer dataset id). - - HTTP Header: - ```diff - - 412 Precondition Failed - + 422 Unprocessable Entity - ``` - - JSON Content - ```diff - - {"error":{"code":"100","message":"Function not valid"}} - + {"detail":[{"loc":["query","_dataset_id"],"msg":"value is not a valid integer","type":"type_error.integer"}]} - ``` - - - For any other error messages, the response is identical except that outer field - will be `"detail"` instead of `"error"`: - ```diff - - {"error":{"code":"112","message":"No access granted"}} - + {"detail":{"code":"112","message":"No access granted"}} - ``` - - - Dataset format names are normalized to be all lower-case - (`"Sparse_ARFF"` -> `"sparse_arff"`). - - Non-`arff` datasets will not incorrectly have a `"parquet_ur"`: - https://github.com/openml/OpenML/issues/1189 - - If `"creator"` contains multiple comma-separated creators it is always returned - as a list, instead of it depending on the quotation used by the original uploader. - - For (some?) datasets that have multiple values in `"ignore_attribute"`, this field - is correctly populated instead of omitted. diff --git a/mkdocs.yml b/mkdocs.yml index f5de7e40..172b5f4c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -7,3 +7,4 @@ theme: nav: - Getting Started: README.md - Contributing: CONTRIBUTING.md + - Migration: migration.md From cda0a98a6f58896bf917741eb966afc3c22fab25 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 13:44:07 +0200 Subject: [PATCH 04/45] Improve installation instructions --- docs/installation.md | 67 ++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 9 +++++- 2 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 docs/installation.md diff --git a/docs/installation.md b/docs/installation.md new file mode 100644 index 00000000..67490384 --- /dev/null +++ b/docs/installation.md @@ -0,0 +1,67 @@ +# Installation + +*Current instructions written for Mac, but can be adapted for other operating systems.* + +The OpenML server will be developed and maintained for the latest minor release of +Python (Python 3.12 as of writing). +You can install the dependencies locally or work from a Docker container (TODO). + +!!! tip "Use `pyenv` to manage Python installations" + + We recommend using [`pyenv`](https://github.com/pyenv/pyenv) if you are working with + multiple local Python versions. + +## Local Installation + + +=== "For Users" + + If you don't plan to make code changes, you can install directly from Github. + We recommend to install the OpenML server and its dependencies into a new virtual + environment. + ```bash title="Installing the project into a new virtual environment" + python -m venv venv + source venv/bin/activate + + python -m pip install git+https://github.com/openml/server-api.git + ``` + If you do plan to make code changes, we recommend you follow the instructions + under the "For Contributors" tab, even if you do not plan to contribute your + changes back into the project. + + +=== "For Contributors" + + If you plan to make changes to this project, it will be useful to install + the project from a cloned fork. To fork the project, go to our + [project page](https://github.com/openml/server-api) and click "fork". + This makes a copy of the repository under your own Github account. + You can then clone your own fork (replace `USER_NAME` with your Github username): + + ```bash title="Cloning your fork" + git clone https://github.com/USER_NAME/server-api.git + cd server-api + ``` + + Then we can install the project into a new virtual environment in edit mode: + + ```bash title="Installing the project into a new virtual environment" + python -m venv venv + source venv/bin/activate + + python -m pip install -e ".[dev,docs]" + ``` + Note that this also installs optional dependencies for development and documentation + tools. We require this for contributors, but we also highly recommend it anyone + that plans to make code changes. + +Before we run the REST API server, we must first set up a database server, and configure +the REST API to connect to it. + +## Setting up a Database Server + +... + +## Configuring the REST API Server + +... diff --git a/mkdocs.yml b/mkdocs.yml index 172b5f4c..5913eb24 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,6 +5,13 @@ theme: name: material nav: - - Getting Started: README.md + - Getting Started: installation.md - Contributing: CONTRIBUTING.md - Migration: migration.md + +markdown_extensions: + - admonition + - pymdownx.details + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true From 5ae72c82069c80bb5f1338ebe5ee20d46b598054 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 14:02:53 +0200 Subject: [PATCH 05/45] Add information on YAML validation --- docs/CONTRIBUTING.md | 83 +++++++++++++++++++++++++++----------------- mkdocs.yml | 1 + 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 486a0dee..fbace94d 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,10 +1,38 @@ -# Installing the Development Environment +# Setting up the Development Environment -*Current instructions written for Mac, but can be adapted for other operating systems.* +First, follow the [installation](installation.md#local-installation) instructions +for contributors to install a local fork with optional development dependencies. -The OpenML server will be developed and maintained for the latest minor release of -Python (Python 3.11 as of writing). -You can install the dependencies locally or work from a Docker container (TODO). +## Pre-commit + +We use [`pre-commit`](https://pre-commit.com) to ensure certain tools and checks are +ran before each commit. These tools perform code-formatting, linting, and more. This +makes the code more consistent across the board, which makes it easier to work with +each other's code and also can catch common errors. After installing it, it will +automatically run when you try to make a commit. Install it now and verify that all +checks pass out of the box: + +```bash title="Install pre-commit and verify it works" +pre-commit install +pre-commit run all-files +``` +Running the tool the first time may be slow as tools need to be installed, +and many tools will build a cache so subsequent runs are faster. +Under normal circumstances, running the pre-commit chain shouldn't take more than a few +seconds. + +## Unit Tests + +Our unit tests are written with the [`pytest`](https://pytest.org) framework. +An invocation could look like this: `pytest -v -x --lf` + +```bash +python -m pytest -v -x --lf -m "not web" +``` + +Where `-v` show the name of each test ran, `-x` ensures testing stops on first failure, +`--lf` will first run the test(s) which failed last, and `-m "not web"` specifies +which tests (not) to run. ## Database The server relies on a database connection, and we recommend to use a containerized @@ -20,35 +48,26 @@ The databases are hardcoded to be accessible by user `root` with password `ok` a TODO: Upload a docker image which has a test database included and can easily be run without any local changes. -## Local Installation -We recommend using [`pyenv`](https://github.com/pyenv/pyenv) if you are working with -multiple local Python versions. -We also recommend installing the dependencies into a virtual environment: +## Working from Docker +TODO -```bash -python -m venv venv -source venv/bin/activate -``` +## YAML Validation +The project contains various [`yaml`](https://yaml.org) files, for example to configure +`mkdocs` or to describe Github workflows. For these `yaml` files we can configure +automatic schema validation, to ensure that the files are valid without having to run +the server. This also helps with other IDE features like autocomplete. Setting this +up is not required, but incredibly useful if you do need to edit these files. +The following `yaml` files have schemas: -Then install the development dependencies: -```bash -python -m pip install -e ".[dev]" -pre-commit install -``` -The [`pre-commit`](https://pre-commit.com) tool will ensure certain tools and checks are -ran before each commit. These tools perform code-formatting, linting, and more. This -makes the code more consistent across the board, which makes it easier to work with -each other's code and also can catch common errors. At this point all tests should pass: -`pre-commit run --all-files`. Running the tool the first time may be slow as tools need -to be installed, and many tools will build a cache so subsequent runs are faster. -Under normal circumstances, running the pre-commit chain shouldn't take more than a few -seconds. +| File(s) | Schema URL | +| -- | -- | +| mkdocs.yml | https://squidfunk.github.io/mkdocs-material/schema.json | +| .pre-commit-config.yaml | https://json.schemastore.org/pre-commit-config.json | +| .github/workflows/*.yaml | https://json.schemastore.org/github-workflow | -Our unit tests are built for the [`pytest`](https://pytest.org) framework. -An invocation could look like this: `pytest -v -x --lf` -Where `-v` show the name of each test ran, `-x` ensures testing stops on first failure, -and `--lf` will first run the test(s) which failed last. +=== "PyCharm" -## Working from Docker -TODO + In PyCharm, these can be configured from `settings` > `languages & frameworks` > + `Schemas and DTDs` > `JSON Schema Mappings`. There, add mappings per file or + file pattern. diff --git a/mkdocs.yml b/mkdocs.yml index 5913eb24..67e98fe0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,3 +15,4 @@ markdown_extensions: - pymdownx.superfences - pymdownx.tabbed: alternate_style: true + - tables From 3a81870daa508a59977aac3076a969e3fd78c769 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 15:01:41 +0200 Subject: [PATCH 06/45] Add info for public database and building documentation --- docs/CONTRIBUTING.md | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index fbace94d..6b38e7e3 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -3,6 +3,12 @@ First, follow the [installation](installation.md#local-installation) instructions for contributors to install a local fork with optional development dependencies. +## Database +In addition to the database setup described in the [installation guide](installation.md#setting-up-a-database-server), +we also host a database on our server which may be connected to that is available +to [OpenML core contributors](https://openml.org/about). If you are a core contributor +and need access, please reach out to one of the engineers in Eindhoven. + ## Pre-commit We use [`pre-commit`](https://pre-commit.com) to ensure certain tools and checks are @@ -34,19 +40,18 @@ Where `-v` show the name of each test ran, `-x` ensures testing stops on first f `--lf` will first run the test(s) which failed last, and `-m "not web"` specifies which tests (not) to run. -## Database -The server relies on a database connection, and we recommend to use a containerized -MySQL server regardless of your local development setup. This way there is no -variability in the database setup, and it is easy to reset to the same database state -other developers use. - -Current workflow is to get a snapshot from the test server at test.openml.org/phpmyadmin, -and then host it in a local mysql container. -The databases are hardcoded to be accessible by user `root` with password `ok` at -`127.0.0.1:3306`. - -TODO: Upload a docker image which has a test database included and can easily be run -without any local changes. +## Building Documentation +We build our documentation with [`mkdocs-material`](https://squidfunk.github.io/mkdocs-material/). +All documentation pages are under the `docs` folder, and the configuration is found in +`mkdocs.yml`. Having installed the `docs` optional dependencies, you should be able +to locally build and serve the documentation: + +```bash title="Serve documentation locally" +python -m mkdocs serve +``` + +You can browse the documentation by visiting `127.0.0.1:8000` in your browser. +The documentation pages will automatically rebuild when there are changes. ## Working from Docker TODO From 1c6888d6b711c262d1984b192bdf7a8ea5032cf6 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 15:02:29 +0200 Subject: [PATCH 07/45] Remove code that was outside of codeblock --- docs/CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 6b38e7e3..719f3468 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -30,7 +30,7 @@ seconds. ## Unit Tests Our unit tests are written with the [`pytest`](https://pytest.org) framework. -An invocation could look like this: `pytest -v -x --lf` +An invocation could look like this: ```bash python -m pytest -v -x --lf -m "not web" From 083321cfb159604cfe4e74f299a0983781b011dc Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 15:32:55 +0200 Subject: [PATCH 08/45] Add github workflow to deploying docs --- .github/workflows/docs.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .github/workflows/docs.yml diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 00000000..6c07fe18 --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,27 @@ +name: Deploy Docs +on: + push: + branches: + - "add/docs" + - main + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: 3.x + - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV + - uses: actions/cache@v3 + with: + key: mkdocs-material-${{ env.cache_id }} + path: .cache + restore-keys: | + mkdocs-material- + - run: pip install mkdocs-material + - run: mkdocs gh-deploy --force From c6fb107fa47a6f1d8f671daabe726547b57d64cc Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 15:40:13 +0200 Subject: [PATCH 09/45] Add source for workflow for later reference --- .github/workflows/docs.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 6c07fe18..690ba33d 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,3 +1,4 @@ +# https://squidfunk.github.io/mkdocs-material/publishing-your-site/?h=#with-github-actions name: Deploy Docs on: push: From 88b19c80a596c62d60c3ccc9195bdd905cdc4e0d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 15:58:03 +0200 Subject: [PATCH 10/45] Add minimal welcome page --- docs/index.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 docs/index.md diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 00000000..d077dd57 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,15 @@ +# OpenML Server API Documentation + +This is the Python-based OpenML REST API server. +It's a rewrite of our [old backend](http://github.com/openml/openml) built with a +modern Python-based stack. + + +!!! question "Looking to access data on OpenML?" + + If you simply want to access data stored on OpenML in a programmatic way, + please have a look at connector packages in + [Python](https://openml.github.io/openml-python/main/), + [Java](https://github.com/openml/openml-java), + or [R](http://openml.github.io/openml-r/). + This documentation is for developing and hosting your own OpenML instance. From ce6d28cb9b2483200967c59aba2cb4762cff4ba9 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 15:58:11 +0200 Subject: [PATCH 11/45] Add project information --- mkdocs.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 67e98fe0..5c554c11 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,11 @@ # yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json site_name: OpenML Server +site_url: https://openml/github.io/server-api + +repo_name: openml/server-api +repo_url: https://github.com/openml/server-api + theme: name: material From 6cc2d195b48de1542fe23fd506871da5b24ecba8 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 16:00:18 +0200 Subject: [PATCH 12/45] Further clarify that this is not REST API documentation itself --- docs/index.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/index.md b/docs/index.md index d077dd57..e8f03d7e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,4 +12,9 @@ modern Python-based stack. [Python](https://openml.github.io/openml-python/main/), [Java](https://github.com/openml/openml-java), or [R](http://openml.github.io/openml-r/). + + If you are looking to interface directly with the REST API, and are looking + for documentation on the REST API endpoints, visit the + [APIs](https://openml.org/apis) page. + This documentation is for developing and hosting your own OpenML instance. From 1f1b99840e628336ac2ac11fa1a175e0b926d3b1 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 16 Oct 2023 16:02:07 +0200 Subject: [PATCH 13/45] Use code-block titles --- docs/migration.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/migration.md b/docs/migration.md index 7a8fc8fb..dace9722 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -9,21 +9,20 @@ The following changes affect all endpoints. When providing input of invalid types (e.g., a non-integer dataset id) the HTTP header and JSON content will be different. -HTTP Header: -```diff +```diff title="HTTP Header" - 412 Precondition Failed + 422 Unprocessable Entity ``` -JSON Content -```diff +```diff title="JSON Content" - {"error":{"code":"100","message":"Function not valid"}} + {"detail":[{"loc":["query","_dataset_id"],"msg":"value is not a valid integer","type":"type_error.integer"}]} ``` ### Other Errors For any other error messages, the response is identical except that outer field will be `"detail"` instead of `"error"`: -```diff + +```diff title="JSON Content" - {"error":{"code":"112","message":"No access granted"}} + {"detail":{"code":"112","message":"No access granted"}} ``` From 4fac2398f6ca6a264fa1d137ed291dc26a16c0d4 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 17 Oct 2023 13:43:54 +0200 Subject: [PATCH 14/45] Add Dockerfile for setting up PHP server --- php/README.md | 2 + php/apache/Dockerfile | 19 + php/apache/README.md | 13 + php/apache/build_docker.sh | 3 + php/apache/config/.htaccess | 17 + php/apache/config/BASE_CONFIG-BLANK.php | 199 +++ php/apache/config/api.conf | 28 + php/apache/config/headers.load | 1 + php/apache/config/php.ini | 1961 +++++++++++++++++++++++ php/apache/config/rewrite.load | 1 + php/apache/run_docker.sh | 9 + 11 files changed, 2253 insertions(+) create mode 100644 php/README.md create mode 100644 php/apache/Dockerfile create mode 100644 php/apache/README.md create mode 100755 php/apache/build_docker.sh create mode 100644 php/apache/config/.htaccess create mode 100644 php/apache/config/BASE_CONFIG-BLANK.php create mode 100644 php/apache/config/api.conf create mode 100644 php/apache/config/headers.load create mode 100644 php/apache/config/php.ini create mode 100644 php/apache/config/rewrite.load create mode 100755 php/apache/run_docker.sh diff --git a/php/README.md b/php/README.md new file mode 100644 index 00000000..5de1bd1c --- /dev/null +++ b/php/README.md @@ -0,0 +1,2 @@ +# PHP REST API +This directory contains files required to build the docker image and host the PHP REST API locally. diff --git a/php/apache/Dockerfile b/php/apache/Dockerfile new file mode 100644 index 00000000..e31fbbcd --- /dev/null +++ b/php/apache/Dockerfile @@ -0,0 +1,19 @@ +FROM php:7.4.33-apache + +RUN docker-php-source extract \ + && docker-php-ext-install mysqli \ + && docker-php-source delete + +RUN apt-get update \ + && apt-get install -y git \ + && git clone https://github.com/openml/openml /var/www/openml + +RUN mkdir /var/www/openml/logs +RUN mkdir /data + +COPY config/*.load /etc/apache2/mods-enabled/ +COPY config/api.conf /etc/apache2/sites-enabled/000-default.conf +COPY config/php.ini /user/local/etc/php/ +COPY config/.htaccess /var/www/openml/.htaccess + +ENTRYPOINT ["apache2-foreground"] diff --git a/php/apache/README.md b/php/apache/README.md new file mode 100644 index 00000000..1c3bb3ec --- /dev/null +++ b/php/apache/README.md @@ -0,0 +1,13 @@ +# Running apache php backend locally + +1. checkout the php code to a directory (/yourpath/to/php_root) +2. update /yourpath/to/php_root/.htaccess: remove the `RedirectMatch 301` +3. create /yourpath/to/php_root/openml_OS/config/BASE_CONFIG.php (copy the BASE_CONFIG_BLANK.php) and configure the database etc. +4. Set `define('ENVIRONMENT', 'development');` in /yourpath/to/php_root/index.php so that you'll get errors + +```bash +./build_docker.sh +./run_docker.sh /yourpath/to/php_root /path/to/a/data/directory +``` + +Go to http://localhost:8001/home diff --git a/php/apache/build_docker.sh b/php/apache/build_docker.sh new file mode 100755 index 00000000..6cf267a8 --- /dev/null +++ b/php/apache/build_docker.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +docker build --tag apache-php -f Dockerfile . diff --git a/php/apache/config/.htaccess b/php/apache/config/.htaccess new file mode 100644 index 00000000..e8918d26 --- /dev/null +++ b/php/apache/config/.htaccess @@ -0,0 +1,17 @@ +RewriteEngine on + +# TODO: specific for main instance of OpenML site. Should do something better +RewriteCond %{HTTP_HOST} ^api_new.openml.org +RewriteRule ^(.*)$ http://www.openml.org/api_new/$1 [L,P] + +RewriteCond %{HTTPS_HOST} ^api_new.openml.org +RewriteRule ^(.*)$ https://www.openml.org/api_new/$1 [L,P] + +RewriteCond $1 !^(questions|SWF|img|docs|downloads|GFX|favicon\.ico|tiny_mce|index\.php|js|css|robots\.txt) +RewriteRule ^(.*)$ index.php/$1 [L] + + + + Header set Access-Control-Allow-Origin "*" + Header set Access-Control-Allow-Headers "Origin, X-Requested-With, Content-Type, Accept" + diff --git a/php/apache/config/BASE_CONFIG-BLANK.php b/php/apache/config/BASE_CONFIG-BLANK.php new file mode 100644 index 00000000..3f1a5e28 --- /dev/null +++ b/php/apache/config/BASE_CONFIG-BLANK.php @@ -0,0 +1,199 @@ + diff --git a/php/apache/config/api.conf b/php/apache/config/api.conf new file mode 100644 index 00000000..14a04b82 --- /dev/null +++ b/php/apache/config/api.conf @@ -0,0 +1,28 @@ +User www-data +Group www-data + +LogFormat "%h %l %u %t \"%r\" %>s %b" common +ErrorLog /var/www/openml/logs/error.log +CustomLog /var/www/openml/logs/access.log common + +HostnameLookups Off + + + + Options FollowSymLinks + AllowOverride None + Require all denied + + + + Options Indexes FollowSymLinks MultiViews + AllowOverride All + Require all granted + + + + DocumentRoot /var/www/openml + RewriteEngine on + RewriteCond %{SERVER_NAME} =api.openml.org + RewriteRule ^ https://%{SERVER_NAME}%{REQUEST_URI} [END,NE,R=permanent] + diff --git a/php/apache/config/headers.load b/php/apache/config/headers.load new file mode 100644 index 00000000..e4497e5a --- /dev/null +++ b/php/apache/config/headers.load @@ -0,0 +1 @@ +LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so diff --git a/php/apache/config/php.ini b/php/apache/config/php.ini new file mode 100644 index 00000000..84ba8f8d --- /dev/null +++ b/php/apache/config/php.ini @@ -0,0 +1,1961 @@ +[PHP] + +;;;;;;;;;;;;;;;;;;; +; About php.ini ; +;;;;;;;;;;;;;;;;;;; +; PHP's initialization file, generally called php.ini, is responsible for +; configuring many of the aspects of PHP's behavior. + +; PHP attempts to find and load this configuration from a number of locations. +; The following is a summary of its search order: +; 1. SAPI module specific location. +; 2. The PHPRC environment variable. +; 3. A number of predefined registry keys on Windows +; 4. Current working directory (except CLI) +; 5. The web server's directory (for SAPI modules), or directory of PHP +; (otherwise in Windows) +; 6. The directory from the --with-config-file-path compile time option, or the +; Windows directory (usually C:\windows) +; See the PHP docs for more specific information. +; https://php.net/configuration.file + +; The syntax of the file is extremely simple. Whitespace and lines +; beginning with a semicolon are silently ignored (as you probably guessed). +; Section headers (e.g. [Foo]) are also silently ignored, even though +; they might mean something in the future. + +; Directives following the section heading [PATH=/www/mysite] only +; apply to PHP files in the /www/mysite directory. Directives +; following the section heading [HOST=www.example.com] only apply to +; PHP files served from www.example.com. Directives set in these +; special sections cannot be overridden by user-defined INI files or +; at runtime. Currently, [PATH=] and [HOST=] sections only work under +; CGI/FastCGI. +; https://php.net/ini.sections + +; Directives are specified using the following syntax: +; directive = value +; Directive names are *case sensitive* - foo=bar is different from FOO=bar. +; Directives are variables used to configure PHP or PHP extensions. +; There is no name validation. If PHP can't find an expected +; directive because it is not set or is mistyped, a default value will be used. + +; The value can be a string, a number, a PHP constant (e.g. E_ALL or M_PI), one +; of the INI constants (On, Off, True, False, Yes, No and None) or an expression +; (e.g. E_ALL & ~E_NOTICE), a quoted string ("bar"), or a reference to a +; previously set variable or directive (e.g. ${foo}) + +; Expressions in the INI file are limited to bitwise operators and parentheses: +; | bitwise OR +; ^ bitwise XOR +; & bitwise AND +; ~ bitwise NOT +; ! boolean NOT + +; Boolean flags can be turned on using the values 1, On, True or Yes. +; They can be turned off using the values 0, Off, False or No. + +; An empty string can be denoted by simply not writing anything after the equal +; sign, or by using the None keyword: + +; foo = ; sets foo to an empty string +; foo = None ; sets foo to an empty string +; foo = "None" ; sets foo to the string 'None' + +; If you use constants in your value, and these constants belong to a +; dynamically loaded extension (either a PHP extension or a Zend extension), +; you may only use these constants *after* the line that loads the extension. + +;;;;;;;;;;;;;;;;;;; +; About this file ; +;;;;;;;;;;;;;;;;;;; +; PHP comes packaged with two INI files. One that is recommended to be used +; in production environments and one that is recommended to be used in +; development environments. + +; php.ini-production contains settings which hold security, performance and +; best practices at its core. But please be aware, these settings may break +; compatibility with older or less security-conscious applications. We +; recommending using the production ini in production and testing environments. + +; php.ini-development is very similar to its production variant, except it is +; much more verbose when it comes to errors. We recommend using the +; development version only in development environments, as errors shown to +; application users can inadvertently leak otherwise secure information. + +; This is the php.ini-production INI file. + +;;;;;;;;;;;;;;;;;;; +; Quick Reference ; +;;;;;;;;;;;;;;;;;;; + +; The following are all the settings which are different in either the production +; or development versions of the INIs with respect to PHP's default behavior. +; Please see the actual settings later in the document for more details as to why +; we recommend these changes in PHP's behavior. + +; display_errors +; Default Value: On +; Development Value: On +; Production Value: Off + +; display_startup_errors +; Default Value: On +; Development Value: On +; Production Value: Off + +; error_reporting +; Default Value: E_ALL +; Development Value: E_ALL +; Production Value: E_ALL & ~E_DEPRECATED & ~E_STRICT + +; log_errors +; Default Value: Off +; Development Value: On +; Production Value: On + +; max_input_time +; Default Value: -1 (Unlimited) +; Development Value: 60 (60 seconds) +; Production Value: 60 (60 seconds) + +; output_buffering +; Default Value: Off +; Development Value: 4096 +; Production Value: 4096 + +; register_argc_argv +; Default Value: On +; Development Value: Off +; Production Value: Off + +; request_order +; Default Value: None +; Development Value: "GP" +; Production Value: "GP" + +; session.gc_divisor +; Default Value: 100 +; Development Value: 1000 +; Production Value: 1000 + +; session.sid_bits_per_character +; Default Value: 4 +; Development Value: 5 +; Production Value: 5 + +; session.sid_length +; Default Value: 32 +; Development Value: 26 +; Production Value: 26 + +; short_open_tag +; Default Value: On +; Development Value: Off +; Production Value: Off + +; variables_order +; Default Value: "EGPCS" +; Development Value: "GPCS" +; Production Value: "GPCS" + +; zend.assertions +; Default Value: 1 +; Development Value: 1 +; Production Value: -1 + +; zend.exception_ignore_args +; Default Value: Off +; Development Value: Off +; Production Value: On + +; zend.exception_string_param_max_len +; Default Value: 15 +; Development Value: 15 +; Production Value: 0 + +;;;;;;;;;;;;;;;;;;;; +; php.ini Options ; +;;;;;;;;;;;;;;;;;;;; +; Name for user-defined php.ini (.htaccess) files. Default is ".user.ini" +;user_ini.filename = ".user.ini" + +; To disable this feature set this option to an empty value +;user_ini.filename = + +; TTL for user-defined php.ini files (time-to-live) in seconds. Default is 300 seconds (5 minutes) +;user_ini.cache_ttl = 300 + +;;;;;;;;;;;;;;;;;;;; +; Language Options ; +;;;;;;;;;;;;;;;;;;;; + +; Enable the PHP scripting language engine under Apache. +; https://php.net/engine +engine = On + +; This directive determines whether or not PHP will recognize code between +; tags as PHP source which should be processed as such. It is +; generally recommended that should be used and that this feature +; should be disabled, as enabling it may result in issues when generating XML +; documents, however this remains supported for backward compatibility reasons. +; Note that this directive does not control the would work. +; https://php.net/syntax-highlighting +;highlight.string = #DD0000 +;highlight.comment = #FF9900 +;highlight.keyword = #007700 +;highlight.default = #0000BB +;highlight.html = #000000 + +; If enabled, the request will be allowed to complete even if the user aborts +; the request. Consider enabling it if executing long requests, which may end up +; being interrupted by the user or a browser timing out. PHP's default behavior +; is to disable this feature. +; https://php.net/ignore-user-abort +;ignore_user_abort = On + +; Determines the size of the realpath cache to be used by PHP. This value should +; be increased on systems where PHP opens many files to reflect the quantity of +; the file operations performed. +; Note: if open_basedir is set, the cache is disabled +; https://php.net/realpath-cache-size +;realpath_cache_size = 4096k + +; Duration of time, in seconds for which to cache realpath information for a given +; file or directory. For systems with rarely changing files, consider increasing this +; value. +; https://php.net/realpath-cache-ttl +;realpath_cache_ttl = 120 + +; Enables or disables the circular reference collector. +; https://php.net/zend.enable-gc +zend.enable_gc = On + +; If enabled, scripts may be written in encodings that are incompatible with +; the scanner. CP936, Big5, CP949 and Shift_JIS are the examples of such +; encodings. To use this feature, mbstring extension must be enabled. +;zend.multibyte = Off + +; Allows to set the default encoding for the scripts. This value will be used +; unless "declare(encoding=...)" directive appears at the top of the script. +; Only affects if zend.multibyte is set. +;zend.script_encoding = + +; Allows to include or exclude arguments from stack traces generated for exceptions. +; In production, it is recommended to turn this setting on to prohibit the output +; of sensitive information in stack traces +; Default Value: Off +; Development Value: Off +; Production Value: On +zend.exception_ignore_args = On + +; Allows setting the maximum string length in an argument of a stringified stack trace +; to a value between 0 and 1000000. +; This has no effect when zend.exception_ignore_args is enabled. +; Default Value: 15 +; Development Value: 15 +; Production Value: 0 +; In production, it is recommended to set this to 0 to reduce the output +; of sensitive information in stack traces. +zend.exception_string_param_max_len = 0 + +;;;;;;;;;;;;;;;;; +; Miscellaneous ; +;;;;;;;;;;;;;;;;; + +; Decides whether PHP may expose the fact that it is installed on the server +; (e.g. by adding its signature to the Web server header). It is no security +; threat in any way, but it makes it possible to determine whether you use PHP +; on your server or not. +; https://php.net/expose-php +expose_php = On + +;;;;;;;;;;;;;;;;;;; +; Resource Limits ; +;;;;;;;;;;;;;;;;;;; + +; Maximum execution time of each script, in seconds +; https://php.net/max-execution-time +; Note: This directive is hardcoded to 0 for the CLI SAPI +max_execution_time = 30 + +; Maximum amount of time each script may spend parsing request data. It's a good +; idea to limit this time on productions servers in order to eliminate unexpectedly +; long running scripts. +; Note: This directive is hardcoded to -1 for the CLI SAPI +; Default Value: -1 (Unlimited) +; Development Value: 60 (60 seconds) +; Production Value: 60 (60 seconds) +; https://php.net/max-input-time +max_input_time = 60 + +; Maximum input variable nesting level +; https://php.net/max-input-nesting-level +;max_input_nesting_level = 64 + +; How many GET/POST/COOKIE input variables may be accepted +;max_input_vars = 1000 + +; How many multipart body parts (combined input variable and file uploads) may +; be accepted. +; Default Value: -1 (Sum of max_input_vars and max_file_uploads) +;max_multipart_body_parts = 1500 + +; Maximum amount of memory a script may consume +; https://php.net/memory-limit +memory_limit = 16G + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; Error handling and logging ; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +; This directive informs PHP of which errors, warnings and notices you would like +; it to take action for. The recommended way of setting values for this +; directive is through the use of the error level constants and bitwise +; operators. The error level constants are below here for convenience as well as +; some common settings and their meanings. +; By default, PHP is set to take action on all errors, notices and warnings EXCEPT +; those related to E_NOTICE and E_STRICT, which together cover best practices and +; recommended coding standards in PHP. For performance reasons, this is the +; recommend error reporting setting. Your production server shouldn't be wasting +; resources complaining about best practices and coding standards. That's what +; development servers and development settings are for. +; Note: The php.ini-development file has this setting as E_ALL. This +; means it pretty much reports everything which is exactly what you want during +; development and early testing. +; +; Error Level Constants: +; E_ALL - All errors and warnings +; E_ERROR - fatal run-time errors +; E_RECOVERABLE_ERROR - almost fatal run-time errors +; E_WARNING - run-time warnings (non-fatal errors) +; E_PARSE - compile-time parse errors +; E_NOTICE - run-time notices (these are warnings which often result +; from a bug in your code, but it's possible that it was +; intentional (e.g., using an uninitialized variable and +; relying on the fact it is automatically initialized to an +; empty string) +; E_STRICT - run-time notices, enable to have PHP suggest changes +; to your code which will ensure the best interoperability +; and forward compatibility of your code +; E_CORE_ERROR - fatal errors that occur during PHP's initial startup +; E_CORE_WARNING - warnings (non-fatal errors) that occur during PHP's +; initial startup +; E_COMPILE_ERROR - fatal compile-time errors +; E_COMPILE_WARNING - compile-time warnings (non-fatal errors) +; E_USER_ERROR - user-generated error message +; E_USER_WARNING - user-generated warning message +; E_USER_NOTICE - user-generated notice message +; E_DEPRECATED - warn about code that will not work in future versions +; of PHP +; E_USER_DEPRECATED - user-generated deprecation warnings +; +; Common Values: +; E_ALL (Show all errors, warnings and notices including coding standards.) +; E_ALL & ~E_NOTICE (Show all errors, except for notices) +; E_ALL & ~E_NOTICE & ~E_STRICT (Show all errors, except for notices and coding standards warnings.) +; E_COMPILE_ERROR|E_RECOVERABLE_ERROR|E_ERROR|E_CORE_ERROR (Show only errors) +; Default Value: E_ALL +; Development Value: E_ALL +; Production Value: E_ALL & ~E_DEPRECATED & ~E_STRICT +; https://php.net/error-reporting +error_reporting = E_ALL & ~E_DEPRECATED & ~E_STRICT + +; This directive controls whether or not and where PHP will output errors, +; notices and warnings too. Error output is very useful during development, but +; it could be very dangerous in production environments. Depending on the code +; which is triggering the error, sensitive information could potentially leak +; out of your application such as database usernames and passwords or worse. +; For production environments, we recommend logging errors rather than +; sending them to STDOUT. +; Possible Values: +; Off = Do not display any errors +; stderr = Display errors to STDERR (affects only CGI/CLI binaries!) +; On or stdout = Display errors to STDOUT +; Default Value: On +; Development Value: On +; Production Value: Off +; https://php.net/display-errors +display_errors = Off + +; The display of errors which occur during PHP's startup sequence are handled +; separately from display_errors. We strongly recommend you set this to 'off' +; for production servers to avoid leaking configuration details. +; Default Value: On +; Development Value: On +; Production Value: Off +; https://php.net/display-startup-errors +display_startup_errors = Off + +; Besides displaying errors, PHP can also log errors to locations such as a +; server-specific log, STDERR, or a location specified by the error_log +; directive found below. While errors should not be displayed on productions +; servers they should still be monitored and logging is a great way to do that. +; Default Value: Off +; Development Value: On +; Production Value: On +; https://php.net/log-errors +log_errors = On + +; Do not log repeated messages. Repeated errors must occur in same file on same +; line unless ignore_repeated_source is set true. +; https://php.net/ignore-repeated-errors +ignore_repeated_errors = Off + +; Ignore source of message when ignoring repeated messages. When this setting +; is On you will not log errors with repeated messages from different files or +; source lines. +; https://php.net/ignore-repeated-source +ignore_repeated_source = Off + +; If this parameter is set to Off, then memory leaks will not be shown (on +; stdout or in the log). This is only effective in a debug compile, and if +; error reporting includes E_WARNING in the allowed list +; https://php.net/report-memleaks +report_memleaks = On + +; This setting is off by default. +;report_zend_debug = 0 + +; Turn off normal error reporting and emit XML-RPC error XML +; https://php.net/xmlrpc-errors +;xmlrpc_errors = 0 + +; An XML-RPC faultCode +;xmlrpc_error_number = 0 + +; When PHP displays or logs an error, it has the capability of formatting the +; error message as HTML for easier reading. This directive controls whether +; the error message is formatted as HTML or not. +; Note: This directive is hardcoded to Off for the CLI SAPI +; https://php.net/html-errors +;html_errors = On + +; If html_errors is set to On *and* docref_root is not empty, then PHP +; produces clickable error messages that direct to a page describing the error +; or function causing the error in detail. +; You can download a copy of the PHP manual from https://php.net/docs +; and change docref_root to the base URL of your local copy including the +; leading '/'. You must also specify the file extension being used including +; the dot. PHP's default behavior is to leave these settings empty, in which +; case no links to documentation are generated. +; Note: Never use this feature for production boxes. +; https://php.net/docref-root +; Examples +;docref_root = "/phpmanual/" + +; https://php.net/docref-ext +;docref_ext = .html + +; String to output before an error message. PHP's default behavior is to leave +; this setting blank. +; https://php.net/error-prepend-string +; Example: +;error_prepend_string = "" + +; String to output after an error message. PHP's default behavior is to leave +; this setting blank. +; https://php.net/error-append-string +; Example: +;error_append_string = "" + +; Log errors to specified file. PHP's default behavior is to leave this value +; empty. +; https://php.net/error-log +; Example: +; error_log = /var/www/openml/logs +; Log errors to syslog (Event Log on Windows). +;error_log = syslog + +; The syslog ident is a string which is prepended to every message logged +; to syslog. Only used when error_log is set to syslog. +;syslog.ident = php + +; The syslog facility is used to specify what type of program is logging +; the message. Only used when error_log is set to syslog. +;syslog.facility = user + +; Set this to disable filtering control characters (the default). +; Some loggers only accept NVT-ASCII, others accept anything that's not +; control characters. If your logger accepts everything, then no filtering +; is needed at all. +; Allowed values are: +; ascii (all printable ASCII characters and NL) +; no-ctrl (all characters except control characters) +; all (all characters) +; raw (like "all", but messages are not split at newlines) +; https://php.net/syslog.filter +;syslog.filter = ascii + +;windows.show_crt_warning +; Default value: 0 +; Development value: 0 +; Production value: 0 + +;;;;;;;;;;;;;;;;; +; Data Handling ; +;;;;;;;;;;;;;;;;; + +; The separator used in PHP generated URLs to separate arguments. +; PHP's default setting is "&". +; https://php.net/arg-separator.output +; Example: +;arg_separator.output = "&" + +; List of separator(s) used by PHP to parse input URLs into variables. +; PHP's default setting is "&". +; NOTE: Every character in this directive is considered as separator! +; https://php.net/arg-separator.input +; Example: +;arg_separator.input = ";&" + +; This directive determines which super global arrays are registered when PHP +; starts up. G,P,C,E & S are abbreviations for the following respective super +; globals: GET, POST, COOKIE, ENV and SERVER. There is a performance penalty +; paid for the registration of these arrays and because ENV is not as commonly +; used as the others, ENV is not recommended on productions servers. You +; can still get access to the environment variables through getenv() should you +; need to. +; Default Value: "EGPCS" +; Development Value: "GPCS" +; Production Value: "GPCS"; +; https://php.net/variables-order +variables_order = "GPCS" + +; This directive determines which super global data (G,P & C) should be +; registered into the super global array REQUEST. If so, it also determines +; the order in which that data is registered. The values for this directive +; are specified in the same manner as the variables_order directive, +; EXCEPT one. Leaving this value empty will cause PHP to use the value set +; in the variables_order directive. It does not mean it will leave the super +; globals array REQUEST empty. +; Default Value: None +; Development Value: "GP" +; Production Value: "GP" +; https://php.net/request-order +request_order = "GP" + +; This directive determines whether PHP registers $argv & $argc each time it +; runs. $argv contains an array of all the arguments passed to PHP when a script +; is invoked. $argc contains an integer representing the number of arguments +; that were passed when the script was invoked. These arrays are extremely +; useful when running scripts from the command line. When this directive is +; enabled, registering these variables consumes CPU cycles and memory each time +; a script is executed. For performance reasons, this feature should be disabled +; on production servers. +; Note: This directive is hardcoded to On for the CLI SAPI +; Default Value: On +; Development Value: Off +; Production Value: Off +; https://php.net/register-argc-argv +register_argc_argv = Off + +; When enabled, the ENV, REQUEST and SERVER variables are created when they're +; first used (Just In Time) instead of when the script starts. If these +; variables are not used within a script, having this directive on will result +; in a performance gain. The PHP directive register_argc_argv must be disabled +; for this directive to have any effect. +; https://php.net/auto-globals-jit +auto_globals_jit = On + +; Whether PHP will read the POST data. +; This option is enabled by default. +; Most likely, you won't want to disable this option globally. It causes $_POST +; and $_FILES to always be empty; the only way you will be able to read the +; POST data will be through the php://input stream wrapper. This can be useful +; to proxy requests or to process the POST data in a memory efficient fashion. +; https://php.net/enable-post-data-reading +;enable_post_data_reading = Off + +; Maximum size of POST data that PHP will accept. +; Its value may be 0 to disable the limit. It is ignored if POST data reading +; is disabled through enable_post_data_reading. +; https://php.net/post-max-size +post_max_size = 8M + +; Automatically add files before PHP document. +; https://php.net/auto-prepend-file +auto_prepend_file = + +; Automatically add files after PHP document. +; https://php.net/auto-append-file +auto_append_file = + +; By default, PHP will output a media type using the Content-Type header. To +; disable this, simply set it to be empty. +; +; PHP's built-in default media type is set to text/html. +; https://php.net/default-mimetype +default_mimetype = "text/html" + +; PHP's default character set is set to UTF-8. +; https://php.net/default-charset +default_charset = "UTF-8" + +; PHP internal character encoding is set to empty. +; If empty, default_charset is used. +; https://php.net/internal-encoding +;internal_encoding = + +; PHP input character encoding is set to empty. +; If empty, default_charset is used. +; https://php.net/input-encoding +;input_encoding = + +; PHP output character encoding is set to empty. +; If empty, default_charset is used. +; See also output_buffer. +; https://php.net/output-encoding +;output_encoding = + +;;;;;;;;;;;;;;;;;;;;;;;;; +; Paths and Directories ; +;;;;;;;;;;;;;;;;;;;;;;;;; + +; UNIX: "/path1:/path2" +;include_path = ".:/php/includes" +; +; Windows: "\path1;\path2" +;include_path = ".;c:\php\includes" +; +; PHP's default setting for include_path is ".;/path/to/php/pear" +; https://php.net/include-path + +; The root of the PHP pages, used only if nonempty. +; if PHP was not compiled with FORCE_REDIRECT, you SHOULD set doc_root +; if you are running php as a CGI under any web server (other than IIS) +; see documentation for security issues. The alternate is to use the +; cgi.force_redirect configuration below +; https://php.net/doc-root +doc_root = + +; The directory under which PHP opens the script using /~username used only +; if nonempty. +; https://php.net/user-dir +user_dir = + +; Directory in which the loadable extensions (modules) reside. +; https://php.net/extension-dir +;extension_dir = "./" +; On windows: +;extension_dir = "ext" + +; Directory where the temporary files should be placed. +; Defaults to the system default (see sys_get_temp_dir) +;sys_temp_dir = "/tmp" + +; Whether or not to enable the dl() function. The dl() function does NOT work +; properly in multithreaded servers, such as IIS or Zeus, and is automatically +; disabled on them. +; https://php.net/enable-dl +enable_dl = Off + +; cgi.force_redirect is necessary to provide security running PHP as a CGI under +; most web servers. Left undefined, PHP turns this on by default. You can +; turn it off here AT YOUR OWN RISK +; **You CAN safely turn this off for IIS, in fact, you MUST.** +; https://php.net/cgi.force-redirect +;cgi.force_redirect = 1 + +; if cgi.nph is enabled it will force cgi to always sent Status: 200 with +; every request. PHP's default behavior is to disable this feature. +;cgi.nph = 1 + +; if cgi.force_redirect is turned on, and you are not running under Apache or Netscape +; (iPlanet) web servers, you MAY need to set an environment variable name that PHP +; will look for to know it is OK to continue execution. Setting this variable MAY +; cause security issues, KNOW WHAT YOU ARE DOING FIRST. +; https://php.net/cgi.redirect-status-env +;cgi.redirect_status_env = + +; cgi.fix_pathinfo provides *real* PATH_INFO/PATH_TRANSLATED support for CGI. PHP's +; previous behaviour was to set PATH_TRANSLATED to SCRIPT_FILENAME, and to not grok +; what PATH_INFO is. For more information on PATH_INFO, see the cgi specs. Setting +; this to 1 will cause PHP CGI to fix its paths to conform to the spec. A setting +; of zero causes PHP to behave as before. Default is 1. You should fix your scripts +; to use SCRIPT_FILENAME rather than PATH_TRANSLATED. +; https://php.net/cgi.fix-pathinfo +;cgi.fix_pathinfo=1 + +; if cgi.discard_path is enabled, the PHP CGI binary can safely be placed outside +; of the web tree and people will not be able to circumvent .htaccess security. +;cgi.discard_path=1 + +; FastCGI under IIS supports the ability to impersonate +; security tokens of the calling client. This allows IIS to define the +; security context that the request runs under. mod_fastcgi under Apache +; does not currently support this feature (03/17/2002) +; Set to 1 if running under IIS. Default is zero. +; https://php.net/fastcgi.impersonate +;fastcgi.impersonate = 1 + +; Disable logging through FastCGI connection. PHP's default behavior is to enable +; this feature. +;fastcgi.logging = 0 + +; cgi.rfc2616_headers configuration option tells PHP what type of headers to +; use when sending HTTP response code. If set to 0, PHP sends Status: header that +; is supported by Apache. When this option is set to 1, PHP will send +; RFC2616 compliant header. +; Default is zero. +; https://php.net/cgi.rfc2616-headers +;cgi.rfc2616_headers = 0 + +; cgi.check_shebang_line controls whether CGI PHP checks for line starting with #! +; (shebang) at the top of the running script. This line might be needed if the +; script support running both as stand-alone script and via PHP CGI<. PHP in CGI +; mode skips this line and ignores its content if this directive is turned on. +; https://php.net/cgi.check-shebang-line +;cgi.check_shebang_line=1 + +;;;;;;;;;;;;;;;; +; File Uploads ; +;;;;;;;;;;;;;;;; + +; Whether to allow HTTP file uploads. +; https://php.net/file-uploads +file_uploads = On + +; Temporary directory for HTTP uploaded files (will use system default if not +; specified). +; https://php.net/upload-tmp-dir +;upload_tmp_dir = + +; Maximum allowed size for uploaded files. +; https://php.net/upload-max-filesize +upload_max_filesize = 2M + +; Maximum number of files that can be uploaded via a single request +max_file_uploads = 20 + +;;;;;;;;;;;;;;;;;; +; Fopen wrappers ; +;;;;;;;;;;;;;;;;;; + +; Whether to allow the treatment of URLs (like http:// or ftp://) as files. +; https://php.net/allow-url-fopen +allow_url_fopen = On + +; Whether to allow include/require to open URLs (like https:// or ftp://) as files. +; https://php.net/allow-url-include +allow_url_include = Off + +; Define the anonymous ftp password (your email address). PHP's default setting +; for this is empty. +; https://php.net/from +;from="john@doe.com" + +; Define the User-Agent string. PHP's default setting for this is empty. +; https://php.net/user-agent +;user_agent="PHP" + +; Default timeout for socket based streams (seconds) +; https://php.net/default-socket-timeout +default_socket_timeout = 60 + +; If your scripts have to deal with files from Macintosh systems, +; or you are running on a Mac and need to deal with files from +; unix or win32 systems, setting this flag will cause PHP to +; automatically detect the EOL character in those files so that +; fgets() and file() will work regardless of the source of the file. +; https://php.net/auto-detect-line-endings +;auto_detect_line_endings = Off + +;;;;;;;;;;;;;;;;;;;;;; +; Dynamic Extensions ; +;;;;;;;;;;;;;;;;;;;;;; + +; If you wish to have an extension loaded automatically, use the following +; syntax: +; +; extension=modulename +; +; For example: +; +; extension=mysqli +; +; When the extension library to load is not located in the default extension +; directory, You may specify an absolute path to the library file: +; +; extension=/path/to/extension/mysqli.so +; +; Note : The syntax used in previous PHP versions ('extension=.so' and +; 'extension='php_.dll') is supported for legacy reasons and may be +; deprecated in a future PHP major version. So, when it is possible, please +; move to the new ('extension=) syntax. +; +; Notes for Windows environments : +; +; - Many DLL files are located in the ext/ +; extension folders as well as the separate PECL DLL download. +; Be sure to appropriately set the extension_dir directive. +; +;extension=bz2 + +; The ldap extension must be before curl if OpenSSL 1.0.2 and OpenLDAP is used +; otherwise it results in segfault when unloading after using SASL. +; See https://github.com/php/php-src/issues/8620 for more info. +;extension=ldap + +;extension=curl +;extension=ffi +;extension=ftp +;extension=fileinfo +;extension=gd +;extension=gettext +;extension=gmp +;extension=intl +;extension=imap +;extension=mbstring +;extension=exif ; Must be after mbstring as it depends on it +extension=mysqli +;extension=oci8_12c ; Use with Oracle Database 12c Instant Client +;extension=oci8_19 ; Use with Oracle Database 19 Instant Client +;extension=odbc +;extension=openssl +;extension=pdo_firebird +;extension=pdo_mysql +;extension=pdo_oci +;extension=pdo_odbc +;extension=pdo_pgsql +;extension=pdo_sqlite +;extension=pgsql +;extension=shmop + +; The MIBS data available in the PHP distribution must be installed. +; See https://www.php.net/manual/en/snmp.installation.php +;extension=snmp + +;extension=soap +;extension=sockets +;extension=sodium +;extension=sqlite3 +;extension=tidy +;extension=xsl +;extension=zip + +;zend_extension=opcache + +;;;;;;;;;;;;;;;;;;; +; Module Settings ; +;;;;;;;;;;;;;;;;;;; + +[CLI Server] +; Whether the CLI web server uses ANSI color coding in its terminal output. +cli_server.color = On + +[Date] +; Defines the default timezone used by the date functions +; https://php.net/date.timezone +;date.timezone = + +; https://php.net/date.default-latitude +;date.default_latitude = 31.7667 + +; https://php.net/date.default-longitude +;date.default_longitude = 35.2333 + +; https://php.net/date.sunrise-zenith +;date.sunrise_zenith = 90.833333 + +; https://php.net/date.sunset-zenith +;date.sunset_zenith = 90.833333 + +[filter] +; https://php.net/filter.default +;filter.default = unsafe_raw + +; https://php.net/filter.default-flags +;filter.default_flags = + +[iconv] +; Use of this INI entry is deprecated, use global input_encoding instead. +; If empty, default_charset or input_encoding or iconv.input_encoding is used. +; The precedence is: default_charset < input_encoding < iconv.input_encoding +;iconv.input_encoding = + +; Use of this INI entry is deprecated, use global internal_encoding instead. +; If empty, default_charset or internal_encoding or iconv.internal_encoding is used. +; The precedence is: default_charset < internal_encoding < iconv.internal_encoding +;iconv.internal_encoding = + +; Use of this INI entry is deprecated, use global output_encoding instead. +; If empty, default_charset or output_encoding or iconv.output_encoding is used. +; The precedence is: default_charset < output_encoding < iconv.output_encoding +; To use an output encoding conversion, iconv's output handler must be set +; otherwise output encoding conversion cannot be performed. +;iconv.output_encoding = + +[imap] +; rsh/ssh logins are disabled by default. Use this INI entry if you want to +; enable them. Note that the IMAP library does not filter mailbox names before +; passing them to rsh/ssh command, thus passing untrusted data to this function +; with rsh/ssh enabled is insecure. +;imap.enable_insecure_rsh=0 + +[intl] +;intl.default_locale = +; This directive allows you to produce PHP errors when some error +; happens within intl functions. The value is the level of the error produced. +; Default is 0, which does not produce any errors. +;intl.error_level = E_WARNING +;intl.use_exceptions = 0 + +[sqlite3] +; Directory pointing to SQLite3 extensions +; https://php.net/sqlite3.extension-dir +;sqlite3.extension_dir = + +; SQLite defensive mode flag (only available from SQLite 3.26+) +; When the defensive flag is enabled, language features that allow ordinary +; SQL to deliberately corrupt the database file are disabled. This forbids +; writing directly to the schema, shadow tables (eg. FTS data tables), or +; the sqlite_dbpage virtual table. +; https://www.sqlite.org/c3ref/c_dbconfig_defensive.html +; (for older SQLite versions, this flag has no use) +;sqlite3.defensive = 1 + +[Pcre] +; PCRE library backtracking limit. +; https://php.net/pcre.backtrack-limit +;pcre.backtrack_limit=100000 + +; PCRE library recursion limit. +; Please note that if you set this value to a high number you may consume all +; the available process stack and eventually crash PHP (due to reaching the +; stack size limit imposed by the Operating System). +; https://php.net/pcre.recursion-limit +;pcre.recursion_limit=100000 + +; Enables or disables JIT compilation of patterns. This requires the PCRE +; library to be compiled with JIT support. +;pcre.jit=1 + +[Pdo] +; Whether to pool ODBC connections. Can be one of "strict", "relaxed" or "off" +; https://php.net/pdo-odbc.connection-pooling +;pdo_odbc.connection_pooling=strict + +[Pdo_mysql] +; Default socket name for local MySQL connects. If empty, uses the built-in +; MySQL defaults. +pdo_mysql.default_socket= + +[Phar] +; https://php.net/phar.readonly +;phar.readonly = On + +; https://php.net/phar.require-hash +;phar.require_hash = On + +;phar.cache_list = + +[mail function] +; For Win32 only. +; https://php.net/smtp +SMTP = localhost +; https://php.net/smtp-port +smtp_port = 25 + +; For Win32 only. +; https://php.net/sendmail-from +;sendmail_from = me@example.com + +; For Unix only. You may supply arguments as well (default: "sendmail -t -i"). +; https://php.net/sendmail-path +;sendmail_path = + +; Force the addition of the specified parameters to be passed as extra parameters +; to the sendmail binary. These parameters will always replace the value of +; the 5th parameter to mail(). +;mail.force_extra_parameters = + +; Add X-PHP-Originating-Script: that will include uid of the script followed by the filename +mail.add_x_header = Off + +; Use mixed LF and CRLF line separators to keep compatibility with some +; RFC 2822 non conformant MTA. +mail.mixed_lf_and_crlf = Off + +; The path to a log file that will log all mail() calls. Log entries include +; the full path of the script, line number, To address and headers. +;mail.log = +; Log mail to syslog (Event Log on Windows). +;mail.log = syslog + +[ODBC] +; https://php.net/odbc.default-db +;odbc.default_db = Not yet implemented + +; https://php.net/odbc.default-user +;odbc.default_user = Not yet implemented + +; https://php.net/odbc.default-pw +;odbc.default_pw = Not yet implemented + +; Controls the ODBC cursor model. +; Default: SQL_CURSOR_STATIC (default). +;odbc.default_cursortype + +; Allow or prevent persistent links. +; https://php.net/odbc.allow-persistent +odbc.allow_persistent = On + +; Check that a connection is still valid before reuse. +; https://php.net/odbc.check-persistent +odbc.check_persistent = On + +; Maximum number of persistent links. -1 means no limit. +; https://php.net/odbc.max-persistent +odbc.max_persistent = -1 + +; Maximum number of links (persistent + non-persistent). -1 means no limit. +; https://php.net/odbc.max-links +odbc.max_links = -1 + +; Handling of LONG fields. Returns number of bytes to variables. 0 means +; passthru. +; https://php.net/odbc.defaultlrl +odbc.defaultlrl = 4096 + +; Handling of binary data. 0 means passthru, 1 return as is, 2 convert to char. +; See the documentation on odbc_binmode and odbc_longreadlen for an explanation +; of odbc.defaultlrl and odbc.defaultbinmode +; https://php.net/odbc.defaultbinmode +odbc.defaultbinmode = 1 + +[MySQLi] + +; Maximum number of persistent links. -1 means no limit. +; https://php.net/mysqli.max-persistent +mysqli.max_persistent = -1 + +; Allow accessing, from PHP's perspective, local files with LOAD DATA statements +; https://php.net/mysqli.allow_local_infile +;mysqli.allow_local_infile = On + +; It allows the user to specify a folder where files that can be sent via LOAD DATA +; LOCAL can exist. It is ignored if mysqli.allow_local_infile is enabled. +;mysqli.local_infile_directory = + +; Allow or prevent persistent links. +; https://php.net/mysqli.allow-persistent +mysqli.allow_persistent = On + +; Maximum number of links. -1 means no limit. +; https://php.net/mysqli.max-links +mysqli.max_links = -1 + +; Default port number for mysqli_connect(). If unset, mysqli_connect() will use +; the $MYSQL_TCP_PORT or the mysql-tcp entry in /etc/services or the +; compile-time value defined MYSQL_PORT (in that order). Win32 will only look +; at MYSQL_PORT. +; https://php.net/mysqli.default-port +mysqli.default_port = 3306 + +; Default socket name for local MySQL connects. If empty, uses the built-in +; MySQL defaults. +; https://php.net/mysqli.default-socket +mysqli.default_socket = + +; Default host for mysqli_connect() (doesn't apply in safe mode). +; https://php.net/mysqli.default-host +mysqli.default_host = + +; Default user for mysqli_connect() (doesn't apply in safe mode). +; https://php.net/mysqli.default-user +mysqli.default_user = + +; Default password for mysqli_connect() (doesn't apply in safe mode). +; Note that this is generally a *bad* idea to store passwords in this file. +; *Any* user with PHP access can run 'echo get_cfg_var("mysqli.default_pw") +; and reveal this password! And of course, any users with read access to this +; file will be able to reveal the password as well. +; https://php.net/mysqli.default-pw +mysqli.default_pw = + +; If this option is enabled, closing a persistent connection will rollback +; any pending transactions of this connection, before it is put back +; into the persistent connection pool. +;mysqli.rollback_on_cached_plink = Off + +[mysqlnd] +; Enable / Disable collection of general statistics by mysqlnd which can be +; used to tune and monitor MySQL operations. +mysqlnd.collect_statistics = On + +; Enable / Disable collection of memory usage statistics by mysqlnd which can be +; used to tune and monitor MySQL operations. +mysqlnd.collect_memory_statistics = Off + +; Records communication from all extensions using mysqlnd to the specified log +; file. +; https://php.net/mysqlnd.debug +;mysqlnd.debug = + +; Defines which queries will be logged. +;mysqlnd.log_mask = 0 + +; Default size of the mysqlnd memory pool, which is used by result sets. +;mysqlnd.mempool_default_size = 16000 + +; Size of a pre-allocated buffer used when sending commands to MySQL in bytes. +;mysqlnd.net_cmd_buffer_size = 2048 + +; Size of a pre-allocated buffer used for reading data sent by the server in +; bytes. +;mysqlnd.net_read_buffer_size = 32768 + +; Timeout for network requests in seconds. +;mysqlnd.net_read_timeout = 31536000 + +; SHA-256 Authentication Plugin related. File with the MySQL server public RSA +; key. +;mysqlnd.sha256_server_public_key = + +[OCI8] + +; Connection: Enables privileged connections using external +; credentials (OCI_SYSOPER, OCI_SYSDBA) +; https://php.net/oci8.privileged-connect +;oci8.privileged_connect = Off + +; Connection: The maximum number of persistent OCI8 connections per +; process. Using -1 means no limit. +; https://php.net/oci8.max-persistent +;oci8.max_persistent = -1 + +; Connection: The maximum number of seconds a process is allowed to +; maintain an idle persistent connection. Using -1 means idle +; persistent connections will be maintained forever. +; https://php.net/oci8.persistent-timeout +;oci8.persistent_timeout = -1 + +; Connection: The number of seconds that must pass before issuing a +; ping during oci_pconnect() to check the connection validity. When +; set to 0, each oci_pconnect() will cause a ping. Using -1 disables +; pings completely. +; https://php.net/oci8.ping-interval +;oci8.ping_interval = 60 + +; Connection: Set this to a user chosen connection class to be used +; for all pooled server requests with Oracle Database Resident +; Connection Pooling (DRCP). To use DRCP, this value should be set to +; the same string for all web servers running the same application, +; the database pool must be configured, and the connection string must +; specify to use a pooled server. +;oci8.connection_class = + +; High Availability: Using On lets PHP receive Fast Application +; Notification (FAN) events generated when a database node fails. The +; database must also be configured to post FAN events. +;oci8.events = Off + +; Tuning: This option enables statement caching, and specifies how +; many statements to cache. Using 0 disables statement caching. +; https://php.net/oci8.statement-cache-size +;oci8.statement_cache_size = 20 + +; Tuning: Enables row prefetching and sets the default number of +; rows that will be fetched automatically after statement execution. +; https://php.net/oci8.default-prefetch +;oci8.default_prefetch = 100 + +; Tuning: Sets the amount of LOB data that is internally returned from +; Oracle Database when an Oracle LOB locator is initially retrieved as +; part of a query. Setting this can improve performance by reducing +; round-trips. +; https://php.net/oci8.prefetch-lob-size +; oci8.prefetch_lob_size = 0 + +; Compatibility. Using On means oci_close() will not close +; oci_connect() and oci_new_connect() connections. +; https://php.net/oci8.old-oci-close-semantics +;oci8.old_oci_close_semantics = Off + +[PostgreSQL] +; Allow or prevent persistent links. +; https://php.net/pgsql.allow-persistent +pgsql.allow_persistent = On + +; Detect broken persistent links always with pg_pconnect(). +; Auto reset feature requires a little overheads. +; https://php.net/pgsql.auto-reset-persistent +pgsql.auto_reset_persistent = Off + +; Maximum number of persistent links. -1 means no limit. +; https://php.net/pgsql.max-persistent +pgsql.max_persistent = -1 + +; Maximum number of links (persistent+non persistent). -1 means no limit. +; https://php.net/pgsql.max-links +pgsql.max_links = -1 + +; Ignore PostgreSQL backends Notice message or not. +; Notice message logging require a little overheads. +; https://php.net/pgsql.ignore-notice +pgsql.ignore_notice = 0 + +; Log PostgreSQL backends Notice message or not. +; Unless pgsql.ignore_notice=0, module cannot log notice message. +; https://php.net/pgsql.log-notice +pgsql.log_notice = 0 + +[bcmath] +; Number of decimal digits for all bcmath functions. +; https://php.net/bcmath.scale +bcmath.scale = 0 + +[browscap] +; https://php.net/browscap +;browscap = extra/browscap.ini + +[Session] +; Handler used to store/retrieve data. +; https://php.net/session.save-handler +session.save_handler = files + +; Argument passed to save_handler. In the case of files, this is the path +; where data files are stored. Note: Windows users have to change this +; variable in order to use PHP's session functions. +; +; The path can be defined as: +; +; session.save_path = "N;/path" +; +; where N is an integer. Instead of storing all the session files in +; /path, what this will do is use subdirectories N-levels deep, and +; store the session data in those directories. This is useful if +; your OS has problems with many files in one directory, and is +; a more efficient layout for servers that handle many sessions. +; +; NOTE 1: PHP will not create this directory structure automatically. +; You can use the script in the ext/session dir for that purpose. +; NOTE 2: See the section on garbage collection below if you choose to +; use subdirectories for session storage +; +; The file storage module creates files using mode 600 by default. +; You can change that by using +; +; session.save_path = "N;MODE;/path" +; +; where MODE is the octal representation of the mode. Note that this +; does not overwrite the process's umask. +; https://php.net/session.save-path +;session.save_path = "/tmp" + +; Whether to use strict session mode. +; Strict session mode does not accept an uninitialized session ID, and +; regenerates the session ID if the browser sends an uninitialized session ID. +; Strict mode protects applications from session fixation via a session adoption +; vulnerability. It is disabled by default for maximum compatibility, but +; enabling it is encouraged. +; https://wiki.php.net/rfc/strict_sessions +session.use_strict_mode = 0 + +; Whether to use cookies. +; https://php.net/session.use-cookies +session.use_cookies = 1 + +; https://php.net/session.cookie-secure +;session.cookie_secure = + +; This option forces PHP to fetch and use a cookie for storing and maintaining +; the session id. We encourage this operation as it's very helpful in combating +; session hijacking when not specifying and managing your own session id. It is +; not the be-all and end-all of session hijacking defense, but it's a good start. +; https://php.net/session.use-only-cookies +session.use_only_cookies = 1 + +; Name of the session (used as cookie name). +; https://php.net/session.name +session.name = PHPSESSID + +; Initialize session on request startup. +; https://php.net/session.auto-start +session.auto_start = 0 + +; Lifetime in seconds of cookie or, if 0, until browser is restarted. +; https://php.net/session.cookie-lifetime +session.cookie_lifetime = 0 + +; The path for which the cookie is valid. +; https://php.net/session.cookie-path +session.cookie_path = / + +; The domain for which the cookie is valid. +; https://php.net/session.cookie-domain +session.cookie_domain = + +; Whether or not to add the httpOnly flag to the cookie, which makes it +; inaccessible to browser scripting languages such as JavaScript. +; https://php.net/session.cookie-httponly +session.cookie_httponly = + +; Add SameSite attribute to cookie to help mitigate Cross-Site Request Forgery (CSRF/XSRF) +; Current valid values are "Strict", "Lax" or "None". When using "None", +; make sure to include the quotes, as `none` is interpreted like `false` in ini files. +; https://tools.ietf.org/html/draft-west-first-party-cookies-07 +session.cookie_samesite = + +; Handler used to serialize data. php is the standard serializer of PHP. +; https://php.net/session.serialize-handler +session.serialize_handler = php + +; Defines the probability that the 'garbage collection' process is started on every +; session initialization. The probability is calculated by using gc_probability/gc_divisor, +; e.g. 1/100 means there is a 1% chance that the GC process starts on each request. +; Default Value: 1 +; Development Value: 1 +; Production Value: 1 +; https://php.net/session.gc-probability +session.gc_probability = 1 + +; Defines the probability that the 'garbage collection' process is started on every +; session initialization. The probability is calculated by using gc_probability/gc_divisor, +; e.g. 1/100 means there is a 1% chance that the GC process starts on each request. +; For high volume production servers, using a value of 1000 is a more efficient approach. +; Default Value: 100 +; Development Value: 1000 +; Production Value: 1000 +; https://php.net/session.gc-divisor +session.gc_divisor = 1000 + +; After this number of seconds, stored data will be seen as 'garbage' and +; cleaned up by the garbage collection process. +; https://php.net/session.gc-maxlifetime +session.gc_maxlifetime = 1440 + +; NOTE: If you are using the subdirectory option for storing session files +; (see session.save_path above), then garbage collection does *not* +; happen automatically. You will need to do your own garbage +; collection through a shell script, cron entry, or some other method. +; For example, the following script is the equivalent of setting +; session.gc_maxlifetime to 1440 (1440 seconds = 24 minutes): +; find /path/to/sessions -cmin +24 -type f | xargs rm + +; Check HTTP Referer to invalidate externally stored URLs containing ids. +; HTTP_REFERER has to contain this substring for the session to be +; considered as valid. +; https://php.net/session.referer-check +session.referer_check = + +; Set to {nocache,private,public,} to determine HTTP caching aspects +; or leave this empty to avoid sending anti-caching headers. +; https://php.net/session.cache-limiter +session.cache_limiter = nocache + +; Document expires after n minutes. +; https://php.net/session.cache-expire +session.cache_expire = 180 + +; trans sid support is disabled by default. +; Use of trans sid may risk your users' security. +; Use this option with caution. +; - User may send URL contains active session ID +; to other person via. email/irc/etc. +; - URL that contains active session ID may be stored +; in publicly accessible computer. +; - User may access your site with the same session ID +; always using URL stored in browser's history or bookmarks. +; https://php.net/session.use-trans-sid +session.use_trans_sid = 0 + +; Set session ID character length. This value could be between 22 to 256. +; Shorter length than default is supported only for compatibility reason. +; Users should use 32 or more chars. +; https://php.net/session.sid-length +; Default Value: 32 +; Development Value: 26 +; Production Value: 26 +session.sid_length = 26 + +; The URL rewriter will look for URLs in a defined set of HTML tags. +;
is special; if you include them here, the rewriter will +; add a hidden field with the info which is otherwise appended +; to URLs. tag's action attribute URL will not be modified +; unless it is specified. +; Note that all valid entries require a "=", even if no value follows. +; Default Value: "a=href,area=href,frame=src,form=" +; Development Value: "a=href,area=href,frame=src,form=" +; Production Value: "a=href,area=href,frame=src,form=" +; https://php.net/url-rewriter.tags +session.trans_sid_tags = "a=href,area=href,frame=src,form=" + +; URL rewriter does not rewrite absolute URLs by default. +; To enable rewrites for absolute paths, target hosts must be specified +; at RUNTIME. i.e. use ini_set() +; tags is special. PHP will check action attribute's URL regardless +; of session.trans_sid_tags setting. +; If no host is defined, HTTP_HOST will be used for allowed host. +; Example value: php.net,www.php.net,wiki.php.net +; Use "," for multiple hosts. No spaces are allowed. +; Default Value: "" +; Development Value: "" +; Production Value: "" +;session.trans_sid_hosts="" + +; Define how many bits are stored in each character when converting +; the binary hash data to something readable. +; Possible values: +; 4 (4 bits: 0-9, a-f) +; 5 (5 bits: 0-9, a-v) +; 6 (6 bits: 0-9, a-z, A-Z, "-", ",") +; Default Value: 4 +; Development Value: 5 +; Production Value: 5 +; https://php.net/session.hash-bits-per-character +session.sid_bits_per_character = 5 + +; Enable upload progress tracking in $_SESSION +; Default Value: On +; Development Value: On +; Production Value: On +; https://php.net/session.upload-progress.enabled +;session.upload_progress.enabled = On + +; Cleanup the progress information as soon as all POST data has been read +; (i.e. upload completed). +; Default Value: On +; Development Value: On +; Production Value: On +; https://php.net/session.upload-progress.cleanup +;session.upload_progress.cleanup = On + +; A prefix used for the upload progress key in $_SESSION +; Default Value: "upload_progress_" +; Development Value: "upload_progress_" +; Production Value: "upload_progress_" +; https://php.net/session.upload-progress.prefix +;session.upload_progress.prefix = "upload_progress_" + +; The index name (concatenated with the prefix) in $_SESSION +; containing the upload progress information +; Default Value: "PHP_SESSION_UPLOAD_PROGRESS" +; Development Value: "PHP_SESSION_UPLOAD_PROGRESS" +; Production Value: "PHP_SESSION_UPLOAD_PROGRESS" +; https://php.net/session.upload-progress.name +;session.upload_progress.name = "PHP_SESSION_UPLOAD_PROGRESS" + +; How frequently the upload progress should be updated. +; Given either in percentages (per-file), or in bytes +; Default Value: "1%" +; Development Value: "1%" +; Production Value: "1%" +; https://php.net/session.upload-progress.freq +;session.upload_progress.freq = "1%" + +; The minimum delay between updates, in seconds +; Default Value: 1 +; Development Value: 1 +; Production Value: 1 +; https://php.net/session.upload-progress.min-freq +;session.upload_progress.min_freq = "1" + +; Only write session data when session data is changed. Enabled by default. +; https://php.net/session.lazy-write +;session.lazy_write = On + +[Assertion] +; Switch whether to compile assertions at all (to have no overhead at run-time) +; -1: Do not compile at all +; 0: Jump over assertion at run-time +; 1: Execute assertions +; Changing from or to a negative value is only possible in php.ini! +; (For turning assertions on and off at run-time, toggle zend.assertions between the values 1 and 0) +; Default Value: 1 +; Development Value: 1 +; Production Value: -1 +; https://php.net/zend.assertions +zend.assertions = -1 + +[COM] +; path to a file containing GUIDs, IIDs or filenames of files with TypeLibs +; https://php.net/com.typelib-file +;com.typelib_file = + +; allow Distributed-COM calls +; https://php.net/com.allow-dcom +;com.allow_dcom = true + +; autoregister constants of a component's typelib on com_load() +; https://php.net/com.autoregister-typelib +;com.autoregister_typelib = true + +; register constants casesensitive +; https://php.net/com.autoregister-casesensitive +;com.autoregister_casesensitive = false + +; show warnings on duplicate constant registrations +; https://php.net/com.autoregister-verbose +;com.autoregister_verbose = true + +; The default character set code-page to use when passing strings to and from COM objects. +; Default: system ANSI code page +;com.code_page= + +; The version of the .NET framework to use. The value of the setting are the first three parts +; of the framework's version number, separated by dots, and prefixed with "v", e.g. "v4.0.30319". +;com.dotnet_version= + +[mbstring] +; language for internal character representation. +; This affects mb_send_mail() and mbstring.detect_order. +; https://php.net/mbstring.language +;mbstring.language = Japanese + +; Use of this INI entry is deprecated, use global internal_encoding instead. +; internal/script encoding. +; Some encoding cannot work as internal encoding. (e.g. SJIS, BIG5, ISO-2022-*) +; If empty, default_charset or internal_encoding or iconv.internal_encoding is used. +; The precedence is: default_charset < internal_encoding < iconv.internal_encoding +;mbstring.internal_encoding = + +; Use of this INI entry is deprecated, use global input_encoding instead. +; http input encoding. +; mbstring.encoding_translation = On is needed to use this setting. +; If empty, default_charset or input_encoding or mbstring.input is used. +; The precedence is: default_charset < input_encoding < mbstring.http_input +; https://php.net/mbstring.http-input +;mbstring.http_input = + +; Use of this INI entry is deprecated, use global output_encoding instead. +; http output encoding. +; mb_output_handler must be registered as output buffer to function. +; If empty, default_charset or output_encoding or mbstring.http_output is used. +; The precedence is: default_charset < output_encoding < mbstring.http_output +; To use an output encoding conversion, mbstring's output handler must be set +; otherwise output encoding conversion cannot be performed. +; https://php.net/mbstring.http-output +;mbstring.http_output = + +; enable automatic encoding translation according to +; mbstring.internal_encoding setting. Input chars are +; converted to internal encoding by setting this to On. +; Note: Do _not_ use automatic encoding translation for +; portable libs/applications. +; https://php.net/mbstring.encoding-translation +;mbstring.encoding_translation = Off + +; automatic encoding detection order. +; "auto" detect order is changed according to mbstring.language +; https://php.net/mbstring.detect-order +;mbstring.detect_order = auto + +; substitute_character used when character cannot be converted +; one from another +; https://php.net/mbstring.substitute-character +;mbstring.substitute_character = none + +; Enable strict encoding detection. +;mbstring.strict_detection = Off + +; This directive specifies the regex pattern of content types for which mb_output_handler() +; is activated. +; Default: mbstring.http_output_conv_mimetypes=^(text/|application/xhtml\+xml) +;mbstring.http_output_conv_mimetypes= + +; This directive specifies maximum stack depth for mbstring regular expressions. It is similar +; to the pcre.recursion_limit for PCRE. +;mbstring.regex_stack_limit=100000 + +; This directive specifies maximum retry count for mbstring regular expressions. It is similar +; to the pcre.backtrack_limit for PCRE. +;mbstring.regex_retry_limit=1000000 + +[gd] +; Tell the jpeg decode to ignore warnings and try to create +; a gd image. The warning will then be displayed as notices +; disabled by default +; https://php.net/gd.jpeg-ignore-warning +;gd.jpeg_ignore_warning = 1 + +[exif] +; Exif UNICODE user comments are handled as UCS-2BE/UCS-2LE and JIS as JIS. +; With mbstring support this will automatically be converted into the encoding +; given by corresponding encode setting. When empty mbstring.internal_encoding +; is used. For the decode settings you can distinguish between motorola and +; intel byte order. A decode setting cannot be empty. +; https://php.net/exif.encode-unicode +;exif.encode_unicode = ISO-8859-15 + +; https://php.net/exif.decode-unicode-motorola +;exif.decode_unicode_motorola = UCS-2BE + +; https://php.net/exif.decode-unicode-intel +;exif.decode_unicode_intel = UCS-2LE + +; https://php.net/exif.encode-jis +;exif.encode_jis = + +; https://php.net/exif.decode-jis-motorola +;exif.decode_jis_motorola = JIS + +; https://php.net/exif.decode-jis-intel +;exif.decode_jis_intel = JIS + +[Tidy] +; The path to a default tidy configuration file to use when using tidy +; https://php.net/tidy.default-config +;tidy.default_config = /usr/local/lib/php/default.tcfg + +; Should tidy clean and repair output automatically? +; WARNING: Do not use this option if you are generating non-html content +; such as dynamic images +; https://php.net/tidy.clean-output +tidy.clean_output = Off + +[soap] +; Enables or disables WSDL caching feature. +; https://php.net/soap.wsdl-cache-enabled +soap.wsdl_cache_enabled=1 + +; Sets the directory name where SOAP extension will put cache files. +; https://php.net/soap.wsdl-cache-dir +soap.wsdl_cache_dir="/tmp" + +; (time to live) Sets the number of second while cached file will be used +; instead of original one. +; https://php.net/soap.wsdl-cache-ttl +soap.wsdl_cache_ttl=86400 + +; Sets the size of the cache limit. (Max. number of WSDL files to cache) +soap.wsdl_cache_limit = 5 + +[sysvshm] +; A default size of the shared memory segment +;sysvshm.init_mem = 10000 + +[ldap] +; Sets the maximum number of open links or -1 for unlimited. +ldap.max_links = -1 + +[dba] +;dba.default_handler= + +[opcache] +; Determines if Zend OPCache is enabled +;opcache.enable=1 + +; Determines if Zend OPCache is enabled for the CLI version of PHP +;opcache.enable_cli=0 + +; The OPcache shared memory storage size. +;opcache.memory_consumption=128 + +; The amount of memory for interned strings in Mbytes. +;opcache.interned_strings_buffer=8 + +; The maximum number of keys (scripts) in the OPcache hash table. +; Only numbers between 200 and 1000000 are allowed. +;opcache.max_accelerated_files=10000 + +; The maximum percentage of "wasted" memory until a restart is scheduled. +;opcache.max_wasted_percentage=5 + +; When this directive is enabled, the OPcache appends the current working +; directory to the script key, thus eliminating possible collisions between +; files with the same name (basename). Disabling the directive improves +; performance, but may break existing applications. +;opcache.use_cwd=1 + +; When disabled, you must reset the OPcache manually or restart the +; webserver for changes to the filesystem to take effect. +;opcache.validate_timestamps=1 + +; How often (in seconds) to check file timestamps for changes to the shared +; memory storage allocation. ("1" means validate once per second, but only +; once per request. "0" means always validate) +;opcache.revalidate_freq=2 + +; Enables or disables file search in include_path optimization +;opcache.revalidate_path=0 + +; If disabled, all PHPDoc comments are dropped from the code to reduce the +; size of the optimized code. +;opcache.save_comments=1 + +; If enabled, compilation warnings (including notices and deprecations) will +; be recorded and replayed each time a file is included. Otherwise, compilation +; warnings will only be emitted when the file is first cached. +;opcache.record_warnings=0 + +; Allow file existence override (file_exists, etc.) performance feature. +;opcache.enable_file_override=0 + +; A bitmask, where each bit enables or disables the appropriate OPcache +; passes +;opcache.optimization_level=0x7FFFBFFF + +;opcache.dups_fix=0 + +; The location of the OPcache blacklist file (wildcards allowed). +; Each OPcache blacklist file is a text file that holds the names of files +; that should not be accelerated. The file format is to add each filename +; to a new line. The filename may be a full path or just a file prefix +; (i.e., /var/www/x blacklists all the files and directories in /var/www +; that start with 'x'). Line starting with a ; are ignored (comments). +;opcache.blacklist_filename= + +; Allows exclusion of large files from being cached. By default all files +; are cached. +;opcache.max_file_size=0 + +; How long to wait (in seconds) for a scheduled restart to begin if the cache +; is not being accessed. +;opcache.force_restart_timeout=180 + +; OPcache error_log file name. Empty string assumes "stderr". +;opcache.error_log= + +; All OPcache errors go to the Web server log. +; By default, only fatal errors (level 0) or errors (level 1) are logged. +; You can also enable warnings (level 2), info messages (level 3) or +; debug messages (level 4). +;opcache.log_verbosity_level=1 + +; Preferred Shared Memory back-end. Leave empty and let the system decide. +;opcache.preferred_memory_model= + +; Protect the shared memory from unexpected writing during script execution. +; Useful for internal debugging only. +;opcache.protect_memory=0 + +; Allows calling OPcache API functions only from PHP scripts which path is +; started from specified string. The default "" means no restriction +;opcache.restrict_api= + +; Mapping base of shared memory segments (for Windows only). All the PHP +; processes have to map shared memory into the same address space. This +; directive allows to manually fix the "Unable to reattach to base address" +; errors. +;opcache.mmap_base= + +; Facilitates multiple OPcache instances per user (for Windows only). All PHP +; processes with the same cache ID and user share an OPcache instance. +;opcache.cache_id= + +; Enables and sets the second level cache directory. +; It should improve performance when SHM memory is full, at server restart or +; SHM reset. The default "" disables file based caching. +;opcache.file_cache= + +; Enables or disables opcode caching in shared memory. +;opcache.file_cache_only=0 + +; Enables or disables checksum validation when script loaded from file cache. +;opcache.file_cache_consistency_checks=1 + +; Implies opcache.file_cache_only=1 for a certain process that failed to +; reattach to the shared memory (for Windows only). Explicitly enabled file +; cache is required. +;opcache.file_cache_fallback=1 + +; Enables or disables copying of PHP code (text segment) into HUGE PAGES. +; Under certain circumstances (if only a single global PHP process is +; started from which all others fork), this can increase performance +; by a tiny amount because TLB misses are reduced. On the other hand, this +; delays PHP startup, increases memory usage and degrades performance +; under memory pressure - use with care. +; Requires appropriate OS configuration. +;opcache.huge_code_pages=0 + +; Validate cached file permissions. +;opcache.validate_permission=0 + +; Prevent name collisions in chroot'ed environment. +;opcache.validate_root=0 + +; If specified, it produces opcode dumps for debugging different stages of +; optimizations. +;opcache.opt_debug_level=0 + +; Specifies a PHP script that is going to be compiled and executed at server +; start-up. +; https://php.net/opcache.preload +;opcache.preload= + +; Preloading code as root is not allowed for security reasons. This directive +; facilitates to let the preloading to be run as another user. +; https://php.net/opcache.preload_user +;opcache.preload_user= + +; Prevents caching files that are less than this number of seconds old. It +; protects from caching of incompletely updated files. In case all file updates +; on your site are atomic, you may increase performance by setting it to "0". +;opcache.file_update_protection=2 + +; Absolute path used to store shared lockfiles (for *nix only). +;opcache.lockfile_path=/tmp + +[curl] +; A default value for the CURLOPT_CAINFO option. This is required to be an +; absolute path. +;curl.cainfo = + +[openssl] +; The location of a Certificate Authority (CA) file on the local filesystem +; to use when verifying the identity of SSL/TLS peers. Most users should +; not specify a value for this directive as PHP will attempt to use the +; OS-managed cert stores in its absence. If specified, this value may still +; be overridden on a per-stream basis via the "cafile" SSL stream context +; option. +;openssl.cafile= + +; If openssl.cafile is not specified or if the CA file is not found, the +; directory pointed to by openssl.capath is searched for a suitable +; certificate. This value must be a correctly hashed certificate directory. +; Most users should not specify a value for this directive as PHP will +; attempt to use the OS-managed cert stores in its absence. If specified, +; this value may still be overridden on a per-stream basis via the "capath" +; SSL stream context option. +;openssl.capath= + +[ffi] +; FFI API restriction. Possible values: +; "preload" - enabled in CLI scripts and preloaded files (default) +; "false" - always disabled +; "true" - always enabled +;ffi.enable=preload + +; List of headers files to preload, wildcard patterns allowed. +;ffi.preload= diff --git a/php/apache/config/rewrite.load b/php/apache/config/rewrite.load new file mode 100644 index 00000000..b32f1626 --- /dev/null +++ b/php/apache/config/rewrite.load @@ -0,0 +1 @@ +LoadModule rewrite_module /usr/lib/apache2/modules/mod_rewrite.so diff --git a/php/apache/run_docker.sh b/php/apache/run_docker.sh new file mode 100755 index 00000000..fade6b6d --- /dev/null +++ b/php/apache/run_docker.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# these need to be absolute paths: +PHP_CONFIG=$1 + +docker run -p 8001:80 --rm -it \ + -v ${PHP_CONFIG}:/var/www/openml/openml_OS/config/BASE_CONFIG.php \ + --network sqlnetwork \ + apache-php From bff897ccdbf62c94083b98ef90da8bb04efc7995 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 18 Oct 2023 14:58:02 +0200 Subject: [PATCH 15/45] Move documentation around, add section index, mark incomplete info --- .../contributing.md} | 5 ++-- docs/contributing/index.md | 3 +++ docs/installation.md | 27 +++++++++++++++++-- mkdocs.yml | 10 +++++-- pyproject.toml | 3 ++- 5 files changed, 41 insertions(+), 7 deletions(-) rename docs/{CONTRIBUTING.md => contributing/contributing.md} (93%) create mode 100644 docs/contributing/index.md diff --git a/docs/CONTRIBUTING.md b/docs/contributing/contributing.md similarity index 93% rename from docs/CONTRIBUTING.md rename to docs/contributing/contributing.md index 719f3468..501a0d8d 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/contributing/contributing.md @@ -1,10 +1,11 @@ # Setting up the Development Environment -First, follow the [installation](installation.md#local-installation) instructions +First, follow the [installation](../installation.md#local-installation) instructions for contributors to install a local fork with optional development dependencies. +When setting up the database, follow the "Setting up a test database" instructions. ## Database -In addition to the database setup described in the [installation guide](installation.md#setting-up-a-database-server), +In addition to the database setup described in the [installation guide](../installation.md#setting-up-a-database-server), we also host a database on our server which may be connected to that is available to [OpenML core contributors](https://openml.org/about). If you are a core contributor and need access, please reach out to one of the engineers in Eindhoven. diff --git a/docs/contributing/index.md b/docs/contributing/index.md new file mode 100644 index 00000000..0a60d196 --- /dev/null +++ b/docs/contributing/index.md @@ -0,0 +1,3 @@ +# Contributing + +There are many ways to contribute. diff --git a/docs/installation.md b/docs/installation.md index 67490384..eb7b0e5a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -59,9 +59,32 @@ Before we run the REST API server, we must first set up a database server, and c the REST API to connect to it. ## Setting up a Database Server +Depending on your use of the server, there are multiple ways to set up your own +OpenML database. To simply connect to an existing database, see +[configuring the REST API Server](#configuring-the-rest-api-server) below. -... + +### Setting up a new database +This sets up an entirely empty database with the expected OpenML tables in place. +This is intended for new deployments of OpenML, for example to host a private OpenML +server. + +!!! Failure "" + + Instructions are incomplete. Please have patience while we are adding more documentation. + +### Setting up a test database +This is a database with only a fraction of the data of the production database, +intended to be used by developers. + +!!! Failure "" + + Instructions are incomplete. Please have patience while we are adding more documentation. ## Configuring the REST API Server -... +The REST API is configured through a [TOML](https://toml.io) file. + +!!! Failure "" + + Instructions are incomplete. Please have patience while we are adding more documentation. diff --git a/mkdocs.yml b/mkdocs.yml index 5c554c11..0b759c7b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,8 +11,11 @@ theme: nav: - Getting Started: installation.md - - Contributing: CONTRIBUTING.md - - Migration: migration.md + - Contributing: + - contributing/index.md + - Development: contributing/contributing.md + - PHP API: contributing/php.md + - Changes: migration.md markdown_extensions: - admonition @@ -21,3 +24,6 @@ markdown_extensions: - pymdownx.tabbed: alternate_style: true - tables + +plugins: + - section-index diff --git a/pyproject.toml b/pyproject.toml index a6000adc..840db1e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,8 @@ dev = [ "httpx", ] docs = [ - "mkdocs-material" + "mkdocs-material", + "mkdocs-section-index", ] [project.urls] From 6c19dd3e34df50753be17fea69a3b1e3e57ca2ae Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 19 Oct 2023 10:16:15 +0200 Subject: [PATCH 16/45] Start dockerizing the test server database --- .gitignore | 2 ++ mysql/Dockerfile | 3 +++ mysql/README.md | 36 ++++++++++++++++++++++++++++++++++++ mysql/data/openml.sql | 1 + mysql/data/openml_expdb.sql | 1 + 5 files changed, 43 insertions(+) create mode 100644 mysql/Dockerfile create mode 100644 mysql/README.md create mode 100644 mysql/data/openml.sql create mode 100644 mysql/data/openml_expdb.sql diff --git a/.gitignore b/.gitignore index 2dc53ca3..f1147cbb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +mysql/data + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/mysql/Dockerfile b/mysql/Dockerfile new file mode 100644 index 00000000..f09546f7 --- /dev/null +++ b/mysql/Dockerfile @@ -0,0 +1,3 @@ +FROM mysql + +COPY ./data /docker-entrypoint-initdb.d diff --git a/mysql/README.md b/mysql/README.md new file mode 100644 index 00000000..4ec60f55 --- /dev/null +++ b/mysql/README.md @@ -0,0 +1,36 @@ +# Test Database + +A + +Files and documentation to create the test database container: + + 1. Create a dump for the current test database: + + ```text + mysqldump -u root --add-drop-database --databases openml_expdb --result-file=openml_expdb.sql -p + mysqldump -u root --add-drop-database --databases openml --ignore_table=openml.login_attempts --result-file=openml.sql -p + ``` + + `login_attempts` is a legacy table which is not used in production but has a few rows in the current test database. + + 2. Copy over the files to the local directory: + + ```bash + scp USERNAME@test.openml.org:/path/to/openml-anonimized.sql data/openml.sql + scp USERNAME@test.openml.org:/path/to/openml_expdb.sql data/openml_expdb.sql + ``` + + 3. Anonimize the sensitive information from the openml database: + ```text + python openml-kube/k8s_manifests/mysql/migration/anonimize-openml-db.py --input=openml.sql + ``` + This produces `openml-anonimized.sql` which has user data replaced by fake data. + + +The test database the following special users: + +| id | API key | Comments | +| -- | -- | -- | +| 1 | AD000000000000000000000000000000 | Administrator rights | +| 2 | 00000000000000000000000000000000 | Normal user | +| 16 | DA1A0000000000000000000000000000 | Normal user with private dataset with id 130 | diff --git a/mysql/data/openml.sql b/mysql/data/openml.sql new file mode 100644 index 00000000..8d96b1bf --- /dev/null +++ b/mysql/data/openml.sql @@ -0,0 +1 @@ +# stub, see readme on how to generate this file. diff --git a/mysql/data/openml_expdb.sql b/mysql/data/openml_expdb.sql new file mode 100644 index 00000000..8d96b1bf --- /dev/null +++ b/mysql/data/openml_expdb.sql @@ -0,0 +1 @@ +# stub, see readme on how to generate this file. From b23181e3de229826856834439f77970e1dc29c6c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 19 Oct 2023 11:00:29 +0200 Subject: [PATCH 17/45] Add usage documentation for the docker image --- mysql/README.md | 47 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/mysql/README.md b/mysql/README.md index 4ec60f55..edb3b48f 100644 --- a/mysql/README.md +++ b/mysql/README.md @@ -1,8 +1,39 @@ # Test Database -A +The test database image is simply a [MySql image](https://hub.docker.com/_/mysql/) with +data already present. For general usage, such as setting a password or persisting data +to disk, see the linked MySQL image documentation. -Files and documentation to create the test database container: +The following command starts the database container: + +```bash +docker run -e MYSQL_ROOT_PASSWORD=ok -p 3306:3306 -d --name testdb openml/test-database:latest +``` +which sets: + + - `-e MYSQL_ROOT_PASSWORD=ok`: the root password is 'ok' + - `-p 3306:3306`: makes the database accessible in the host on port 3306 + +You should be able to connect to it using `mysql`: +```bash + +``` +If you do not have `mysql` installed, you may refer to the MySQL image documentation on +how to use the image instead to connect over a docker network if you want to connect +with `mysql`. + +The test database the following special users: + +| id | API key | Comments | +| -- | -- | -- | +| 1 | AD000000000000000000000000000000 | Administrator rights | +| 2 | 00000000000000000000000000000000 | Normal user | +| 16 | DA1A0000000000000000000000000000 | Normal user with private dataset with id 130 | + + +## Creating the `openml/test-database` image + +The following steps were taken to create the image: 1. Create a dump for the current test database: @@ -26,11 +57,9 @@ Files and documentation to create the test database container: ``` This produces `openml-anonimized.sql` which has user data replaced by fake data. + 4. Build and publish the docker image: -The test database the following special users: - -| id | API key | Comments | -| -- | -- | -- | -| 1 | AD000000000000000000000000000000 | Administrator rights | -| 2 | 00000000000000000000000000000000 | Normal user | -| 16 | DA1A0000000000000000000000000000 | Normal user with private dataset with id 130 | + ```bash + docker build --tag openml/test-database:latest -f Dockerfile . + docker push openml/test-database:latest + ``` From 846a89b20a02989c7233cccff764152869bed8c5 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 19 Oct 2023 16:57:09 +0200 Subject: [PATCH 18/45] Add additional files for building PHP docker container --- php/apache/Dockerfile | 9 ++++++++- php/apache/run_docker.sh | 3 +++ php/apache/set_configuration.sh | 25 +++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 php/apache/set_configuration.sh diff --git a/php/apache/Dockerfile b/php/apache/Dockerfile index e31fbbcd..3adcd365 100644 --- a/php/apache/Dockerfile +++ b/php/apache/Dockerfile @@ -8,12 +8,19 @@ RUN apt-get update \ && apt-get install -y git \ && git clone https://github.com/openml/openml /var/www/openml +RUN mv /var/www/openml/openml_OS/config/BASE_CONFIG-BLANK.php /var/www/openml/openml_OS/config/BASE_CONFIG.php + RUN mkdir /var/www/openml/logs RUN mkdir /data + COPY config/*.load /etc/apache2/mods-enabled/ COPY config/api.conf /etc/apache2/sites-enabled/000-default.conf COPY config/php.ini /user/local/etc/php/ COPY config/.htaccess /var/www/openml/.htaccess -ENTRYPOINT ["apache2-foreground"] +RUN mkdir /scripts +COPY set_configuration.sh /scripts/ + +ENTRYPOINT ["bash", "/scripts/set_configuration.sh"] +#ENTRYPOINT ["apache2-foreground"] diff --git a/php/apache/run_docker.sh b/php/apache/run_docker.sh index fade6b6d..bae7ce32 100755 --- a/php/apache/run_docker.sh +++ b/php/apache/run_docker.sh @@ -7,3 +7,6 @@ docker run -p 8001:80 --rm -it \ -v ${PHP_CONFIG}:/var/www/openml/openml_OS/config/BASE_CONFIG.php \ --network sqlnetwork \ apache-php + + +docker run -p 8001:80 --rm -it apache-php diff --git a/php/apache/set_configuration.sh b/php/apache/set_configuration.sh new file mode 100644 index 00000000..1fc8f3a6 --- /dev/null +++ b/php/apache/set_configuration.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# TODO: read credentials from secrets instead +OPENML_PATH=${OPENML_PATH:-/var/www/} +BASE_CONFIG_PATH=${OPENML_PATH}openml/openml_OS/config/BASE_CONFIG.php + +# We expect some paths/urls to contain '/' characters, so we use '*' instead +sed "s*'BASE_URL', 'FILL_IN'*'BASE_URL', '${BASE_URL:-https://test.openml.org/}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'MINIO_URL', 'FILL_IN'*'MINIO_URL', '${MINIO_URL:-https://openml1.win.tue.nl/}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'PATH', 'FILL_IN'*'PATH', '${OPENML_PATH:-/var/www/}'*g" --in-place ${BASE_CONFIG_PATH} + + +sed "s*'DB_NAME_EXPDB', 'FILL_IN'*'DB_NAME_EXPDB', '${DB_NAME_EXPDB:-openml_expdb}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_HOST_EXPDB', 'FILL_IN'*'DB_HOST_EXPDB', '${DB_HOST_EXPDB:-openml-test-database:3306}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_USER_EXPDB_READ', 'FILL_IN'*'DB_USER_EXPDB_READ', '${DB_USER_EXPDB_READ:-root}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_PASS_EXPDB_READ', 'FILL_IN'*'DB_PASS_EXPDB_READ', '${DB_PASS_EXPDB_READ:-ok}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_USER_EXPDB_WRITE', 'FILL_IN'*'DB_USER_EXPDB_WRITE', '${DB_USER_EXPDB_WRITE:-root}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_PASS_EXPDB_WRITE', 'FILL_IN'*'DB_PASS_EXPDB_WRITE', '${DB_PASS_EXPDB_WRITE:-ok}'*g" --in-place ${BASE_CONFIG_PATH} + +sed "s*'DB_NAME_OPENML', 'FILL_IN'*'DB_NAME_OPENML', '${DB_NAME_OPENML:-openml}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_HOST_OPENML', 'FILL_IN'*'DB_HOST_OPENML', '${DB_HOST_OPENML:-openml-test-database:3306}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_USER_OPENML', 'FILL_IN'*'DB_USER_OPENML', '${DB_USER_OPENML:-root}'*g" --in-place ${BASE_CONFIG_PATH} +sed "s*'DB_PASS_OPENML', 'FILL_IN'*'DB_PASS_OPENML', '${DB_PASS_OPENML:-ok}'*g" --in-place ${BASE_CONFIG_PATH} + +apache2-foreground From 88428198a199adf23e8d22807ff4e5e2c5ddfdca Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 19 Oct 2023 17:11:04 +0200 Subject: [PATCH 19/45] Rename image and add brief updated README --- php/apache/README.md | 22 +++++++++++++++------- php/apache/build_docker.sh | 2 +- php/apache/run_docker.sh | 12 ------------ 3 files changed, 16 insertions(+), 20 deletions(-) delete mode 100755 php/apache/run_docker.sh diff --git a/php/apache/README.md b/php/apache/README.md index 1c3bb3ec..2e1440a5 100644 --- a/php/apache/README.md +++ b/php/apache/README.md @@ -1,13 +1,21 @@ # Running apache php backend locally -1. checkout the php code to a directory (/yourpath/to/php_root) -2. update /yourpath/to/php_root/.htaccess: remove the `RedirectMatch 301` -3. create /yourpath/to/php_root/openml_OS/config/BASE_CONFIG.php (copy the BASE_CONFIG_BLANK.php) and configure the database etc. -4. Set `define('ENVIRONMENT', 'development');` in /yourpath/to/php_root/index.php so that you'll get errors +In most cases, you probably want to run through docker compose. +This file contains instructions for running it on its own. ```bash -./build_docker.sh -./run_docker.sh /yourpath/to/php_root /path/to/a/data/directory +docker run -p 8001:80 --rm -it openml/php-rest-api ``` -Go to http://localhost:8001/home +Runs the PHP REST API server and exposes it to `http://localhost:8001/`. +Some `BASE_CONFIG.php` variables can be overwritten with environment variables, +these can be passed in the run command with the `-e` option, e.g.: `-e BASE_URL=http://localhost/`. +See `set_configuration.sh` for the variables which can be overwritten out-of-the-box. +Alternatively, mount your own `BASE_CONFIG.php` into the container at `/var/www/openml/openml_OS/config/BASE_CONFIG.php`. +The `set_configuration.sh` script will only overwrite unset variables. +To avoid overwriting altogether, also change the entrypoint: `--entrypoint=apache2-foreground`. + +To connect to a separate container running a MySQL server, they need to be on the same docker network. +For both, specify the network with `--network NETWORK_NAME`, which can be any network you create with `docker network create NETWORK_NAME`. +Assuming a connection to the database can be established, to get a dataset description go to `http://127.0.0.1:8001/api/v1/json/data/1`. +Note that the protocol is `http` not `https`. diff --git a/php/apache/build_docker.sh b/php/apache/build_docker.sh index 6cf267a8..b4f91ba5 100755 --- a/php/apache/build_docker.sh +++ b/php/apache/build_docker.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker build --tag apache-php -f Dockerfile . +docker build --tag openml/php-rest-api -f Dockerfile . diff --git a/php/apache/run_docker.sh b/php/apache/run_docker.sh deleted file mode 100755 index bae7ce32..00000000 --- a/php/apache/run_docker.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# these need to be absolute paths: -PHP_CONFIG=$1 - -docker run -p 8001:80 --rm -it \ - -v ${PHP_CONFIG}:/var/www/openml/openml_OS/config/BASE_CONFIG.php \ - --network sqlnetwork \ - apache-php - - -docker run -p 8001:80 --rm -it apache-php From 9c9789e78a0cf2537f37d3c790d5832459e3cee1 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 19 Oct 2023 17:12:26 +0200 Subject: [PATCH 20/45] Move files up one level --- php/{apache => }/Dockerfile | 1 - php/README.md | 23 +++++++++++++++++-- php/apache/README.md | 21 ----------------- php/{apache => }/build_docker.sh | 0 php/{apache => }/config/.htaccess | 0 php/{apache => }/config/BASE_CONFIG-BLANK.php | 0 php/{apache => }/config/api.conf | 0 php/{apache => }/config/headers.load | 0 php/{apache => }/config/php.ini | 0 php/{apache => }/config/rewrite.load | 0 php/{apache => }/set_configuration.sh | 0 11 files changed, 21 insertions(+), 24 deletions(-) rename php/{apache => }/Dockerfile (95%) delete mode 100644 php/apache/README.md rename php/{apache => }/build_docker.sh (100%) rename php/{apache => }/config/.htaccess (100%) rename php/{apache => }/config/BASE_CONFIG-BLANK.php (100%) rename php/{apache => }/config/api.conf (100%) rename php/{apache => }/config/headers.load (100%) rename php/{apache => }/config/php.ini (100%) rename php/{apache => }/config/rewrite.load (100%) rename php/{apache => }/set_configuration.sh (100%) diff --git a/php/apache/Dockerfile b/php/Dockerfile similarity index 95% rename from php/apache/Dockerfile rename to php/Dockerfile index 3adcd365..5e5c246a 100644 --- a/php/apache/Dockerfile +++ b/php/Dockerfile @@ -23,4 +23,3 @@ RUN mkdir /scripts COPY set_configuration.sh /scripts/ ENTRYPOINT ["bash", "/scripts/set_configuration.sh"] -#ENTRYPOINT ["apache2-foreground"] diff --git a/php/README.md b/php/README.md index 5de1bd1c..2e1440a5 100644 --- a/php/README.md +++ b/php/README.md @@ -1,2 +1,21 @@ -# PHP REST API -This directory contains files required to build the docker image and host the PHP REST API locally. +# Running apache php backend locally + +In most cases, you probably want to run through docker compose. +This file contains instructions for running it on its own. + +```bash +docker run -p 8001:80 --rm -it openml/php-rest-api +``` + +Runs the PHP REST API server and exposes it to `http://localhost:8001/`. +Some `BASE_CONFIG.php` variables can be overwritten with environment variables, +these can be passed in the run command with the `-e` option, e.g.: `-e BASE_URL=http://localhost/`. +See `set_configuration.sh` for the variables which can be overwritten out-of-the-box. +Alternatively, mount your own `BASE_CONFIG.php` into the container at `/var/www/openml/openml_OS/config/BASE_CONFIG.php`. +The `set_configuration.sh` script will only overwrite unset variables. +To avoid overwriting altogether, also change the entrypoint: `--entrypoint=apache2-foreground`. + +To connect to a separate container running a MySQL server, they need to be on the same docker network. +For both, specify the network with `--network NETWORK_NAME`, which can be any network you create with `docker network create NETWORK_NAME`. +Assuming a connection to the database can be established, to get a dataset description go to `http://127.0.0.1:8001/api/v1/json/data/1`. +Note that the protocol is `http` not `https`. diff --git a/php/apache/README.md b/php/apache/README.md deleted file mode 100644 index 2e1440a5..00000000 --- a/php/apache/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Running apache php backend locally - -In most cases, you probably want to run through docker compose. -This file contains instructions for running it on its own. - -```bash -docker run -p 8001:80 --rm -it openml/php-rest-api -``` - -Runs the PHP REST API server and exposes it to `http://localhost:8001/`. -Some `BASE_CONFIG.php` variables can be overwritten with environment variables, -these can be passed in the run command with the `-e` option, e.g.: `-e BASE_URL=http://localhost/`. -See `set_configuration.sh` for the variables which can be overwritten out-of-the-box. -Alternatively, mount your own `BASE_CONFIG.php` into the container at `/var/www/openml/openml_OS/config/BASE_CONFIG.php`. -The `set_configuration.sh` script will only overwrite unset variables. -To avoid overwriting altogether, also change the entrypoint: `--entrypoint=apache2-foreground`. - -To connect to a separate container running a MySQL server, they need to be on the same docker network. -For both, specify the network with `--network NETWORK_NAME`, which can be any network you create with `docker network create NETWORK_NAME`. -Assuming a connection to the database can be established, to get a dataset description go to `http://127.0.0.1:8001/api/v1/json/data/1`. -Note that the protocol is `http` not `https`. diff --git a/php/apache/build_docker.sh b/php/build_docker.sh similarity index 100% rename from php/apache/build_docker.sh rename to php/build_docker.sh diff --git a/php/apache/config/.htaccess b/php/config/.htaccess similarity index 100% rename from php/apache/config/.htaccess rename to php/config/.htaccess diff --git a/php/apache/config/BASE_CONFIG-BLANK.php b/php/config/BASE_CONFIG-BLANK.php similarity index 100% rename from php/apache/config/BASE_CONFIG-BLANK.php rename to php/config/BASE_CONFIG-BLANK.php diff --git a/php/apache/config/api.conf b/php/config/api.conf similarity index 100% rename from php/apache/config/api.conf rename to php/config/api.conf diff --git a/php/apache/config/headers.load b/php/config/headers.load similarity index 100% rename from php/apache/config/headers.load rename to php/config/headers.load diff --git a/php/apache/config/php.ini b/php/config/php.ini similarity index 100% rename from php/apache/config/php.ini rename to php/config/php.ini diff --git a/php/apache/config/rewrite.load b/php/config/rewrite.load similarity index 100% rename from php/apache/config/rewrite.load rename to php/config/rewrite.load diff --git a/php/apache/set_configuration.sh b/php/set_configuration.sh similarity index 100% rename from php/apache/set_configuration.sh rename to php/set_configuration.sh From af4096b58ea89773591bc08f8d80e3ebe527056c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 19 Oct 2023 17:23:07 +0200 Subject: [PATCH 21/45] Add docker compose file to automatically spin up db and php api --- docker-compose.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 docker-compose.yaml diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 00000000..344d3380 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,12 @@ +services: + php-api: + image: "openml/php-rest-api" + ports: + - "8001:80" + database: + image: "openml/test-database" + container_name: "openml-test-database" + environment: + MYSQL_ROOT_PASSWORD: ok + ports: + - "3306:3306" From 9012cf5a9360917d7a97a9b05a4b84ce0f1b2807 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 10:49:43 +0200 Subject: [PATCH 22/45] Minor text clarifications --- docs/index.md | 4 ++-- docs/installation.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index e8f03d7e..569e3889 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -# OpenML Server API Documentation +# OpenML Server API Software Documentation This is the Python-based OpenML REST API server. It's a rewrite of our [old backend](http://github.com/openml/openml) built with a @@ -17,4 +17,4 @@ modern Python-based stack. for documentation on the REST API endpoints, visit the [APIs](https://openml.org/apis) page. - This documentation is for developing and hosting your own OpenML instance. + This documentation is for developing and hosting your own OpenML REST API. diff --git a/docs/installation.md b/docs/installation.md index eb7b0e5a..02a925c2 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,6 +1,6 @@ # Installation -*Current instructions written for Mac, but can be adapted for other operating systems.* +*Current instructions tested on Mac, but likely work on most Unix systems.* The OpenML server will be developed and maintained for the latest minor release of Python (Python 3.12 as of writing). From 56eb32dc56c6e4e8f2fee028b4208555fb733b13 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 10:50:04 +0200 Subject: [PATCH 23/45] Bump to 3.12 --- .python-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.python-version b/.python-version index 2c073331..e4fba218 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11 +3.12 From de281065c00cae2df288bfd0cb952df108dd191a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 10:56:34 +0200 Subject: [PATCH 24/45] Move php and database docker files to docker directory --- .gitignore | 2 +- {mysql => docker/mysql}/Dockerfile | 0 {mysql => docker/mysql}/README.md | 0 {php => docker/php}/Dockerfile | 0 {php => docker/php}/README.md | 0 {php => docker/php}/build_docker.sh | 0 {php => docker/php}/config/.htaccess | 0 {php => docker/php}/config/BASE_CONFIG-BLANK.php | 0 {php => docker/php}/config/api.conf | 0 {php => docker/php}/config/headers.load | 0 {php => docker/php}/config/php.ini | 0 {php => docker/php}/config/rewrite.load | 0 {php => docker/php}/set_configuration.sh | 0 13 files changed, 1 insertion(+), 1 deletion(-) rename {mysql => docker/mysql}/Dockerfile (100%) rename {mysql => docker/mysql}/README.md (100%) rename {php => docker/php}/Dockerfile (100%) rename {php => docker/php}/README.md (100%) rename {php => docker/php}/build_docker.sh (100%) rename {php => docker/php}/config/.htaccess (100%) rename {php => docker/php}/config/BASE_CONFIG-BLANK.php (100%) rename {php => docker/php}/config/api.conf (100%) rename {php => docker/php}/config/headers.load (100%) rename {php => docker/php}/config/php.ini (100%) rename {php => docker/php}/config/rewrite.load (100%) rename {php => docker/php}/set_configuration.sh (100%) diff --git a/.gitignore b/.gitignore index f1147cbb..75dd10da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -mysql/data +docker/mysql/data # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/mysql/Dockerfile b/docker/mysql/Dockerfile similarity index 100% rename from mysql/Dockerfile rename to docker/mysql/Dockerfile diff --git a/mysql/README.md b/docker/mysql/README.md similarity index 100% rename from mysql/README.md rename to docker/mysql/README.md diff --git a/php/Dockerfile b/docker/php/Dockerfile similarity index 100% rename from php/Dockerfile rename to docker/php/Dockerfile diff --git a/php/README.md b/docker/php/README.md similarity index 100% rename from php/README.md rename to docker/php/README.md diff --git a/php/build_docker.sh b/docker/php/build_docker.sh similarity index 100% rename from php/build_docker.sh rename to docker/php/build_docker.sh diff --git a/php/config/.htaccess b/docker/php/config/.htaccess similarity index 100% rename from php/config/.htaccess rename to docker/php/config/.htaccess diff --git a/php/config/BASE_CONFIG-BLANK.php b/docker/php/config/BASE_CONFIG-BLANK.php similarity index 100% rename from php/config/BASE_CONFIG-BLANK.php rename to docker/php/config/BASE_CONFIG-BLANK.php diff --git a/php/config/api.conf b/docker/php/config/api.conf similarity index 100% rename from php/config/api.conf rename to docker/php/config/api.conf diff --git a/php/config/headers.load b/docker/php/config/headers.load similarity index 100% rename from php/config/headers.load rename to docker/php/config/headers.load diff --git a/php/config/php.ini b/docker/php/config/php.ini similarity index 100% rename from php/config/php.ini rename to docker/php/config/php.ini diff --git a/php/config/rewrite.load b/docker/php/config/rewrite.load similarity index 100% rename from php/config/rewrite.load rename to docker/php/config/rewrite.load diff --git a/php/set_configuration.sh b/docker/php/set_configuration.sh similarity index 100% rename from php/set_configuration.sh rename to docker/php/set_configuration.sh From f31eddd53313c84117f56ff39bf1404341e696f4 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 11:00:00 +0200 Subject: [PATCH 25/45] Move stub files --- {mysql => docker/mysql}/data/openml.sql | 0 {mysql => docker/mysql}/data/openml_expdb.sql | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {mysql => docker/mysql}/data/openml.sql (100%) rename {mysql => docker/mysql}/data/openml_expdb.sql (100%) diff --git a/mysql/data/openml.sql b/docker/mysql/data/openml.sql similarity index 100% rename from mysql/data/openml.sql rename to docker/mysql/data/openml.sql diff --git a/mysql/data/openml_expdb.sql b/docker/mysql/data/openml_expdb.sql similarity index 100% rename from mysql/data/openml_expdb.sql rename to docker/mysql/data/openml_expdb.sql From ad048b20849020b7782bca5ec718e3953411bde4 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 12:19:45 +0200 Subject: [PATCH 26/45] Add docs service for serving mkdocs documentation --- docker-compose.yaml | 18 ++++++++++++++---- docker/docs/Dockerfile | 4 ++++ 2 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 docker/docs/Dockerfile diff --git a/docker-compose.yaml b/docker-compose.yaml index 344d3380..773955be 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,8 +1,4 @@ services: - php-api: - image: "openml/php-rest-api" - ports: - - "8001:80" database: image: "openml/test-database" container_name: "openml-test-database" @@ -10,3 +6,17 @@ services: MYSQL_ROOT_PASSWORD: ok ports: - "3306:3306" + - + docs: + build: + context: . + dockerfile: docker/docs/Dockerfile + ports: + - "8000:8000" + volumes: + - .:/docs + + php-api: + image: "openml/php-rest-api" + ports: + - "8001:80" diff --git a/docker/docs/Dockerfile b/docker/docs/Dockerfile new file mode 100644 index 00000000..ae6d376e --- /dev/null +++ b/docker/docs/Dockerfile @@ -0,0 +1,4 @@ +FROM squidfunk/mkdocs-material +RUN python -m pip install mkdocs-section-index +ENTRYPOINT ["mkdocs"] +CMD ["serve", "--dev-addr=0.0.0.0:8000"] From 9007610d7f178cff191aeb39a2066daa07db4f7f Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 12:32:53 +0200 Subject: [PATCH 27/45] Add Dockerfile for running Python-based REST API --- docker-compose.yaml | 14 ++++++++++++-- docker/python/Dockerfile | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 docker/python/Dockerfile diff --git a/docker-compose.yaml b/docker-compose.yaml index 773955be..43f27e7d 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -6,7 +6,7 @@ services: MYSQL_ROOT_PASSWORD: ok ports: - "3306:3306" - - + docs: build: context: . @@ -19,4 +19,14 @@ services: php-api: image: "openml/php-rest-api" ports: - - "8001:80" + - "8002:80" + + python-api: + container_name: "python-api" + build: + context: . + dockerfile: docker/python/Dockerfile + ports: + - "8001:8000" + volumes: + - .:/python-api diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile new file mode 100644 index 00000000..ba1abf95 --- /dev/null +++ b/docker/python/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.12-bookworm + +RUN apt-get update \ + && apt-get install -y python3-dev default-libmysqlclient-dev build-essential pkg-config + +COPY . /python-api +WORKDIR /python-api + +RUN python -m pip install --upgrade pip +RUN python -m pip install -e ".[dev]" + +EXPOSE 8000 +ENTRYPOINT ["uvicorn"] +CMD ["src.main:app", "--host", "0.0.0.0"] From fc037f80ebfa35644872ff2322a6311f77bcd196 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 12:33:16 +0200 Subject: [PATCH 28/45] Set ignored files for docker builds --- .dockerignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..48429a30 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +.github +.mypy_cache +.pytest_cache +.ruff_cache +venv From 6a61982eeb84b9d582e7f05b50fbd43bee236441 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 14:20:26 +0200 Subject: [PATCH 29/45] Correctly load database configuration for users database --- src/config.toml | 2 +- src/database/users.py | 7 ++++++- tests/config_test.py | 5 +++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/config.toml b/src/config.toml index f85d9a06..a268e485 100644 --- a/src/config.toml +++ b/src/config.toml @@ -1,5 +1,5 @@ [databases.defaults] -host="127.0.0.1" +host="openml-test-database" port="3306" # SQLAlchemy `dialect` and `driver`: https://docs.sqlalchemy.org/en/20/dialects/index.html drivername="mysql" diff --git a/src/database/users.py b/src/database/users.py index 2bf8641c..f8f58d50 100644 --- a/src/database/users.py +++ b/src/database/users.py @@ -1,12 +1,17 @@ from typing import Annotated +from config import load_database_configuration from pydantic import StringConstraints from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL from database.meta import get_column_names +_database_configuration = load_database_configuration() + +openml_url = URL.create(**_database_configuration["openml"]) openml = create_engine( - "mysql://root:ok@127.0.0.1:3306/openml", + openml_url, echo=True, pool_recycle=3600, ) diff --git a/tests/config_test.py b/tests/config_test.py index 412606d1..287b128d 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,5 +1,6 @@ import os from pathlib import Path +from unittest import mock from config import _apply_defaults_to_siblings, load_database_configuration @@ -30,6 +31,6 @@ def test_load_configuration_adds_environment_variables(default_configuration_fil assert database_configuration["openml"]["username"] == "root" load_database_configuration.cache_clear() - os.environ["OPENML_DATABASES_OPENML_USERNAME"] = "foo" - database_configuration = load_database_configuration(default_configuration_file) + with mock.patch.dict(os.environ, {"OPENML_DATABASES_OPENML_USERNAME": "foo"}): + database_configuration = load_database_configuration(default_configuration_file) assert database_configuration["openml"]["username"] == "foo" From 8bc78a90a1108cd258ab7fd178d567f509231010 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 14:38:35 +0200 Subject: [PATCH 30/45] Old server serves http minio instead of https --- src/routers/old/datasets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/routers/old/datasets.py b/src/routers/old/datasets.py index 6c7d7f49..8a757a0a 100644 --- a/src/routers/old/datasets.py +++ b/src/routers/old/datasets.py @@ -32,6 +32,8 @@ def get_dataset_wrapped( dataset["processing_date"] = str(processing_data).replace("T", " ") if parquet_url := dataset.get("parquet_url"): dataset["parquet_url"] = str(parquet_url).replace("https", "http") + if minio_url := dataset.get("minio_url"): + dataset["minio_url"] = str(minio_url).replace("https", "http") manual = [] # ref test.openml.org/d/33 (contributor) and d/34 (creator) From ddc8fe61a797db0b3f7a123fd083ae723476d18b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 14:39:07 +0200 Subject: [PATCH 31/45] Change connection address, minio can now be compared With the docker compose setup, we have a version of the server software that serves minio_url, so we can include it in our comparison --- tests/identical_test.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/identical_test.py b/tests/identical_test.py index afb0faac..c026368c 100644 --- a/tests/identical_test.py +++ b/tests/identical_test.py @@ -13,7 +13,7 @@ range(1, 9078), ) def test_dataset_response_is_identical(dataset_id: int, api_client: FastAPI) -> None: - original = httpx.get(f"https://test.openml.org/api/v1/json/data/{dataset_id}") + original = httpx.get(f"http://server-api-php-api-1:80/api/v1/json/data/{dataset_id}") new = cast(httpx.Response, api_client.get(f"/old/datasets/{dataset_id}")) assert original.status_code == new.status_code assert new.json() @@ -61,9 +61,6 @@ def test_dataset_response_is_identical(dataset_id: int, api_client: FastAPI) -> if field in new: del new[field] - if "minio_url" in new: - del new["minio_url"] # not served from the test server (and not for sparse) - # There is odd behavior in the live server that I don't want to recreate: # when the creator is a list of csv names, it can either be a str or a list # depending on whether the names are quoted. E.g.: From 5cf754884f27d87e7e8c21103b31d2f6dd2d246d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 14:46:37 +0200 Subject: [PATCH 32/45] URL is no longer str, needs explicit cast --- src/routers/old/datasets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/routers/old/datasets.py b/src/routers/old/datasets.py index 8a757a0a..df58cf33 100644 --- a/src/routers/old/datasets.py +++ b/src/routers/old/datasets.py @@ -45,7 +45,7 @@ def get_dataset_wrapped( manual.append(field) if isinstance(dataset["original_data_url"], list): - dataset["original_data_url"] = ", ".join(dataset["original_data_url"]) + dataset["original_data_url"] = ", ".join(str(url) for url in dataset["original_data_url"]) for field, value in list(dataset.items()): if field in manual: From 2e4caa2a5cccb78e60b702b6884ce1e6e9e2ad5f Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 14:47:14 +0200 Subject: [PATCH 33/45] Only 131 datasets in the test database --- tests/identical_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/identical_test.py b/tests/identical_test.py index c026368c..c397e096 100644 --- a/tests/identical_test.py +++ b/tests/identical_test.py @@ -10,7 +10,7 @@ @pytest.mark.web() @pytest.mark.parametrize( "dataset_id", - range(1, 9078), + range(1, 132), ) def test_dataset_response_is_identical(dataset_id: int, api_client: FastAPI) -> None: original = httpx.get(f"http://server-api-php-api-1:80/api/v1/json/data/{dataset_id}") From d9ca0ea571fae2fb3e007f85574cc288399d2be3 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 16:11:13 +0200 Subject: [PATCH 34/45] Add test database setup instructions --- docs/installation.md | 55 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 9 deletions(-) diff --git a/docs/installation.md b/docs/installation.md index 02a925c2..1ee6943a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -4,7 +4,7 @@ The OpenML server will be developed and maintained for the latest minor release of Python (Python 3.12 as of writing). -You can install the dependencies locally or work from a Docker container (TODO). +You can install the dependencies locally or work with docker containers. !!! tip "Use `pyenv` to manage Python installations" @@ -13,6 +13,15 @@ You can install the dependencies locally or work from a Docker container (TODO). ## Local Installation +These instructions assume [Python 3.12](https://www.python.org/downloads/) +and [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) are already installed. + +!!! info "You may need to install Python3 and MySQL development headers." + + It may be necessary to first install additional headers before proceeding with a + local installation of the `mysqlclient` dependency. They are documented under + ["Installation"](https://github.com/PyMySQL/mysqlclient#linux) of the `mysqlclient` + documentation. === "For Users" @@ -55,9 +64,6 @@ You can install the dependencies locally or work from a Docker container (TODO). tools. We require this for contributors, but we also highly recommend it anyone that plans to make code changes. -Before we run the REST API server, we must first set up a database server, and configure -the REST API to connect to it. - ## Setting up a Database Server Depending on your use of the server, there are multiple ways to set up your own OpenML database. To simply connect to an existing database, see @@ -71,15 +77,46 @@ server. !!! Failure "" - Instructions are incomplete. Please have patience while we are adding more documentation. + Instructions are incomplete. See [issue#78](https://github.com/openml/server-api/issues/78). ### Setting up a test database -This is a database with only a fraction of the data of the production database, -intended to be used by developers. -!!! Failure "" +We provide a prebuilt docker image that already contains test data. - Instructions are incomplete. Please have patience while we are adding more documentation. +=== "Docker Compose" + To start the database through `docker compose`, run: + + ```bash + docker compose up database + ``` + + which starts a database. + +=== "Docker Run" + + To start a test database as stand-alone container, run: + + ```bash + docker run --rm -e MYSQL_ROOT_PASSWORD=ok -p 3306:3306 -d --name openml-test-database openml/test-database:latest + ``` + + You may opt to add the container to a network instead, to make it reachable + from other docker containers: + + ```bash + docker network create openml + docker run --rm -e MYSQL_ROOT_PASSWORD=ok -p 3306:3306 -d --name openml-test-database --network openml openml/test-database:latest + ``` + +The container may take a minute to initialise, but afterwards you can connect to it. +Either from a local `mysql` client at `127.0.0.1:3306` or from a docker container +on the same network. For example: + +```bash +docker run --network NETWORK --rm -it mysql mysql -hopenml-test-database -uroot -pok +``` +where `NETWORK` is `server-api_default` if you used `docker compose`, and `openml` +when using `docker run` (or whatever network you specified instead). ## Configuring the REST API Server From 48dd659c377d5864719138a3065a684bdcce5a50 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 16:43:09 +0200 Subject: [PATCH 35/45] Add information about using docker compose to run services --- docs/contributing/contributing.md | 90 ++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 13 deletions(-) diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 501a0d8d..84c11ca2 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -1,14 +1,8 @@ -# Setting up the Development Environment +# Setting up the development environment First, follow the [installation](../installation.md#local-installation) instructions for contributors to install a local fork with optional development dependencies. -When setting up the database, follow the "Setting up a test database" instructions. - -## Database -In addition to the database setup described in the [installation guide](../installation.md#setting-up-a-database-server), -we also host a database on our server which may be connected to that is available -to [OpenML core contributors](https://openml.org/about). If you are a core contributor -and need access, please reach out to one of the engineers in Eindhoven. +Stop when you reach the section "Setting up a Database Server". ## Pre-commit @@ -28,7 +22,69 @@ and many tools will build a cache so subsequent runs are faster. Under normal circumstances, running the pre-commit chain shouldn't take more than a few seconds. -## Unit Tests + +## Docker + +With the projected forked, cloned, and installed, the easiest way to set up all +required services for development is through [`docker compose`](https://docs.docker.com/compose/). + +### Starting containers + +```bash +docker compose up +``` + +This will spin up 4 containers, as defined in the `docker-compose.yaml` file: + + - `openml-test-database`: this is a mysql database prepopulated with test data. + It is reachable from the host machine with port `3306`, by default it is configured + to have a root user with password `"ok"`. + - `server-api-docs-1`: this container serves project documentation at `localhost:8000`. + These pages are built from the documents in the `docs/` directory of this repository, + whenever you edit and save a file there, the page will immediately be updated. + - `server-api-php-api-1`: this container serves the old PHP REST API at `localhost:8002`. + For example, visit [http://localhost:8002/api/v1/json/data/1](http://localhost:8002/api/v1/json/data/1) + to fetch a JSON description of dataset 1. + - `python-api`: this container serves the new Python-based REST API at `localhost:8001`. + For example, visit [http://localhost:8001/docs](http://localhost:8001/docs) to see + the REST API documentation. Changes to the code in `src/` will be reflected in this + container. + +You don't always need every container, often just having a database and the Python-based +REST API may be enough. In that case, only specify those services: + +```bash +docker compose up database python-api +``` + +Refer to the `docker compose` documentation for more uses. + +### Connecting to containers + +To connect to a container, run: + +```bash +docker exec -it CONTAINER_NAME /bin/bash +``` + +where `CONTAINER_NAME` is the name of the container. If you are unsure of your container +name, then `docker container ls` may help you find it. Assuming the default container +names are used, you may connect to the Python-based REST API container using: + +```bash +docker exec -it python-api /bin/bash +``` + +This is useful, for example, to run unit tests in the container: + +```bash +python -m pytest -x -v -m "not web" +``` + +More on unit tests in a later section. + + +## Unit tests Our unit tests are written with the [`pytest`](https://pytest.org) framework. An invocation could look like this: @@ -41,7 +97,7 @@ Where `-v` show the name of each test ran, `-x` ensures testing stops on first f `--lf` will first run the test(s) which failed last, and `-m "not web"` specifies which tests (not) to run. -## Building Documentation +## Building documentation We build our documentation with [`mkdocs-material`](https://squidfunk.github.io/mkdocs-material/). All documentation pages are under the `docs` folder, and the configuration is found in `mkdocs.yml`. Having installed the `docs` optional dependencies, you should be able @@ -54,10 +110,8 @@ python -m mkdocs serve You can browse the documentation by visiting `127.0.0.1:8000` in your browser. The documentation pages will automatically rebuild when there are changes. -## Working from Docker -TODO -## YAML Validation +## YAML validation The project contains various [`yaml`](https://yaml.org) files, for example to configure `mkdocs` or to describe Github workflows. For these `yaml` files we can configure automatic schema validation, to ensure that the files are valid without having to run @@ -77,3 +131,13 @@ The following `yaml` files have schemas: In PyCharm, these can be configured from `settings` > `languages & frameworks` > `Schemas and DTDs` > `JSON Schema Mappings`. There, add mappings per file or file pattern. + +## Connecting to another database +In addition to the database setup described in the [installation guide](../installation.md#setting-up-a-database-server), +we also host a database on our server which may be connected to that is available +to [OpenML core contributors](https://openml.org/about). If you are a core contributor +and need access, please reach out to one of the engineers in Eindhoven. + +!!! Failure "" + + Instructions are incomplete. Please have patience while we are adding more documentation. From e1603436ea1194b21a6c008ea3e18efe33220563 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 19:32:47 +0200 Subject: [PATCH 36/45] Add instructions to contribute documentation changes --- docs/contributing/documentation.md | 41 ++++++++++++++++++++++++++++++ mkdocs.yml | 2 +- 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 docs/contributing/documentation.md diff --git a/docs/contributing/documentation.md b/docs/contributing/documentation.md new file mode 100644 index 00000000..25718ba3 --- /dev/null +++ b/docs/contributing/documentation.md @@ -0,0 +1,41 @@ +# Documentation with mkdocs-material + +Our documentation is built using [mkdocs](https://www.mkdocs.org) and the +[mkdocs-material](https://squidfunk.github.io/mkdocs-material/) theme. We also use some +[plugins](https://squidfunk.github.io/mkdocs-material/plugins/). +Please refer to the ["Getting Started"](https://squidfunk.github.io/mkdocs-material/getting-started/) +pages of `mkdocs-material` for a general overview on how to work with `mkdocs-material`. + +For minor changes, it should be fine to edit the page directly on Github. +That should commit to a separate branch (or fork), and you can set up a pull request. +For larger changes, clone a fork of the repository as described in the +["Local Installation"](../installation.md#local-installation) section. + + +=== "Docker" + + After cloning the repository, you may also build and serve the documentation through Docker: + + ``` + docker compose up docs + ``` + + +=== "Local installation" + + Instead of installing all dependencies (with `python -m pip install -e ".[docs]"`), + you may also install just the documentation dependencies: + + ```bash + python -m pip install mkdocs-material mkdocs-section-index + ``` + + You can then build and serve the documentation with + + ```bash + python -m mkdocs serve + ``` + +This will serve the documentation from the `docs/` directory to [https://localhost:8000/](https://localhost:8000/). +Any updates you make to files in that directory will be reflected on the website. +When you are happy with your changes, just commit and set up a pull request! diff --git a/mkdocs.yml b/mkdocs.yml index 0b759c7b..3341d6bd 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,7 +14,7 @@ nav: - Contributing: - contributing/index.md - Development: contributing/contributing.md - - PHP API: contributing/php.md + - Documentation: contributing/documentation.md - Changes: migration.md markdown_extensions: From 53f074409e024e6090c595f1ebf9b8649961516a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 20:13:09 +0200 Subject: [PATCH 37/45] Add emoji extension The documentation also says to specify an index using a "!!" syntax but this seems to be unsafe yaml, which does not get past the validator. So for now I will leave it out - emojis seem to render just fine, if not a bit flat. --- mkdocs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 3341d6bd..81b6afc2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,11 +18,13 @@ nav: - Changes: migration.md markdown_extensions: + - attr_list - admonition - pymdownx.details - pymdownx.superfences - pymdownx.tabbed: alternate_style: true + - pymdownx.emoji - tables plugins: From b384433e723484f272bddb0de2f8a22391217135 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 20:14:33 +0200 Subject: [PATCH 38/45] Add general contribution introduction --- docs/contributing/index.md | 47 +++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/docs/contributing/index.md b/docs/contributing/index.md index 0a60d196..b0250fe9 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -1,3 +1,48 @@ # Contributing -There are many ways to contribute. +Like many other open source projects, contributions from the community are an essential +piece in making OpenML successful. We very much appreciate that you want to make a +contribution. To try to make this as smooth as possible for everyone involved, +we wrote this contribution guide. There are many kinds of contributions, such as +bug reports, participating in discussions, making code contributions, or updating +documentation. + +### Bug reports + +When you encounter an issue using or deploying the Python-based REST API, the first +is to search our [issue tracker](https://github.com/openml/server-api/issues) to see if others +have already reported the issue. If this is the case, please indicate that you encountered +the issue by leaving a :+1: on the issue. Feel free to leave a comment if you have +additional information that may be useful to fix the bug. + +If the bug has not been reported before, please open a new issue to report it. +When you open a bug report, please include: + + - a [minimal reproducible example](https://stackoverflow.com/help/minimal-reproducible-example), + - a description of the expected behavior, + - a description of the encountered behavior, + - and any additional information that you think may be relevant, such as which environment you encountered it in, or when you first encountered the bug. + +### Code + +If you want to make code contributions, please first make sure there is an open issue +that describes what the contribution should look like. This may be a proposal that you +write yourself, or an existing open issue that you want to help us with. Please indicate +on that issue that you would like to contribute to it. This way, we can ensure that +everything is clear, it's not out of date, and "officially" assign you to the issue so +that others know its being worked on! Making sure there is a clear description and clear +assignment helps prevent a situation where someone makes a large contribution that is +unwanted, or is simultaneously developed by someone else. With an issue assigned, +please head over to the "[Development](contributing.md)" section. + +### Documentation + +For minor fixes, it's fine to make the changes and submit them through a pull request. +For larger changes, please make sure you are assigned to an issue first as is described +in the "[Code](#code)" section of this page. Then, visit the "[Documentation](documentation.md)" +page to learn how to contribute documentation changes. + +### Other + +If you are looking to contribute to OpenML in general, visit +"[Contributing](https://www.openml.org/contribute)" on the OpenML website. From 78f64745a8ffae8a76036dbef7a3fef242f9e905 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 20:19:41 +0200 Subject: [PATCH 39/45] Make sure all documentation information is in dedicated section --- docs/contributing/contributing.md | 13 ------------- docs/contributing/documentation.md | 2 ++ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 84c11ca2..5030b922 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -97,19 +97,6 @@ Where `-v` show the name of each test ran, `-x` ensures testing stops on first f `--lf` will first run the test(s) which failed last, and `-m "not web"` specifies which tests (not) to run. -## Building documentation -We build our documentation with [`mkdocs-material`](https://squidfunk.github.io/mkdocs-material/). -All documentation pages are under the `docs` folder, and the configuration is found in -`mkdocs.yml`. Having installed the `docs` optional dependencies, you should be able -to locally build and serve the documentation: - -```bash title="Serve documentation locally" -python -m mkdocs serve -``` - -You can browse the documentation by visiting `127.0.0.1:8000` in your browser. -The documentation pages will automatically rebuild when there are changes. - ## YAML validation The project contains various [`yaml`](https://yaml.org) files, for example to configure diff --git a/docs/contributing/documentation.md b/docs/contributing/documentation.md index 25718ba3..aed4cadb 100644 --- a/docs/contributing/documentation.md +++ b/docs/contributing/documentation.md @@ -5,6 +5,8 @@ Our documentation is built using [mkdocs](https://www.mkdocs.org) and the [plugins](https://squidfunk.github.io/mkdocs-material/plugins/). Please refer to the ["Getting Started"](https://squidfunk.github.io/mkdocs-material/getting-started/) pages of `mkdocs-material` for a general overview on how to work with `mkdocs-material`. +All documentation files are in the `docs/` folder, except for the configuration file +which is `mkdocs.yml` at the root of the repository. For minor changes, it should be fine to edit the page directly on Github. That should commit to a separate branch (or fork), and you can set up a pull request. From 9aafcdcf863943d45766c4a22fbfd906f2dd111c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 20:21:15 +0200 Subject: [PATCH 40/45] Add mkdocs-section-index requirement --- .github/workflows/docs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 690ba33d..5fcec20e 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,5 +24,5 @@ jobs: path: .cache restore-keys: | mkdocs-material- - - run: pip install mkdocs-material + - run: pip install mkdocs-material mkdocs-section-index - run: mkdocs gh-deploy --force From 1f8aabe26f4f8afd5246fabd03ce12d059ea5c3d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Fri, 20 Oct 2023 20:27:43 +0200 Subject: [PATCH 41/45] Fix typo in site_url --- mkdocs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 81b6afc2..02b2cc92 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,7 @@ # yaml-language-server: $schema=https://squidfunk.github.io/mkdocs-material/schema.json site_name: OpenML Server -site_url: https://openml/github.io/server-api +site_url: https://openml.github.io/server-api repo_name: openml/server-api repo_url: https://github.com/openml/server-api From 74cf33c09b48885f067d8d731a807dd70356372b Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 21 Oct 2023 10:50:30 +0200 Subject: [PATCH 42/45] Minor text changes and corrections --- docs/contributing/contributing.md | 2 +- docs/contributing/documentation.md | 2 +- docs/contributing/index.md | 6 +++--- docs/installation.md | 6 ++++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 5030b922..6922be41 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -15,7 +15,7 @@ checks pass out of the box: ```bash title="Install pre-commit and verify it works" pre-commit install -pre-commit run all-files +pre-commit run --all-files ``` Running the tool the first time may be slow as tools need to be installed, and many tools will build a cache so subsequent runs are faster. diff --git a/docs/contributing/documentation.md b/docs/contributing/documentation.md index aed4cadb..5f319805 100644 --- a/docs/contributing/documentation.md +++ b/docs/contributing/documentation.md @@ -38,6 +38,6 @@ For larger changes, clone a fork of the repository as described in the python -m mkdocs serve ``` -This will serve the documentation from the `docs/` directory to [https://localhost:8000/](https://localhost:8000/). +This will serve the documentation from the `docs/` directory to [http://localhost:8000/](http://localhost:8000/). Any updates you make to files in that directory will be reflected on the website. When you are happy with your changes, just commit and set up a pull request! diff --git a/docs/contributing/index.md b/docs/contributing/index.md index b0250fe9..ecefbf50 100644 --- a/docs/contributing/index.md +++ b/docs/contributing/index.md @@ -9,11 +9,11 @@ documentation. ### Bug reports -When you encounter an issue using or deploying the Python-based REST API, the first +When you encounter an issue using or deploying the Python-based REST API, the first step is to search our [issue tracker](https://github.com/openml/server-api/issues) to see if others have already reported the issue. If this is the case, please indicate that you encountered the issue by leaving a :+1: on the issue. Feel free to leave a comment if you have -additional information that may be useful to fix the bug. +additional information that may be useful to track down or fix the bug. If the bug has not been reported before, please open a new issue to report it. When you open a bug report, please include: @@ -30,7 +30,7 @@ that describes what the contribution should look like. This may be a proposal th write yourself, or an existing open issue that you want to help us with. Please indicate on that issue that you would like to contribute to it. This way, we can ensure that everything is clear, it's not out of date, and "officially" assign you to the issue so -that others know its being worked on! Making sure there is a clear description and clear +that others know its being worked on. Making sure there is a clear description and clear assignment helps prevent a situation where someone makes a large contribution that is unwanted, or is simultaneously developed by someone else. With an issue assigned, please head over to the "[Development](contributing.md)" section. diff --git a/docs/installation.md b/docs/installation.md index 1ee6943a..f1371892 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -115,8 +115,10 @@ on the same network. For example: ```bash docker run --network NETWORK --rm -it mysql mysql -hopenml-test-database -uroot -pok ``` -where `NETWORK` is `server-api_default` if you used `docker compose`, and `openml` -when using `docker run` (or whatever network you specified instead). +where `NETWORK` is `openml` when using `docker run` when following the example, +and `NETWORK` is `server-api_default` if you used `docker compose` (specifically, +it is `DIRECTORY_NAME` + `_default`, so if you renamed the `server-api` directory to +something else, the network name reflects that). ## Configuring the REST API Server From eafce20427525afb7edfb3e30e3841af007db3fa Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 21 Oct 2023 10:50:41 +0200 Subject: [PATCH 43/45] Make codeblocks copyable --- mkdocs.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mkdocs.yml b/mkdocs.yml index 02b2cc92..86f76e4a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,9 @@ repo_url: https://github.com/openml/server-api theme: name: material + features: + - content.code.copy + nav: - Getting Started: installation.md - Contributing: From 8a37572cae95487d7c9cf0b2fc1620fb77a684e7 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sat, 21 Oct 2023 11:05:15 +0200 Subject: [PATCH 44/45] Add project overview and move roadmap content --- docs/contributing/contributing.md | 12 +++++++++--- docs/contributing/project_overview.md | 15 ++++++++++++++ docs/index.md | 24 +++++++++++++++++++++++ docs/roadmap.md | 28 --------------------------- mkdocs.yml | 1 + 5 files changed, 49 insertions(+), 31 deletions(-) create mode 100644 docs/contributing/project_overview.md delete mode 100644 docs/roadmap.md diff --git a/docs/contributing/contributing.md b/docs/contributing/contributing.md index 6922be41..60213691 100644 --- a/docs/contributing/contributing.md +++ b/docs/contributing/contributing.md @@ -81,9 +81,6 @@ This is useful, for example, to run unit tests in the container: python -m pytest -x -v -m "not web" ``` -More on unit tests in a later section. - - ## Unit tests Our unit tests are written with the [`pytest`](https://pytest.org) framework. @@ -97,6 +94,15 @@ Where `-v` show the name of each test ran, `-x` ensures testing stops on first f `--lf` will first run the test(s) which failed last, and `-m "not web"` specifies which tests (not) to run. +The directory structure of our tests follows the structure of the `src/` directory. +For files, we follow the convention of _appending_ `_test`. +Try to keep tests as small as possible, and only rely on database and/or web connections +when absolutely necessary. + +!!! Failure "" + + Instructions are incomplete. Please have patience while we are adding more documentation. + ## YAML validation The project contains various [`yaml`](https://yaml.org) files, for example to configure diff --git a/docs/contributing/project_overview.md b/docs/contributing/project_overview.md new file mode 100644 index 00000000..7264fd7d --- /dev/null +++ b/docs/contributing/project_overview.md @@ -0,0 +1,15 @@ +# Project overview + +The Python-based REST API serves several groups of endpoints: + + - `/old/`: serves the old-style JSON format, this should mimic the PHP responses exactly with the only deviations recorded in the [migration guide](../migration.md). + - `/mldcat_ap/`: serves datasets in [MLDCAT_AP](https://semiceu.github.io/MLDCAT-AP/releases/1.0.0/) format. + - `/*`: serves new-style JSON format. At this point it is intentionally similar to the old-style format. + +The endpoints are specified in subdirectories of `src/routers`. +They pull data from the database through the `src/database` module. +The schemas for each entity, and possible conversions between them, are defined in the `src/schemas` directory. + +!!! Failure "" + + Instructions are incomplete. Please have patience while we are adding more documentation. diff --git a/docs/index.md b/docs/index.md index 569e3889..68a6631d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -18,3 +18,27 @@ modern Python-based stack. [APIs](https://openml.org/apis) page. This documentation is for developing and hosting your own OpenML REST API. + +## Development Roadmap + +First we will mimic current server functionality, relying on many implementation details +present in the current production server. We will implement all endpoints using the SQL +text queries based on PHP implementation, which should give near-identical responses to +the current JSON endpoints. Minor exceptions are permitted but will be documented. + +At the same time we may also provide a work-in-progress "new" endpoint, but there won't +be official support for it at this stage. After we verify the output of the endpoints +are identical (minus any intentional documented differences), we will officially release +the new API. The old API will remain available. After that, we can start working on a +new version of the JSON API which is more standardized, leverages typing, and so on: + + - Clean up the database: standardize value formats where possible (e.g., (un)quoting + contributor names in the dataset's contributor field), and add database level + constraints on new values. + - Redesign what the new API responses should look like and implement them, + API will be available to the public as it is developed. + - Refactor code-base to use ORM (using `SQLAlchemy`, `SQLModel`, or similar). + - Officially release the modernized API. + +There is no planned sunset date for the old API. This will depend on the progress with +the new API as well as the usage numbers of the old API. diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 57e3d27e..00000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,28 +0,0 @@ - -## Getting Started - -## Development Roadmap -First we will mimic current server functionality, relying on many implementation details -present in the current production server: - - - Implement all GET endpoints using the SQL text queries based on PHP implementation, - which should give near-identical responses to the current JSON endpoints. Minor - exceptions are permitted but will be documented. - - Implement non-GET endpoints in similar fashion. - -At the same time we may also provide a work-in-progress "new" endpoint, but there won't -be official support for it at this stage. After we verify the output of the endpoints -are identical (minus any intentional documented differences), we will officially release -the new API. The old API will remain available. After that, we can start working on a -new version of the JSON API which is more standardized, leverages typing, and so on: - - - Clean up the database: standardize value formats where possible (e.g., (un)quoting - contributor names in the dataset's contributor field), and add database level - constraints on new values. - - Redesign what the new API responses should look like and implement them, - API will be available to the public as it is developed. - - Refactor code-base to use ORM (using `SQLAlchemy`, `SQLModel`, or similar). - - Officially release the modernized API. - -There is no planned sunset date for the old API. This will depend on the progress with -the new API as well as the usage numbers of the old API. diff --git a/mkdocs.yml b/mkdocs.yml index 86f76e4a..27681b1e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,6 +18,7 @@ nav: - contributing/index.md - Development: contributing/contributing.md - Documentation: contributing/documentation.md + - Project Overview: contributing/project_overview.md - Changes: migration.md markdown_extensions: From 2919fe2a7d003cf3c3950b4bf5ee1296ec73e5c3 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 31 Oct 2023 17:27:08 +0100 Subject: [PATCH 45/45] Update unit tests to be more strict Since we now have an easily reproducible and identical test database behind the PHP API, we can be more strict about ensuring there are no differences in the data. Only changed interfaces are now accounted for in the unit tests. --- tests/identical_test.py | 33 ++------------------------------- 1 file changed, 2 insertions(+), 31 deletions(-) diff --git a/tests/identical_test.py b/tests/identical_test.py index c397e096..1ad49350 100644 --- a/tests/identical_test.py +++ b/tests/identical_test.py @@ -34,32 +34,8 @@ def test_dataset_response_is_identical(dataset_id: int, api_client: FastAPI) -> if "div" in original: pytest.skip("A PHP error occurred on the test server.") - # In case the test environment is set up to communicate with a snapshot of - # the test server database, we expect some fields to be outdated: - assert int(original["description_version"]) >= int(new["description_version"]) - if "tag" in original and "tag" in new: - # TODO: Ask Jan why some datasets don't have tags. - assert set(original["tag"]) >= set(new["tag"]) - - assert original["format"].lower() == new["format"] - - # Sometimes the test server provides `parquet_url`s, sometimes not. - if "parquet_url" in original: - if original["format"] == "sparse_arff": - # https://github.com/openml/OpenML/issues/1189 - # The test server incorrectly thinks there is an associated parquet file: - del original["parquet_url"] - elif "parquet_url" in new: - del new["parquet_url"] - - # Format is tested in normalized form above, tags and description_version may - # be out of sync with the snapshot of the database that we use to generate - # new responses. - for field in ["description_version", "tag", "format"]: - if field in original: - del original[field] - if field in new: - del new[field] + # The new API has normalized `format` field: + original["format"] = original["format"].lower() # There is odd behavior in the live server that I don't want to recreate: # when the creator is a list of csv names, it can either be a str or a list @@ -73,11 +49,6 @@ def test_dataset_response_is_identical(dataset_id: int, api_client: FastAPI) -> ): original["creator"] = [name.strip() for name in original["creator"].split(",")] - # For some reason, the TALLO dataset has multiple 'ignore attribute' but the - # live server is not able to parse that and provides no 'ignore attribute' field: - if dataset_id in range(8592, 8606): - del new["ignore_attribute"] - # The remainder of the fields should be identical: assert original == new