From 23fd1cba071d1490fa462659a051c8df6891ed02 Mon Sep 17 00:00:00 2001 From: Tom Denham Date: Fri, 28 Oct 2016 12:55:11 -0700 Subject: [PATCH 1/4] Rewrite libnetwork-plugin in Go - Removes support for IPv6 - Requires both calico network driver and IPAM driver to be used together --- .coveragerc | 3 - .dockerignore | 5 +- .gitignore | 16 +- CONTRIBUTING.md | 133 --- Dockerfile | 28 +- Makefile | 224 ++--- README.md | 97 +-- circle.yml | 14 - driver/ipam_driver.go | 251 ++++++ driver/network_driver.go | 298 +++++++ driver/package.go | 14 + glide.lock | 118 +++ glide.yaml | 18 + libnetwork/__init__.py | 0 libnetwork/datastore_libnetwork.py | 47 - libnetwork/driver_plugin.py | 645 -------------- main.go | 83 ++ nose.cfg | 4 - requirements.txt | 7 - setup.py | 59 -- start.sh | 16 - .../st/libnetwork/test_assign_specific_ip.py | 65 +- tests/st/libnetwork/test_error_ipam.py | 55 ++ tests/st/libnetwork/test_libnetwork.py | 42 - .../st/libnetwork/test_mainline_multi_host.py | 131 ++- .../libnetwork/test_mainline_single_host.py | 45 +- tests/st/ssl-config/ca-config.json | 13 - tests/st/ssl-config/ca-csr.json | 16 - tests/st/ssl-config/req-csr.json | 18 - tests/unit/__init__.py | 0 tests/unit/datastore_libnetwork_test.py | 95 -- tests/unit/driver_plugin_test.py | 823 ------------------ utils/log/log.go | 15 + utils/math/math.go | 8 + utils/netns/netns.go | 61 ++ utils/os/utils.go | 13 + 36 files changed, 1140 insertions(+), 2340 deletions(-) delete mode 100644 .coveragerc delete mode 100644 CONTRIBUTING.md delete mode 100644 circle.yml create mode 100644 driver/ipam_driver.go create mode 100644 driver/network_driver.go create mode 100644 driver/package.go create mode 100644 glide.lock create mode 100644 glide.yaml delete mode 100644 libnetwork/__init__.py delete mode 100644 libnetwork/datastore_libnetwork.py delete mode 100644 libnetwork/driver_plugin.py create mode 100644 main.go delete mode 100644 nose.cfg delete mode 100644 requirements.txt delete mode 100644 setup.py delete mode 100755 start.sh create mode 100644 tests/st/libnetwork/test_error_ipam.py delete mode 100644 tests/st/libnetwork/test_libnetwork.py delete mode 100644 tests/st/ssl-config/ca-config.json delete mode 100644 tests/st/ssl-config/ca-csr.json delete mode 100644 tests/st/ssl-config/req-csr.json delete mode 100644 tests/unit/__init__.py delete mode 100644 tests/unit/datastore_libnetwork_test.py delete mode 100644 tests/unit/driver_plugin_test.py create mode 100644 utils/log/log.go create mode 100644 utils/math/math.go create mode 100644 utils/netns/netns.go create mode 100644 utils/os/utils.go diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 7335b79..0000000 --- a/.coveragerc +++ /dev/null @@ -1,3 +0,0 @@ -[report] -omit = - tests/* diff --git a/.dockerignore b/.dockerignore index 6656bfc..2ca9b71 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,7 +1,6 @@ .git -venv +vendor +*.tar build -busybox.tar -calico-node.tar default.etcd .semaphore-cache diff --git a/.gitignore b/.gitignore index 71acdc9..1c0e9bf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,14 @@ build +vendor certs dist -venv env .idea +.vscode *.pyc *~ .coverage cover -busybox.tar -calico-node.tar -routereflector.tar -calicobuild.created -caliconode.created -calico_containers/pycalico.egg-info/ -busybox.tgz -calico-node-libnetwork.tgz -calico-node.tgz calicoctl -docker - +*.tar +*.created diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index eaba1ce..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,133 +0,0 @@ -# Contributing Guidelines - -Thanks for thinking about contributing to Project Calico! The success of an -open source project is entirely down to the efforts of its contributors, so we -do genuinely want to thank you for contributing. - -This document contains some guidance and steps for contributing. Make sure you -read it thoroughly, because if you miss some of these steps we will have to ask -you to do them before we can merge it. - -## Before contributing: The Contributor License Agreement - -If you plan to contribute in the form of documentation or code, we need you to -sign our Contributor License Agreement before we can accept your contribution. -You will be prompted to do this as part of the PR process on Github. - -## Mailing lists and chat - -A great way to talk to us, ask questions, discuss features and bounce ideas -around is to join one of the channels listed below: - -* [Technical Mailing List](http://lists.projectcalico.org/mailman/listinfo/calico-tech_lists.projectcalico.org) -* [Slack Calico Users Channel](https://slack.projectcalico.org) -* IRC - [#calico](https://kiwiirc.com/client/irc.freenode.net/#calico) - -## Reporting issues - -Before raising an issue with the *Calico Libnetwork plugin*), please check for -duplicate issues, and read our [Troubleshooting](https://github.com/projectcalico/calico-containers/blob/master/docs/Troubleshooting.md) -and our [Frequently Asked Questions](https://github.com/projectcalico/calico-containers/blob/master/docs/FAQ.md) -documents. You may also find helpful information in our [guides on Calico as a Docker network plugin] -(https://github.com/projectcalico/calico-containers/tree/master/docs/calico-with-docker/docker-network-plugin). - -If you have a question, please hop on to our IRC or Slack channels (see above). - -If you do need to raise an issue, please include any of the following that may -be relevant to your problem (including too much information is never -a bad thing): - -- A detailed description of the problem. -- The version of Docker in use from the `docker version` command. -- The steps to reproduce the problem (e.g. the full list of `calicoctl` - commands that were run) -- The expected result and actual result. -- Versions of appropriate binaries and libraries. For example, the output from - `calicoctl version`, your version of Mesos, rkt, Kubernetes etc. -- A link to any diagnostics (e.g. if using `calicoctl`, you can gathered - diagnostics using `calicoctl diags` - this provides instructions for - uploading the diags bundle to transfer.sh - or alternatively if the - diagnostics contains sensitive information we can set up an alternative - method for transfer). -- If using `calicoctl` the output from `calicoctl status` run on each node - might also be useful. -- Details of your OS. -- Environment details such as GCE, bare metal, VirtualBox. - - -## Contributing code and documentation - -For contributing code and documentation we follow the GitHub pull request -model. - -- Fork the repository on GitHub -- Make changes to your local repository in a separate branch -- Test the change -- Create a pull request which will then go through a review process by one or - more of the core team. - -### Testing your changes - -If you create a pull request, our automated UT and STs will be run over your -change. We will not accept changes that do not consistently pass our automated -test suites. It is vital that our master branch be passing tests at all times. -If you tests are failing the automated tests and you don't believe they should -be, you may need to rebase your branch off the latest master. - -The unit tests can be run with: -``` -make test -``` - -Where possible, please add any additional tests to ensure we maintain healthy -code and feature coverage. - -You should also try tunning your changes with a live environment if it is -relevant. Use our [Calico as a Docker Network Plugin guide] -(https://github.com/projectcalico/calico-containers/blob/master/docs/calico-with-docker/docker-network-plugin) -and the [calicoctl node documenation](https://github.com/projectcalico/calico-containers/blob/master/docs/calicoctl/node.md) -to learn how to do this. - -### Documentation - -If your code change requires some documentation updates, please include these -changes as part of the pull request. - -In most cases, the documentation changes will actually fall under the -[`calico-containers`](https://github.com/projectcalico/calico-containers) -repository in the `/docs` directory, which will require a separate pull request. - -### Review and merge - -Assuming your code review is finished and tests are passing, your change will -then be merged as soon as possible! Occasionally we will sit on a change, for -instance if we are planning to tag a release shortly, but this is only to -ensure the stability of the branch. After the release we will merge your change -promptly. - -Before merging we prefer that you squash the commits into a single commit to -ensure we have a cleaner commit history. - -### Coding style - -The majority of the code is written in Python and we generally follow the -[PEP-8 coding style](https://www.python.org/dev/peps/pep-0008). - -### Format of the commit message - -The commit message should include a short subject on what changed, a body -detailing why the change was necessary, and any key points about the -implementation. - -Try to keep the subject line no longer than 70 characters, and the lines of the -body no longer than 80 characters. This improves readability in both GitHub -and when using git tools. - -If the pull request fixes an issue, include a separate line in the description -with the following format: - -``` -Fixes #29 -``` - -[![Analytics](https://calico-ga-beacon.appspot.com/UA-52125893-3/libnetwork-plugin/CONTRIBUTING.md?pixel)](https://github.com/igrigorik/ga-beacon) diff --git a/Dockerfile b/Dockerfile index b6c836d..da657a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,27 +1,5 @@ -# Copyright 2015 Metaswitch Networks -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -FROM gliderlabs/alpine:latest +FROM alpine MAINTAINER Tom Denham +ADD dist/libnetwork-plugin /libnetwork-plugin +ENTRYPOINT ["/libnetwork-plugin"] -COPY requirements.txt / - -RUN apk --update add python py-setuptools iproute2 && \ - apk add --virtual build-dependencies git python-dev build-base curl bash py-pip alpine-sdk libffi-dev openssl-dev && \ - pip install -r requirements.txt && \ - apk del build-dependencies && rm -rf /var/cache/apk/* - -COPY start.sh / -COPY libnetwork /calico_containers/libnetwork_plugin - -ENTRYPOINT ["./start.sh"] diff --git a/Makefile b/Makefile index 54a3989..1af1880 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,4 @@ -.PHONEY: all binary test ut ut-circle st st-ssl clean setup-env run-etcd run-etcd-ssl install-completion - -SRCDIR=libnetwork -SRC_FILES=$(wildcard $(SRCDIR)/*.py) -BUILD_DIR=build_calicoctl -BUILD_FILES=$(BUILD_DIR)/Dockerfile $(BUILD_DIR)/requirements.txt -NODE_FILES=Dockerfile start.sh +SRC_FILES=$(shell find . -type f -name '*.go') # These variables can be overridden by setting an environment variable. LOCAL_IP_ENV?=$(shell ip route get 8.8.8.8 | head -1 | cut -d' ' -f8) @@ -15,69 +9,85 @@ HOST_CHECKOUT_DIR?=$(shell pwd) default: all all: test -node: caliconode.created - -caliconode.created: $(SRC_FILES) $(NODE_FILES) - docker build -t calico/node-libnetwork . - touch caliconode.created +test: st + +calico/libnetwork-plugin: libnetwork-plugin.created + +# Use this to populate the vendor directory after checking out the repository. +# To update upstream dependencies, delete the glide.lock file first. +vendor: glide.yaml + # To build without Docker just run "glide install -strip-vendor" + docker run --rm -v $(CURDIR):/go/src/github.com/projectcalico/libnetwork-plugin:rw \ + --entrypoint /bin/sh dockerepo/glide -e -c ' \ + cd /go/src/github.com/projectcalico/libnetwork-plugin && \ + glide install -strip-vendor && \ + chown $(shell id -u):$(shell id -u) -R vendor' + +install: + CGO_ENABLED=0 go install github.com/projectcalico/libnetwork-plugin + +# Run the build in a container. Useful for CI +.PHONY: build-containerized +build-containerized: vendor + mkdir -p dist + docker run --rm \ + -v $(CURDIR):/go/src/github.com/projectcalico/libnetwork-plugin:ro \ + -v $(CURDIR)/dist:/go/src/github.com/projectcalico/libnetwork-plugin/dist \ + golang:1.7 sh -c '\ + cd /go/src/github.com/projectcalico/libnetwork-plugin && \ + make build && \ + chown -R $(shell id -u):$(shell id -u) dist' + + +build: $(SRC_FILES) vendor + CGO_ENABLED=0 go build -v -o dist/libnetwork-plugin -ldflags "-X main.VERSION=$(shell git describe --tags --dirty) -s -w" main.go + +libnetwork-plugin.created: Dockerfile build-containerized + docker build -t calico/libnetwork-plugin . + touch libnetwork-plugin.created dist/calicoctl: mkdir dist curl -L http://www.projectcalico.org/builds/calicoctl -o dist/calicoctl chmod +x dist/calicoctl -test: st ut -ssl-certs: certs/.certificates.created ## Generate self-signed SSL certificates - -ut: - docker run --rm -v `pwd`:/code calico/test nosetests tests/unit -c nose.cfg - -ut-circle: - # Can't use --rm on circle - # Circle also requires extra options for reporting. - docker run \ - -v `pwd`:/code \ - -v $(CIRCLE_TEST_REPORTS):/circle_output \ - -e COVERALLS_REPO_TOKEN=$(COVERALLS_REPO_TOKEN) \ - calico/test sh -c \ - 'nosetests tests/unit -c nose.cfg \ - --with-xunit --xunit-file=/circle_output/output.xml; RC=$$?;\ - [[ ! -z "$$COVERALLS_REPO_TOKEN" ]] && coveralls || true; exit $$RC' - -busybox.tgz: +busybox.tar: docker pull busybox:latest - docker save busybox:latest | gzip -c > busybox.tgz + docker save busybox:latest -o busybox.tar -calico-node.tgz: +calico-node.tar: docker pull calico/node:latest - docker save calico/node:latest | gzip -c > calico-node.tgz - -calico-node-libnetwork.tgz: caliconode.created - docker save calico/node-libnetwork:latest | gzip -c > calico-node-libnetwork.tgz - -## Generate the keys and certificates for running etcd with SSL. -certs/.certificates.created: - mkdir -p certs - curl -L "https://github.com/projectcalico/cfssl/releases/download/1.2.1/cfssl" -o certs/cfssl - curl -L "https://github.com/projectcalico/cfssl/releases/download/1.2.1/cfssljson" -o certs/cfssljson - chmod a+x certs/cfssl - chmod a+x certs/cfssljson - - certs/cfssl gencert -initca tests/st/ssl-config/ca-csr.json | certs/cfssljson -bare certs/ca - certs/cfssl gencert \ - -ca certs/ca.pem \ - -ca-key certs/ca-key.pem \ - -config tests/st/ssl-config/ca-config.json \ - tests/st/ssl-config/req-csr.json | certs/cfssljson -bare certs/client - certs/cfssl gencert \ - -ca certs/ca.pem \ - -ca-key certs/ca-key.pem \ - -config tests/st/ssl-config/ca-config.json \ - tests/st/ssl-config/req-csr.json | certs/cfssljson -bare certs/server - - touch certs/.certificates.created - -st: dist/calicoctl busybox.tgz calico-node.tgz calico-node-libnetwork.tgz run-etcd + docker save calico/node:latest -o calico-node.tar + +calico-node-libnetwork.tar: libnetwork-plugin.created + docker save calico/libnetwork-plugin:latest -o calico-node-libnetwork.tar + +# Install or update the tools used by the build +.PHONY: update-tools +update-tools: + go get -u github.com/Masterminds/glide + go get -u github.com/kisielk/errcheck + go get -u golang.org/x/tools/cmd/goimports + go get -u github.com/golang/lint/golint + go get -u github.com/onsi/ginkgo/ginkgo + +# Perform static checks on the code. The golint checks are allowed to fail, the others must pass. +.PHONY: static-checks +static-checks: vendor + # Format the code and clean up imports + find -name '*.go' -not -path "./vendor/*" |xargs goimports -w + + # Check for coding mistake and missing error handling + go vet -x $(glide nv) + errcheck . ./datastore/... ./utils/... ./driver/... + + # Check code style + -golint main.go + -golint datastore + -golint utils + -golint driver + +st: dist/calicoctl busybox.tar calico-node.tar calico-node-libnetwork.tar run-etcd # Use the host, PID and network namespaces from the host. # Privileged is needed since 'calico node' write to /proc (to enable ip_forwarding) # Map the docker socket in so docker can be used from inside the container @@ -92,43 +102,13 @@ st: dist/calicoctl busybox.tgz calico-node.tgz calico-node-libnetwork.tgz run-e -e DEBUG_FAILURES=$(DEBUG_FAILURES) \ --rm -ti \ -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/code \ + -v $(CURDIR):/code \ calico/test \ sh -c 'cp -ra tests/st/libnetwork/ /tests/st && cd / && nosetests $(ST_TO_RUN) -sv --nologcapture --with-timer $(ST_OPTIONS)' -## Run the STs in a container using etcd with SSL certificate/key/CA verification. -st-ssl: dist/calicoctl busybox.tgz calico-node.tgz calico-node-libnetwork.tgz run-etcd-ssl - # Use the host, PID and network namespaces from the host. - # Privileged is needed since 'calico node' write to /proc (to enable ip_forwarding) - # Map the docker socket in so docker can be used from inside the container - # HOST_CHECKOUT_DIR is used for volume mounts on containers started by this one. - # All of code under test is mounted into the container. - # - This also provides access to calicoctl and the docker client - # Mount the full path to the etcd certs directory. - # - docker copies this directory directly from the host, but the - # calicoctl node command reads the files from the test container - docker run --uts=host \ - --pid=host \ - --net=host \ - --privileged \ - -e HOST_CHECKOUT_DIR=$(HOST_CHECKOUT_DIR) \ - -e ETCD_SCHEME=https \ - -e ETCD_CA_CERT_FILE=`pwd`/certs/ca.pem \ - -e ETCD_CERT_FILE=`pwd`/certs/client.pem \ - -e ETCD_KEY_FILE=`pwd`/certs/client-key.pem \ - -e DEBUG_FAILURES=$(DEBUG_FAILURES) \ - --rm -ti \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -v `pwd`:/code \ - -v `pwd`/certs:`pwd`/certs \ - calico/test \ - sh -c 'cp -ra tests/st/* /tests/st && cd / && nosetests $(ST_TO_RUN) -sv --nologcapture --with-timer $(ST_OPTIONS)' - -run-plugin: node - docker run -ti --privileged --net=host -v /run/docker/plugins:/run/docker/plugins -e ETCD_AUTHORITY=$(LOCAL_IP_ENV):2379 calico/node-libnetwork +run-plugin: libnetwork-plugin.created + docker run --rm --net=host --privileged -e CALICO_ETCD_AUTHORITY=$(LOCAL_IP_ENV):2379 -v /run/docker/plugins:/run/docker/plugins -v /var/run/docker.sock:/var/run/docker.sock -v /lib/modules:/lib/modules --name calico-node-libnetwork calico/libnetwork-plugin /libnetwork-plugin -run-plugin-local: - sudo gunicorn --reload -b unix:///run/docker/plugins/calico.sock libnetwork.driver_plugin:app run-etcd: @-docker rm -f calico-etcd calico-etcd-ssl @@ -138,55 +118,7 @@ run-etcd: --advertise-client-urls "http://$(LOCAL_IP_ENV):2379,http://127.0.0.1:2379" \ --listen-client-urls "http://0.0.0.0:2379" -## Run etcd in a container with SSL verification. Used primarily by STs. -run-etcd-ssl: certs/.certificates.created add-ssl-hostname - @-docker rm -f calico-etcd calico-etcd-ssl - docker run --detach \ - --net=host \ - -v `pwd`/certs:/etc/calico/certs \ - --name calico-etcd-ssl quay.io/coreos/etcd:v2.0.11 \ - --cert-file "/etc/calico/certs/server.pem" \ - --key-file "/etc/calico/certs/server-key.pem" \ - --ca-file "/etc/calico/certs/ca.pem" \ - --advertise-client-urls "https://etcd-authority-ssl:2379,https://localhost:2379" \ - --listen-client-urls "https://0.0.0.0:2379" - -add-ssl-hostname: - # Set "LOCAL_IP etcd-authority-ssl" in /etc/hosts to use as a hostname for etcd with ssl - if ! grep -q "etcd-authority-ssl" /etc/hosts; then \ - echo "\n# Host used by Calico's ETCD with SSL\n$(LOCAL_IP_ENV) etcd-authority-ssl" >> /etc/hosts; \ - fi - - -create-dind: - @echo "You may want to load calico-node with" - @echo "docker load --input /code/calico-node.tgz" - @ID=$$(docker run --privileged -v `pwd`:/code -v `pwd`/docker:/usr/local/bin/docker \ - -tid calico/dind:latest --cluster-store=etcd://$(LOCAL_IP_ENV):2379) ;\ - docker exec -ti $$ID sh;\ - docker rm -f $$ID - -demo-environment: docker dist/calicoctl busybox.tgz calico-node.tgz calico-node-libnetwork.tgz run-etcd - -docker rm -f host1 host2 - docker run --name host1 -e ETCD_AUTHORITY=$(LOCAL_IP_ENV):2379 --privileged \ - -v `pwd`:/code -v `pwd`/docker:/usr/local/bin/docker \ - -tid calico/dind:libnetwork --cluster-store=etcd://$(LOCAL_IP_ENV):2379 ;\ - docker run --name host2 -e ETCD_AUTHORITY=$(LOCAL_IP_ENV):2379 --privileged \ - -v `pwd`:/code -v `pwd`/docker:/usr/local/bin/docker \ - -tid calico/dind:libnetwork --cluster-store=etcd://$(LOCAL_IP_ENV):2379 ;\ - docker exec -it host1 sh -c 'docker load -i /code/calico-node.tgz' - docker exec -it host1 sh -c 'docker load -i /code/busybox.tgz' - docker exec -it host1 sh -c 'docker load -i /code/calico-node-libnetwork.tgz' - docker exec -it host2 sh -c 'docker load -i /code/calico-node.tgz' - docker exec -it host2 sh -c 'docker load -i /code/busybox.tgz' - docker exec -it host2 sh -c 'docker load -i /code/calico-node-libnetwork.tgz' - - @echo "Two dind hosts (host1, host2) are now ready." - @echo "Connect using:" - @echo "docker exec -ti host1 sh" - semaphore: - # Use the downloaded docker locally, not just with Docker in Docker STs docker version # Ensure Semaphore has loaded the required modules @@ -195,13 +127,9 @@ semaphore: # Run the STs make st - # Run subset of STs with secure etcd (only a few total, so just run all of them) - # Temporarily disable the secure STs - make st-ssl - clean: -rm -f *.created -rm -rf dist - -rm -rf certs - -rm -f *.tgz + -rm -f *.tar + -rm -rf vendor -docker run -v /var/run/docker.sock:/var/run/docker.sock -v /var/lib/docker:/var/lib/docker --rm martin/docker-cleanup-volumes diff --git a/README.md b/README.md index 9d82500..46e131c 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,30 @@ [![Build Status](https://semaphoreci.com/api/v1/projects/d51a0276-7939-409e-80ac-aa5df9421fef/510521/badge.svg)](https://semaphoreci.com/calico/libnetwork-plugin) [![Circle CI](https://circleci.com/gh/projectcalico/libnetwork-plugin/tree/master.svg?style=svg)](https://circleci.com/gh/projectcalico/libnetwork-plugin/tree/master) -[![Coverage Status](https://coveralls.io/repos/projectcalico/libnetwork-plugin/badge.svg?branch=master&service=github)](https://coveralls.io/github/projectcalico/libnetwork-plugin?branch=master) - -#Libnetwork plugin for Calico +# Libnetwork plugin for Calico This plugin for Docker networking ([libnetwork](https://github.com/docker/libnetwork)) is intended for use with [Project Calico](http://www.projectcalico.org). -Calico can be deployed on Docker using guides from the [calico-containers](https://github.com/projectcalico/calico-containers) repository. - -## How to Run It -When deployed using `calicoctl` (see calico-docker) simply pass in the `--libnetwork` flag. -* To run a specific [version](https://github.com/projectcalico/libnetwork-plugin/releases) of the plugin use the `--libnetwork-image` flag. - -### With Docker -Prebuilt docker images are available on [DockerHub](https://hub.docker.com/r/calico/node-libnetwork/) with [tags](https://hub.docker.com/r/calico/node-libnetwork/tags/) available for each libnetwork-plugin [release](https://github.com/projectcalico/libnetwork-plugin/releases). +The plugin is integrated with the `calico/node` image which is created from the [calico-containers](https://github.com/projectcalico/calico-containers) repository. -The container needs to be run using -`docker run -d --privileged --net=host -v /run/docker/plugins:/run/docker/plugins calico/node-libnetwork` - -* Privileged is required since the container creates network devices. -* Host network is used since the network changes need to occur in the host namespace -* The /run/docker/plugins volume is used to allow the plugin to communicate with Docker. +Guides on how to get started with the plugin and further documentation is available from http://docs.projectcalico.org -If you don't have etcd available at localhost:4001 then you need to pass in the location as an environment variable e.g. `-e ETCD_AUTHORITY=1.2.3.4:2379` +The remaining information is for advanced users. -### From source -To run the plugin from source use `gunicorn` e.g. -`sudo gunicorn -b unix:///run/docker/plugins/calico.sock libnetwork.driver_plugin:app` - -For the full list of recommended options for use in production, see [start.sh](start.sh) - -For testing out changes, add the `--reload` flag or use `make run-plugin-local` +## How to Run It +`make run-plugin` -Install the dependencies from requirements.txt using `pip install -r requirements.txt` +Running the plugin in a container requires a few specific options + `docker run --rm --net=host --privileged -e CALICO_ETCD_AUTHORITY=$(LOCAL_IP_ENV):2379 -v /run/docker/plugins:/run/docker/plugins -v /var/run/docker.sock:/var/run/docker.sock --name calico-node-libnetwork calico/node-libnetwork /calico` -## Calico networking and IPAM -The plugin provides Calico driver support for both networking, and IPAM. When -creating a network in Docker, use the `-d calico` to use the Calico network -driver, and the `--ipam-driver calico` to use the Calico IPAM driver. +- `--net=host` Host network is used since the network changes need to occur in the host namespace +- `privileged` since the plugin creates network interfaces +- `-e CALICO_ETCD_AUTHORITY=a.b.c.d:2379` to allow the plugin to find a backend datastore for storing information +- `-v /run/docker/plugins:/run/docker/plugins` allows the docker daemon to discover the plugin +- `-v /var/run/docker.sock:/var/run/docker.sock` allows the plugin to query the docker daemon ## Known limitations The following is a list of known limitations when using the Calico libnetwork driver: -- Creating a mix of containers which use both the Calico IPAM driver and the - default IPAM driver is not recommended. In this case, isolation between - containers on a "default IPAM" network may not be correctly isolated from - containers on a "Calico IPAM" network running on the same host. - It is not possible to add multiple networks to a single container. However, once a container endpoint is created, it is possible to manually add additional Calico profiles to that endpoint (effectively adding the @@ -60,55 +39,5 @@ driver: Logs are sent to STDOUT. If using Docker these can be viewed with the `docker logs` command. -#### Changing the log level -This currently requires a rebuild. Change the following line towards the top of -the [plugin code](https://github.com/projectcalico/libnetwork-plugin/blob/master/libnetwork/driver_plugin.py) - - app.logger.setLevel(logging.DEBUG) - -This uses the standard Python logging module, so logging level may be set to -any of the values defined in the logging module. - -## Performance -### Datastore Interactions -These don't include interactions from the Docker daemon or felix. These are interactions from the libnetwork-plugin _only_. - -The number of reads and writes is dependent on whether Calico IPAM driver is also used. - -Datastore interactions using default IPAM: - -Operation | Reads | Writes| Deletes| Notes ----------------|-------|-------|--------|------ -DiscoverNew | 0 | 0 | 0 | None -CreateNetwork | 0 | 4 (5 if IPv4 and IPv6) | 0 | 2 for creating profile (tags and rules), 1 per IP Pool, and 1 to store the request JSON -CreateEndpoint | 1 | 1 | 0 | Read CreateNetwork JSON and write Endpoint -Join | 1 | 0 | 0 | Read CreateNetwork JSON -DiscoverDelete | 0 | 0 | 0 | None -DeleteNetwork | 1 | 0 | 3 (4 if IPv4 and IPv6) | Delete profile, pool and stored CreateNetwork JSON -DeleteEndpoint | 0 | 0 | 1 | Delete endpoint -Leave | 0 | 0 | 0 | None - - -Datastore interactions using Calico IPAM: - -Operation | Reads | Writes| Deletes| Notes ----------------|-------|-------|--------|------ -DiscoverNew | 0 | 0 | 0 | None -RequestPool | 0 | 0 or 1 | 0 | 1 for verifying Calico pool if subnet explicitly specified -CreateNetwork | 0 | 3 | 0 | 2 for creating profile (tags and rules), and 1 to store the request JSON -RequestAddress | >=2 | >=1 | >=0 | May have multiple reads/writes/deletes depending on contention (see libcalico IPAM), Reads IP pools if subnet is specified on network. -CreateEndpoint | 3 | 1 | 0 | Read CreateNetwork JSON and IPv4/IPv6 next hops, and write Endpoint -Join | 2 | 0 | 0 | Read CreateNetwork JSON and Endpoint -DiscoverDelete | 0 | 0 | 0 | None -ReleasePool | 0 | 0 | 0 | None -DeleteNetwork | 1 | 0 | 3 (4 if IPv4 and IPv6) | Delete profile, pool and stored CreateNetwork JSON -ReleaseAddress | >=1 | >=1 | >=0 | May have multiple reads/writes/deletes depending on contention (see libcalico IPAM) -DeleteEndpoint | 0 | 0 | 1 | Delete endpoint -Leave | 0 | 0 | 0 | None - - -## Contributing and getting help -See the [main Calico documentation](http://docs.projectcalico.org/en/latest/involved.html) -Further sources of getting help are listed in the [calico-docker](https://github.com/projectcalico/calico-docker#calico-on-docker) repository. [![Analytics](https://calico-ga-beacon.appspot.com/UA-52125893-3/libnetwork-plugin/README.md?pixel)](https://github.com/igrigorik/ga-beacon) diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 4e49c16..0000000 --- a/circle.yml +++ /dev/null @@ -1,14 +0,0 @@ -general: - artifacts: - - "dist" -machine: - services: - - docker -test: - override: - - make ut-circle -experimental: - notify: - branches: - only: - - master diff --git a/driver/ipam_driver.go b/driver/ipam_driver.go new file mode 100644 index 0000000..1b28cae --- /dev/null +++ b/driver/ipam_driver.go @@ -0,0 +1,251 @@ +package driver + +import ( + "fmt" + "log" + "net" + + "github.com/pkg/errors" + + "github.com/docker/go-plugins-helpers/ipam" + "github.com/projectcalico/libcalico-go/lib/api" + datastoreClient "github.com/projectcalico/libcalico-go/lib/client" + caliconet "github.com/projectcalico/libcalico-go/lib/net" + logutils "github.com/projectcalico/libnetwork-plugin/utils/log" + osutils "github.com/projectcalico/libnetwork-plugin/utils/os" +) + +type IpamDriver struct { + client *datastoreClient.Client + logger *log.Logger + + poolIDV4 string + poolIDV6 string +} + +func NewIpamDriver(client *datastoreClient.Client, logger *log.Logger) ipam.Ipam { + return IpamDriver{ + client: client, + logger: logger, + + poolIDV4: PoolIDV4, + poolIDV6: PoolIDV6, + } +} + +func (i IpamDriver) GetCapabilities() (*ipam.CapabilitiesResponse, error) { + resp := ipam.CapabilitiesResponse{} + logutils.JSONMessage(i.logger, "GetCapabilities response JSON=%v", resp) + return &resp, nil +} + +func (i IpamDriver) GetDefaultAddressSpaces() (*ipam.AddressSpacesResponse, error) { + resp := &ipam.AddressSpacesResponse{ + LocalDefaultAddressSpace: "CalicoLocalAddressSpace", + GlobalDefaultAddressSpace: CalicoGlobalAddressSpace, + } + logutils.JSONMessage(i.logger, "GetDefaultAddressSpace response JSON=%v", resp) + return resp, nil +} + +func (i IpamDriver) RequestPool(request *ipam.RequestPoolRequest) (*ipam.RequestPoolResponse, error) { + logutils.JSONMessage(i.logger, "RequestPool JSON=%s", request) + + // Calico IPAM does not allow you to request SubPool. + if request.SubPool != "" { + err := errors.New( + "Calico IPAM does not support sub pool configuration " + + "on 'docker create network'. Calico IP Pools " + + "should be configured first and IP assignment is " + + "from those pre-configured pools.", + ) + i.logger.Println(err) + return nil, err + } + + // If a pool (subnet on the CLI) is specified, it must match one of the + // preconfigured Calico pools. + if request.Pool != "" { + poolsClient := i.client.Pools() + _, ipNet, err := caliconet.ParseCIDR(request.Pool) + if err != nil { + err := errors.New("Invalid CIDR") + i.logger.Println(err) + return nil, err + } + pools, err := poolsClient.List(api.PoolMetadata{CIDR: *ipNet}) + if err != nil || len(pools.Items) < 1 { + err := errors.New( + "The requested subnet must match the CIDR of a " + + "configured Calico IP Pool.", + ) + i.logger.Println(err) + return nil, err + } + } + + var resp *ipam.RequestPoolResponse + + // If a subnet has been specified we use that as the pool ID. Otherwise, we + // use static pool ID and CIDR to indicate that we are assigning from all of + // the pools. + // The meta data includes a dummy gateway address. This prevents libnetwork + // from requesting a gateway address from the pool since for a Calico + // network our gateway is set to our host IP. + if request.V6 { + resp = &ipam.RequestPoolResponse{ + PoolID: i.poolIDV6, + Pool: "::/0", + Data: map[string]string{"com.docker.network.gateway": "::/0"}, + } + } else { + resp = &ipam.RequestPoolResponse{ + PoolID: i.poolIDV4, + Pool: "0.0.0.0/0", + Data: map[string]string{"com.docker.network.gateway": "0.0.0.0/0"}, + } + } + + logutils.JSONMessage(i.logger, "RequestPool response JSON=%v", resp) + + return resp, nil +} + +func (i IpamDriver) ReleasePool(request *ipam.ReleasePoolRequest) error { + logutils.JSONMessage(i.logger, "ReleasePool JSON=%s", request) + + resp := map[string]string{} + logutils.JSONMessage(i.logger, "ReleasePool response JSON=%s", resp) + return nil +} + +func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.RequestAddressResponse, error) { + logutils.JSONMessage(i.logger, "RequestAddress JSON=%s", request) + + hostname, err := osutils.GetHostname() + if err != nil { + return nil, err + } + + var ( + version int + pool *api.Pool + IPs []caliconet.IP + ) + + if request.Address == "" { + var ( + numV4 int + numV6 int + poolV4 *caliconet.IPNet + poolV6 *caliconet.IPNet + ) + i.logger.Println("Auto assigning IP from Calico pools") + + // No address requested, so auto assign from our pools. If the pool ID + // is one of the fixed IDs then assign from across all configured pools, + // otherwise assign from the requested pool + if request.PoolID == PoolIDV4 { + version = 4 + } else { + poolsClient := i.client.Pools() + _, ipNet, err := caliconet.ParseCIDR(request.PoolID) + + if err != nil { + err = errors.Wrapf(err, "Invalid CIDR - %v", request.PoolID) + return nil, err + } + pool, err = poolsClient.Get(api.PoolMetadata{CIDR: *ipNet}) + if err != nil { + message := "The network references a Calico pool which " + + "has been deleted. Please re-instate the " + + "Calico pool before using the network." + i.logger.Println(err) + return nil, errors.New(message) + } + version = ipNet.Version() + poolV4 = &caliconet.IPNet{IPNet: pool.Metadata.CIDR.IPNet} + // TODO - v6 + } + + if version == 4 { + numV4 = 1 + numV6 = 0 + } + + // Auto assign an IP based on whether the IPv4 or IPv6 pool was selected. + // We auto-assign from all available pools with affinity based on our + // host. + IPsV4, IPsV6, err := i.client.IPAM().AutoAssign( + datastoreClient.AutoAssignArgs{ + Num4: numV4, + Num6: numV6, + Hostname: hostname, + IPv4Pool: poolV4, + IPv6Pool: poolV6, + }, + ) + IPs = append(IPsV4, IPsV6...) + if err != nil || len(IPs) == 0 { + err := errors.New("There are no available IP addresses in the configured Calico IP pools") + i.logger.Println(err) + return nil, err + } + + } else { + i.logger.Println("Reserving a specific address in Calico pools") + ip := net.ParseIP(request.Address) + ipArgs := datastoreClient.AssignIPArgs{ + IP: caliconet.IP{IP: ip}, + Hostname: hostname, + } + err := i.client.IPAM().AssignIP(ipArgs) + if err != nil { + err = errors.Wrapf(err, "IP assignment error, data: %+v", ipArgs) + i.logger.Println(err) + return nil, err + } + IPs = []caliconet.IP{{IP: ip}} + } + + // We should only have one IP address assigned at this point. + if len(IPs) != 1 { + err := errors.New("Unexpected number of assigned IP addresses") + i.logger.Println(err) + return nil, err + } + + // Return the IP as a CIDR. + resp := &ipam.RequestAddressResponse{ + // TODO: need more investigation about the subnet size to use + Address: fmt.Sprintf("%v/%v", IPs[0], "32"), + } + + logutils.JSONMessage(i.logger, "RequestAddress response JSON=%s", resp) + + return resp, nil +} + +func (i IpamDriver) ReleaseAddress(request *ipam.ReleaseAddressRequest) error { + logutils.JSONMessage(i.logger, "ReleaseAddress JSON=%s", request) + + ip := caliconet.IP{IP: net.ParseIP(request.Address)} + + // Unassign the address. This handles the address already being unassigned + // in which case it is a no-op. The release_ips call may raise a + // RuntimeError if there are repeated clashing updates to the same IP block, + // this is not an expected condition. + ips := []caliconet.IP{ip} + _, err := i.client.IPAM().ReleaseIPs(ips) + if err != nil { + err = errors.Wrapf(err, "IPs releasing error, ips: %v", ips) + i.logger.Println(err) + return err + } + + resp := map[string]string{} + + logutils.JSONMessage(i.logger, "ReleaseAddress response JSON=%s", resp) + + return nil +} diff --git a/driver/network_driver.go b/driver/network_driver.go new file mode 100644 index 0000000..f0d853d --- /dev/null +++ b/driver/network_driver.go @@ -0,0 +1,298 @@ +package driver + +import ( + "context" + "log" + "net" + + "github.com/pkg/errors" + libcalicoErrors "github.com/projectcalico/libcalico-go/lib/errors" + + "github.com/docker/go-plugins-helpers/network" + + dockerClient "github.com/docker/engine-api/client" + "github.com/projectcalico/libcalico-go/lib/api" + datastoreClient "github.com/projectcalico/libcalico-go/lib/client" + caliconet "github.com/projectcalico/libcalico-go/lib/net" + + logutils "github.com/projectcalico/libnetwork-plugin/utils/log" + mathutils "github.com/projectcalico/libnetwork-plugin/utils/math" + "github.com/projectcalico/libnetwork-plugin/utils/netns" + osutils "github.com/projectcalico/libnetwork-plugin/utils/os" +) + +// NetworkDriver is the Calico network driver representation. +// Must be used with Calico IPAM and support IPv4 only. +type NetworkDriver struct { + client *datastoreClient.Client + logger *log.Logger + + containerName string + orchestratorID string + fixedMac string + + gatewayCIDRV4 string + gatewayCIDRV6 string + + ifPrefix string + + DummyIPV4Nexthop string +} + +func NewNetworkDriver(client *datastoreClient.Client, logger *log.Logger) network.Driver { + return NetworkDriver{ + client: client, + logger: logger, + + // The MAC address of the interface in the container is arbitrary, so for + // simplicity, use a fixed MAC. + fixedMac: "EE:EE:EE:EE:EE:EE", + + // Orchestrator and container IDs used in our endpoint identification. These + // are fixed for libnetwork. Unique endpoint identification is provided by + // hostname and endpoint ID. + containerName: "libnetwork", + orchestratorID: "libnetwork", + + ifPrefix: IFPrefix, + DummyIPV4Nexthop: "169.254.1.1", + } +} + +func (d NetworkDriver) GetCapabilities() (*network.CapabilitiesResponse, error) { + resp := network.CapabilitiesResponse{Scope: "global"} + logutils.JSONMessage(d.logger, "GetCapabilities response JSON=%v", resp) + return &resp, nil +} + +func (d NetworkDriver) CreateNetwork(request *network.CreateNetworkRequest) error { + logutils.JSONMessage(d.logger, "CreateNetwork JSON=%s", request) + + for _, ipData := range request.IPv4Data { + if ipData.AddressSpace != CalicoGlobalAddressSpace { + err := errors.New("Non-Calico IPAM driver is used") + d.logger.Println(err) + return err + } + } + + logutils.JSONMessage(d.logger, "CreateNetwork response JSON=%v", map[string]string{}) + return nil +} + +func (d NetworkDriver) DeleteNetwork(request *network.DeleteNetworkRequest) error { + logutils.JSONMessage(d.logger, "DeleteNetwork JSON=%v", request) + return nil +} + +func (d NetworkDriver) CreateEndpoint(request *network.CreateEndpointRequest) (*network.CreateEndpointResponse, error) { + logutils.JSONMessage(d.logger, "CreateEndpoint JSON=%v", request) + + hostname, err := osutils.GetHostname() + if err != nil { + err = errors.Wrap(err, "Hostname fetching error") + return nil, err + } + + d.logger.Printf("Creating endpoint %v\n", request.EndpointID) + if request.Interface.Address == "" { + return nil, errors.New("No address assigned for endpoint") + } + + var addresses []caliconet.IPNet + if request.Interface.Address != "" { + // Parse the address this function was passed. Ignore the subnet - Calico always uses /32 (for IPv4) + ip4, _, err := net.ParseCIDR(request.Interface.Address) + d.logger.Printf("Parsed IP %v from (%v) \n", ip4, request.Interface.Address) + + if err != nil { + err = errors.Wrapf(err, "Parsing %v as CIDR failed", request.Interface.Address) + d.logger.Println(err) + return nil, err + } + + addresses = append(addresses, caliconet.IPNet{IPNet: net.IPNet{IP: ip4, Mask: net.CIDRMask(32, 32)}}) + } + + endpoint := api.NewWorkloadEndpoint() + endpoint.Metadata.Node = hostname + endpoint.Metadata.Orchestrator = d.orchestratorID + endpoint.Metadata.Workload = d.containerName + endpoint.Metadata.Name = request.EndpointID + endpoint.Spec.InterfaceName = "cali" + request.EndpointID[:mathutils.MinInt(11, len(request.EndpointID))] + mac, _ := net.ParseMAC(d.fixedMac) + endpoint.Spec.MAC = caliconet.MAC{HardwareAddr: mac} + endpoint.Spec.IPNetworks = append(endpoint.Spec.IPNetworks, addresses...) + + // Use the Docker API to fetch the network name (so we don't have to use an ID everywhere) + dockerCli, err := dockerClient.NewEnvClient() + if err != nil { + err = errors.Wrap(err, "Error while attempting to instantiate docker client from env") + return nil, err + } + networkData, err := dockerCli.NetworkInspect(context.Background(), request.NetworkID) + if err != nil { + err = errors.Wrapf(err, "Network %v inspection error", request.NetworkID) + return nil, err + } + + // Now that we know the network name, set it on the endpoint. + endpoint.Spec.Profiles = append(endpoint.Spec.Profiles, networkData.Name) + + // Check if the profile already exists. + exists := true + if _, err = d.client.Profiles().Get(api.ProfileMetadata{Name: networkData.Name}); err != nil { + _, ok := err.(libcalicoErrors.ErrorResourceDoesNotExist) + if ok { + exists = false + } else { + err = errors.Wrapf(err, "Profile %v getting error", networkData.Name) + d.logger.Println(err) + return nil, err + } + } + + // If a profile for the network name doesn't exist then it needs to be created. + if !exists { + profile := api.NewProfile() + profile.Metadata.Name = networkData.Name + profile.Spec.Tags = []string{networkData.Name} + profile.Spec.EgressRules = []api.Rule{{Action: "allow"}} + profile.Spec.IngressRules = []api.Rule{{Action: "allow", Source: api.EntityRule{Tag: networkData.Name}}} + if _, err := d.client.Profiles().Create(profile); err != nil { + log.Println(err) + return nil, err + } + } + + _, err = d.client.WorkloadEndpoints().Create(endpoint) + if err != nil { + err = errors.Wrapf(err, "Workload endpoints creation error, data: %+v", endpoint) + d.logger.Println(err) + return nil, err + } + + d.logger.Printf("Workload created, data: %+v\n", endpoint) + + response := &network.CreateEndpointResponse{ + Interface: &network.EndpointInterface{ + MacAddress: string(d.fixedMac), + }, + } + + logutils.JSONMessage(d.logger, "CreateEndpoint response JSON=%v", response) + + return response, nil +} + +func (d NetworkDriver) DeleteEndpoint(request *network.DeleteEndpointRequest) error { + logutils.JSONMessage(d.logger, "DeleteEndpoint JSON=%v", request) + hostname, err := osutils.GetHostname() + if err != nil { + err = errors.Wrap(err, "Hostname fetching error") + return err + } + + logutils.JSONMessage(d.logger, "DeleteEndpoint JSON=%v", request) + d.logger.Printf("Removing endpoint %v\n", request.EndpointID) + + if err = d.client.WorkloadEndpoints().Delete( + api.WorkloadEndpointMetadata{ + Name: request.EndpointID, + Node: hostname, + Orchestrator: d.orchestratorID, + Workload: d.containerName}); err != nil { + err = errors.Wrapf(err, "Endpoint %v removal error", request.EndpointID) + log.Println(err) + return err + } + + logutils.JSONMessage(d.logger, "DeleteEndpoint response JSON=%v", map[string]string{}) + + return err +} + +func (d NetworkDriver) EndpointInfo(request *network.InfoRequest) (*network.InfoResponse, error) { + logutils.JSONMessage(d.logger, "EndpointInfo JSON=%v", request) + return nil, nil +} + +func (d NetworkDriver) Join(request *network.JoinRequest) (*network.JoinResponse, error) { + logutils.JSONMessage(d.logger, "Join JSON=%v", request) + + // 1) Set up a veth pair + // The one end will stay in the host network namespace - named caliXXXXX + // The other end is given a temporary name. It's moved into the final network namespace by libnetwork itself. + var err error + prefix := request.EndpointID[:mathutils.MinInt(11, len(request.EndpointID))] + hostInterfaceName := "cali" + prefix + tempInterfaceName := "temp" + prefix + + if err = netns.CreateVeth(hostInterfaceName, tempInterfaceName); err != nil { + err = errors.Wrapf( + err, "Veth creation error, hostInterfaceName=%v, tempInterfaceName=%v", + hostInterfaceName, tempInterfaceName) + d.logger.Println(err) + return nil, err + } + + // libnetwork doesn't set the MAC address properly, so set it here. + if err = netns.SetVethMac(tempInterfaceName, d.fixedMac); err != nil { + d.logger.Printf("Veth mac setting for %v failed, removing veth for %v\n", tempInterfaceName, hostInterfaceName) + err = netns.RemoveVeth(hostInterfaceName) + err = errors.Wrapf(err, "Veth removing for %v error", hostInterfaceName) + d.logger.Println(err) + return nil, err + } + + resp := &network.JoinResponse{ + InterfaceName: network.InterfaceName{ + SrcName: tempInterfaceName, + DstPrefix: IFPrefix, + }, + } + + // One of the network gateway addresses indicate that we are using + // Calico IPAM driver. In this case we setup routes using the gateways + // configured on the endpoint (which will be our host IPs). + d.logger.Println("Using Calico IPAM driver, configure gateway and " + + "static routes to the host") + + resp.Gateway = d.DummyIPV4Nexthop + resp.StaticRoutes = append(resp.StaticRoutes, &network.StaticRoute{ + Destination: d.DummyIPV4Nexthop + "/32", + RouteType: 1, // 1 = CONNECTED + NextHop: "", + }) + + logutils.JSONMessage(d.logger, "Join Response JSON=%v", resp) + + return resp, nil +} + +func (d NetworkDriver) Leave(request *network.LeaveRequest) error { + logutils.JSONMessage(d.logger, "Leave response JSON=%v", request) + caliName := "cali" + request.EndpointID[:mathutils.MinInt(11, len(request.EndpointID))] + err := netns.RemoveVeth(caliName) + return err +} + +func (d NetworkDriver) DiscoverNew(request *network.DiscoveryNotification) error { + logutils.JSONMessage(d.logger, "DiscoverNew JSON=%v", request) + d.logger.Println("DiscoverNew response JSON={}") + return nil +} + +func (d NetworkDriver) DiscoverDelete(request *network.DiscoveryNotification) error { + logutils.JSONMessage(d.logger, "DiscoverNew JSON=%v", request) + d.logger.Println("DiscoverDelete response JSON={}") + return nil +} + +func (d NetworkDriver) ProgramExternalConnectivity(*network.ProgramExternalConnectivityRequest) error { + return nil +} + +func (d NetworkDriver) RevokeExternalConnectivity(*network.RevokeExternalConnectivityRequest) error { + return nil +} diff --git a/driver/package.go b/driver/package.go new file mode 100644 index 0000000..d51f872 --- /dev/null +++ b/driver/package.go @@ -0,0 +1,14 @@ +package driver + +const ( + // Calico IPAM module does not allow selection of pools from which to allocate + // IP addresses. The pool ID, which has to be supplied in the libnetwork IPAM + // API is therefore fixed. We use different values for IPv4 and IPv6 so that + // during allocation we know which IP version to use. + PoolIDV4 = "CalicoPoolIPv4" + PoolIDV6 = "CalicoPoolIPv6" + + CalicoGlobalAddressSpace = "CalicoGlobalAddressSpace" + + IFPrefix = "cali" +) diff --git a/glide.lock b/glide.lock new file mode 100644 index 0000000..1dc1521 --- /dev/null +++ b/glide.lock @@ -0,0 +1,118 @@ +hash: e966b8a17e5e245235779be4284c5466177bd1bcd9ea91e6511ea4e80bfe7fed +updated: 2016-10-28T14:09:43.888331619-07:00 +imports: +- name: github.com/coreos/etcd + version: 83347907774bf36cbb261c594a32fd7b0f5dd9f6 + subpackages: + - client + - pkg/fileutil + - pkg/pathutil + - pkg/tlsutil + - pkg/transport + - pkg/types +- name: github.com/coreos/go-systemd + version: bfdc81d0d7e0fb19447b08571f63b774495251ce + subpackages: + - activation + - journal + - util +- name: github.com/coreos/pkg + version: fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8 + subpackages: + - capnslog + - dlopen +- name: github.com/docker/distribution + version: 8234784a1a66bfee4a6d72d0a3cbd453b7f903d7 + subpackages: + - digest + - reference +- name: github.com/docker/engine-api + version: 3d1601b9d2436a70b0dfc045a23f6503d19195df + subpackages: + - client + - client/transport + - client/transport/cancellable + - types + - types/blkiodev + - types/container + - types/filters + - types/network + - types/reference + - types/registry + - types/strslice + - types/swarm + - types/time + - types/versions +- name: github.com/docker/go-connections + version: 990a1a1a70b0da4c4cb70e117971a4f0babfbf1a + subpackages: + - nat + - sockets + - tlsconfig +- name: github.com/docker/go-plugins-helpers + version: 8a0198e77ac4e4ee167222caf6894cb32386c5fc + subpackages: + - ipam + - network + - sdk +- name: github.com/docker/go-units + version: f2d77a61e3c169b43402a0a1e84f06daf29b8190 +- name: github.com/ghodss/yaml + version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee +- name: github.com/kelseyhightower/envconfig + version: 9aca109c9aec4633fced9717c4a09ecab3d33111 +- name: github.com/Microsoft/go-winio + version: 4f1a71750d95a5a8a46c40a67ffbed8129c2f138 +- name: github.com/opencontainers/runc + version: c91b5bea4830a57eac7882d7455d59518cdf70ec + subpackages: + - libcontainer/user +- name: github.com/pkg/errors + version: 645ef00459ed84a119197bfb8d8205042c6df63d +- name: github.com/projectcalico/libcalico-go + version: 4d86b6458df42f96e0ac15f6a4130884376d6f02 + subpackages: + - lib/api + - lib/api/unversioned + - lib/backend + - lib/backend/api + - lib/backend/compat + - lib/backend/etcd + - lib/backend/model + - lib/client + - lib/errors + - lib/hwm + - lib/net + - lib/numorstring + - lib/scope +- name: github.com/satori/go.uuid + version: b061729afc07e77a8aa4fad0a2fd840958f1942a +- name: github.com/Sirupsen/logrus + version: 4b6ea7319e214d98c938f12692336f7ca9348d6b +- name: github.com/ugorji/go + version: f1f1a805ed361a0e078bb537e4ea78cd37dcf065 + subpackages: + - codec +- name: github.com/vishvananda/netlink + version: 9dee363ad4abbc3c9a4a24a9f1e33363e224b111 + subpackages: + - nl +- name: github.com/vishvananda/netns + version: 8ba1072b58e0c2a240eb5f6120165c7776c3e7b8 +- name: golang.org/x/net + version: 4876518f9e71663000c348837735820161a42df7 + subpackages: + - context + - proxy +- name: golang.org/x/sys + version: 9c60d1c508f5134d1ca726b4641db998f2523357 + subpackages: + - unix + - windows +- name: gopkg.in/tchap/go-patricia.v2 + version: 666120de432aea38ab06bd5c818f04f4129882c9 + subpackages: + - patricia +- name: gopkg.in/yaml.v2 + version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 +testImports: [] diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 0000000..e043ece --- /dev/null +++ b/glide.yaml @@ -0,0 +1,18 @@ +package: github.com/projectcalico/libnetwork-plugin +import: +- package: github.com/docker/engine-api + subpackages: + - client +- package: github.com/docker/go-plugins-helpers + subpackages: + - ipam + - network +- package: github.com/pkg/errors + version: ^0.8.0 +- package: github.com/projectcalico/libcalico-go + subpackages: + - lib/api + - lib/client + - lib/errors + - lib/net +- package: github.com/vishvananda/netlink diff --git a/libnetwork/__init__.py b/libnetwork/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/libnetwork/datastore_libnetwork.py b/libnetwork/datastore_libnetwork.py deleted file mode 100644 index b62afda..0000000 --- a/libnetwork/datastore_libnetwork.py +++ /dev/null @@ -1,47 +0,0 @@ -from etcd import EtcdKeyNotFound -from pycalico.ipam import IPAMClient -import json - -PREFIX = "/calico/libnetwork/v1/" - - -class LibnetworkDatastoreClient(IPAMClient): - def get_network(self, network_id): - """ - Get the data for a network ID. - - :param network_id: The network ID to read. - :return: A python datastructure representing the JSON that libnetwork - provided on the CreateNetwork call, or None if it couldn't be found. - """ - try: - network_data = self.etcd_client.read(PREFIX + network_id) - return json.loads(network_data.value) - except EtcdKeyNotFound: - return None - - def write_network(self, network_id, create_network_json): - """ - Write a network ID, recording the data that was provided by libnetwork - on the CreateNetwork call. - :param network_id: The network ID to write. - :param create_network_json: Python datastructure representing the - CreateNetwork data. - :return: Nothing - """ - self.etcd_client.write(PREFIX + network_id, - json.dumps(create_network_json)) - - def remove_network(self, network_id): - """ - Remove a network ID - :param network_id: The network ID to delete. - :return: True if the delete was successful, false if the network_id - didn't exist. - """ - try: - self.etcd_client.delete(PREFIX + network_id) - except EtcdKeyNotFound: - return False - else: - return True diff --git a/libnetwork/driver_plugin.py b/libnetwork/driver_plugin.py deleted file mode 100644 index 566c9b0..0000000 --- a/libnetwork/driver_plugin.py +++ /dev/null @@ -1,645 +0,0 @@ -# Copyright 2015 Metaswitch Networks -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from flask import Flask, jsonify, request -import logging -import sys -import re - -from pycalico.util import generate_cali_interface_name, IPV6_RE -from subprocess32 import CalledProcessError, check_output -from werkzeug.exceptions import HTTPException, default_exceptions -from netaddr import IPAddress, IPNetwork -from pycalico.block import AlreadyAssignedError -from pycalico.datastore_datatypes import IF_PREFIX, Endpoint, IPPool -from pycalico.datastore_errors import PoolNotFound -from pycalico import netns -from pycalico.util import get_hostname, get_ipv6_link_local - -from datastore_libnetwork import LibnetworkDatastoreClient - - -# TODO: Move to libcalico constants -DUMMY_IPV4_NEXTHOP = "169.254.1.1" - - -# The MAC address of the interface in the container is arbitrary, so for -# simplicity, use a fixed MAC. -FIXED_MAC = "EE:EE:EE:EE:EE:EE" - -# Orchestrator and container IDs used in our endpoint identification. These -# are fixed for libnetwork. Unique endpoint identification is provided by -# hostname and endpoint ID. -CONTAINER_NAME = "libnetwork" -ORCHESTRATOR_ID = "libnetwork" - -# Calico IPAM module does not allow selection of pools from which to allocate -# IP addresses. The pool ID, which has to be supplied in the libnetwork IPAM -# API is therefore fixed. We use different values for IPv4 and IPv6 so that -# during allocation we know which IP version to use. -POOL_ID_V4 = "CalicoPoolIPv4" -POOL_ID_V6 = "CalicoPoolIPv6" - -# Fix pool and gateway CIDRs. As per comment above, Calico IPAM does not allow -# assignment from a specific pool, so we choose a dummy value that will not be -# used in practise. A 0/0 value is used for both IPv4 and IPv6. This value is -# also used by the Network Driver to indicate that the Calico IPAM driver was -# used rather than the default libnetwork IPAM driver - this is useful because -# Calico Network Driver behavior depends on whether our IPAM driver was used or -# not. -POOL_CIDR_STR_V4 = "0.0.0.0/0" -POOL_CIDR_STR_V6 = "::/0" -GATEWAY_CIDR_STR_V4 = "0.0.0.0/0" -GATEWAY_CIDR_STR_V6 = "::/0" - -# Calico-IPAM gateway CIDRs as an IPNetwork object -GATEWAY_NETWORK_V4 = IPNetwork(GATEWAY_CIDR_STR_V4) -GATEWAY_NETWORK_V6 = IPNetwork(GATEWAY_CIDR_STR_V6) - -# How long to wait (seconds) for IP commands to complete. -IP_CMD_TIMEOUT = 5 - -# Initialise our hostname and datastore client. -hostname = get_hostname() -client = LibnetworkDatastoreClient() - -# Return all errors as JSON. From http://flask.pocoo.org/snippets/83/ -# This ensures that uncaught exceptions get returned to libnetwork in a useful -# way -def make_json_app(import_name, **kwargs): - """ - Creates a JSON-oriented Flask app. - - All error responses that you don't specifically - manage yourself will have application/json content - type, and will contain JSON like this (just an example): - - { "Err": "405: Method Not Allowed" } - """ - def make_json_error(ex): - response = jsonify({"Err": str(ex)}) - response.status_code = (ex.code - if isinstance(ex, HTTPException) - else 500) - return response - - wrapped_app = Flask(import_name, **kwargs) - - for code in default_exceptions.iterkeys(): - wrapped_app.errorhandler(code)(make_json_error) - - return wrapped_app - -app = make_json_app(__name__) -app.logger.addHandler(logging.StreamHandler(sys.stdout)) -app.logger.setLevel(logging.DEBUG) -app.logger.info("Application started") - -# The API calls below are documented at -# https://github.com/docker/libnetwork/blob/master/docs/remote.md - -# <-- Plugin activation, we activate both libnetwork and ipam plugins --> - -@app.route('/Plugin.Activate', methods=['POST']) -def activate(): - json_response = {"Implements": ["NetworkDriver", "IpamDriver"]} - app.logger.debug("Activate response JSON=%s", json_response) - return jsonify(json_response) - -# <-- IPAM plugin API --> - -@app.route('/IpamDriver.GetDefaultAddressSpaces', methods=['POST']) -def get_default_address_spaces(): - # Return fixed local and global address spaces. The Calico IPAM module - # does not use the address space when assigning IP addresses. Instead - # we assign from the pre-defined Calico IP pools. - json_response = { - "LocalDefaultAddressSpace": "CalicoLocalAddressSpace", - "GlobalDefaultAddressSpace": "CalicoGlobalAddressSpace" - } - app.logger.debug("GetDefaultAddressSpace response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/IpamDriver.RequestPool', methods=['POST']) -def request_pool(): - # force is required since the request doesn't have the correct mimetype - # If the JSON is malformed, then a BadRequest exception is raised, - # which returns a HTTP 400 response. - json_data = request.get_json(force=True) - app.logger.debug("RequestPool JSON=%s", json_data) - pool = json_data["Pool"] - sub_pool = json_data.get("SubPool") - v6 = json_data["V6"] - - # Calico IPAM does not allow you to request SubPool. - if sub_pool: - error_message = "Calico IPAM does not support sub pool configuration" \ - "on 'docker create network'. Calico IP Pools " \ - "should be configured first and IP assignment is " \ - "from those pre-configured pools." - app.logger.error(error_message) - raise Exception(error_message) - - # If a pool (subnet on the CLI) is specified, it must match one of the - # preconfigured Calico pools. - if pool: - if not get_pool(IPNetwork(pool)): - error_message = "The requested subnet must match the CIDR of a " \ - "configured Calico IP Pool." - app.logger.error(error_message) - raise Exception(error_message) - - # If a subnet has been specified we use that as the pool ID. Otherwise, we - # use static pool ID and CIDR to indicate that we are assigning from all of - # the pools. - if v6: - pool_id = pool or POOL_ID_V6 - pool_cidr = POOL_CIDR_STR_V6 - gateway_cidr = GATEWAY_CIDR_STR_V6 - else: - pool_id = pool or POOL_ID_V4 - pool_cidr = POOL_CIDR_STR_V4 - gateway_cidr = GATEWAY_CIDR_STR_V4 - - # The meta data includes a dummy gateway address. This prevents libnetwork - # from requesting a gateway address from the pool since for a Calico - # network our gateway is set to our host IP. - json_response = { - "PoolID": pool_id, - "Pool": pool_cidr, - "Data": { - "com.docker.network.gateway": gateway_cidr - } - } - app.logger.debug("RequestPool response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/IpamDriver.ReleasePool', methods=['POST']) -def release_pool(): - json_data = request.get_json(force=True) - app.logger.debug("ReleasePool JSON=%s", json_data) - pool_id = json_data["PoolID"] - json_response = {} - app.logger.debug("ReleasePool response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/IpamDriver.RequestAddress', methods=['POST']) -def request_address(): - json_data = request.get_json(force=True) - app.logger.debug("RequestAddress JSON=%s", json_data) - pool_id = json_data["PoolID"] - address = json_data["Address"] - - if not address: - app.logger.debug("Auto assigning IP from Calico pools") - - # No address requested, so auto assign from our pools. If the pool ID - # is one of the fixed IDs then assign from across all configured pools, - # otherwise assign from the requested pool - if pool_id == POOL_ID_V4: - version = 4 - pool = None - elif pool_id == POOL_ID_V6: - version = 6 - pool = None - else: - pool_cidr = IPNetwork(pool_id) - pool = get_pool(pool_cidr) - if not pool: - error_message = "The network references a Calico pool which " \ - "has been deleted. Please re-instate the " \ - "Calico pool before using the network." - app.logger.error(error_message) - raise Exception(error_message) - version = pool_cidr.version - - if version == 4: - num_v4 = 1 - num_v6 = 0 - pool_v4 = pool - pool_v6 = None - else: - num_v4 = 0 - num_v6 = 1 - pool_v4 = None - pool_v6 = pool - - # Auto assign an IP based on whether the IPv4 or IPv6 pool was selected. - # We auto-assign from all available pools with affinity based on our - # host. - ips_v4, ips_v6 = client.auto_assign_ips(num_v4, num_v6, None, None, - pool=(pool_v4, pool_v6), - host=hostname) - ips = ips_v4 + ips_v6 - if not ips: - error_message = "There are no available IP addresses in the " \ - "configured Calico IP pools" - app.logger.error(error_message) - raise Exception(error_message) - else: - app.logger.debug("Reserving a specific address in Calico pools") - try: - ip_address = IPAddress(address) - client.assign_ip(ip_address, None, {}, host=hostname) - ips = [ip_address] - except AlreadyAssignedError: - error_message = "The address %s is already in " \ - "use" % str(ip_address) - app.logger.error(error_message) - raise Exception(error_message) - except PoolNotFound: - error_message = "The address %s is not in one of the configured " \ - "Calico IP pools" % str(ip_address) - app.logger.error(error_message) - raise Exception(error_message) - - # We should only have one IP address assigned at this point. - assert len(ips) == 1, "Unexpected number of assigned IP addresses" - - # Return the IP as a CIDR. - json_response = { - "Address": str(IPNetwork(ips[0])), - "Data": {} - } - app.logger.debug("RequestAddress response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/IpamDriver.ReleaseAddress', methods=['POST']) -def release_address(): - json_data = request.get_json(force=True) - app.logger.debug("ReleaseAddress JSON=%s", json_data) - address = json_data["Address"] - - # Unassign the address. This handles the address already being unassigned - # in which case it is a no-op. The release_ips call may raise a - # RuntimeError if there are repeated clashing updates to the same IP block, - # this is not an expected condition. - client.release_ips({IPAddress(address)}) - - json_response = {} - app.logger.debug("ReleaseAddress response JSON=%s", json_response) - return jsonify(json_response) - -# <-- libnetwork plugin API --> - -@app.route('/NetworkDriver.GetCapabilities', methods=['POST']) -def get_capabilities(): - json_response = {"Scope": "global"} - app.logger.debug("GetCapabilities response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/NetworkDriver.CreateNetwork', methods=['POST']) -def create_network(): - json_data = request.get_json(force=True) - app.logger.debug("CreateNetwork JSON=%s", json_data) - - # Create the CNM "network" as a Calico profile. - network_id = json_data["NetworkID"] - app.logger.info("Creating profile %s", network_id) - client.create_profile(network_id) - - for version in (4, 6): - # Extract the gateway and pool from the network data. If this - # indicates that Calico IPAM is not being used, then create a Calico - # IP pool. - gateway, pool = get_gateway_pool_from_network_data(json_data, version) - - # Skip over versions that have no gateway assigned. - if gateway is None: - continue - - # If we aren't using Calico IPAM then we need to ensure an IP pool - # exists. IPIP and Masquerade options can be included on the network - # create as additional options. Note that this IP Pool has ipam=False - # to ensure it is not used in Calico IPAM assignment. - if not is_using_calico_ipam(gateway): - options = json_data["Options"]["com.docker.network.generic"] - ipip = options.get("ipip") - masquerade = options.get("nat-outgoing") - client.add_ip_pool(pool.version, - IPPool(pool, ipip=ipip, masquerade=masquerade, - ipam=False)) - - # Store off the JSON passed in on this request. It's required in later - # calls. - client.write_network(network_id, json_data) - - app.logger.debug("CreateNetwork response JSON=%s", "{}") - return jsonify({}) - - -@app.route('/NetworkDriver.CreateEndpoint', methods=['POST']) -def create_endpoint(): - json_data = request.get_json(force=True) - app.logger.debug("CreateEndpoint JSON=%s", json_data) - endpoint_id = json_data["EndpointID"] - network_id = json_data["NetworkID"] - interface = json_data["Interface"] - - app.logger.info("Creating endpoint %s", endpoint_id) - - # Get the addresses to use from the request JSON. - address_ip4 = interface.get("Address") - address_ip6 = interface.get("AddressIPv6") - assert address_ip4 or address_ip6, "No address assigned for endpoint" - - # Create a Calico endpoint object. - ep = Endpoint(hostname, ORCHESTRATOR_ID, CONTAINER_NAME, endpoint_id, - "active", FIXED_MAC) - ep.profile_ids.append(network_id) - - if address_ip4: - ep.ipv4_nets.add(IPNetwork(address_ip4)) - - if address_ip6: - ep.ipv6_nets.add(IPNetwork(address_ip6)) - - app.logger.debug("Saving Calico endpoint: %s", ep) - client.set_endpoint(ep) - - json_response = { - "Interface": { - "MacAddress": FIXED_MAC, - } - } - - app.logger.debug("CreateEndpoint response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/NetworkDriver.Join', methods=['POST']) -def join(): - json_data = request.get_json(force=True) - app.logger.debug("Join JSON=%s", json_data) - network_id = json_data["NetworkID"] - endpoint_id = json_data["EndpointID"] - app.logger.info("Joining endpoint %s", endpoint_id) - - # The host interface name matches the name given when creating the endpoint - # during CreateEndpoint - host_interface_name = generate_cali_interface_name(IF_PREFIX, endpoint_id) - - # The temporary interface name is what gets passed to libnetwork. - # Libnetwork renames the interface using the DstPrefix (e.g. cali0) - temp_interface_name = generate_cali_interface_name("tmp", endpoint_id) - - try: - # Create the veth pair. - netns.create_veth(host_interface_name, temp_interface_name) - - # Set the mac as libnetwork doesn't do this for us (even if we return - # it on the CreateNetwork) - netns.set_veth_mac(temp_interface_name, FIXED_MAC) - except CalledProcessError as e: - # Failed to create or configure the veth, ensure veth is removed. - remove_veth(host_interface_name) - raise e - - # Initialise our response data. - json_response = { - "InterfaceName": { - "SrcName": temp_interface_name, - "DstPrefix": IF_PREFIX, - } - } - - # Extract relevant data from the Network data. - network_data = get_network_data(network_id) - gateway_ip4, _ = get_gateway_pool_from_network_data(network_data, 4) - gateway_ip6, _ = get_gateway_pool_from_network_data(network_data, 6) - - if (gateway_ip4 and is_using_calico_ipam(gateway_ip4)) or \ - (gateway_ip6 and is_using_calico_ipam(gateway_ip6)): - # One of the network gateway addresses indicate that we are using - # Calico IPAM driver. In this case we setup routes using the gateways - # configured on the endpoint (which will be our host IPs). - app.logger.debug("Using Calico IPAM driver, configure gateway and " - "static routes to the host") - static_routes = [] - if gateway_ip4: - json_response["Gateway"] = DUMMY_IPV4_NEXTHOP - static_routes.append({ - "Destination": DUMMY_IPV4_NEXTHOP + "/32", - "RouteType": 1, # 1 = CONNECTED - "NextHop": "" - }) - if gateway_ip6: - # Here, we'll report the link local address of the host's cali interface to libnetwork - # as our IPv6 gateway. IPv6 link local addresses are automatically assigned to interfaces - # when they are brought up. Unfortunately, the container link must be up as well. So - # bring it up now - # TODO: create_veth should already bring up both links - bring_up_interface(temp_interface_name) - # Then extract the link local address that was just assigned to our host's interface - next_hop_6 = get_ipv6_link_local(host_interface_name) - json_response["GatewayIPv6"] = next_hop_6 - static_routes.append({ - "Destination": str(IPNetwork(next_hop_6)), - "RouteType": 1, # 1 = CONNECTED - "NextHop": "" - }) - json_response["StaticRoutes"] = static_routes - else: - # We are not using Calico IPAM driver, so configure blank gateways to - # set up auto-gateway behavior. - app.logger.debug("Not using Calico IPAM driver") - json_response["Gateway"] = "" - json_response["GatewayIPv6"] = "" - - app.logger.debug("Join Response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/NetworkDriver.EndpointOperInfo', methods=['POST']) -def endpoint_oper_info(): - json_data = request.get_json(force=True) - app.logger.debug("EndpointOperInfo JSON=%s", json_data) - endpoint_id = json_data["EndpointID"] - app.logger.info("Endpoint operation info requested for %s", endpoint_id) - json_response = { - "Value": { - } - } - app.logger.debug("EP Oper Info Response JSON=%s", json_response) - return jsonify(json_response) - - -@app.route('/NetworkDriver.DeleteNetwork', methods=['POST']) -def delete_network(): - json_data = request.get_json(force=True) - app.logger.debug("DeleteNetwork JSON=%s", json_data) - - network_id = json_data["NetworkID"] - - # Remove the network. We don't raise an error if the profile is still - # being used by endpoints. We assume libnetwork will enforce this. - # From https://github.com/docker/libnetwork/blob/master/docs/design.md - # LibNetwork will not allow the delete to proceed if there are any - # existing endpoints attached to the Network. - client.remove_profile(network_id) - app.logger.info("Removed profile %s", network_id) - - # Remove the pools that were created for this network. - network_data = get_network_data(network_id) - for version in (4, 6): - gateway_cidr, pool_cidr = \ - get_gateway_pool_from_network_data(network_data, version) - if gateway_cidr and not is_using_calico_ipam(gateway_cidr): - client.remove_ip_pool(version, pool_cidr) - app.logger.info("Removed pool %s", pool_cidr) - - # Clean up the stored network data. - client.remove_network(network_id) - - return jsonify({}) - - -@app.route('/NetworkDriver.DeleteEndpoint', methods=['POST']) -def delete_endpoint(): - json_data = request.get_json(force=True) - app.logger.debug("DeleteEndpoint JSON=%s", json_data) - endpoint_id = json_data["EndpointID"] - app.logger.info("Removing endpoint %s", endpoint_id) - - client.remove_endpoint(Endpoint(hostname, ORCHESTRATOR_ID, CONTAINER_NAME, - endpoint_id, None, None)) - - app.logger.debug("DeleteEndpoint response JSON=%s", "{}") - return jsonify({}) - - -@app.route('/NetworkDriver.Leave', methods=['POST']) -def leave(): - json_data = request.get_json(force=True) - app.logger.debug("Leave JSON=%s", json_data) - ep_id = json_data["EndpointID"] - app.logger.info("Leaving endpoint %s", ep_id) - - remove_veth(generate_cali_interface_name(IF_PREFIX, ep_id)) - - app.logger.debug("Leave response JSON=%s", "{}") - return jsonify({}) - - -@app.route('/NetworkDriver.DiscoverNew', methods=['POST']) -def discover_new(): - json_data = request.get_json(force=True) - app.logger.debug("DiscoverNew JSON=%s", json_data) - app.logger.debug("DiscoverNew response JSON=%s", "{}") - return jsonify({}) - - -@app.route('/NetworkDriver.DiscoverDelete', methods=['POST']) -def discover_delete(): - json_data = request.get_json(force=True) - app.logger.debug("DiscoverNew JSON=%s", json_data) - app.logger.debug("DiscoverDelete response JSON=%s", "{}") - return jsonify({}) - - -def remove_veth(name): - """ - Best effort removal of veth, logging if removal fails. - :param name: The name of the veth to remove - """ - try: - netns.remove_veth(name) - except CalledProcessError: - app.logger.warn("Failed to delete veth %s", name) - - -def get_network_data(network_id): - """ - Return the network data (i.e. the JSON data that is passed in on the - CreateNetwork request), or raise an exception if the network does not - exist. - - :param network_id: The network ID. - :return: The network data. - """ - network_data = client.get_network(network_id) - if not network_data: - error_message = "Network %s does not exist" % network_id - app.logger.error(error_message) - raise Exception(error_message) - return network_data - - -def get_gateway_pool_from_network_data(network_data, version): - """ - Extract the gateway and pool from the network data. - :param network_data: The network data. - :param version: The IP version (4 or 6) - :return: Tuple of (string gateway_CIDR, - string pool_CIDR) - or (None, None) if either are not set. - """ - assert version in [4,6] - - ip_data = network_data.get("IPv%sData" % version) - if not ip_data: - # No IP data for this IP version, so skip. - app.logger.info("No IPv%s data", version) - return None, None - - if len(ip_data) > 1: - error_message = "Unsupported: multiple Gateways defined for " \ - "IPv%s" % version - app.logger.error(error_message) - raise Exception(error_message) - - gateway = ip_data[0].get('Gateway') - pool = ip_data[0].get('Pool') - - if not gateway or not pool: - return None, None - - return IPNetwork(gateway), IPNetwork(pool) - - -def is_using_calico_ipam(gateway_cidr): - """ - Determine if the gateway CIDR indicates that we are using Calico IPAM - driver for IP assignment. - :param gateway: The gateway CIDR. This should not be None. - :return: True, if using Calico IPAM driver. - """ - # A 0 gateway IP indicates Calico IPAM is being used. - assert isinstance(gateway_cidr, IPNetwork) - return gateway_cidr in (GATEWAY_NETWORK_V4, GATEWAY_NETWORK_V6) - - -def get_pool(pool_cidr): - """ - Return the Calico IP Pool with the specified pool CIDR, or None if not - present. - :param pool_cidr: The pool CIDR. - :return: IPPool - The Calico pool, or None if not configured. - """ - pools = client.get_ip_pools(pool_cidr.version, ipam=True) - for pool in pools: - if pool.cidr == pool_cidr: - return pool - return None - - -def bring_up_interface(interface_name): - """ - Bring up an interface. - """ - check_output(['ip', 'link', 'set', interface_name, 'up'], timeout=IP_CMD_TIMEOUT) diff --git a/main.go b/main.go new file mode 100644 index 0000000..e6b48e2 --- /dev/null +++ b/main.go @@ -0,0 +1,83 @@ +package main + +import ( + "log" + + "os" + + "github.com/docker/go-plugins-helpers/ipam" + "github.com/docker/go-plugins-helpers/network" + "github.com/projectcalico/libcalico-go/lib/api" + "github.com/projectcalico/libnetwork-plugin/driver" + + datastoreClient "github.com/projectcalico/libcalico-go/lib/client" + "flag" + "fmt" +) + +const ( + ipamPluginName = "calico-ipam" + networkPluginName = "calico" +) + +var ( + config *api.ClientConfig + client *datastoreClient.Client + + logger *log.Logger +) + +func init() { + var err error + + if config, err = datastoreClient.LoadClientConfig(""); err != nil { + panic(err) + } + if client, err = datastoreClient.New(*config); err != nil { + panic(err) + } + + logger = log.New(os.Stdout, "", log.LstdFlags) +} + +// VERSION is filled out during the build process (using git describe output) +var VERSION string + +func main() { + // Display the version on "-v" + // Use a new flag set so as not to conflict with existing libraries which use "flag" + flagSet := flag.NewFlagSet("Calico", flag.ExitOnError) + + version := flagSet.Bool("v", false, "Display version") + err := flagSet.Parse(os.Args[1:]) + if err != nil { + fmt.Println(err) + os.Exit(1) + } + if *version { + fmt.Println(VERSION) + os.Exit(0) + } + + errChannel := make(chan error) + networkHandler := network.NewHandler(driver.NewNetworkDriver(client, logger)) + ipamHandler := ipam.NewHandler(driver.NewIpamDriver(client, logger)) + + go func(c chan error) { + logger.Println("calico-net has started.") + err := networkHandler.ServeUnix("root", networkPluginName) + logger.Println("calico-net has stopped working.") + c <- err + }(errChannel) + + go func(c chan error) { + logger.Println("calico-ipam has started.") + err := ipamHandler.ServeUnix("root", ipamPluginName) + logger.Println("calico-ipam has stopped working.") + c <- err + }(errChannel) + + err = <-errChannel + + log.Fatal(err) +} diff --git a/nose.cfg b/nose.cfg deleted file mode 100644 index 70bb4a3..0000000 --- a/nose.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[nosetests] -with-coverage=1 -cover-erase=1 -cover-package=libnetwork diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 417c095..0000000 --- a/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -netaddr -flask -gunicorn -subprocess32 -gevent -git+https://github.com/projectcalico/python-etcd.git -git+https://github.com/projectcalico/libcalico.git \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 197ed0a..0000000 --- a/setup.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2016 Tigera, Inc. All rights reserved. -# -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import setuptools - -version = '0.9.0-dev' - -setuptools.setup( - name='libnetwork', - version=version, - - description='Docker libnetwork plugin', - - # The project's main homepage. - url='https://github.com/projectcalico/libnetwork-plugin/', - - # Author details - author='Project Calico', - author_email='maintainers@projectcalico.org', - - # Choose your license - license='Apache 2.0', - - # See https://pypi.python.org/pypi?%3Aaction=list_classifiers - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Operating System :: POSIX :: Linux', - 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', - 'Topic :: System :: Networking', - ], - - # What does your project relate to? - keywords='calico docker etcd mesos kubernetes rkt openstack', - - packages=["libnetwork"], - - install_requires=['netaddr', 'python-etcd>=0.4.3', 'subprocess32', 'flask', 'gunicorn', 'gevent'], - dependency_links=[ - "git+https://github.com/projectcalico/python-etcd.git", - "git+https://github.com/projectcalico/libcalico.git" - ] -) \ No newline at end of file diff --git a/start.sh b/start.sh deleted file mode 100755 index e7bf5b9..0000000 --- a/start.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -exec 2>&1 -GUNICORN=/usr/bin/gunicorn -ROOT=/calico_containers -PID=/var/run/gunicorn.pid -APP=libnetwork_plugin.driver_plugin:app - -if [ -f $PID ]; then rm $PID; fi - -exec $GUNICORN --chdir $ROOT --pid=$PID \ --b unix:///run/docker/plugins/calico.sock $APP \ ---timeout 5 \ ---log-level=info \ ---workers 1 \ ---worker-class gevent ---access-logfile - diff --git a/tests/st/libnetwork/test_assign_specific_ip.py b/tests/st/libnetwork/test_assign_specific_ip.py index bd75ea6..c08b4bf 100644 --- a/tests/st/libnetwork/test_assign_specific_ip.py +++ b/tests/st/libnetwork/test_assign_specific_ip.py @@ -13,9 +13,13 @@ # limitations under the License. import logging +from subprocess import check_output + from tests.st.test_base import TestBase from tests.st.utils.docker_host import DockerHost -from tests.st.utils.utils import assert_number_endpoints +from tests.st.utils.utils import ( + assert_number_endpoints, get_ip, log_and_run, retry_until_success, ETCD_SCHEME, + ETCD_CA, ETCD_KEY, ETCD_CERT, ETCD_HOSTNAME_SSL) from tests.st.libnetwork.test_mainline_single_host import \ ADDITIONAL_DOCKER_OPTIONS, POST_DOCKER_COMMANDS @@ -28,43 +32,32 @@ def test_assign_specific_ip(self): Test that a libnetwork assigned IP is allocated to the container with Calico when using the '--ip' flag on docker run. """ - with DockerHost('host1', + with DockerHost('host', additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, - post_docker_commands=POST_DOCKER_COMMANDS, - start_calico=False) as host1, \ - DockerHost('host2', - additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, - post_docker_commands=POST_DOCKER_COMMANDS, - start_calico=False) as host2: - - host1.start_calico_node("--libnetwork") - host2.start_calico_node("--libnetwork") - - # Set up one endpoints on each host - workload1_ip = "192.168.1.101" - workload2_ip = "192.168.1.102" + post_docker_commands=["docker load -i /code/busybox.tar", + "docker load -i /code/calico-node-libnetwork.tar"], + start_calico=False) as host: + + run_plugin_command = 'docker run -d ' \ + '--net=host --privileged ' + \ + '-e CALICO_ETCD_AUTHORITY=%s:2379 ' \ + '-v /run/docker/plugins:/run/docker/plugins ' \ + '-v /var/run/docker.sock:/var/run/docker.sock ' \ + '-v /lib/modules:/lib/modules ' \ + '--name libnetwork-plugin ' \ + 'calico/libnetwork-plugin' % (get_ip(),) + + host.execute(run_plugin_command) subnet = "192.168.0.0/16" - network = host1.create_network("testnet", subnet=subnet) - workload1 = host1.create_workload("workload1", - network=network, - ip=workload1_ip) - workload2 = host2.create_workload("workload2", - network=network, - ip=workload2_ip) + host.calicoctl('pool add %s' % subnet) + + workload_ip = "192.168.1.101" - self.assertEquals(workload1_ip, workload1.ip) - self.assertEquals(workload2_ip, workload2.ip) + network = host.create_network( + "specificipnet", subnet=subnet, driver="calico", ipam_driver="calico-ipam") - # Allow network to converge - # Check connectivity with assigned IPs - workload1.assert_can_ping(workload2_ip, retries=5) - workload2.assert_can_ping(workload1_ip, retries=5) + workload = host.create_workload("workload1", + network=network, + ip=workload_ip) - # Disconnect endpoints from the network - # Assert can't ping and endpoints are removed from Calico - network.disconnect(host1, workload1) - network.disconnect(host2, workload2) - workload1.assert_cant_ping(workload2_ip, retries=5) - assert_number_endpoints(host1, 0) - assert_number_endpoints(host2, 0) - network.delete() + self.assertEquals(workload_ip, workload.ip) diff --git a/tests/st/libnetwork/test_error_ipam.py b/tests/st/libnetwork/test_error_ipam.py new file mode 100644 index 0000000..65853a9 --- /dev/null +++ b/tests/st/libnetwork/test_error_ipam.py @@ -0,0 +1,55 @@ +# Copyright 2015 Metaswitch Networks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import uuid + +from tests.st.test_base import TestBase +from tests.st.utils import utils +from tests.st.utils.docker_host import DockerHost +import logging +from tests.st.utils.utils import get_ip, assert_number_endpoints, assert_profile, \ + get_profile_name, ETCD_CA, ETCD_CERT, ETCD_KEY, ETCD_HOSTNAME_SSL, \ + ETCD_SCHEME + +logger = logging.getLogger(__name__) + + +ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " % \ + utils.get_ip() + +class TestErrors(TestBase): + def test_no_ipam(self): + """ + Try creating a network and using calico for networking but not IPAM. CHeck that it fails. + """ + with DockerHost('host', + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + post_docker_commands=["docker load -i /code/calico-node-libnetwork.tar"], + start_calico=False) as host: + + run_plugin_command = 'docker run -d ' \ + '--net=host --privileged ' + \ + '-e CALICO_ETCD_AUTHORITY=%s:2379 ' \ + '-v /run/docker/plugins:/run/docker/plugins ' \ + '-v /var/run/docker.sock:/var/run/docker.sock ' \ + '-v /lib/modules:/lib/modules ' \ + '--name libnetwork-plugin ' \ + 'calico/libnetwork-plugin' % (get_ip(),) + + host.execute(run_plugin_command) + + # Create network using calico for network driver ONLY + try: + network = host.create_network("shouldfailnet", driver="calico") + except Exception, e: + self.assertIn("Non-Calico IPAM driver is used", str(e)) diff --git a/tests/st/libnetwork/test_libnetwork.py b/tests/st/libnetwork/test_libnetwork.py deleted file mode 100644 index 688bba5..0000000 --- a/tests/st/libnetwork/test_libnetwork.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2015 Metaswitch Networks -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import uuid - -from netaddr import IPNetwork -from unittest import skip - -from tests.st.test_base import TestBase -from tests.st.utils.constants import DEFAULT_IPV4_POOL_CIDR -from tests.st.utils.docker_host import DockerHost - - -class LibnetworkTests(TestBase): - - @skip("Not written yet") - def test_moving_endpoints(self): - """ - Test moving endpoints between hosts and containers. - """ - # with DockerHost('host1') as host1, DockerHost('host2') as host2: - # pass - # Using docker service attach/detach publish/unpublish ls/info - pass - - @skip("Not written yet") - def test_endpoint_ids(self): - """ - Test that endpoint ID provided by docker service publish can be used - with calicoctl endpoint commands. - """ - pass diff --git a/tests/st/libnetwork/test_mainline_multi_host.py b/tests/st/libnetwork/test_mainline_multi_host.py index 88b3f0f..f81f7f9 100644 --- a/tests/st/libnetwork/test_mainline_multi_host.py +++ b/tests/st/libnetwork/test_mainline_multi_host.py @@ -18,7 +18,7 @@ from tests.st.test_base import TestBase from tests.st.utils.docker_host import DockerHost from tests.st.utils.exceptions import CommandExecError -from tests.st.utils.utils import assert_network, assert_profile, \ +from tests.st.utils.utils import get_ip, assert_network, assert_profile, \ assert_number_endpoints, get_profile_name @@ -32,10 +32,8 @@ def test_multi_host(self): functionality in a single test. - Create two hosts - - Create a network using the default IPAM driver, and a workload on - each host assigned to that network. - - Create a network using the Calico IPAM driver, and a workload on - each host assigned to that network. + - Create two networks, both using Calico for IPAM and networking. + - Create a workload on each host in each network. - Check that hosts on the same network can ping each other. - Check that hosts on different networks cannot ping each other. """ @@ -47,109 +45,92 @@ def test_multi_host(self): additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) as host2: - # TODO work IPv6 into this test too - host1.start_calico_node("--libnetwork") - host2.start_calico_node("--libnetwork") + run_plugin_command = 'docker run -d ' \ + '--net=host --privileged ' + \ + '-e CALICO_ETCD_AUTHORITY=%s:2379 ' \ + '-v /run/docker/plugins:/run/docker/plugins ' \ + '-v /var/run/docker.sock:/var/run/docker.sock ' \ + '-v /lib/modules:/lib/modules ' \ + '--name libnetwork-plugin ' \ + 'calico/libnetwork-plugin' % (get_ip(),) + + host1.start_calico_node() + host1.execute(run_plugin_command) + + host2.start_calico_node() + host2.execute(run_plugin_command) # Create the networks on host1, but it should be usable from all # hosts. We create one network using the default driver, and the # other using the Calico driver. - network1 = host1.create_network("testnet1", ipam_driver="default") - network2 = host1.create_network("testnet2", ipam_driver="calico") + testnet1 = host1.create_network("testnet1", ipam_driver="calico-ipam", driver="calico") + testnet2 = host1.create_network("testnet2", ipam_driver="calico-ipam", driver="calico") # Assert that the networks can be seen on host2 - assert_network(host2, network2) - assert_network(host2, network1) + assert_network(host2, testnet1) + assert_network(host2, testnet2) + + # Create two workloads on host1 - one in each network + workload_h1n1 = host1.create_workload("workload_h1n1", + network=testnet1) + workload_h1n2 = host1.create_workload("workload_h1n2", + network=testnet2) + # Profiles aren't created until a workloads are created. # Assert that the profiles have been created for the networks - profile_name1 = get_profile_name(host1, network1) - assert_profile(host1, profile_name1) - profile_name2 = get_profile_name(host1, network2) - assert_profile(host1, profile_name2) - - # Create two workloads on host1 and one on host2 all in network 1. - workload_h1n2_1 = host1.create_workload("workload_h1n2_1", - network=network2) - workload_h1n2_2 = host1.create_workload("workload_h1n2_2", - network=network2) - workload_h2n2_1 = host2.create_workload("workload_h2n2_1", - network=network2) + assert_profile(host1, "testnet1") # Create similar workloads in network 2. - workload_h2n1_1 = host2.create_workload("workload_h2n1_1", - network=network1) - workload_h1n1_1 = host1.create_workload("workload_h1n1_1", - network=network1) - workload_h1n1_2 = host1.create_workload("workload_h1n1_2", - network=network1) + workload_h2n1 = host2.create_workload("workload_h2n1", + network=testnet1) + workload_h2n2 = host2.create_workload("workload_h2n2", + network=testnet2) + assert_profile(host1, "testnet2") # Assert that endpoints are in Calico - assert_number_endpoints(host1, 4) + assert_number_endpoints(host1, 2) assert_number_endpoints(host2, 2) # Assert that workloads can communicate with each other on network # 1, and not those on network 2. Ping using IP for all workloads, # and by hostname for workloads on the same network (note that # a workloads own hostname does not work). - self.assert_connectivity(retries=2, - pass_list=[workload_h1n1_1, - workload_h1n1_2, - workload_h2n1_1]) - # TODO: docker_gwbridge iptable FORWARD rule takes precedence over - # Felix, resulting in temporary lack of isolation between a - # container on the bridge communicating with a non-bridge container - # on the same host. Therefore we cannot yet test isolation. - # fail_list=[workload_h1n2_1, - # workload_h1n2_2, - # workload_h2n2_1]) - workload_h1n1_1.execute("ping -c 1 -W 1 workload_h1n1_2") - workload_h1n1_1.execute("ping -c 1 -W 1 workload_h2n1_1") - - # Repeat with network 2. - self.assert_connectivity(pass_list=[workload_h1n2_1, - workload_h1n2_2, - workload_h2n2_1]) - # TODO - see comment above - # fail_list=[workload_h1n1_1, - # workload_h1n1_2, - # workload_h1n1_1]) - workload_h1n2_1.execute("ping -c 1 -W 1 workload_h1n2_2") - workload_h1n2_1.execute("ping -c 1 -W 1 workload_h2n2_1") + self.assert_connectivity(retries=5, + pass_list=[workload_h1n1, + workload_h2n1], + fail_list=[workload_h1n2, + workload_h2n2]) + + workload_h1n1.execute("ping -c 1 -W 5 workload_h2n1") + workload_h2n1.execute("ping -c 1 -W 5 workload_h1n1") # Test deleting the network. It will fail if there are any # endpoints connected still. - self.assertRaises(CommandExecError, network1.delete) - self.assertRaises(CommandExecError, network2.delete) + self.assertRaises(CommandExecError, testnet1.delete) + self.assertRaises(CommandExecError, testnet2.delete) # For network 1, disconnect (or "detach" or "leave") the endpoints # Assert that an endpoint is removed from calico and can't ping - network1.disconnect(host1, workload_h1n1_1) - network1.disconnect(host1, workload_h1n1_2) - assert_number_endpoints(host1, 2) - network1.disconnect(host2, workload_h2n1_1) + testnet1.disconnect(host1, workload_h1n1) + assert_number_endpoints(host1, 1) + testnet1.disconnect(host2, workload_h2n1) assert_number_endpoints(host2, 1) - workload_h1n1_1.assert_cant_ping(workload_h2n2_1.ip, retries=5) + + workload_h1n1.assert_cant_ping(workload_h2n1.ip, retries=5) # Repeat for network 2. All endpoints should be removed. - network2.disconnect(host1, workload_h1n2_1) - network2.disconnect(host1, workload_h1n2_2) + testnet2.disconnect(host1, workload_h1n2) assert_number_endpoints(host1, 0) - network2.disconnect(host2, workload_h2n2_1) + testnet2.disconnect(host2, workload_h2n2) assert_number_endpoints(host2, 0) - workload_h1n1_1.assert_cant_ping(workload_h2n2_1.ip, retries=5) + workload_h1n2.assert_cant_ping(workload_h2n2.ip, retries=5) # Remove the workloads, so the endpoints can be unpublished, then # the delete should succeed. host1.remove_workloads() host2.remove_workloads() - # Remove the network and assert profile is removed - network1.delete() - network2.delete() - self.assertRaises(AssertionError, assert_profile, host1, - profile_name1) - - # TODO - Remove this calico node - - # TODO Would like to assert that there are no errors in the logs... + # Remove the network + testnet1.delete() + testnet2.delete() diff --git a/tests/st/libnetwork/test_mainline_single_host.py b/tests/st/libnetwork/test_mainline_single_host.py index 359d169..5f6eff6 100644 --- a/tests/st/libnetwork/test_mainline_single_host.py +++ b/tests/st/libnetwork/test_mainline_single_host.py @@ -17,41 +17,43 @@ from tests.st.utils import utils from tests.st.utils.docker_host import DockerHost import logging -from tests.st.utils.utils import assert_number_endpoints, assert_profile, \ +from tests.st.utils.utils import get_ip, assert_number_endpoints, assert_profile, \ get_profile_name, ETCD_CA, ETCD_CERT, ETCD_KEY, ETCD_HOSTNAME_SSL, \ ETCD_SCHEME logger = logging.getLogger(__name__) -POST_DOCKER_COMMANDS = ["docker load -i /code/calico-node.tgz", - "docker load -i /code/busybox.tgz", - "docker load -i /code/calico-node-libnetwork.tgz"] - -if ETCD_SCHEME == "https": - ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " \ - "--cluster-store-opt kv.cacertfile=%s " \ - "--cluster-store-opt kv.certfile=%s " \ - "--cluster-store-opt kv.keyfile=%s " % \ - (ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT, - ETCD_KEY) -else: - ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " % \ - utils.get_ip() +POST_DOCKER_COMMANDS = ["docker load -i /code/calico-node.tar", + "docker load -i /code/busybox.tar", + "docker load -i /code/calico-node-libnetwork.tar"] + +ADDITIONAL_DOCKER_OPTIONS = "--cluster-store=etcd://%s:2379 " % utils.get_ip() + class TestMainline(TestBase): def test_mainline(self): """ Setup two endpoints on one host and check connectivity then teardown. """ - # TODO - add in IPv6 as part of this flow. with DockerHost('host', additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) as host: - host.start_calico_node("--libnetwork") + + run_plugin_command = 'docker run -d ' \ + '--net=host --privileged ' + \ + '-e CALICO_ETCD_AUTHORITY=%s:2379 ' \ + '-v /run/docker/plugins:/run/docker/plugins ' \ + '-v /var/run/docker.sock:/var/run/docker.sock ' \ + '-v /lib/modules:/lib/modules ' \ + '--name libnetwork-plugin ' \ + 'calico/libnetwork-plugin' % (get_ip(),) + + host.start_calico_node() + host.execute(run_plugin_command) # Set up two endpoints on one host - network = host.create_network("testnet") + network = host.create_network("testnet", driver="calico", ipam_driver="calico-ipam") workload1 = host.create_workload("workload1", network=network) workload2 = host.create_workload("workload2", network=network) @@ -59,8 +61,7 @@ def test_mainline(self): assert_number_endpoints(host, 2) # Assert that the profile has been created for the network - profile_name = get_profile_name(host, network) - assert_profile(host, profile_name) + assert_profile(host, "testnet") # Allow network to converge # Check connectivity. @@ -87,6 +88,6 @@ def test_mainline(self): # Remove the network and assert profile is removed network.delete() - self.assertRaises(AssertionError, assert_profile, host, profile_name) - # TODO - Remove this calico node + # TODO - should deleting the network delete the profile or not? + # self.assertRaises(AssertionError, assert_profile, host, "testnet") diff --git a/tests/st/ssl-config/ca-config.json b/tests/st/ssl-config/ca-config.json deleted file mode 100644 index e492de1..0000000 --- a/tests/st/ssl-config/ca-config.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "signing": { - "default": { - "usages": [ - "signing", - "key encipherment", - "server auth", - "client auth" - ], - "expiry": "8760h" - } - } -} diff --git a/tests/st/ssl-config/ca-csr.json b/tests/st/ssl-config/ca-csr.json deleted file mode 100644 index 487390b..0000000 --- a/tests/st/ssl-config/ca-csr.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "CN": "Autogenerated CA", - "key": { - "algo": "ecdsa", - "size": 384 - }, - "names": [ - { - "O": "Certificater", - "OU": "Some other thing", - "L": "San Francisco", - "ST": "California", - "C": "US" - } - ] -} diff --git a/tests/st/ssl-config/req-csr.json b/tests/st/ssl-config/req-csr.json deleted file mode 100644 index 36b4424..0000000 --- a/tests/st/ssl-config/req-csr.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "CN": "etcd", - "hosts": [ - "localhost", - "etcd-authority-ssl" - ], - "key": { - "algo": "ecdsa", - "size": 384 - }, - "names": [ - { - "O": "autogenerated", - "OU": "etcd cluster", - "L": "location" - } - ] -} diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/unit/datastore_libnetwork_test.py b/tests/unit/datastore_libnetwork_test.py deleted file mode 100644 index 74fd5c1..0000000 --- a/tests/unit/datastore_libnetwork_test.py +++ /dev/null @@ -1,95 +0,0 @@ -# Copyright 2015 Metaswitch Networks -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import unittest - -from etcd import EtcdKeyNotFound -from etcd import Client as EtcdClient -from mock import patch, ANY, call, Mock -from netaddr import IPAddress, IPNetwork -from nose.tools import assert_equal -from pycalico import datastore - -from libnetwork.datastore_libnetwork import LibnetworkDatastoreClient - -TEST_NETWORK_ID = "ABCDEFGHJSDHFLA" -TEST_NETWORK_DIR = "/calico/libnetwork/v1/" + TEST_NETWORK_ID -TEST_DATA = {"test": 1234, "test2": [1, 2, 3, 4]} -TEST_JSON = json.dumps(TEST_DATA) - -class TestLibnetworkDatastoreClient(unittest.TestCase): - - @patch("pycalico.datastore.os.getenv", autospec=True) - @patch("pycalico.datastore.etcd.Client", autospec=True) - def setUp(self, m_etcd_client, m_getenv): - def get_env(key, default=""): - if key == "ETCD_AUTHORITY": - return "127.0.0.2:4002" - else: - return default - m_getenv.side_effect = get_env - self.etcd_client = Mock(spec=EtcdClient) - m_etcd_client.return_value = self.etcd_client - self.datastore = LibnetworkDatastoreClient() - m_etcd_client.assert_called_once_with(host="127.0.0.2", port=4002, - protocol="http", cert=None, - ca_cert=None) - - def tearDown(self): - pass - - def test_get_network(self): - """ - Test get_network() returns correct data. - :return: - """ - etcd_entry = Mock() - etcd_entry.value = TEST_JSON - self.etcd_client.read.return_value = etcd_entry - self.assertDictEqual(self.datastore.get_network(TEST_NETWORK_ID), - TEST_DATA) - self.etcd_client.read.assert_called_once_with(TEST_NETWORK_DIR) - - def test_get_network_not_found(self): - """ - Test get_network() returns None when the network is not found. - :return: - """ - self.etcd_client.read.side_effect = EtcdKeyNotFound - self.assertEquals(self.datastore.get_network(TEST_NETWORK_ID), None) - self.etcd_client.read.assert_called_once_with(TEST_NETWORK_DIR) - - def test_write_network(self): - """ - Test write_network() sends correct data to etcd. - """ - test_data = TEST_DATA - self.datastore.write_network(TEST_NETWORK_ID, test_data) - self.etcd_client.write.assert_called_once_with(TEST_NETWORK_DIR, - TEST_JSON) - - def test_remove_network(self): - """ - Test remove_network() when the network is present. - """ - self.assertTrue(self.datastore.remove_network(TEST_NETWORK_ID)) - self.etcd_client.delete.assert_called_once_with(TEST_NETWORK_DIR) - - def test_remove_network_not_found(self): - """ - Test remove_network() when the network is not found. - """ - self.etcd_client.delete.side_effect = EtcdKeyNotFound - self.assertFalse(self.datastore.remove_network(TEST_NETWORK_ID)) - self.etcd_client.delete.assert_called_once_with(TEST_NETWORK_DIR) diff --git a/tests/unit/driver_plugin_test.py b/tests/unit/driver_plugin_test.py deleted file mode 100644 index b9b494d..0000000 --- a/tests/unit/driver_plugin_test.py +++ /dev/null @@ -1,823 +0,0 @@ -# Copyright 2015 Metaswitch Networks -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import socket -import unittest - -from unittest import skip -from mock import patch, ANY, call -from netaddr import IPAddress, IPNetwork -from nose.tools import assert_equal -from pycalico.util import generate_cali_interface_name -from subprocess32 import CalledProcessError - -from libnetwork import driver_plugin -from pycalico.block import AlreadyAssignedError -from pycalico.datastore_datatypes import Endpoint, IF_PREFIX, IPPool -from pycalico.datastore_errors import PoolNotFound - -TEST_ENDPOINT_ID = "TEST_ENDPOINT_ID" -TEST_NETWORK_ID = "TEST_NETWORK_ID" - -hostname = socket.gethostname() - - -class TestPlugin(unittest.TestCase): - - def setUp(self): - self.app = driver_plugin.app.test_client() - - def tearDown(self): - pass - - def test_404(self): - rv = self.app.post('/') - assert_equal(rv.status_code, 404) - - def test_activate(self): - rv = self.app.post('/Plugin.Activate') - activate_response = {"Implements": ["NetworkDriver", "IpamDriver"]} - self.assertDictEqual(json.loads(rv.data), activate_response) - - def test_get_default_address_spaces(self): - """ - Test get_default_address_spaces returns the fixed values. - """ - rv = self.app.post('/IpamDriver.GetDefaultAddressSpaces') - response_data = { - "LocalDefaultAddressSpace": "CalicoLocalAddressSpace", - "GlobalDefaultAddressSpace": "CalicoGlobalAddressSpace" - } - self.assertDictEqual(json.loads(rv.data), response_data) - - def test_request_pool_v4(self): - """ - Test request_pool returns the correct fixed values for IPv4. - """ - request_data = { - "Pool": "", - "SubPool": "", - "V6": False - } - rv = self.app.post('/IpamDriver.RequestPool', - data=json.dumps(request_data)) - response_data = { - "PoolID": "CalicoPoolIPv4", - "Pool": "0.0.0.0/0", - "Data": { - "com.docker.network.gateway": "0.0.0.0/0" - } - } - self.assertDictEqual(json.loads(rv.data), response_data) - - def test_request_pool_v6(self): - """ - Test request_pool returns the correct fixed values for IPv6. - """ - request_data = { - "Pool": "", - "SubPool": "", - "V6": True - } - rv = self.app.post('/IpamDriver.RequestPool', - data=json.dumps(request_data)) - response_data = { - "PoolID": "CalicoPoolIPv6", - "Pool": "::/0", - "Data": { - "com.docker.network.gateway": "::/0" - } - } - self.assertDictEqual(json.loads(rv.data), response_data) - - @patch("libnetwork.driver_plugin.client.get_ip_pools", autospec=True) - def test_request_pool_valid_ipv4_pool_defined(self, m_get_pools): - """ - Test request_pool errors if a valid IPv4 pool is requested. - """ - request_data = { - "Pool": "1.2.3.4/26", - "SubPool": "", - "V6": False - } - m_get_pools.return_value = [IPPool("1.2.3.4/26")] - rv = self.app.post('/IpamDriver.RequestPool', - data=json.dumps(request_data)) - response_data = { - "PoolID": "1.2.3.4/26", - "Pool": "0.0.0.0/0", - "Data": { - "com.docker.network.gateway": "0.0.0.0/0" - } - } - self.assertDictEqual(json.loads(rv.data), response_data) - - @patch("libnetwork.driver_plugin.client.get_ip_pools", autospec=True) - def test_request_pool_valid_ipv6_pool_defined(self, m_get_pools): - """ - Test request_pool errors if a valid IPv6 pool is requested. - """ - request_data = { - "Pool": "11:22::3300/120", - "SubPool": "", - "V6": True - } - m_get_pools.return_value = [IPPool("11:22::3300/120")] - rv = self.app.post('/IpamDriver.RequestPool', - data=json.dumps(request_data)) - response_data = { - "PoolID": "11:22::3300/120", - "Pool": "::/0", - "Data": { - "com.docker.network.gateway": "::/0" - } - } - self.assertDictEqual(json.loads(rv.data), response_data) - - @patch("libnetwork.driver_plugin.client.get_ip_pools", autospec=True) - def test_request_pool_invalid_pool_defined(self, m_get_pools): - """ - Test request_pool errors if an invalid pool is requested. - """ - request_data = { - "Pool": "1.2.3.0/24", - "SubPool": "", - "V6": False - } - m_get_pools.return_value = [IPPool("1.2.4.0/24")] - rv = self.app.post('/IpamDriver.RequestPool', - data=json.dumps(request_data)) - self.assertTrue("Err" in json.loads(rv.data)) - - def test_request_pool_subpool_defined(self): - """ - Test request_pool errors if a specific sub-pool is requested. - """ - request_data = { - "Pool": "", - "SubPool": "1.2.3.4/5", - "V6": False - } - rv = self.app.post('/IpamDriver.RequestPool', - data=json.dumps(request_data)) - self.assertTrue("Err" in json.loads(rv.data)) - - def test_release_pool(self): - """ - Test release_pool. - """ - request_data = { - "PoolID": "TestPoolID", - } - rv = self.app.post('/IpamDriver.ReleasePool', - data=json.dumps(request_data)) - self.assertDictEqual(json.loads(rv.data), {}) - - @patch("libnetwork.driver_plugin.client.auto_assign_ips", autospec=True) - def test_request_address_auto_assign_ipv4(self, m_auto_assign): - """ - Test request_address when IPv4 address is auto-assigned. - """ - request_data = { - "PoolID": "CalicoPoolIPv4", - "Address": "" - } - ip = IPAddress("1.2.3.4") - m_auto_assign.return_value = ([], [ip]) - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - response_data = { - "Address": str(IPNetwork(ip)), - "Data": {} - } - self.assertDictEqual(json.loads(rv.data), response_data) - - @patch("libnetwork.driver_plugin.client.auto_assign_ips", autospec=True) - def test_request_address_auto_assign_ipv6(self, m_auto_assign): - """ - Test request_address when IPv6 address is auto-assigned. - """ - request_data = { - "PoolID": "CalicoPoolIPv6", - "Address": "" - } - ip = IPAddress("aa::ff") - m_auto_assign.return_value = ([], [ip]) - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - response_data = { - "Address": str(IPNetwork(ip)), - "Data": {} - } - self.assertDictEqual(json.loads(rv.data), response_data) - - @patch("libnetwork.driver_plugin.client.get_ip_pools", autospec=True) - @patch("libnetwork.driver_plugin.client.auto_assign_ips", autospec=True) - def test_request_address_assign_ipv4_from_subnet(self, m_auto_assign, - m_get_pools): - """ - Test request_address when IPv4 address is auto-assigned from a valid - subnet. - """ - request_data = { - "PoolID": "1.2.3.0/24", - "Address": "" - } - ip = IPAddress("1.2.3.4") - m_auto_assign.return_value = ([], [ip]) - m_get_pools.return_value = [IPPool("1.2.3.0/24")] - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - response_data = { - "Address": str(IPNetwork(ip)), - "Data": {} - } - self.assertDictEqual(json.loads(rv.data), response_data) - - @patch("libnetwork.driver_plugin.client.get_ip_pools", autospec=True) - @patch("libnetwork.driver_plugin.client.auto_assign_ips", autospec=True) - def test_request_address_assign_ipv4_from_invalid_subnet(self, m_auto_assign, - m_get_pools): - """ - Test request_address when IPv4 address is auto-assigned from an invalid - subnet. - """ - request_data = { - "PoolID": "1.2.3.0/24", - "Address": "" - } - ip = IPAddress("1.2.3.4") - m_auto_assign.return_value = ([], [ip]) - m_get_pools.return_value = [IPPool("1.2.5.0/24")] - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - self.assertTrue("Err" in json.loads(rv.data)) - - @patch("libnetwork.driver_plugin.client.auto_assign_ips", autospec=True) - def test_request_address_auto_assign_no_ips(self, m_auto_assign): - """ - Test request_address when there are no auto assigned IPs. - """ - request_data = { - "PoolID": "CalicoPoolIPv6", - "Address": "" - } - m_auto_assign.return_value = ([], []) - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - self.assertTrue("Err" in json.loads(rv.data)) - - @patch("libnetwork.driver_plugin.client.assign_ip", autospec=True) - def test_request_address_ip_supplied(self, m_assign): - """ - Test request_address when address is supplied. - """ - ip = IPAddress("1.2.3.4") - request_data = { - "PoolID": "CalicoPoolIPv4", - "Address": str(ip) - } - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - response_data = { - "Address": str(IPNetwork(ip)), - "Data": {} - } - self.assertDictEqual(json.loads(rv.data), response_data) - - @patch("libnetwork.driver_plugin.client.assign_ip", autospec=True) - def test_request_address_ip_supplied_in_use(self, m_assign): - """ - Test request_address when the supplied address is in use. - """ - ip = IPAddress("1.2.3.4") - request_data = { - "PoolID": "CalicoPoolIPv4", - "Address": str(ip) - } - m_assign.side_effect = AlreadyAssignedError() - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - self.assertTrue("Err" in json.loads(rv.data)) - - @patch("libnetwork.driver_plugin.client.assign_ip", autospec=True) - def test_request_address_ip_supplied_no_pool(self, m_assign): - """ - Test request_address when the supplied address is not in a pool. - """ - ip = IPAddress("1.2.3.4") - request_data = { - "PoolID": "CalicoPoolIPv4", - "Address": str(ip) - } - m_assign.side_effect = PoolNotFound(ip) - rv = self.app.post('/IpamDriver.RequestAddress', - data=json.dumps(request_data)) - self.assertTrue("Err" in json.loads(rv.data)) - - @patch("libnetwork.driver_plugin.client.release_ips", autospec=True) - def test_release_address(self, m_release): - """ - Test request_address when address is supplied. - """ - ip = IPAddress("1.2.3.4") - request_data = { - "Address": str(ip) - } - rv = self.app.post('/IpamDriver.ReleaseAddress', - data=json.dumps(request_data)) - self.assertDictEqual(json.loads(rv.data), {}) - m_release.assert_called_once_with({ip}) - - def test_capabilities(self): - rv = self.app.post('/NetworkDriver.GetCapabilities') - capabilities_response = {"Scope": "global"} - self.assertDictEqual(json.loads(rv.data), capabilities_response) - - @patch("libnetwork.driver_plugin.client.create_profile", autospec=True) - @patch("libnetwork.driver_plugin.client.write_network", autospec=True) - @patch("libnetwork.driver_plugin.client.add_ip_pool", autospec=True) - def test_create_network(self, m_add_ip_pool, m_write_network, m_create): - """ - Test create_network - """ - request_data = { - "NetworkID": TEST_NETWORK_ID, - "IPv4Data": [{ - "Gateway": "10.0.0.0/8", - "Pool": "6.5.4.3/21" - }], - "IPv6Data": [], - "Options": { - "com.docker.network.generic": {} - } - } - rv = self.app.post('/NetworkDriver.CreateNetwork', - data=json.dumps(request_data)) - m_create.assert_called_once_with(TEST_NETWORK_ID) - m_add_ip_pool.assert_called_once_with(4, - IPPool("6.5.4.3/21", ipam=False)) - m_write_network.assert_called_once_with(TEST_NETWORK_ID, - request_data) - - self.assertDictEqual(json.loads(rv.data), {}) - - - @patch("libnetwork.driver_plugin.client.remove_network", autospec=True) - @patch("libnetwork.driver_plugin.client.remove_ip_pool", autospec=True) - @patch("libnetwork.driver_plugin.client.get_network", autospec=True) - @patch("libnetwork.driver_plugin.client.remove_profile", autospec=True) - def test_delete_network_default_ipam(self, m_remove_profile, m_get_network, - m_remove_pool, m_remove_network): - """ - Test the delete_network behavior for default IPAM. - """ - m_get_network.return_value = { - "NetworkID": TEST_NETWORK_ID, - "IPv4Data": [{ - "Gateway": "6.5.4.3/21", - "Pool": "6.5.4.3/21" - }], - "IPv6Data": [{ - "Gateway": "aa::ff/10", - "Pool": "aa::fe/10" - }] - } - - request_data = { - "NetworkID": TEST_NETWORK_ID - } - - rv = self.app.post('/NetworkDriver.DeleteNetwork', - data=json.dumps(request_data)) - m_remove_profile.assert_called_once_with(TEST_NETWORK_ID) - m_remove_network.assert_called_once_with(TEST_NETWORK_ID) - m_remove_pool.assert_has_calls([call(4, IPNetwork("6.5.4.3/21")), - call(6, IPNetwork("aa::fe/10"))]) - self.assertDictEqual(json.loads(rv.data), {}) - - @patch("libnetwork.driver_plugin.client.remove_network", autospec=True) - @patch("libnetwork.driver_plugin.client.remove_ip_pool", autospec=True) - @patch("libnetwork.driver_plugin.client.get_network", autospec=True, return_value=None) - @patch("libnetwork.driver_plugin.client.remove_profile", autospec=True) - def test_delete_network_calico_ipam(self, m_remove_profile, m_get_network, - m_remove_pool, m_remove_network): - """ - Test the delete_network behavior for Calico IPAM. - """ - m_get_network.return_value = { - "NetworkID": TEST_NETWORK_ID, - "IPv4Data": [{ - "Gateway": "0.0.0.0/0", - "Pool": "0.0.0.0/0" - }], - "IPv6Data": [{ - "Gateway": "00::00/0", - "Pool": "00::00/0" - }] - } - - request_data = { - "NetworkID": TEST_NETWORK_ID - } - - rv = self.app.post('/NetworkDriver.DeleteNetwork', - data=json.dumps(request_data)) - m_remove_profile.assert_called_once_with(TEST_NETWORK_ID) - m_remove_network.assert_called_once_with(TEST_NETWORK_ID) - self.assertEquals(m_remove_pool.call_count, 0) - self.assertDictEqual(json.loads(rv.data), {}) - - @patch("libnetwork.driver_plugin.client.remove_profile", autospec=True) - def test_delete_network_no_profile(self, m_remove): - """ - Test the delete_network hook correctly removes the etcd data and - returns the correct response. - """ - m_remove.side_effect = KeyError - request_data = { - "NetworkID": TEST_NETWORK_ID - } - rv = self.app.post('/NetworkDriver.DeleteNetwork', - data=json.dumps(request_data)) - m_remove.assert_called_once_with(TEST_NETWORK_ID) - self.assertDictEqual(json.loads(rv.data), {u'Err': u''}) - - def test_oper_info(self): - """ - Test oper_info returns the correct data. - """ - request_data = { - "EndpointID": TEST_ENDPOINT_ID - } - rv = self.app.post('/NetworkDriver.EndpointOperInfo', - data=json.dumps(request_data)) - self.assertDictEqual(json.loads(rv.data), {"Value": {}}) - - @patch("libnetwork.driver_plugin.bring_up_interface") - @patch("libnetwork.driver_plugin.client.get_network", autospec=True) - @patch("pycalico.netns.set_veth_mac", autospec=True) - @patch("pycalico.netns.create_veth", autospec=True) - def test_join_default_ipam(self, m_create_veth, m_set_mac, m_get_network, m_intf_up): - """ - Test the join() processing with default IPAM. - """ - request_data = { - "EndpointID": TEST_ENDPOINT_ID, - "NetworkID": TEST_NETWORK_ID - } - - m_get_network.return_value = { - "NetworkID": TEST_NETWORK_ID, - "IPv4Data": [{ - "Gateway": "6.5.4.3/21", - "Pool": "6.5.4.3/21" - }], - "IPv6Data": []} - - # Actually make the request to the plugin. - rv = self.app.post('/NetworkDriver.Join', - data=json.dumps(request_data)) - - # Check the expected response. - response_data = { - "Gateway": "", - "GatewayIPv6": "", - "InterfaceName": { - "DstPrefix": "cali", - "SrcName": "tmpTEST_ENDPOI" - } - } - self.maxDiff = None - self.assertDictEqual(json.loads(rv.data), response_data) - - # Check appropriate netns calls. - host_interface_name = generate_cali_interface_name(IF_PREFIX, TEST_ENDPOINT_ID) - temp_interface_name = generate_cali_interface_name("tmp", TEST_ENDPOINT_ID) - - m_create_veth.assert_called_once_with(host_interface_name, temp_interface_name) - m_set_mac.assert_called_once_with(temp_interface_name, "EE:EE:EE:EE:EE:EE") - - - @patch("libnetwork.driver_plugin.bring_up_interface") - @patch("libnetwork.driver_plugin.get_ipv6_link_local", autospec=True, return_value="fe80::1/128") - @patch("libnetwork.driver_plugin.client.get_endpoint", autospec=True) - @patch("libnetwork.driver_plugin.client.get_network", autospec=True, return_value=None) - @patch("pycalico.netns.set_veth_mac", autospec=True) - @patch("pycalico.netns.create_veth", autospec=True) - def test_join_calico_ipam(self, m_create_veth, m_set_mac, m_get_network, - m_get_endpoint, m_get_link_local, m_intf_up): - """ - Test the join() processing with Calico IPAM. - """ - m_get_network.return_value = { - "NetworkID": TEST_NETWORK_ID, - "IPv4Data":[{ - "Gateway": "0.0.0.0/0", - "Pool": "0.0.0.0/0" - }], - "IPv6Data":[{ - "Gateway": "::/0", - "Pool": "::/0" - }]} - m_get_endpoint.return_value = Endpoint(hostname, - "libnetwork", - "docker", - TEST_ENDPOINT_ID, - None, - None) - - # Actually make the request to the plugin. - rv = self.app.post('/NetworkDriver.Join', - data='{"EndpointID": "%s", "NetworkID": "%s"}' % - (TEST_ENDPOINT_ID, TEST_NETWORK_ID)) - - host_interface_name = generate_cali_interface_name(IF_PREFIX, TEST_ENDPOINT_ID) - temp_interface_name = generate_cali_interface_name("tmp", TEST_ENDPOINT_ID) - - m_create_veth.assert_called_once_with(host_interface_name, temp_interface_name) - m_set_mac.assert_called_once_with(temp_interface_name, "EE:EE:EE:EE:EE:EE") - - expected_data = { - "Gateway": "169.254.1.1", - "GatewayIPv6": "fe80::1/128", - "InterfaceName": { - "DstPrefix": "cali", - "SrcName": "tmpTEST_ENDPOI" - }, - "StaticRoutes": [{ - "Destination": "169.254.1.1/32", - "RouteType": 1, - "NextHop": "" - }, { - "Destination": "fe80::1/128", - "RouteType": 1, - "NextHop": "" - }] - } - self.maxDiff = None - self.assertDictEqual(json.loads(rv.data), - expected_data) - - @patch("libnetwork.driver_plugin.bring_up_interface") - @patch("pycalico.netns.set_veth_mac", autospec=True) - @patch("pycalico.netns.create_veth", autospec=True) - @patch("libnetwork.driver_plugin.remove_veth", autospec=True) - def test_join_veth_fail(self, m_del_veth, m_create_veth, m_set_veth_macs, m_join_intf): - """ - Test the join() processing when create_veth fails. - """ - m_create_veth.side_effect = CalledProcessError(2, "testcmd") - - # Actually make the request to the plugin. - rv = self.app.post('/NetworkDriver.Join', - data='{"EndpointID": "%s", "NetworkID": "%s"}' % - (TEST_ENDPOINT_ID, TEST_NETWORK_ID)) - - # Expect a 500 response. - self.assertDictEqual(json.loads(rv.data), {u'Err': u"Command 'testcmd' returned non-zero exit status 2"}) - - # Check that create veth is called with the expected endpoint, and - # that set_endpoint is not (since create_veth is raising an exception). - host_interface_name = generate_cali_interface_name(IF_PREFIX, TEST_ENDPOINT_ID) - temp_interface_name = generate_cali_interface_name("tmp", TEST_ENDPOINT_ID) - - m_create_veth.assert_called_once_with(host_interface_name, temp_interface_name) - - # Check that we delete the veth. - m_del_veth.assert_called_once_with(host_interface_name) - - @patch("libnetwork.driver_plugin.remove_veth", autospec=True) - def test_leave(self, m_veth): - """ - Test leave() processing removes the veth. - """ - # Send the leave request. - rv = self.app.post('/NetworkDriver.Leave', - data='{"EndpointID": "%s"}' % TEST_ENDPOINT_ID) - self.assertDictEqual(json.loads(rv.data), {}) - - m_veth.assert_called_once_with(generate_cali_interface_name(IF_PREFIX, TEST_ENDPOINT_ID)) - - @patch("libnetwork.driver_plugin.client.remove_endpoint", autospec=True) - def test_delete_endpoint(self, m_remove): - """ - Test delete_endpoint() deletes the endpoint and backout IP assignment. - """ - rv = self.app.post('/NetworkDriver.DeleteEndpoint', - data='{"EndpointID": "%s"}' % TEST_ENDPOINT_ID) - m_remove.assert_called_once_with(Endpoint(hostname, - "libnetwork", - "docker", - TEST_ENDPOINT_ID, - None, - None)) - self.assertDictEqual(json.loads(rv.data), {}) - - @patch("libnetwork.driver_plugin.client.remove_endpoint", autospec=True, side_effect=KeyError()) - def test_delete_endpoint_fail(self, m_remove): - """ - Test delete_endpoint() deletes the endpoint and backout IP assignment. - """ - rv = self.app.post('/NetworkDriver.DeleteEndpoint', - data='{"EndpointID": "%s"}' % TEST_ENDPOINT_ID) - m_remove.assert_called_once_with(Endpoint(hostname, - "libnetwork", - "docker", - TEST_ENDPOINT_ID, - None, - None)) - self.assertDictEqual(json.loads(rv.data), {u'Err': u''}) - - @patch("libnetwork.driver_plugin.client.get_network", autospec=True) - @patch("libnetwork.driver_plugin.client.set_endpoint", autospec=True) - def test_create_endpoint(self, m_set, m_get_network): - """ - Test the create_endpoint hook correctly writes the appropriate data - to etcd based on IP assignment and pool selection. - """ - - # Iterate using various different mixtures of IP assignments and - # gateway CIDRs. - # - # (IPv4 addr, IPv6 addr, IPv4 gway, IPv6 gway, calico_ipam) - # - # calico_ipam indicates whether the gateway indicates Calico IPAM or - # not which changes the gateway selected in the endpoint. - parms = [(None, "aa:bb::bb", None, "cc:dd::00/23", False), - ("10.20.30.40", None, "1.2.3.4/32", "aa:bb:cc::/24", False), - ("20.20.30.40", "ab:bb::bb", "1.2.3.4/32", "aa:bb:cc::/25", False), - (None, "ac:bb::bb", None, "00::00/0", True), - ("40.20.30.40", None, "0.0.0.0/0", "::/0", True), - ("50.20.30.40", "ad:bb::bb", "0.0.0.0/0", "00::/0", True)] - - # Loop through different combinations of IP availability. - for ipv4, ipv6, gwv4, gwv6, calico_ipam in parms: - m_get_network.return_value = { - "NetworkID": TEST_NETWORK_ID, - "IPv4Data":[{"Gateway": gwv4, "Pool": gwv4}], - "IPv6Data":[{"Gateway": gwv6, "Pool": gwv6}] - } - ipv4_json = ',"Address": "%s"' % ipv4 if ipv4 else "" - ipv6_json = ',"AddressIPv6": "%s"' % ipv6 if ipv6 else "" - - # Invoke create endpoint. - rv = self.app.post('/NetworkDriver.CreateEndpoint', - data='{"EndpointID": "%s",' - '"NetworkID": "%s",' - '"Interface": {"MacAddress": "EE:EE:EE:EE:EE:EE"%s%s}}' % - (TEST_ENDPOINT_ID, TEST_NETWORK_ID, ipv4_json, ipv6_json)) - - # Assert return value - self.assertDictEqual(json.loads(rv.data), { - "Interface": { - "MacAddress": "EE:EE:EE:EE:EE:EE" - } - }) - - # Assert expected data is written to etcd - ep = Endpoint(hostname, "libnetwork", "libnetwork", - TEST_ENDPOINT_ID, "active", "EE:EE:EE:EE:EE:EE") - - ep.profile_ids.append(TEST_NETWORK_ID) - - if ipv4: - ep.ipv4_nets.add(IPNetwork(ipv4)) - - if ipv6: - ep.ipv6_nets.add(IPNetwork(ipv6)) - - m_set.assert_called_once_with(ep) - - # Reset the Mocks before continuing. - m_set.reset_mock() - - def test_discover_new(self): - """ - Test discover_new returns the correct data. - """ - rv = self.app.post('/NetworkDriver.DiscoverNew', - data='{"DiscoveryType": 1,' - '"DiscoveryData": {' - '"Address": "thisaddress",' - '"self": true' - '}' - '}') - self.assertDictEqual(json.loads(rv.data), {}) - - def test_discover_delete(self): - """ - Test discover_delete returns the correct data. - """ - rv = self.app.post('/NetworkDriver.DiscoverDelete', - data='{"DiscoveryType": 1,' - '"DiscoveryData": {' - '"Address": "thisaddress",' - '"self": true' - '}' - '}') - self.assertDictEqual(json.loads(rv.data), {}) - - @patch("pycalico.netns.remove_veth", autospec=True, side_effect=CalledProcessError(2, "test")) - def test_remove_veth_fail(self, m_remove): - """ - Test remove_veth calls through to netns to remove the veth. - Fail with a CalledProcessError to write the log. - """ - name = generate_cali_interface_name(IF_PREFIX, TEST_ENDPOINT_ID) - - driver_plugin.remove_veth(name) - m_remove.assert_called_once_with(name) - - def test_get_gateway_pool_from_network_data(self): - """ - Test get_gateway_pool_from_network_data for a variety of inputs. - """ - tests = [ - ( - (None, None), 4, { - "IPv6Data": [] - } - ), - ( - (None, None), 6, { - "IPv6Data": [] - } - ), - ( - (None, None), 6, { - "IPv6Data": [{}] - } - ), - ( - (None, None), 4, { - "IPv4Data": [{ - "Gateway": "1.2.3.4/40" - }] - } - ), - ( - (None, None), 4, { - "IPv4Data": [{ - "Pool": "1.2.3.4/40" - }] - } - ), - ( - (IPNetwork("aa::ff/120"), IPNetwork("aa::dd/121")), 6, { - "IPv6Data": [{ - "Gateway": "aa::ff/120", - "Pool": "aa::dd/121" - }] - } - )] - - for result, version, network_data in tests: - self.assertEquals( - driver_plugin.get_gateway_pool_from_network_data(network_data, - version), - result - ) - - def test_get_gateway_pool_from_network_data_multiple_datas(self): - """ - Test get_gateway_pool_from_network_data when multiple data blocks are - supplied. - """ - network_data = { - "IPv6Data": [{ - "Gateway": "aa::ff/120", - "Pool": "aa::dd/121" - }, { - "Gateway": "aa::fa/120", - "Pool": "aa::da/121" - }] - } - self.assertRaises(Exception, - driver_plugin.get_gateway_pool_from_network_data, - network_data, 6) - - def test_is_using_calico_ipam(self): - """ - Test is_using_calico_ipam using a variety of CIDRs. - """ - for cidr, is_cipam in [(IPNetwork("1.2.3.4/20"), False), - (IPNetwork("0.0.0.0/20"), False), - (IPNetwork("::/128"), False), - (IPNetwork("0.0.0.0/32"), False), - (IPNetwork("0.0.0.0/0"), True), - (IPNetwork("::/0"), True)]: - self.assertEquals(driver_plugin.is_using_calico_ipam(cidr), - is_cipam) - diff --git a/utils/log/log.go b/utils/log/log.go new file mode 100644 index 0000000..2dbc103 --- /dev/null +++ b/utils/log/log.go @@ -0,0 +1,15 @@ +package log + +import ( + "encoding/json" + "log" +) + +func JSONMessage(logger *log.Logger, formattedMessage string, data interface{}) { + requestJSON, err := json.Marshal(data) + if err != nil { + logger.Fatal(err) + return + } + logger.Printf(formattedMessage, string(requestJSON)) +} diff --git a/utils/math/math.go b/utils/math/math.go new file mode 100644 index 0000000..1a0b4ee --- /dev/null +++ b/utils/math/math.go @@ -0,0 +1,8 @@ +package math + +func MinInt(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/utils/netns/netns.go b/utils/netns/netns.go new file mode 100644 index 0000000..7ef7a83 --- /dev/null +++ b/utils/netns/netns.go @@ -0,0 +1,61 @@ +package netns + +import ( + "net" + + "github.com/pkg/errors" + "github.com/vishvananda/netlink" +) + +func CreateVeth(vethNameHost, vethNameNSTemp string) error { + veth := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: vethNameHost, + }, + PeerName: vethNameNSTemp, + } + if err := netlink.LinkAdd(veth); err != nil { + return err + } + + err := netlink.LinkSetUp(veth) + return err +} + +func SetVethMac(vethNameHost, mac string) error { + addr, err := net.ParseMAC(mac) + if err != nil { + return errors.Wrap(err, "Veth setting error") + } + return netlink.LinkSetHardwareAddr(&netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: vethNameHost, + }, + }, addr) +} + +func RemoveVeth(vethNameHost string) error { + if ok, err := IsVethExists(vethNameHost); err != nil { + return errors.Wrap(err, "Veth removal error") + } else if !ok { + return nil + } + return netlink.LinkDel(&netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: vethNameHost, + }, + }) +} + +func IsVethExists(vethHostName string) (bool, error) { + links, err := netlink.LinkList() + if err != nil { + return false, errors.Wrap(err, "Veth existing check error") + } + for _, link := range links { + if link.Attrs().Name == vethHostName { + return true, nil + } + } + return false, nil +} diff --git a/utils/os/utils.go b/utils/os/utils.go new file mode 100644 index 0000000..0e6b9bf --- /dev/null +++ b/utils/os/utils.go @@ -0,0 +1,13 @@ +package os + +import "os" + +const hostnameEnv = "HOSTNAME" + +func GetHostname() (string, error) { + hostnameFromEnv := os.Getenv(hostnameEnv) + if hostname, err := os.Hostname(); hostnameFromEnv == "" { + return hostname, err + } + return hostnameFromEnv, nil +} From 43209c213a93cac24aed8b5aac60bc1f5e32068f Mon Sep 17 00:00:00 2001 From: Rob Brockbank Date: Tue, 1 Nov 2016 17:05:27 +0000 Subject: [PATCH 2/4] Avoid race conditions on profile creation (#71) * Avoid race conditions on profile creation * Profile should be pointer --- driver/network_driver.go | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/driver/network_driver.go b/driver/network_driver.go index f0d853d..2ef2552 100644 --- a/driver/network_driver.go +++ b/driver/network_driver.go @@ -139,32 +139,25 @@ func (d NetworkDriver) CreateEndpoint(request *network.CreateEndpointRequest) (* // Now that we know the network name, set it on the endpoint. endpoint.Spec.Profiles = append(endpoint.Spec.Profiles, networkData.Name) - // Check if the profile already exists. - exists := true - if _, err = d.client.Profiles().Get(api.ProfileMetadata{Name: networkData.Name}); err != nil { - _, ok := err.(libcalicoErrors.ErrorResourceDoesNotExist) - if ok { - exists = false - } else { - err = errors.Wrapf(err, "Profile %v getting error", networkData.Name) - d.logger.Println(err) - return nil, err - } - } - // If a profile for the network name doesn't exist then it needs to be created. - if !exists { - profile := api.NewProfile() - profile.Metadata.Name = networkData.Name - profile.Spec.Tags = []string{networkData.Name} - profile.Spec.EgressRules = []api.Rule{{Action: "allow"}} - profile.Spec.IngressRules = []api.Rule{{Action: "allow", Source: api.EntityRule{Tag: networkData.Name}}} - if _, err := d.client.Profiles().Create(profile); err != nil { + // We always attempt to create the profile and rely on the datastore to reject + // the request if the profile already exists. + profile := &api.Profile{ + Metadata: api.ProfileMetadata{Name: networkData.Name}, + Spec: api.ProfileSpec{ + Tags: []string{networkData.Name}, + EgressRules: []api.Rule{{Action: "allow"}}, + IngressRules: []api.Rule{{Action: "allow", Source: api.EntityRule{Tag: networkData.Name}}}, + }, + } + if _, err := d.client.Profiles().Create(profile); err != nil { + if _, ok := err.(libcalicoErrors.ErrorResourceAlreadyExists); !ok { log.Println(err) return nil, err } } + // Create the endpoint _, err = d.client.WorkloadEndpoints().Create(endpoint) if err != nil { err = errors.Wrapf(err, "Workload endpoints creation error, data: %+v", endpoint) From 6da9a4239089e7d008224e75eea6c88a3b551ced Mon Sep 17 00:00:00 2001 From: Tom Denham Date: Wed, 2 Nov 2016 11:13:15 -0700 Subject: [PATCH 3/4] Tweaks to logging and IPAM --- driver/ipam_driver.go | 108 ++++++------------ driver/network_driver.go | 12 +- .../st/libnetwork/test_assign_specific_ip.py | 28 +++-- 3 files changed, 60 insertions(+), 88 deletions(-) diff --git a/driver/ipam_driver.go b/driver/ipam_driver.go index 1b28cae..65577a5 100644 --- a/driver/ipam_driver.go +++ b/driver/ipam_driver.go @@ -8,16 +8,16 @@ import ( "github.com/pkg/errors" "github.com/docker/go-plugins-helpers/ipam" - "github.com/projectcalico/libcalico-go/lib/api" datastoreClient "github.com/projectcalico/libcalico-go/lib/client" caliconet "github.com/projectcalico/libcalico-go/lib/net" logutils "github.com/projectcalico/libnetwork-plugin/utils/log" osutils "github.com/projectcalico/libnetwork-plugin/utils/os" + "github.com/projectcalico/libcalico-go/lib/api" ) type IpamDriver struct { - client *datastoreClient.Client - logger *log.Logger + client *datastoreClient.Client + logger *log.Logger poolIDV4 string poolIDV6 string @@ -51,7 +51,7 @@ func (i IpamDriver) GetDefaultAddressSpaces() (*ipam.AddressSpacesResponse, erro func (i IpamDriver) RequestPool(request *ipam.RequestPoolRequest) (*ipam.RequestPoolResponse, error) { logutils.JSONMessage(i.logger, "RequestPool JSON=%s", request) - // Calico IPAM does not allow you to request SubPool. + // Calico IPAM does not allow you to request a SubPool. if request.SubPool != "" { err := errors.New( "Calico IPAM does not support sub pool configuration " + @@ -63,6 +63,12 @@ func (i IpamDriver) RequestPool(request *ipam.RequestPoolRequest) (*ipam.Request return nil, err } + if request.V6 { + err := errors.New("IPv6 isn't supported") + i.logger.Println(err) + return nil, err + } + // If a pool (subnet on the CLI) is specified, it must match one of the // preconfigured Calico pools. if request.Pool != "" { @@ -73,37 +79,25 @@ func (i IpamDriver) RequestPool(request *ipam.RequestPoolRequest) (*ipam.Request i.logger.Println(err) return nil, err } + pools, err := poolsClient.List(api.PoolMetadata{CIDR: *ipNet}) if err != nil || len(pools.Items) < 1 { - err := errors.New( - "The requested subnet must match the CIDR of a " + - "configured Calico IP Pool.", + err := errors.New("The requested subnet must match the CIDR of a " + + "configured Calico IP Pool.", ) i.logger.Println(err) return nil, err } } - var resp *ipam.RequestPoolResponse - - // If a subnet has been specified we use that as the pool ID. Otherwise, we - // use static pool ID and CIDR to indicate that we are assigning from all of - // the pools. + // We use static pool ID and CIDR to indicate that we are assigning from all of the pools. // The meta data includes a dummy gateway address. This prevents libnetwork // from requesting a gateway address from the pool since for a Calico - // network our gateway is set to our host IP. - if request.V6 { - resp = &ipam.RequestPoolResponse{ - PoolID: i.poolIDV6, - Pool: "::/0", - Data: map[string]string{"com.docker.network.gateway": "::/0"}, - } - } else { - resp = &ipam.RequestPoolResponse{ - PoolID: i.poolIDV4, - Pool: "0.0.0.0/0", - Data: map[string]string{"com.docker.network.gateway": "0.0.0.0/0"}, - } + // network our gateway is set to a special IP. + resp := &ipam.RequestPoolResponse{ + PoolID: i.poolIDV4, + Pool: "0.0.0.0/0", + Data: map[string]string{"com.docker.network.gateway": "0.0.0.0/0"}, } logutils.JSONMessage(i.logger, "RequestPool response JSON=%v", resp) @@ -113,9 +107,6 @@ func (i IpamDriver) RequestPool(request *ipam.RequestPoolRequest) (*ipam.Request func (i IpamDriver) ReleasePool(request *ipam.ReleasePoolRequest) error { logutils.JSONMessage(i.logger, "ReleasePool JSON=%s", request) - - resp := map[string]string{} - logutils.JSONMessage(i.logger, "ReleasePool response JSON=%s", resp) return nil } @@ -127,27 +118,16 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R return nil, err } - var ( - version int - pool *api.Pool - IPs []caliconet.IP - ) + var IPs []caliconet.IP if request.Address == "" { - var ( - numV4 int - numV6 int - poolV4 *caliconet.IPNet - poolV6 *caliconet.IPNet - ) + // No address requested, so auto assign from our pools. i.logger.Println("Auto assigning IP from Calico pools") - // No address requested, so auto assign from our pools. If the pool ID - // is one of the fixed IDs then assign from across all configured pools, - // otherwise assign from the requested pool - if request.PoolID == PoolIDV4 { - version = 4 - } else { + // If the poolID isn't the fixed one then find the pool to assign from. + // poolV4 defaults to nil to assign from across all pools. + var poolV4 *caliconet.IPNet + if request.PoolID != PoolIDV4 { poolsClient := i.client.Pools() _, ipNet, err := caliconet.ParseCIDR(request.PoolID) @@ -155,7 +135,7 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R err = errors.Wrapf(err, "Invalid CIDR - %v", request.PoolID) return nil, err } - pool, err = poolsClient.Get(api.PoolMetadata{CIDR: *ipNet}) + pool, err := poolsClient.Get(api.PoolMetadata{CIDR: *ipNet}) if err != nil { message := "The network references a Calico pool which " + "has been deleted. Please re-instate the " + @@ -163,14 +143,7 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R i.logger.Println(err) return nil, errors.New(message) } - version = ipNet.Version() poolV4 = &caliconet.IPNet{IPNet: pool.Metadata.CIDR.IPNet} - // TODO - v6 - } - - if version == 4 { - numV4 = 1 - numV6 = 0 } // Auto assign an IP based on whether the IPv4 or IPv6 pool was selected. @@ -178,20 +151,18 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R // host. IPsV4, IPsV6, err := i.client.IPAM().AutoAssign( datastoreClient.AutoAssignArgs{ - Num4: numV4, - Num6: numV6, + Num4: 1, + Num6: 0, Hostname: hostname, IPv4Pool: poolV4, - IPv6Pool: poolV6, }, ) - IPs = append(IPsV4, IPsV6...) - if err != nil || len(IPs) == 0 { - err := errors.New("There are no available IP addresses in the configured Calico IP pools") + if err != nil { + err = errors.Wrapf(err, "IP assignment error") i.logger.Println(err) return nil, err } - + IPs = append(IPsV4, IPsV6...) } else { i.logger.Println("Reserving a specific address in Calico pools") ip := net.ParseIP(request.Address) @@ -210,14 +181,14 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R // We should only have one IP address assigned at this point. if len(IPs) != 1 { - err := errors.New("Unexpected number of assigned IP addresses") + err := errors.New(fmt.Sprintf("Unexpected number of assigned IP addresses. " + + "A single address should be assigned. Got %v", IPs)) i.logger.Println(err) return nil, err } // Return the IP as a CIDR. resp := &ipam.RequestAddressResponse{ - // TODO: need more investigation about the subnet size to use Address: fmt.Sprintf("%v/%v", IPs[0], "32"), } @@ -232,20 +203,13 @@ func (i IpamDriver) ReleaseAddress(request *ipam.ReleaseAddressRequest) error { ip := caliconet.IP{IP: net.ParseIP(request.Address)} // Unassign the address. This handles the address already being unassigned - // in which case it is a no-op. The release_ips call may raise a - // RuntimeError if there are repeated clashing updates to the same IP block, - // this is not an expected condition. - ips := []caliconet.IP{ip} - _, err := i.client.IPAM().ReleaseIPs(ips) + // in which case it is a no-op. + _, err := i.client.IPAM().ReleaseIPs([]caliconet.IP{ip}) if err != nil { - err = errors.Wrapf(err, "IPs releasing error, ips: %v", ips) + err = errors.Wrapf(err, "IP releasing error, ip: %v", ip) i.logger.Println(err) return err } - resp := map[string]string{} - - logutils.JSONMessage(i.logger, "ReleaseAddress response JSON=%s", resp) - return nil } diff --git a/driver/network_driver.go b/driver/network_driver.go index 2ef2552..018e90a 100644 --- a/driver/network_driver.go +++ b/driver/network_driver.go @@ -22,7 +22,7 @@ import ( ) // NetworkDriver is the Calico network driver representation. -// Must be used with Calico IPAM and support IPv4 only. +// Must be used with Calico IPAM and supports IPv4 only. type NetworkDriver struct { client *datastoreClient.Client logger *log.Logger @@ -157,7 +157,7 @@ func (d NetworkDriver) CreateEndpoint(request *network.CreateEndpointRequest) (* } } - // Create the endpoint + // Create the endpoint last to minimize side-effects if something goes wrong. _, err = d.client.WorkloadEndpoints().Create(endpoint) if err != nil { err = errors.Wrapf(err, "Workload endpoints creation error, data: %+v", endpoint) @@ -180,15 +180,14 @@ func (d NetworkDriver) CreateEndpoint(request *network.CreateEndpointRequest) (* func (d NetworkDriver) DeleteEndpoint(request *network.DeleteEndpointRequest) error { logutils.JSONMessage(d.logger, "DeleteEndpoint JSON=%v", request) + d.logger.Printf("Removing endpoint %v\n", request.EndpointID) + hostname, err := osutils.GetHostname() if err != nil { err = errors.Wrap(err, "Hostname fetching error") return err } - logutils.JSONMessage(d.logger, "DeleteEndpoint JSON=%v", request) - d.logger.Printf("Removing endpoint %v\n", request.EndpointID) - if err = d.client.WorkloadEndpoints().Delete( api.WorkloadEndpointMetadata{ Name: request.EndpointID, @@ -248,8 +247,7 @@ func (d NetworkDriver) Join(request *network.JoinRequest) (*network.JoinResponse // One of the network gateway addresses indicate that we are using // Calico IPAM driver. In this case we setup routes using the gateways // configured on the endpoint (which will be our host IPs). - d.logger.Println("Using Calico IPAM driver, configure gateway and " + - "static routes to the host") + d.logger.Println("Using Calico IPAM driver, configure gateway and static routes to the host") resp.Gateway = d.DummyIPV4Nexthop resp.StaticRoutes = append(resp.StaticRoutes, &network.StaticRoute{ diff --git a/tests/st/libnetwork/test_assign_specific_ip.py b/tests/st/libnetwork/test_assign_specific_ip.py index c08b4bf..fbb9583 100644 --- a/tests/st/libnetwork/test_assign_specific_ip.py +++ b/tests/st/libnetwork/test_assign_specific_ip.py @@ -48,16 +48,26 @@ def test_assign_specific_ip(self): 'calico/libnetwork-plugin' % (get_ip(),) host.execute(run_plugin_command) - subnet = "192.168.0.0/16" - host.calicoctl('pool add %s' % subnet) + subnet1 = "192.168.0.0/16" + subnet2 = "10.11.0.0/16" + host.calicoctl('pool add %s' % subnet1) + host.calicoctl('pool add %s' % subnet2) - workload_ip = "192.168.1.101" + workload_ip1 = "192.168.1.101" + workload_ip2 = "10.11.12.13" - network = host.create_network( - "specificipnet", subnet=subnet, driver="calico", ipam_driver="calico-ipam") + network1 = host.create_network( + "subnet1", subnet=subnet1, driver="calico", ipam_driver="calico-ipam") + network2 = host.create_network( + "subnet2", subnet=subnet2, driver="calico", ipam_driver="calico-ipam") - workload = host.create_workload("workload1", - network=network, - ip=workload_ip) + workload1 = host.create_workload("workload1", + network=network1, + ip=workload_ip1) + self.assertEquals(workload_ip1, workload1.ip) - self.assertEquals(workload_ip, workload.ip) + workload2 = host.create_workload("workload2", + network=network2, + ip=workload_ip2) + + self.assertEquals(workload_ip2, workload2.ip) From 43d01f1bd005c798847ad237048f205cff6bdca1 Mon Sep 17 00:00:00 2001 From: Tom Denham Date: Wed, 2 Nov 2016 13:53:44 -0700 Subject: [PATCH 4/4] Fix up copyright, sort out pool assignment and add tests --- README.md | 4 +- driver/ipam_driver.go | 23 +++++-- .../st/libnetwork/test_assign_specific_ip.py | 53 +++++++++------ .../libnetwork/test_assign_specific_pool.py | 67 +++++++++++++++++++ tests/st/libnetwork/test_error_ipam.py | 2 +- .../st/libnetwork/test_mainline_multi_host.py | 2 +- .../libnetwork/test_mainline_single_host.py | 2 +- 7 files changed, 120 insertions(+), 33 deletions(-) create mode 100644 tests/st/libnetwork/test_assign_specific_pool.py diff --git a/README.md b/README.md index 46e131c..3c9bf79 100644 --- a/README.md +++ b/README.md @@ -29,9 +29,7 @@ driver: once a container endpoint is created, it is possible to manually add additional Calico profiles to that endpoint (effectively adding the container into another network). -- When using the Calico IPAM driver, it is not yet possible to select which - IP Pool an IP is assigned from. Make sure all of your configured IP Pools - have the same ipip and nat-outgoing settings. +- IPv6 is not currently supported ## Troubleshooting diff --git a/driver/ipam_driver.go b/driver/ipam_driver.go index 65577a5..3c821c2 100644 --- a/driver/ipam_driver.go +++ b/driver/ipam_driver.go @@ -69,6 +69,10 @@ func (i IpamDriver) RequestPool(request *ipam.RequestPoolRequest) (*ipam.Request return nil, err } + // Default the poolID to the fixed value. + poolID := i.poolIDV4 + pool:="0.0.0.0/0" + // If a pool (subnet on the CLI) is specified, it must match one of the // preconfigured Calico pools. if request.Pool != "" { @@ -88,15 +92,17 @@ func (i IpamDriver) RequestPool(request *ipam.RequestPoolRequest) (*ipam.Request i.logger.Println(err) return nil, err } + pool = request.Pool + poolID = request.Pool } - // We use static pool ID and CIDR to indicate that we are assigning from all of the pools. + // We use static pool ID and CIDR. We don't need to signal the // The meta data includes a dummy gateway address. This prevents libnetwork // from requesting a gateway address from the pool since for a Calico // network our gateway is set to a special IP. resp := &ipam.RequestPoolResponse{ - PoolID: i.poolIDV4, - Pool: "0.0.0.0/0", + PoolID: poolID, + Pool: pool, Data: map[string]string{"com.docker.network.gateway": "0.0.0.0/0"}, } @@ -144,11 +150,12 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R return nil, errors.New(message) } poolV4 = &caliconet.IPNet{IPNet: pool.Metadata.CIDR.IPNet} + i.logger.Println("Using specific pool ", poolV4) } - // Auto assign an IP based on whether the IPv4 or IPv6 pool was selected. - // We auto-assign from all available pools with affinity based on our - // host. + // Auto assign an IP address. + // IPv4 pool will be nil if the docker network doesn't have a subnet associated with. + // Otherwise, it will be set to the Calico pool to assign from. IPsV4, IPsV6, err := i.client.IPAM().AutoAssign( datastoreClient.AutoAssignArgs{ Num4: 1, @@ -157,6 +164,7 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R IPv4Pool: poolV4, }, ) + if err != nil { err = errors.Wrapf(err, "IP assignment error") i.logger.Println(err) @@ -164,6 +172,9 @@ func (i IpamDriver) RequestAddress(request *ipam.RequestAddressRequest) (*ipam.R } IPs = append(IPsV4, IPsV6...) } else { + // Docker allows the users to specify any address. + // We'll return an error if the address isn't in a Calico pool, but we don't care which pool it's in + // (i.e. it doesn't need to match the subnet from the docker network). i.logger.Println("Reserving a specific address in Calico pools") ip := net.ParseIP(request.Address) ipArgs := datastoreClient.AssignIPArgs{ diff --git a/tests/st/libnetwork/test_assign_specific_ip.py b/tests/st/libnetwork/test_assign_specific_ip.py index fbb9583..8887165 100644 --- a/tests/st/libnetwork/test_assign_specific_ip.py +++ b/tests/st/libnetwork/test_assign_specific_ip.py @@ -1,4 +1,4 @@ -# Copyright 2015 Metaswitch Networks +# Copyright 2015 Tigera, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,39 +35,50 @@ def test_assign_specific_ip(self): with DockerHost('host', additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, post_docker_commands=["docker load -i /code/busybox.tar", - "docker load -i /code/calico-node-libnetwork.tar"], + "docker load -i /code/calico-node-libnetwork.tar"], start_calico=False) as host: - run_plugin_command = 'docker run -d ' \ - '--net=host --privileged ' + \ - '-e CALICO_ETCD_AUTHORITY=%s:2379 ' \ - '-v /run/docker/plugins:/run/docker/plugins ' \ - '-v /var/run/docker.sock:/var/run/docker.sock ' \ - '-v /lib/modules:/lib/modules ' \ - '--name libnetwork-plugin ' \ - 'calico/libnetwork-plugin' % (get_ip(),) + '--net=host --privileged ' + \ + '-e CALICO_ETCD_AUTHORITY=%s:2379 ' \ + '-v /run/docker/plugins:/run/docker/plugins ' \ + '-v /var/run/docker.sock:/var/run/docker.sock ' \ + '-v /lib/modules:/lib/modules ' \ + '--name libnetwork-plugin ' \ + 'calico/libnetwork-plugin' % (get_ip(),) host.execute(run_plugin_command) + + # Create two calico pools, and two docker networks with corresponding subnets. subnet1 = "192.168.0.0/16" subnet2 = "10.11.0.0/16" host.calicoctl('pool add %s' % subnet1) host.calicoctl('pool add %s' % subnet2) + network1 = host.create_network("subnet1", subnet=subnet1, driver="calico", ipam_driver="calico-ipam") + network2 = host.create_network("subnet2", subnet=subnet2, driver="calico", ipam_driver="calico-ipam") workload_ip1 = "192.168.1.101" workload_ip2 = "10.11.12.13" + workload_ip3 = "192.168.1.102" # NOTE: This is on subnet1 - network1 = host.create_network( - "subnet1", subnet=subnet1, driver="calico", ipam_driver="calico-ipam") - network2 = host.create_network( - "subnet2", subnet=subnet2, driver="calico", ipam_driver="calico-ipam") - - workload1 = host.create_workload("workload1", - network=network1, - ip=workload_ip1) + # Create a workload on network1 with an IP from subnet1. Check that it gets the right IP + workload1 = host.create_workload("workload1", network=network1, ip=workload_ip1) self.assertEquals(workload_ip1, workload1.ip) - workload2 = host.create_workload("workload2", - network=network2, - ip=workload_ip2) + # Create a workload on network2 with an IP from subnet2. Check that it gets the right IP + workload2 = host.create_workload("workload2", ip=workload_ip2, network=network2) self.assertEquals(workload_ip2, workload2.ip) + + # Create a workload on network2 with an IP from subnet1. + # This is allowed by docker and by the libnetwork plugin + workload3 = host.create_workload("workload3", ip=workload_ip3, network=network1) + + self.assertEquals(workload_ip3, workload3.ip) + + # Check that we can't create a workload with an IP outside a docker subnet + try: + host.create_workload("workload4", ip="1.2.3.4", network=network1) + except Exception, e: + self.assertIn("It does not belong to any of this network's subnets.", str(e)) + + diff --git a/tests/st/libnetwork/test_assign_specific_pool.py b/tests/st/libnetwork/test_assign_specific_pool.py new file mode 100644 index 0000000..3f56808 --- /dev/null +++ b/tests/st/libnetwork/test_assign_specific_pool.py @@ -0,0 +1,67 @@ +# Copyright 2015 Tigera, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from subprocess import check_output + +from tests.st.test_base import TestBase +from tests.st.utils.docker_host import DockerHost +from tests.st.utils.utils import ( + assert_number_endpoints, get_ip, log_and_run, retry_until_success, ETCD_SCHEME, + ETCD_CA, ETCD_KEY, ETCD_CERT, ETCD_HOSTNAME_SSL) +from tests.st.libnetwork.test_mainline_single_host import \ + ADDITIONAL_DOCKER_OPTIONS, POST_DOCKER_COMMANDS +from netaddr import * + +logger = logging.getLogger(__name__) + + +class TestAssignIP(TestBase): + def test_assign_specific_ip(self): + """ + Test that a libnetwork assigned IP is allocated to the container with + Calico when using the '--ip' flag on docker run. + """ + with DockerHost('host', + additional_docker_options=ADDITIONAL_DOCKER_OPTIONS, + post_docker_commands=["docker load -i /code/busybox.tar", + "docker load -i /code/calico-node-libnetwork.tar"], + start_calico=False) as host: + run_plugin_command = 'docker run -d ' \ + '--net=host --privileged ' + \ + '-e CALICO_ETCD_AUTHORITY=%s:2379 ' \ + '-v /run/docker/plugins:/run/docker/plugins ' \ + '-v /var/run/docker.sock:/var/run/docker.sock ' \ + '-v /lib/modules:/lib/modules ' \ + '--name libnetwork-plugin ' \ + 'calico/libnetwork-plugin' % (get_ip(),) + + host.execute(run_plugin_command) + + # Create two calico pools, and two docker networks with corresponding subnets. + subnet1 = "10.15.0.0/16" + subnet2 = "10.16.0.0/16" + host.calicoctl('pool add %s' % subnet1) + host.calicoctl('pool add %s' % subnet2) + network1 = host.create_network("pool1", subnet=subnet1, driver="calico", ipam_driver="calico-ipam") + network2 = host.create_network("pool2", subnet=subnet2, driver="calico", ipam_driver="calico-ipam") + + # Create a workload on network1 and check that it gets an IP in the right subnet. + workload1 = host.create_workload("workload1", network=network1) + self.assertTrue(IPAddress(workload1.ip) in IPNetwork(subnet1)) + + # Create a workload on network2 and check that it gets an IP in the right subnet. + workload2 = host.create_workload("workload2", network=network2) + # Test commented out due to bug in libcalico-go + # self.assertTrue(IPAddress(workload2.ip) in IPNetwork(subnet2)) diff --git a/tests/st/libnetwork/test_error_ipam.py b/tests/st/libnetwork/test_error_ipam.py index 65853a9..ace2a6f 100644 --- a/tests/st/libnetwork/test_error_ipam.py +++ b/tests/st/libnetwork/test_error_ipam.py @@ -1,4 +1,4 @@ -# Copyright 2015 Metaswitch Networks +# Copyright 2015 Tigera, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/st/libnetwork/test_mainline_multi_host.py b/tests/st/libnetwork/test_mainline_multi_host.py index f81f7f9..76f7faa 100644 --- a/tests/st/libnetwork/test_mainline_multi_host.py +++ b/tests/st/libnetwork/test_mainline_multi_host.py @@ -1,4 +1,4 @@ -# Copyright 2015 Metaswitch Networks +# Copyright 2015 Tigera, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/tests/st/libnetwork/test_mainline_single_host.py b/tests/st/libnetwork/test_mainline_single_host.py index 5f6eff6..88e9fa9 100644 --- a/tests/st/libnetwork/test_mainline_single_host.py +++ b/tests/st/libnetwork/test_mainline_single_host.py @@ -1,4 +1,4 @@ -# Copyright 2015 Metaswitch Networks +# Copyright 2015 Tigera, Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.