Add logic to support vxlan on Windows

plugins/meta/flannel/README.md

@@ -86,3 +86,59 @@ flannel plugin will set the following fields in the delegated plugin configurati
 * `mtu`: `$FLANNEL_MTU`
 
 Additionally, for the bridge plugin, `isGateway` will be set to `true`, if not present.
+
+## Windows Support (Experimental)
+This plugin supports delegating to the windows CNI plugin (wincni.exe) to work in conjunction with [Flannel on Windows](https://github.com/coreos/flannel/issues/833). 
+Flannel sets up an [HNS Network](https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-networking) in L2Bridge mode for host-gw and 
+in Overlay mode for vxlan. 
+
+The following fields must be set in the delegated plugin configuration:
+* `name` (string, required): the name of the network (must match the name in Flannel config / name of the HNS network)
+* `type` (string, optional): for custom scenarios can be set to delegate to a plugin other than WINCNI
+* `backendType` (string, optional): set to the Flannel backend mode being used, "host-gw" (default) or "vxlan" 
+* `endpointMacPrefix` (string, optional): required for vxlan mode, set to the MAC prefix configured for Flannel  
+
+For host-gw, the Flannel CNI plugin will set:
+* `ipam` (string, required): subnet to `$FLANNEL_SUBNET` and GW to the .2 address in the `$FLANNEL_SUBNET` (this is required by HNS). IPAM type is left empty to allow Windows HNS to do IPAM
+
+For vxlan, the Flannel CNI plugin will set:
+* `ipam`: "host-local" type will be used with "subnet" set to `$FLANNEL_NETWORK` but limited to a range per `$FLANNEL_SUBNET` and gateway as the .1 address in `$FLANNEL_NETWORK`
+
+If IPMASQ is true, the Flannel CNI plugin will setup an OutBoundNAT policy and add FLANNEL_SUBNET to any existing exclusions.
+
+All other delegate config e.g. other HNS endpoint policis in AdditionalArgs will be passed to WINCNI as-is.    
+
+Example VXLAN Flannel CNI config
+```
+{
+	"name": "vxlan0",
+	"type": "flannel",
+	"delegate": {
+		"backendType": "vxlan",
+		"endpointMacPrefix": "0E-2A"
+	}
+}
+```
+
+For this example, Flannel CNI would generate the following config to delegate to WINCNI when FLANNEL_NETWORK=10.244.0.0/16, FLANNEL_SUBNET=10.244.1.0/24 and IPMASQ=true
+```
+{
+	"name": "vxlan0",
+	"type": "wincni.exe",
+	"endpointMacPrefix": "0E-2A",
+	"ipam": {
+		"gateway": "10.244.0.1",
+		"rangeEnd": "10.244.1.254",
+		"rangeStart": "10.244.1.2",
+		"subnet": "10.244.0.0/16",
+		"type": "host-local"
+	},
+	"AdditionalArgs": [{
+		"Name": "EndpointPolicy",
+		"Value": {
+			"ExceptionList": ["10.244.0.0/16"],
+			"Type": "OutBoundNAT"
+		}
+	}]
+}
+```

plugins/meta/flannel/flannel.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"bufio"
+	"encoding/binary"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
@@ -30,11 +31,11 @@ import (
 	"strconv"
 	"strings"
 
+	"errors"
 	"github.com/containernetworking/cni/pkg/invoke"
 	"github.com/containernetworking/cni/pkg/skel"
 	"github.com/containernetworking/cni/pkg/types"
 	"github.com/containernetworking/cni/pkg/version"
-	"math/big"
 )
 
 const (
@@ -262,19 +263,63 @@ func cmdAddWindows(containerID string, n *NetConf, fenv *subnetEnv) error {
 		n.Delegate["cniVersion"] = n.CNIVersion
 	}
 
-	// for now get Windows HNS to do IPAM
-	n.Delegate["ipam"] = map[string]interface{}{
-		"subnet": fenv.sn.String(),
-		"routes": []interface{}{
-			map[string]interface{}{
-				"GW": calcGatewayIPforWindows(fenv.sn),
+	backendType := "host-gw"
+	if hasKey(n.Delegate, "backendType") {
+		backendType = n.Delegate["backendType"].(string)
+	}
+
+	switch backendType {
+	case "host-gw":
+		// let HNS do IPAM for hostgw (L2 bridge) mode
+		gw := fenv.sn.IP.Mask(fenv.sn.Mask)
+		gw[len(gw)-1] += 2
+
+		n.Delegate["ipam"] = map[string]interface{}{
+			"subnet": fenv.sn.String(),
+			"routes": []interface{}{
+				map[string]interface{}{
+					"GW": gw.String(),
+				},
 			},
-		},
+		}
+	case "vxlan":
+		// for vxlan (Overlay) mode the gw is on the cluster CIDR
+		gw := fenv.nw.IP.Mask(fenv.nw.Mask)
+		gw[len(gw)-1] += 1
+
+		// but restrict allocation to the node's pod CIDR
+		rs := fenv.sn.IP.Mask(fenv.sn.Mask).To4()
+		rs[len(rs)-1] += 2
+		re, err := lastAddr(fenv.sn)
+		if err != nil {
+			return err
+		}
+		re[len(re)-1] -= 1
+		n.Delegate["ipam"] = map[string]interface{}{
+			"type":       "host-local",
+			"subnet":     fenv.nw.String(),
+			"rangeStart": rs.String(),
+			"rangeEnd":   re.String(),
+			"gateway":    gw.String(),
+		}
+
+	default:
+		return fmt.Errorf("backendType [%v] is not supported on windows", backendType)
 	}
 
 	return delegateAdd(containerID, n.DataDir, n.Delegate)
 }
 
+// https://stackoverflow.com/questions/36166791/how-to-get-broadcast-address-of-ipv4-net-ipnet
+func lastAddr(n *net.IPNet) (net.IP, error) { // works when the n is a prefix, otherwise...
+	if n.IP.To4() == nil {
+		return net.IP{}, errors.New("does not support IPv6 addresses.")
+	}
+	ip := make(net.IP, len(n.IP.To4()))
+	binary.BigEndian.PutUint32(ip, binary.BigEndian.Uint32(n.IP.To4())|^binary.BigEndian.Uint32(net.IP(n.Mask).To4()))
+	return ip, nil
+}
+
 func updateOutboundNat(delegate map[string]interface{}, fenv *subnetEnv) {
 	if !*fenv.ipmasq {
 		return
@@ -332,24 +377,6 @@ func updateOutboundNat(delegate map[string]interface{}, fenv *subnetEnv) {
 	delegate["AdditionalArgs"] = append(addlArgs, natEntry)
 }
 
-func calcGatewayIPforWindows(ipn *net.IPNet) net.IP {
-	// HNS currently requires x.x.x.2
-	nid := ipn.IP.Mask(ipn.Mask)
-	nid[len(nid)-1] += 2
-	return nid
-}
-
-func ipToInt(ip net.IP) *big.Int {
-	if v := ip.To4(); v != nil {
-		return big.NewInt(0).SetBytes(v)
-	}
-	return big.NewInt(0).SetBytes(ip.To16())
-}
-
-func intToIP(i *big.Int) net.IP {
-	return net.IP(i.Bytes())
-}
-
 func cmdDel(args *skel.CmdArgs) error {
 	nc, err := loadFlannelNetConf(args.StdinData)
 	if err != nil {