From 3507e2f0cb7c076aa70ded97839955bb3d43bc27 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Thu, 20 Apr 2023 21:35:23 +0800 Subject: [PATCH 1/2] Enable no uplink OVN-K gateway bridge mode Use no uplink OVN-K gateway bridge to decouple br-ex from host physical interface. Use ovs-vsctl to create br-ex bridge instead of NM, as we don't need run DHCP client for br-ex interface anymore. Signed-off-by: Peng Liu --- assets/components/ovn/common/clusterrole.yaml | 6 + .../ovn/single-node/master/daemonset.yaml | 1 + .../microshift/pkg/config/ovn/ovn.go | 184 ------ .../openshift/microshift/pkg/util/net.go | 19 +- etcd/vendor/modules.txt | 1 - packaging/systemd/configure-ovs.sh | 624 +----------------- packaging/systemd/microshift-ovs-init.service | 2 +- pkg/config/ovn/ovn.go | 17 - pkg/mdns/controller.go | 8 +- pkg/node/netconfig.go | 25 +- pkg/util/net.go | 19 +- 11 files changed, 57 insertions(+), 849 deletions(-) delete mode 100644 etcd/vendor/github.com/openshift/microshift/pkg/config/ovn/ovn.go diff --git a/assets/components/ovn/common/clusterrole.yaml b/assets/components/ovn/common/clusterrole.yaml index 9d43f0bb81..a37236ed6a 100644 --- a/assets/components/ovn/common/clusterrole.yaml +++ b/assets/components/ovn/common/clusterrole.yaml @@ -153,12 +153,18 @@ rules: - egressfirewalls - egressips - egressqoses + - adminpolicybasedexternalroutes verbs: - get - list - watch - update - patch +- apiGroups: + - k8s.ovn.org + resources: + - adminpolicybasedexternalroutes/status + verbs: [ "update"] - apiGroups: ["cloud.network.openshift.io"] resources: - cloudprivateipconfigs diff --git a/assets/components/ovn/single-node/master/daemonset.yaml b/assets/components/ovn/single-node/master/daemonset.yaml index df8858802e..2a680c5021 100644 --- a/assets/components/ovn/single-node/master/daemonset.yaml +++ b/assets/components/ovn/single-node/master/daemonset.yaml @@ -355,6 +355,7 @@ spec: exec /usr/bin/ovnkube \ --init-master "${K8S_NODE}" \ --init-node "${K8S_NODE}" \ + --allow-no-uplink \ --config-file=/run/ovnkube-config/ovnkube.conf \ --loglevel "${OVN_KUBE_LOG_LEVEL}" \ ${gateway_mode_flags} \ diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/config/ovn/ovn.go b/etcd/vendor/github.com/openshift/microshift/pkg/config/ovn/ovn.go deleted file mode 100644 index 85562346c8..0000000000 --- a/etcd/vendor/github.com/openshift/microshift/pkg/config/ovn/ovn.go +++ /dev/null @@ -1,184 +0,0 @@ -package ovn - -import ( - "errors" - "fmt" - "net" - "os" - "path/filepath" - "regexp" - - "k8s.io/klog/v2" - "sigs.k8s.io/yaml" -) - -const ( - ovnConfigFileName = "ovn.yaml" - OVNGatewayInterface = "br-ex" - defaultMTU = 1500 - OVNKubernetesV4MasqueradeIP = "169.254.169.2" - OVNKubernetesV6MasqueradeIP = "fd69::2" - - // used for multinode ovn database transport - OVN_NB_PORT = "9641" - OVN_SB_PORT = "9642" - - // geneve header length for IPv4 - GeneveHeaderLengthIPv4 = 58 - // geneve header length for IPv6 - GeneveHeaderLengthIPv6 = GeneveHeaderLengthIPv4 + 20 -) - -type OVNKubernetesConfig struct { - // Configuration for microshift-ovs-init.service - OVSInit OVSInitConfig `json:"ovsInit,omitempty"` - // MTU to use for the pod interface. Default is 1500. - MTU int `json:"mtu,omitempty"` -} - -type OVSInitConfig struct { - // disable microshift-ovs-init.service. - // OVS bridge "br-ex" needs to be configured manually when disableOVSInit is true. - DisableOVSInit bool `json:"disableOVSInit,omitempty"` - // Uplink interface for OVS bridge "br-ex" - GatewayInterface string `json:"gatewayInterface,omitempty"` -} - -func (o *OVNKubernetesConfig) Validate() error { - // br-ex is required to run ovn-kubernetes - err := o.validateOVSBridge() - if err != nil { - return err - } - err = o.validateConfig() - if err != nil { - return err - } - return nil -} - -// validateOVSBridge validates the existence of ovn-kubernetes br-ex bridge -func (o *OVNKubernetesConfig) validateOVSBridge() error { - _, err := net.InterfaceByName(OVNGatewayInterface) - return err -} - -// validateConfig validates the user defined configuration in /etc/microshift/ovn.yaml -func (o *OVNKubernetesConfig) validateConfig() error { - // validate gateway interfaces conf - if o.OVSInit.GatewayInterface != "" { - _, err := net.InterfaceByName(o.OVSInit.GatewayInterface) - if err != nil { - return fmt.Errorf("gateway interface %s not found", o.OVSInit.GatewayInterface) - } - } - - // validate MTU conf - iface, err := net.InterfaceByName(OVNGatewayInterface) - if err != nil { - return err - } - - if iface.MTU < o.MTU { - return fmt.Errorf("interface MTU (%d) is too small for specified overlay (%d)", iface.MTU, o.MTU) - } - return nil -} - -// getClusterMTU retrieves MTU from ovn-kubernetes gateway interface "br-ex", -// and falls back to use 1500 when "br-ex" mtu is unable to get or less than 0. -func (o *OVNKubernetesConfig) getClusterMTU(multinode bool) { - link, err := net.InterfaceByName(OVNGatewayInterface) - if err == nil && link.MTU > 0 { - o.MTU = link.MTU - } else { - o.MTU = defaultMTU - } - - if multinode { - o.MTU = o.MTU - GeneveHeaderLengthIPv6 - } -} - -// withDefaults returns the default values when ovn.yaml is not provided -func (o *OVNKubernetesConfig) withDefaults(multinode bool) *OVNKubernetesConfig { - o.OVSInit.DisableOVSInit = false - o.getClusterMTU(multinode) - return o -} - -func newOVNKubernetesConfigFromFile(path string, multinode bool) (*OVNKubernetesConfig, error) { - o := new(OVNKubernetesConfig) - buf, err := os.ReadFile(path) - if err != nil { - return nil, err - } - - err = yaml.Unmarshal(buf, &o) - if err != nil { - return nil, fmt.Errorf("parsing OVNKubernetes config: %v", err) - } - // in case mtu is not defined - if o.MTU == 0 { - o.getClusterMTU(multinode) - } - klog.Infof("parsed OVNKubernetes config from file %q: %+v", path, o) - - return o, nil -} - -func NewOVNKubernetesConfigFromFileOrDefault(dir string, multinode bool) (*OVNKubernetesConfig, error) { - path := filepath.Join(dir, ovnConfigFileName) - if _, err := os.Stat(path); err != nil { - if errors.Is(err, os.ErrNotExist) { - klog.Infof("OVNKubernetes config file not found, assuming default values") - return new(OVNKubernetesConfig).withDefaults(multinode), nil - } - return nil, fmt.Errorf("failed to get OVNKubernetes config file: %v", err) - } - - o, err := newOVNKubernetesConfigFromFile(path, multinode) - if err == nil { - return o, nil - } - return nil, fmt.Errorf("getting OVNKubernetes config: %v", err) -} - -func GetOVNGatewayIP() (string, error) { - iface, err := net.InterfaceByName(OVNGatewayInterface) - if err != nil { - return "", err - } - addrs, err := iface.Addrs() - if err != nil { - return "", err - } - for _, addr := range addrs { - ip := addr.(*net.IPNet).IP - // return the first available addr, ipv4 takes precedence in ip.String() - return ip.String(), nil - } - return "", fmt.Errorf("failed to get ovn gateway IP address") -} - -func ExcludeOVNKubernetesMasqueradeIPs(addrs []net.Addr) []net.Addr { - var netAddrs []net.Addr - for _, a := range addrs { - ipNet, _, _ := net.ParseCIDR(a.String()) - if ipNet.String() != OVNKubernetesV4MasqueradeIP && ipNet.String() != OVNKubernetesV6MasqueradeIP { - netAddrs = append(netAddrs, a) - } - } - return netAddrs -} - -func IsOVNKubernetesInternalInterface(name string) bool { - excludedInterfacesRegexp := regexp.MustCompile( - "^[A-Fa-f0-9]{15}|" + // OVN pod interfaces - "ovn.*|" + // OVN ovn-k8s-mp0 and similar interfaces - "br-int|" + // OVN integration bridge - "veth.*|cni.*|" + // Interfaces used in bridge-cni or flannel - "ovs-system$") // Internal OVS interface - - return excludedInterfacesRegexp.MatchString(name) -} diff --git a/etcd/vendor/github.com/openshift/microshift/pkg/util/net.go b/etcd/vendor/github.com/openshift/microshift/pkg/util/net.go index 3c1ade909d..1e16c10f66 100644 --- a/etcd/vendor/github.com/openshift/microshift/pkg/util/net.go +++ b/etcd/vendor/github.com/openshift/microshift/pkg/util/net.go @@ -27,7 +27,6 @@ import ( "strings" "time" - "github.com/openshift/microshift/pkg/config/ovn" "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" @@ -36,23 +35,17 @@ import ( var previousGatewayIP string = "" func GetHostIP() (string, error) { - // Prefer OVN-K gateway IP if it is the CNI - gatewayIP, err := ovn.GetOVNGatewayIP() - if err != nil && !strings.Contains(err.Error(), "no such network interface") { - return "", err - } - if gatewayIP != previousGatewayIP { - previousGatewayIP = gatewayIP - klog.V(2).Infof("ovn gateway IP address: %s", gatewayIP) - } - ip, err := net.ChooseHostInterface() if err == nil { return ip.String(), nil } - klog.V(2).Infof("could not find default route IP address, using ovn gateway IP %q as host IP: %v", gatewayIP, err) + hostIP := ip.String() + if hostIP != previousGatewayIP { + previousGatewayIP = hostIP + klog.V(2).Infof("host gateway IP address: %s", hostIP) + } - return gatewayIP, nil + return hostIP, nil } func RetryInsecureGet(ctx context.Context, url string) int { diff --git a/etcd/vendor/modules.txt b/etcd/vendor/modules.txt index af14d5edc0..5a4e730af1 100644 --- a/etcd/vendor/modules.txt +++ b/etcd/vendor/modules.txt @@ -199,7 +199,6 @@ github.com/openshift/build-machinery-go/scripts # github.com/openshift/microshift v0.0.0 => ../ ## explicit; go 1.20 github.com/openshift/microshift/pkg/config -github.com/openshift/microshift/pkg/config/ovn github.com/openshift/microshift/pkg/util github.com/openshift/microshift/pkg/util/cryptomaterial # github.com/peterbourgon/diskv v2.0.1+incompatible diff --git a/packaging/systemd/configure-ovs.sh b/packaging/systemd/configure-ovs.sh index c8331f0e23..e8010dbd23 100755 --- a/packaging/systemd/configure-ovs.sh +++ b/packaging/systemd/configure-ovs.sh @@ -1,7 +1,5 @@ #!/bin/bash -# source: https://github.com/openshift/machine-config-operator/blob/master/templates/common/_base/files/configure-ovs-network.yaml - set -eux # This file is not needed anymore in 4.7+, but when rolling back to 4.6 @@ -56,260 +54,6 @@ rm_nm_conn_files() { fi done } -# Used to clone a slave connection by uuid, returns new uuid -clone_slave_connection() { - local uuid="$1" - local old_name - old_name="$(nmcli -g connection.id connection show uuid "$uuid")" - local new_name="${old_name}${MANAGED_NM_CONN_SUFFIX}" - if nmcli connection show id "${new_name}" &> /dev/null; then - echo "WARN: existing ovs slave ${new_name} connection profile file found, overwriting..." >&2 - nmcli connection delete id "${new_name}" &> /dev/null - fi - nmcli connection clone $uuid "${new_name}" &> /dev/null - nmcli -g connection.uuid connection show "${new_name}" -} -# Used to replace an old master connection uuid with a new one on all connections -replace_connection_master() { - local old="$1" - local new="$2" - for conn_uuid in $(nmcli -g UUID connection show) ; do - if [ "$(nmcli -g connection.master connection show uuid "$conn_uuid")" != "$old" ]; then - continue - fi - local active_state=$(nmcli -g GENERAL.STATE connection show "$conn_uuid") - local autoconnect=$(nmcli -g connection.autoconnect connection show "$conn_uuid") - if [ "$active_state" != "activated" ] && [ "$autoconnect" != "yes" ]; then - # Assume that slave profiles intended to be used are those that are: - # - active - # - or inactive (which might be due to link being down) but to be autoconnected. - # Otherwise, ignore them. - continue - fi - # make changes for slave profiles in a new clone - local new_uuid - new_uuid=$(clone_slave_connection $conn_uuid) - nmcli conn mod uuid $new_uuid connection.master "$new" - nmcli conn mod $new_uuid connection.autoconnect-priority 100 - nmcli conn mod $new_uuid connection.autoconnect no - echo "Replaced master $old with $new for slave profile $new_uuid" - done -} -# when creating the bridge, we use a value lower than NM's ethernet device default route metric -# (we pick 48 and 49 to be lower than anything that NM chooses by default) -BRIDGE_METRIC="48" -BRIDGE1_METRIC="49" -# Given an interface, generates NM configuration to add to an OVS bridge -convert_to_bridge() { - local iface=${1} - local bridge_name=${2} - local port_name=${3} - local bridge_metric=${4} - local ovs_port="ovs-port-${bridge_name}" - local ovs_interface="ovs-if-${bridge_name}" - local default_port_name="ovs-port-${port_name}" # ovs-port-phys0 - local bridge_interface_name="ovs-if-${port_name}" # ovs-if-phys0 - if [ "$iface" = "$bridge_name" ]; then - # handle vlans and bonds etc if they have already been - # configured via nm key files and br-ex is already up - ifaces=$(ovs-vsctl list-ifaces ${iface}) - for intf in $ifaces; do configure_driver_options $intf; done - echo "Networking already configured and up for ${bridge-name}!" - return - fi - # flag to reload NM to account for all the configuration changes - # going forward - nm_config_changed=1 - if [ -z "$iface" ]; then - echo "ERROR: Unable to find default gateway interface" - exit 1 - fi - # find the MAC from OVS config or the default interface to use for OVS internal port - # this prevents us from getting a different DHCP lease and dropping connection - if ! iface_mac=$(<"/sys/class/net/${iface}/address"); then - echo "Unable to determine default interface MAC" - exit 1 - fi - echo "MAC address found for iface: ${iface}: ${iface_mac}" - # find MTU from original iface - iface_mtu=$(ip link show "$iface" | awk '{print $5; exit}') - if [[ -z "$iface_mtu" ]]; then - echo "Unable to determine default interface MTU, defaulting to 1500" - iface_mtu=1500 - else - echo "MTU found for iface: ${iface}: ${iface_mtu}" - fi - # store old conn for later - old_conn=$(nmcli --fields UUID,DEVICE conn show --active | awk "/\s${iface}\s*\$/ {print \$1}") - # create bridge - if ! nmcli connection show "$bridge_name" &> /dev/null; then - ovs-vsctl --timeout=30 --if-exists del-br "$bridge_name" - add_nm_conn type ovs-bridge con-name "$bridge_name" conn.interface "$bridge_name" 802-3-ethernet.mtu ${iface_mtu} - fi - # find default port to add to bridge - if ! nmcli connection show "$default_port_name" &> /dev/null; then - ovs-vsctl --timeout=30 --if-exists del-port "$bridge_name" ${iface} - add_nm_conn type ovs-port conn.interface ${iface} master "$bridge_name" con-name "$default_port_name" - fi - if ! nmcli connection show "$ovs_port" &> /dev/null; then - ovs-vsctl --timeout=30 --if-exists del-port "$bridge_name" "$bridge_name" - add_nm_conn type ovs-port conn.interface "$bridge_name" master "$bridge_name" con-name "$ovs_port" - fi - extra_phys_args=() - # check if this interface is a vlan, bond, team, or ethernet type - if [ $(nmcli --get-values connection.type conn show ${old_conn}) == "vlan" ]; then - iface_type=vlan - vlan_id=$(nmcli --get-values vlan.id conn show ${old_conn}) - if [ -z "$vlan_id" ]; then - echo "ERROR: unable to determine vlan_id for vlan connection: ${old_conn}" - exit 1 - fi - vlan_parent=$(nmcli --get-values vlan.parent conn show ${old_conn}) - if [ -z "$vlan_parent" ]; then - echo "ERROR: unable to determine vlan_parent for vlan connection: ${old_conn}" - exit 1 - fi - extra_phys_args=( dev "${vlan_parent}" id "${vlan_id}" ) - elif [ $(nmcli --get-values connection.type conn show ${old_conn}) == "bond" ]; then - iface_type=bond - # check bond options - bond_opts=$(nmcli --get-values bond.options conn show ${old_conn}) - if [ -n "$bond_opts" ]; then - extra_phys_args+=( bond.options "${bond_opts}" ) - MODE_REGEX="(^|,)mode=active-backup(,|$)" - MAC_REGEX="(^|,)fail_over_mac=(1|active|2|follow)(,|$)" - if [[ $bond_opts =~ $MODE_REGEX ]] && [[ $bond_opts =~ $MAC_REGEX ]]; then - clone_mac=0 - fi - fi - elif [ $(nmcli --get-values connection.type conn show ${old_conn}) == "team" ]; then - iface_type=team - # check team config options - team_config_opts=$(nmcli --get-values team.config -e no conn show ${old_conn}) - if [ -n "$team_config_opts" ]; then - # team.config is json, remove spaces to avoid problems later on - extra_phys_args+=( team.config "${team_config_opts//[[:space:]]/}" ) - team_mode=$(echo "${team_config_opts}" | jq -r ".runner.name // empty") - team_mac_policy=$(echo "${team_config_opts}" | jq -r ".runner.hwaddr_policy // empty") - MAC_REGEX="(by_active|only_active)" - if [ "$team_mode" = "activebackup" ] && [[ "$team_mac_policy" =~ $MAC_REGEX ]]; then - clone_mac=0 - fi - fi - elif [ $(nmcli --get-values connection.type conn show ${old_conn}) == "tun" ]; then - iface_type=tun - tun_mode=$(nmcli --get-values tun.mode -e no connection show ${old_conn}) - extra_phys_args+=( tun.mode "${tun_mode}" ) - else - iface_type=802-3-ethernet - fi - if [ ! "${clone_mac:-}" = "0" ]; then - # In active-backup link aggregation, with fail_over_mac mode enabled, - # cloning the mac address is not supported. It is possible then that - # br-ex has a different mac address than the bond which might be - # troublesome on some platforms where the nic won't accept packets with - # a different destination mac. But nobody has complained so far so go on - # with what we got. - - # Do set it though for other link aggregation configurations where the - # mac address would otherwise depend on enslave order for which we have - # no control going forward. - extra_phys_args+=( 802-3-ethernet.cloned-mac-address "${iface_mac}" ) - fi - # use ${extra_phys_args[@]+"${extra_phys_args[@]}"} instead of ${extra_phys_args[@]} to be compatible with bash 4.2 in RHEL7.9 - if ! nmcli connection show "$bridge_interface_name" &> /dev/null; then - ovs-vsctl --timeout=30 --if-exists destroy interface ${iface} - add_nm_conn type ${iface_type} conn.interface ${iface} master "$default_port_name" con-name "$bridge_interface_name" \ - connection.autoconnect-priority 100 802-3-ethernet.mtu ${iface_mtu} ${extra_phys_args[@]+"${extra_phys_args[@]}"} - fi - # Get the new connection uuids - new_conn=$(nmcli -g connection.uuid conn show "$bridge_interface_name") - ovs_port_conn=$(nmcli -g connection.uuid conn show "$ovs_port") - # Update connections with master property set to use the new connection - replace_connection_master $old_conn $new_conn - replace_connection_master $iface $new_conn - if ! nmcli connection show "$ovs_interface" &> /dev/null; then - ovs-vsctl --timeout=30 --if-exists destroy interface "$bridge_name" - if nmcli --fields ipv4.method,ipv6.method conn show $old_conn | grep manual; then - echo "Static IP addressing detected on default gateway connection: ${old_conn}" - # clone the old connection to get the address settings - # prefer cloning vs copying the connection file to avoid problems with selinux - nmcli conn clone "${old_conn}" "${ovs_interface}" - shopt -s nullglob - new_conn_files=(${NM_CONN_PATH}/"${ovs_interface}"*) - shopt -u nullglob - if [ ${#new_conn_files[@]} -ne 1 ] || [ ! -f "${new_conn_files[0]}" ]; then - echo "ERROR: could not find ${ovs_interface} conn file after cloning from ${old_conn}" - exit 1 - fi - new_conn_file="${new_conn_files[0]}" - # modify basic connection settings, some of which can't be modified through nmcli - sed -i '/^multi-connect=.*$/d' ${new_conn_file} - sed -i '/^autoconnect=.*$/d' ${new_conn_file} - sed -i '/^\[connection\]$/a autoconnect=false' ${new_conn_file} - sed -i '/^\[connection\]$/,/^\[/ s/^type=.*$/type=ovs-interface/' ${new_conn_file} - sed -i '/^\[connection\]$/a slave-type=ovs-port' ${new_conn_file} - sed -i '/^\[connection\]$/a master='"$ovs_port_conn" ${new_conn_file} - cat <> ${new_conn_file} -[ovs-interface] -type=internal -EOF - # reload the connection and modify some more settings through nmcli - nmcli c load ${new_conn_file} - nmcli c mod "${ovs_interface}" conn.interface "$bridge_name" \ - 802-3-ethernet.mtu ${iface_mtu} 802-3-ethernet.cloned-mac-address ${iface_mac} \ - ipv4.route-metric "${bridge_metric}" ipv6.route-metric "${bridge_metric}" - echo "Loaded new $ovs_interface connection file: ${new_conn_file}" - else - extra_if_brex_args="" - # check if interface had ipv4/ipv6 addresses assigned - num_ipv4_addrs=$(ip -j a show dev ${iface} | jq ".[0].addr_info | map(. | select(.family == \"inet\")) | length") - if [ "$num_ipv4_addrs" -gt 0 ]; then - extra_if_brex_args+="ipv4.may-fail no " - fi - # IPV6 should have at least a link local address. Check for more than 1 to see if there is an - # assigned address. - num_ip6_addrs=$(ip -j a show dev ${iface} | jq ".[0].addr_info | map(. | select(.family == \"inet6\" and .scope != \"link\")) | length") - if [ "$num_ip6_addrs" -gt 0 ]; then - extra_if_brex_args+="ipv6.may-fail no " - fi - # check for dhcp client ids - dhcp_client_id=$(nmcli --get-values ipv4.dhcp-client-id conn show ${old_conn}) - if [ -n "$dhcp_client_id" ]; then - extra_if_brex_args+="ipv4.dhcp-client-id ${dhcp_client_id} " - fi - dhcp6_client_id=$(nmcli --get-values ipv6.dhcp-duid conn show ${old_conn}) - if [ -n "$dhcp6_client_id" ]; then - extra_if_brex_args+="ipv6.dhcp-duid ${dhcp6_client_id} " - fi - ipv6_addr_gen_mode=$(nmcli --get-values ipv6.addr-gen-mode conn show ${old_conn}) - if [ -n "$ipv6_addr_gen_mode" ]; then - extra_if_brex_args+="ipv6.addr-gen-mode ${ipv6_addr_gen_mode} " - fi - ipv4_ignore_auto_dns=$(nmcli --get-values ipv4.ignore-auto-dns conn show ${old_conn}) - if [ -n "$ipv4_ignore_auto_dns" ]; then - extra_if_brex_args+="ipv4.ignore-auto-dns ${ipv4_ignore_auto_dns} " - fi - ipv6_ignore_auto_dns=$(nmcli --get-values ipv6.ignore-auto-dns conn show ${old_conn}) - if [ -n "$ipv6_ignore_auto_dns" ]; then - extra_if_brex_args+="ipv6.ignore-auto-dns ${ipv6_ignore_auto_dns} " - fi - ipv4_dns=$(nmcli --get-values ipv4.dns conn show ${old_conn}) - if [ -n "$ipv4_dns" ]; then - extra_if_brex_args+="ipv4.dns ${ipv4_dns} " - fi - ipv6_dns=$(nmcli --get-values ipv6.dns conn show ${old_conn}) - if [ -n "$ipv6_dns" ]; then - extra_if_brex_args+="ipv6.dns ${ipv6_dns} " - fi - add_nm_conn type ovs-interface slave-type ovs-port conn.interface "$bridge_name" master "$ovs_port_conn" con-name \ - "$ovs_interface" 802-3-ethernet.mtu ${iface_mtu} 802-3-ethernet.cloned-mac-address ${iface_mac} \ - ipv4.route-metric "${bridge_metric}" ipv6.route-metric "${bridge_metric}" ${extra_if_brex_args} - fi - fi - configure_driver_options "${iface}" - update_nm_conn_files "$bridge_name" "$port_name" -} # Used to remove a bridge remove_ovn_bridges() { bridge_name=${1} @@ -318,6 +62,7 @@ remove_ovn_bridges() { # should be auto-activated update_nm_conn_files ${bridge_name} ${port_name} rm_nm_conn_files + nmcli connection reload # NetworkManager will not remove ${bridge_name} if it has the patch port created by ovn-kubernetes # so remove explicitly ovs-vsctl --timeout=30 --if-exists del-br ${bridge_name} @@ -392,83 +137,6 @@ rollback_nm() { # reload profiles so that NM notices that some were removed reload_profiles_nm "$phys0" "$phys1" } -# Add a deactivated connection profile -add_nm_conn() { - nmcli c add "$@" connection.autoconnect no -} -# Activates an ordered set of NM connection profiles -activate_nm_connections() { - local connections=("$@") - - # make sure to set bond or team slaves autoconnect, otherwise as we - # activate one slave, the other slave might get implicitly re-activated - # with the old profile, activating the old master, interfering and - # causing the former activation to fail. - # we don't want to set autoconnect on all the other profiles just yet - # though as that would activate them which is what we want to do next. - # note that these slaves should already be activated with their original - # profiles and setting autoconnect won't implicitly activate them again. - for conn in "${connections[@]}"; do - local slave_type=$(nmcli -g connection.slave-type connection show "$conn") - if [ "$slave_type" = "team" ] || [ "$slave_type" = "bond" ]; then - nmcli c mod "$conn" connection.autoconnect yes - fi - done - # Then activate all the connections - for conn in "${connections[@]}"; do - local active_state=$(nmcli -g GENERAL.STATE conn show "$conn") - if [ "$active_state" != "activated" ]; then - for i in {1..10}; do - echo "Attempt $i to bring up connection $conn" - nmcli conn up "$conn" && s=0 && break || s=$? - sleep 5 - done - if [ $s -eq 0 ]; then - echo "Brought up connection $conn successfully" - else - echo "ERROR: Cannot bring up connection $conn after $i attempts" - return $s - fi - else - echo "Connection $conn already activated" - fi - nmcli c mod "$conn" connection.autoconnect yes - done -} - -# Accepts parameters $iface_default_hint_file, $iface -# Writes content of $iface into $iface_default_hint_file -write_iface_default_hint() { - local iface_default_hint_file="$1" - local iface="$2" - echo "${iface}" >| "${iface_default_hint_file}" -} - -# Accepts parameters $extra_bridge_file, $iface -# Writes content of $iface into $extra_bridge_file -write_iface_external_hint() { - local iface_default_hint_file="$1" - local iface="$2" - echo "${iface}" >| "${extra_bridge_file}" -} - -# Accepts parameters $iface_default_hint_file -# Returns the stored interface default hint if the hint is non-empty, -# not br-ex, not br-ex1 and if the interface can be found in /sys/class/net -get_iface_default_hint() { - local iface_default_hint_file=$1 - if [ -f "${iface_default_hint_file}" ]; then - local iface_default_hint=$(cat "${iface_default_hint_file}") - if [ "${iface_default_hint}" != "" ] && - [ "${iface_default_hint}" != "br-ex" ] && - [ "${iface_default_hint}" != "br-ex1" ] && - [ -d "/sys/class/net/${iface_default_hint}" ]; then - echo "${iface_default_hint}" - return - fi - fi - echo "" -} # Accepts parameters $bridge_interface (e.g. ovs-port-phys0) # Returns the physical interface name if $bridge_interface exists, "" otherwise get_bridge_physical_interface() { @@ -477,284 +145,24 @@ get_bridge_physical_interface() { physical_interface=$(nmcli -g connection.interface-name conn show "${bridge_interface}" 2>/dev/null || echo "") echo "${physical_interface}" } -# Accepts parameters $iface, $iface_default_hint_file -# Finds the default interface. If the default interface is br-ex, use that and return. -# Never use the interface that is provided inside extra_bridge_file for br-ex1. -# Never use br-ex1. -# If the default interface is not br-ex: -# Check if there is a valid hint inside iface_default_hint_file. If so, use that hint. -# If there is no valid hint, use the default interface that we found during the step -# earlier. Write the default interface to the hint file. -get_default_interface() { - local iface="" - local counter=0 - local iface_default_hint_file="$1" - local extra_bridge_file="$2" - local extra_bridge="" - if [ -f "${extra_bridge_file}" ]; then - extra_bridge=$(cat ${extra_bridge_file}) - fi - # find default interface - # the default interface might be br-ex, so check this before looking at the hint - while [ ${counter} -lt 12 ]; do - # check ipv4 - # never use the interface that's specified in extra_bridge_file - # never use br-ex1 - if [ "${extra_bridge}" != "" ]; then - iface=$(ip route show default | grep -v "br-ex1" | grep -v "${extra_bridge}" | awk '{ if ($4 == "dev") { print $5; exit } }') - else - iface=$(ip route show default | grep -v "br-ex1" | awk '{ if ($4 == "dev") { print $5; exit } }') - fi - if [[ -n "${iface}" ]]; then - break - fi - # check ipv6 - # never use the interface that's specified in extra_bridge_file - # never use br-ex1 - if [ "${extra_bridge}" != "" ]; then - iface=$(ip -6 route show default | grep -v "br-ex1" | grep -v "${extra_bridge}" | awk '{ if ($4 == "dev") { print $5; exit } }') - else - iface=$(ip -6 route show default | grep -v "br-ex1" | awk '{ if ($4 == "dev") { print $5; exit } }') - fi - if [[ -n "${iface}" ]]; then - break - fi - # TODO: sync with openshift configure-ovs.sh when it supports non default route - # pick the first ipv4 route in the case that default route doesn't exist - # - # Interface may not get default route immediately after boot: https://issues.redhat.com/browse/USHIFT-340 - # Give it 8*5=40 seconds to detect default route interface, otherwise try to get non-default route interface - if [ ${counter} -gt 8 ]; then - if [ "${extra_bridge}" != "" ]; then - iface=$(ip route show | grep -v "br-ex1" | grep -v "${extra_bridge}" | awk '{ if ($2 == "dev") { print $3; exit } }') - else - iface=$(ip route show | grep -v "br-ex1" | awk '{ if ($2 == "dev") { print $3; exit } }') - fi - if [[ -n "${iface}" ]]; then - break - fi - # pick the first ipv6 route in the case that default route doesn't exist - if [ "${extra_bridge}" != "" ]; then - iface=$(ip -6 route show | grep -v "br-ex1" | grep -v "${extra_bridge}" | awk '{ if ($2 == "dev") { print $3; exit } }') - else - iface=$(ip -6 route show | grep -v "br-ex1" | grep -w -v "lo" | awk '{ if ($2 == "dev") { print $3; exit } }') - fi - if [[ -n "${iface}" ]]; then - break - fi - fi - - counter=$((counter+1)) - sleep 5 - done - # if the default interface does not point out of br-ex or br-ex1 - if [ "${iface}" != "br-ex" ] && [ "${iface}" != "br-ex1" ]; then - # determine if an interface default hint exists from a previous run - # and if the interface has a valid default route - iface_default_hint=$(get_iface_default_hint "${iface_default_hint_file}") - if [ "${iface_default_hint}" != "" ] && - [ "${iface_default_hint}" != "${iface}" ]; then - # start wherever count left off in the previous loop - # allow this for one more iteration than the previous loop - while [ ${counter} -le 12 ]; do - # check ipv4 - if [ "$(ip route show default dev "${iface_default_hint}")" != "" ]; then - iface="${iface_default_hint}" - break - fi - # check ipv6 - if [ "$(ip -6 route show default dev "${iface_default_hint}")" != "" ]; then - iface="${iface_default_hint}" - break - fi - - # TODO: sync with openshift configure-ovs.sh when it supports non default route - # check ipv4 - # use iface hint when default route doesn't exist - if [ "$(ip route show dev "${iface_default_hint}")" != "" ]; then - iface="${iface_default_hint}" - break - fi - # check ipv6 - # use iface hint when default route doesn't exist - if [ "$(ip -6 route show dev "${iface_default_hint}")" != "" ]; then - iface="${iface_default_hint}" - break - fi - - counter=$((counter+1)) - sleep 5 - done - fi - # store what was determined was the (new) default interface inside - # the default hint file for future reference - if [ "${iface}" != "" ]; then - write_iface_default_hint "${iface_default_hint_file}" "${iface}" - fi - fi - echo "${iface}" -} -# Used to print network state -print_state() { - echo "Current device, connection, interface and routing state:" - nmcli -g all device | grep -v unmanaged - nmcli -g all connection - ip -d address show - ip route show - ip -6 route show -} -# Setup an exit trap to rollback on error -handle_exit() { - e=$? - [ $e -eq 0 ] && print_state && exit 0 - echo "ERROR: configure-ovs exited with error: $e" - print_state - # copy configuration to tmp - dir=$(mktemp -d -t "configure-ovs-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX") - update_nm_conn_files br-ex phys0 - copy_nm_conn_files "$dir" +nm_need_rollback=false +update_nm_conn_files br-ex phys0 +if [ -d "/sys/class/net/br-ex1" ]; then update_nm_conn_files br-ex1 phys1 - copy_nm_conn_files "$dir" - echo "Copied OVS configuration to $dir for troubleshooting" - # attempt to restore the previous network state - echo "Attempting to restore previous configuration..." - rollback_nm - print_state - exit $e -} -trap "handle_exit" EXIT -# Clean up old config on behalf of mtu-migration -if [ ! -f /etc/cno/mtu-migration/config ]; then - echo "Cleaning up left over mtu migration configuration" - rm -rf /etc/cno/mtu-migration fi -if ! rpm -qa | grep -q openvswitch; then - echo "Warning: Openvswitch package is not installed!" - exit 1 -fi -# print initial state -print_state -if [ "$1" == "OVNKubernetes" ]; then - - ovnk_config_dir='/etc/ovnk' - ovnk_var_dir='/var/lib/ovnk' - extra_bridge_file="${ovnk_config_dir}/extra_bridge" - iface_default_hint_file="${ovnk_var_dir}/iface_default_hint" - # make sure to create ovnk_config_dir if it does not exist, yet - mkdir -p "${ovnk_config_dir}" - # make sure to create ovnk_var_dir if it does not exist, yet - mkdir -p "${ovnk_var_dir}" - - # Parse MicroShift OVNKubernetes CNI config file - MICROSHIFT_OVN_CONFIG_FILE_PATH="/etc/microshift/ovn.yaml" - function get_cni_config { - ovnkey=$1 - echo "$(cat "$MICROSHIFT_OVN_CONFIG_FILE_PATH" | awk "/$ovnkey:/ && ! /#.*$ovnkey:/ {print \$2}")" - } - if [ -f "$MICROSHIFT_OVN_CONFIG_FILE_PATH" ]; then - # Skip configuring NICs onto OVS bridge "br-ex" or "br-ex1" when disableOVSInit is true - disableOVSInit=$(get_cni_config "disableOVSInit") - if [ "$disableOVSInit" == "true" ]; then - echo "disableOVSInit is true, skipped configure-ovs.sh " - exit 0 - fi - # Store user provided gateway interfaces - gatewayInterface=$(get_cni_config "gatewayInterface") - if [ "$gatewayInterface" != "" ]; then - write_iface_default_hint "${iface_default_hint_file}" "${gatewayInterface}" - else - rm -f "${iface_default_hint_file}" - fi - fi - # Configures NICs onto OVS bridge "br-ex" - # Configuration is either auto-detected or provided through a config file written already in Network Manager - # key files under /etc/NetworkManager/system-connections/ - # Managing key files is outside of the scope of this script - # if the interface is of type vmxnet3 add multicast capability for that driver - # REMOVEME: Once BZ:1854355 is fixed, this needs to get removed. - function configure_driver_options { - intf=$1 - if [ ! -f "/sys/class/net/${intf}/device/uevent" ]; then - echo "Device file doesn't exist, skipping setting multicast mode" - else - driver=$(cat "/sys/class/net/${intf}/device/uevent" | grep DRIVER | awk -F "=" '{print $2}') - echo "Driver name is" $driver - if [ "$driver" = "vmxnet3" ]; then - ifconfig "$intf" allmulti - fi - fi - } - # For upgrade scenarios, make sure that we stabilize what we already configured - # before. If we do not have a valid interface hint, find the physical interface - # that's attached to ovs-if-phys0. - # If we find such an interface, write it to the hint file. - iface_default_hint=$(get_iface_default_hint "${iface_default_hint_file}") - if [ "${iface_default_hint}" == "" ]; then - current_interface=$(get_bridge_physical_interface ovs-if-phys0) - if [ "${current_interface}" != "" ]; then - write_iface_default_hint "${iface_default_hint_file}" "${current_interface}" - fi - fi - # delete iface_default_hint_file if it has the same content as extra_bridge_file - # in that case, we must also force a reconfiguration of our network interfaces - # to make sure that we reconcile this conflict - if [ -f "${iface_default_hint_file}" ] && - [ -f "${extra_bridge_file}" ] && - [ "$(cat "${iface_default_hint_file}")" == "$(cat "${extra_bridge_file}")" ]; then - echo "${iface_default_hint_file} and ${extra_bridge_file} share the same content" - echo "Deleting file ${iface_default_hint_file} to choose a different interface" - rm -f "${iface_default_hint_file}" - rm -f /run/configure-ovs-boot-done - fi - # on every boot we rollback and generate the configuration again, to take - # in any changes that have possibly been applied in the standard - # configuration sources - if [ ! -f /run/configure-ovs-boot-done ]; then - echo "Running on boot, restoring previous configuration before proceeding..." - rollback_nm - print_state - fi - touch /run/configure-ovs-boot-done - iface=$(get_default_interface "${iface_default_hint_file}" "$extra_bridge_file") - if [ "$iface" != "br-ex" ]; then - # Default gateway is not br-ex. - # Some deployments use a temporary solution where br-ex is moved out from the default gateway interface - # and bound to a different nic (https://github.com/trozet/openshift-ovn-migration). - # This is now supported through an extra bridge if requested. If that is the case, we rollback. - # We also rollback if it looks like we need to configure things, just in case there are any leftovers - # from previous attempts. - if [ -f "$extra_bridge_file" ] || [ -z "$(nmcli connection show --active br-ex 2> /dev/null)" ]; then - echo "Bridge br-ex is not active, restoring previous configuration before proceeding..." - rollback_nm - print_state - fi - fi - convert_to_bridge "$iface" "br-ex" "phys0" "${BRIDGE_METRIC}" - # Remove the second bridge - if nmcli connection show br-ex1 &> /dev/null || nmcli connection show ovs-if-phys1 &> /dev/null; then - update_nm_conn_files br-ex1 phys1 - rm_nm_conn_files +for file in "${MANAGED_NM_CONN_FILES[@]}"; do + if [ -f "$file" ]; then + nm_need_rollback=true fi - # Remove the second bridge hint file - if [ -f "$extra_bridge_file" ]; then - rm -f "$extra_bridge_file" - fi - # Remove bridges created by openshift-sdn - ovs-vsctl --timeout=30 --if-exists del-br br0 - # Make sure everything is activated - connections=() - while IFS= read -r connection; do - if [[ $connection == *"$MANAGED_NM_CONN_SUFFIX" ]]; then - connections+=("$connection") - fi - done < <(nmcli -g NAME c) - connections+=(ovs-if-phys0 ovs-if-br-ex) - activate_nm_connections "${connections[@]}" -elif [ "$1" == "OpenShiftSDN" ]; then - # Revert changes made by /usr/local/bin/configure-ovs.sh during SDN migration. +done + +if [ "${nm_need_rollback}" = "true" ]; then + # Remove NM config files create by previous release when doing upgrade rollback_nm - - # Remove bridges created by ovn-kubernetes - ovs-vsctl --timeout=30 --if-exists del-br br-int -- --if-exists del-br br-local fi + +# use a locally administered addresse as the static MAC of br-ex +static_mac="0a:59:00:00:00:01" +ovs-vsctl --timeout=15 --may-exist add-br br-ex -- br-set-external-id br-ex bridge-id br-ex -- set bridge br-ex fail-mode=standalone other_config:hwaddr="${static_mac}" +nmcli device set br-ex managed no diff --git a/packaging/systemd/microshift-ovs-init.service b/packaging/systemd/microshift-ovs-init.service index 2addbe0b3c..4e25402eac 100644 --- a/packaging/systemd/microshift-ovs-init.service +++ b/packaging/systemd/microshift-ovs-init.service @@ -6,6 +6,6 @@ Requires=openvswitch.service NetworkManager.service [Service] Type=oneshot -ExecStart=/bin/bash /usr/bin/configure-ovs.sh OVNKubernetes +ExecStart=/bin/bash /usr/bin/configure-ovs.sh ExecStart=/bin/bash /usr/bin/configure-ovs-microshift.sh TimeoutSec=2m diff --git a/pkg/config/ovn/ovn.go b/pkg/config/ovn/ovn.go index 85562346c8..018c8b2e8b 100644 --- a/pkg/config/ovn/ovn.go +++ b/pkg/config/ovn/ovn.go @@ -144,23 +144,6 @@ func NewOVNKubernetesConfigFromFileOrDefault(dir string, multinode bool) (*OVNKu return nil, fmt.Errorf("getting OVNKubernetes config: %v", err) } -func GetOVNGatewayIP() (string, error) { - iface, err := net.InterfaceByName(OVNGatewayInterface) - if err != nil { - return "", err - } - addrs, err := iface.Addrs() - if err != nil { - return "", err - } - for _, addr := range addrs { - ip := addr.(*net.IPNet).IP - // return the first available addr, ipv4 takes precedence in ip.String() - return ip.String(), nil - } - return "", fmt.Errorf("failed to get ovn gateway IP address") -} - func ExcludeOVNKubernetesMasqueradeIPs(addrs []net.Addr) []net.Addr { var netAddrs []net.Addr for _, a := range addrs { diff --git a/pkg/mdns/controller.go b/pkg/mdns/controller.go index 0d87ec9f3b..af20a7146f 100644 --- a/pkg/mdns/controller.go +++ b/pkg/mdns/controller.go @@ -48,14 +48,10 @@ func (c *MicroShiftmDNSController) Run(ctx context.Context, ready chan<- struct{ ifs, _ := net.Interfaces() - // NOTE: this will listen on both br-ex and the physical interface attached to it - // i.e. eth0 . We don't believe it's worth going into the complexities (and coupling) - // of talking to OpenvSwitch to discover the physical interface(s) on br-ex. And - // we have also verified that no duplicate mDNS answers will happen because of this, - // if those were to happened it would be harmless. + // NOTE: this will listen the physical interface of the host. for n := range ifs { name := ifs[n].Name - if ovn.IsOVNKubernetesInternalInterface(name) { + if ovn.IsOVNKubernetesInternalInterface(name) || name == ovn.OVNGatewayInterface { continue } klog.Infof("mDNS: Starting server on interface %q, NodeIP %q, NodeName %q", name, c.NodeIP, c.NodeName) diff --git a/pkg/node/netconfig.go b/pkg/node/netconfig.go index 0d7e138602..bd67d9a515 100644 --- a/pkg/node/netconfig.go +++ b/pkg/node/netconfig.go @@ -24,6 +24,7 @@ import ( "github.com/vishvananda/netlink" "github.com/openshift/microshift/pkg/config" + "github.com/openshift/microshift/pkg/config/ovn" ) const ( @@ -76,9 +77,15 @@ func (n *NetworkConfiguration) Run(ctx context.Context, ready chan<- struct{}, s } func (n *NetworkConfiguration) addServiceIPLoopback() error { - link, err := netlink.LinkByName(loopbackInterface) - if err != nil { - return err + var link netlink.Link + var err error + + link, err = netlink.LinkByName(ovn.OVNGatewayInterface) + if _, ok := err.(netlink.LinkNotFoundError); ok { + link, err = netlink.LinkByName(loopbackInterface) + if err != nil { + return err + } } address, err := netlink.ParseAddr(fmt.Sprintf("%s/32", n.kasAdvertiseAddress)) if err != nil { @@ -97,9 +104,15 @@ func (n *NetworkConfiguration) addServiceIPLoopback() error { } func (n *NetworkConfiguration) removeServiceIPLoopback() error { - link, err := netlink.LinkByName(loopbackInterface) - if err != nil { - return err + var link netlink.Link + var err error + + link, err = netlink.LinkByName(ovn.OVNGatewayInterface) + if _, ok := err.(netlink.LinkNotFoundError); ok { + link, err = netlink.LinkByName(loopbackInterface) + if err != nil { + return err + } } address, err := netlink.ParseAddr(fmt.Sprintf("%s/32", n.kasAdvertiseAddress)) if err != nil { diff --git a/pkg/util/net.go b/pkg/util/net.go index 3c1ade909d..1e16c10f66 100644 --- a/pkg/util/net.go +++ b/pkg/util/net.go @@ -27,7 +27,6 @@ import ( "strings" "time" - "github.com/openshift/microshift/pkg/config/ovn" "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/klog/v2" @@ -36,23 +35,17 @@ import ( var previousGatewayIP string = "" func GetHostIP() (string, error) { - // Prefer OVN-K gateway IP if it is the CNI - gatewayIP, err := ovn.GetOVNGatewayIP() - if err != nil && !strings.Contains(err.Error(), "no such network interface") { - return "", err - } - if gatewayIP != previousGatewayIP { - previousGatewayIP = gatewayIP - klog.V(2).Infof("ovn gateway IP address: %s", gatewayIP) - } - ip, err := net.ChooseHostInterface() if err == nil { return ip.String(), nil } - klog.V(2).Infof("could not find default route IP address, using ovn gateway IP %q as host IP: %v", gatewayIP, err) + hostIP := ip.String() + if hostIP != previousGatewayIP { + previousGatewayIP = hostIP + klog.V(2).Infof("host gateway IP address: %s", hostIP) + } - return gatewayIP, nil + return hostIP, nil } func RetryInsecureGet(ctx context.Context, url string) int { From e3fc6b94fe9f901245f64fe862056729eba07124 Mon Sep 17 00:00:00 2001 From: Peng Liu Date: Sun, 25 Jun 2023 13:58:47 +0800 Subject: [PATCH 2/2] Remove OVN-K config flag disableOVSInit and gatewayInterface Signed-off-by: Peng Liu --- docs/network/default_cni_plugin.md | 50 +----------------------------- packaging/microshift/ovn.yaml | 7 ----- pkg/config/ovn/ovn.go | 19 ------------ pkg/config/ovn/test/ovn.yaml | 2 -- 4 files changed, 1 insertion(+), 77 deletions(-) diff --git a/docs/network/default_cni_plugin.md b/docs/network/default_cni_plugin.md index 0983bb6eb7..ee2270407e 100644 --- a/docs/network/default_cni_plugin.md +++ b/docs/network/default_cni_plugin.md @@ -65,59 +65,17 @@ The following configs are supported in ovn-kubernetes config file: |Field |Required |Type |Default |Description |Example| |:--------------------------------|:--------|:-------|:-------|:----------------------------------------------------------------------------|:------| -|ovsInit.disableOVSInit |N |bool |false |Skip configuring OVS bridge "br-ex" in microshift-ovs-init.service |true | -|ovsInit.gatewayInterface |N |string |"" |Interface to be added in OVS gateway bridge "br-ex" |eth0 | |mtu |N |int |*auto* |MTU value to be used for the Pods, must be less than or equal to the MTU of default route interface|1500| -> When `disableOVSInit` is true, OVS bridge "br-ex" needs to be configured manually. This OVS bridge is required by ovn-kubernetes CNI. See section [OVS bridge](#ovs-bridge) for guidance on configuring the OVS gateway bridge manually. -> When `gatewayInterface` is not provided, it defaults to the default route interface. -> When `mtu` is not provided, it defaults to the MTU of `gatewayInterface` interface. In the case that `gatewayInterface` is not specified, it is set to the default route MTU. +> When `mtu` is not provided, it is set to the default route MTU. Below is an example of `ovn.yaml`: ```yaml -ovsInit: - disableOVSInit: true - gatewayInterface: eth0 mtu: 1500 ``` **NOTE:* The change of `mtu` configuration in `ovn.yaml` requires node reboot to take effect.
-### Configuring Host - -#### OVS bridge - -When `disableOVSInit` is set to true in ovn-kubernetes CNI config file, OVS bridge "br-ex" needs to be manually configured: - -```bash -nmcli con add type ovs-bridge con-name br-ex conn.interface br-ex 802-3-ethernet.mtu 1500 connection.autoconnect no -nmcli con add type ovs-port conn.interface enp1s0 master br-ex con-name ovs-port-phys0 connection.autoconnect no -nmcli con add type ovs-port conn.interface br-ex master br-ex con-name ovs-port-br-ex connection.autoconnect no - -nmcli con add type 802-3-ethernet conn.interface enp1s0 master ovs-port-phys0 con-name ovs-if-phys0 \ - connection.autoconnect-priority 100 802-3-ethernet.mtu 1500 connection.autoconnect no - -ovs_port_conn=$(nmcli -g connection.uuid conn show ovs-port-br-ex) -iface_mac=$(<"/sys/class/net/enp1s0/address") - -nmcli con add type ovs-interface slave-type ovs-port conn.interface br-ex master "$ovs_port_conn" con-name \ - ovs-if-br-ex 802-3-ethernet.mtu 1500 802-3-ethernet.cloned-mac-address ${iface_mac} \ - ipv4.route-metric 48 ipv6.route-metric 48 connection.autoconnect no - -nmcli con up ovs-if-phys0 -nmcli con up ovs-if-br-ex -nmcli con mod ovs-if-phys0 connection.autoconnect yes -nmcli con mod ovs-if-br-ex connection.autoconnect yes -``` - -Replace `enp1s0` with the network interface name where node IP address is assigned to.
-Replace `1500` with the actual MTU on the network interface.
- -**NOTE:* Copy the above NetworkManager command in a script and execute them at once.
-**NOTE:* Execution of the above commands will cause transient network disconnection from the node IP.
- -[comment]: # (TODO: replace OVS commands with nmcli which can be easily installed under /etc) - ## Network Features A wide range of networking features are available with MicroShift and ovn-kubernetes, including but not limited to: @@ -139,12 +97,6 @@ See [ovn-kubernetes network policy](https://github.com/ovn-org/ovn-kubernetes/bl MicroShift is able to detect node IP change and restarts itself to take in the new IP address. Upon restarting, it recreates ovnkube-master daemonset with updated IP address in openshift-ovn-kubernetes namespace. -### Custom gateway interface - -microshift-ovs-init.service is able to use user specified host interface for cluster network. -This is done by specifying the `gatewayInterface` in the CNI config file `/etc/microshift/ovn.yaml`. -The specified interface will be added in OVS bridge `br-ex` which acts as gateway bridge for ovn-kubernetes CNI network. - ### Blocking external access to NodePort service on specific host interfaces ovn-kubernetes doesn't restrict the host interfaces where NodePort service can be accessed from outside MicroShift node. The following `nft` instructions block NodePort service on a specific host interface.
diff --git a/packaging/microshift/ovn.yaml b/packaging/microshift/ovn.yaml index f52d94bf8f..7adc84eac3 100644 --- a/packaging/microshift/ovn.yaml +++ b/packaging/microshift/ovn.yaml @@ -1,10 +1,3 @@ -ovsInit: - # Skip configuring OVS bridge "br-ex" in microshift-ovs-init.service - #disableOVSInit: false - - # Interface to be added in OVS gateway bridge "br-ex" - #gatewayInterface: "" - # MTU value to be used for the Pods, must be less than or equal to the MTU of # default route interface. #mtu: 1500 diff --git a/pkg/config/ovn/ovn.go b/pkg/config/ovn/ovn.go index 018c8b2e8b..7e9b3a38de 100644 --- a/pkg/config/ovn/ovn.go +++ b/pkg/config/ovn/ovn.go @@ -30,20 +30,10 @@ const ( ) type OVNKubernetesConfig struct { - // Configuration for microshift-ovs-init.service - OVSInit OVSInitConfig `json:"ovsInit,omitempty"` // MTU to use for the pod interface. Default is 1500. MTU int `json:"mtu,omitempty"` } -type OVSInitConfig struct { - // disable microshift-ovs-init.service. - // OVS bridge "br-ex" needs to be configured manually when disableOVSInit is true. - DisableOVSInit bool `json:"disableOVSInit,omitempty"` - // Uplink interface for OVS bridge "br-ex" - GatewayInterface string `json:"gatewayInterface,omitempty"` -} - func (o *OVNKubernetesConfig) Validate() error { // br-ex is required to run ovn-kubernetes err := o.validateOVSBridge() @@ -65,14 +55,6 @@ func (o *OVNKubernetesConfig) validateOVSBridge() error { // validateConfig validates the user defined configuration in /etc/microshift/ovn.yaml func (o *OVNKubernetesConfig) validateConfig() error { - // validate gateway interfaces conf - if o.OVSInit.GatewayInterface != "" { - _, err := net.InterfaceByName(o.OVSInit.GatewayInterface) - if err != nil { - return fmt.Errorf("gateway interface %s not found", o.OVSInit.GatewayInterface) - } - } - // validate MTU conf iface, err := net.InterfaceByName(OVNGatewayInterface) if err != nil { @@ -102,7 +84,6 @@ func (o *OVNKubernetesConfig) getClusterMTU(multinode bool) { // withDefaults returns the default values when ovn.yaml is not provided func (o *OVNKubernetesConfig) withDefaults(multinode bool) *OVNKubernetesConfig { - o.OVSInit.DisableOVSInit = false o.getClusterMTU(multinode) return o } diff --git a/pkg/config/ovn/test/ovn.yaml b/pkg/config/ovn/test/ovn.yaml index b6da9f2e26..b807164241 100644 --- a/pkg/config/ovn/test/ovn.yaml +++ b/pkg/config/ovn/test/ovn.yaml @@ -1,3 +1 @@ -ovsInit: - disableOVSInit: true mtu: 1300