Add an ENV to control ipv6 behavior in the router

images/router/haproxy/conf/haproxy-config.template

@@ -6,6 +6,8 @@
 {{- define "/var/lib/haproxy/conf/haproxy.config" }}
 {{- $workingDir := .WorkingDir }}
 {{- $defaultDestinationCA := .DefaultDestinationCA }}
+{{- $router_ip_v4_v6_mode := env "ROUTER_IP_V4_V6_MODE" "v4" }}
+
 
 {{/* A bunch of regular expressions.  Each should be wrapped in (?:) so that it is safe to include bare */}}
 {{/* quadPattern: Match a quad in an IP address; e.g. 123 */}}
@@ -163,7 +165,14 @@ listen stats :1936
 
 {{ if .BindPorts -}}
 frontend public
-  bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80" }} v4v6 {{ if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
+    {{ if eq "v4v6" $router_ip_v4_v6_mode }}
+  bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v4v6
+    {{- else if eq "v6" $router_ip_v4_v6_mode }}
+  bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v6only
+    {{- else }}
+  bind :{{env "ROUTER_SERVICE_HTTP_PORT" "80"}}
+    {{- end }}
+    {{- if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
   mode http
   tcp-request inspect-delay 5s
   tcp-request content accept if HTTP
@@ -199,7 +208,14 @@ frontend public
 # determined by the next backend in the chain which may be an app backend (passthrough termination) or a backend
 # that terminates encryption in this router (edge)
 frontend public_ssl
-  bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v4v6 {{ if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
+    {{ if eq "v4v6" $router_ip_v4_v6_mode }}
+  bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v4v6
+    {{- else if eq "v6" $router_ip_v4_v6_mode }}
+  bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v6only
+    {{- else }}
+  bind :{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}}
+    {{- end }}
+    {{- if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }}
   tcp-request  inspect-delay 5s
   tcp-request content accept if { req_ssl_hello_type 1 }
 
@@ -394,7 +410,13 @@ backend be_secure:{{$cfgIdx}}
   http-request set-header X-Forwarded-Port %[dst_port]
   http-request set-header X-Forwarded-Proto http if !{ ssl_fc }
   http-request set-header X-Forwarded-Proto https if { ssl_fc }
+  {{- if matchPattern "v6" $router_ip_v4_v6_mode }}
+  # See the quoting rules in https://tools.ietf.org/html/rfc7239 for IPv6 addresses (v4 addresses get translated to v6 when in hybrid mode)
+  http-request set-header Forwarded for="[%[src]]";host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)]
+  {{- else }}
   http-request set-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)]
+  {{- end }}
+
   {{- if not (matchPattern "true|TRUE" (index $cfg.Annotations "haproxy.router.openshift.io/disable_cookies")) }}
   cookie {{$cfg.RoutingKeyName}} insert indirect nocache httponly
     {{- if and (or (eq $cfg.TLSTermination "edge") (eq $cfg.TLSTermination "reencrypt")) (ne $cfg.InsecureEdgeTerminationPolicy "Allow") }} secure

test/extended/router/headers.go

@@ -0,0 +1,132 @@
+package images
+
+import (
+	"bufio"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+
+	g "github.com/onsi/ginkgo"
+	o "github.com/onsi/gomega"
+
+	kapierrs "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+
+	exutil "github.com/openshift/origin/test/extended/util"
+)
+
+var _ = g.Describe("[Conformance][networking][router] router headers", func() {
+	defer g.GinkgoRecover()
+	var (
+		configPath = exutil.FixturePath("testdata", "router-http-echo-server.yaml")
+		oc         = exutil.NewCLI("router-headers", exutil.KubeConfigPath())
+
+		routerIP, routerNs string
+	)
+
+	g.BeforeEach(func() {
+		svc, err := oc.AdminKubeClient().Core().Services("default").Get("router", metav1.GetOptions{})
+		if kapierrs.IsNotFound(err) {
+			g.Skip("no router installed on the cluster")
+			return
+		}
+		o.Expect(err).NotTo(o.HaveOccurred())
+		routerIP = svc.Spec.ClusterIP
+		routerNs = oc.KubeFramework().Namespace.Name
+	})
+
+	g.Describe("The HAProxy router", func() {
+		g.It("should set Forwarded headers appropriately", func() {
+			defer func() {
+				// This should be done if the test fails but
+				// for now always dump the logs.
+				// if g.CurrentGinkgoTestDescription().Failed
+				dumpRouterHeadersLogs(oc, g.CurrentGinkgoTestDescription().FullTestText)
+			}()
+			oc.SetOutputDir(exutil.TestContext.OutputDir)
+			ns := oc.KubeFramework().Namespace.Name
+			execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod")
+			defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }()
+
+			g.By(fmt.Sprintf("creating an http echo server from a config file %q", configPath))
+
+			err := oc.Run("create").Args("-f", configPath).Execute()
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			var clientIP string
+			err = wait.Poll(time.Second, changeTimeoutSeconds*time.Second, func() (bool, error) {
+				pod, err := oc.KubeFramework().ClientSet.Core().Pods(ns).Get("execpod", metav1.GetOptions{})
+				if err != nil {
+					return false, err
+				}
+				if len(pod.Status.PodIP) == 0 {
+					return false, nil
+				}
+
+				clientIP = pod.Status.PodIP
+				return true, nil
+			})
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			// router expected to listen on port 80
+			routerURL := fmt.Sprintf("http://%s", routerIP)
+
+			g.By("waiting for the healthz endpoint to respond")
+			healthzURI := fmt.Sprintf("http://%s:1936/healthz", routerIP)
+			err = waitForRouterOKResponseExec(ns, execPodName, healthzURI, routerIP, changeTimeoutSeconds)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			host := "router-headers.example.com"
+			g.By(fmt.Sprintf("waiting for the route to become active"))
+			err = waitForRouterOKResponseExec(ns, execPodName, routerURL, host, changeTimeoutSeconds)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			g.By(fmt.Sprintf("making a request and reading back the echoed headers"))
+			var payload string
+			payload, err = getRoutePayloadExec(ns, execPodName, routerURL, host)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			// The trailing \n is being stripped, so add it back
+			payload = payload + "\n"
+
+			// parse the echoed request
+			reader := bufio.NewReader(strings.NewReader(payload))
+			req, err := http.ReadRequest(reader)
+			o.Expect(err).NotTo(o.HaveOccurred())
+
+			// check that the header is what we expect
+			g.By(fmt.Sprintf("inspecting the echoed headers"))
+			ffHeader := req.Header.Get("X-Forwarded-For")
+			if ffHeader != clientIP {
+				e2e.Failf("Unexpected header: '%s' (expected %s); All headers: %#v", ffHeader, clientIP, req.Header)
+			}
+		})
+	})
+})
+
+func dumpRouterHeadersLogs(oc *exutil.CLI, name string) {
+	log, _ := e2e.GetPodLogs(oc.AdminKubeClient(), oc.KubeFramework().Namespace.Name, "router-headers", "router")
+	e2e.Logf("Weighted Router test %s logs:\n %s", name, log)
+}
+
+func getRoutePayloadExec(ns, execPodName, url, host string) (string, error) {
+	cmd := fmt.Sprintf(`
+		set -e
+		payload=$( curl -s --header 'Host: %s' %q ) || rc=$?
+		if [[ "${rc:-0}" -eq 0 ]]; then
+			printf "${payload}"
+			exit 0
+		else
+			echo "error ${rc}" 1>&2
+			exit 1
+		fi
+		`, host, url)
+	output, err := e2e.RunHostCmd(ns, execPodName, cmd)
+	if err != nil {
+		return "", fmt.Errorf("host command failed: %v\n%s", err, output)
+	}
+	return output, nil
+}

test/extended/testdata/router-http-echo-server.yaml

@@ -0,0 +1,56 @@
+apiVersion: v1
+kind: List
+metadata: {}
+items:
+- apiVersion: v1
+  kind: DeploymentConfig
+  metadata:
+    name: router-http-echo
+  spec:
+    replicas: 1
+    selector:
+      app: router-http-echo
+      deploymentconfig: router-http-echo
+    strategy:
+      type: Rolling
+    template:
+      metadata:
+        labels:
+          app: router-http-echo
+          deploymentconfig: router-http-echo
+      spec:
+        containers:
+        - image: openshift/origin-base
+          name: router-http-echo
+          command:
+            - /usr/bin/socat
+            - TCP4-LISTEN:8676,reuseaddr,fork
+            - EXEC:'perl -e \"print qq(HTTP/1.0 200 OK\r\n\r\n); while (<>) { print; last if /^\r/}\"'
+          ports:
+          - containerPort: 8676
+            protocol: TCP
+        dnsPolicy: ClusterFirst
+        restartPolicy: Always
+        securityContext: {}
+- apiVersion: v1
+  kind: Service
+  metadata:
+    name: router-http-echo
+    labels:
+      app: router-http-echo
+  spec:
+    selector:
+      app: router-http-echo
+    ports:
+      - port: 8676
+        name: router-http-echo
+        protocol: TCP
+- apiVersion: v1
+  kind: Route
+  metadata:
+    name: router-http-echo
+  spec:
+    host: router-headers.example.com
+    to:
+      kind: Service
+      name: router-http-echo