diff --git a/images/router/haproxy/conf/haproxy-config.template b/images/router/haproxy/conf/haproxy-config.template index a0850342314f..8a814b4b1896 100644 --- a/images/router/haproxy/conf/haproxy-config.template +++ b/images/router/haproxy/conf/haproxy-config.template @@ -6,6 +6,8 @@ {{- define "/var/lib/haproxy/conf/haproxy.config" }} {{- $workingDir := .WorkingDir }} {{- $defaultDestinationCA := .DefaultDestinationCA }} +{{- $router_ip_v4_v6_mode := env "ROUTER_IP_V4_V6_MODE" "v4" }} + {{/* A bunch of regular expressions. Each should be wrapped in (?:) so that it is safe to include bare */}} {{/* quadPattern: Match a quad in an IP address; e.g. 123 */}} @@ -163,7 +165,14 @@ listen stats :1936 {{ if .BindPorts -}} frontend public - bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80" }} v4v6 {{ if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }} + {{ if eq "v4v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v4v6 + {{- else if eq "v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} v6only + {{- else }} + bind :{{env "ROUTER_SERVICE_HTTP_PORT" "80"}} + {{- end }} + {{- if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }} mode http tcp-request inspect-delay 5s tcp-request content accept if HTTP @@ -199,7 +208,14 @@ frontend public # determined by the next backend in the chain which may be an app backend (passthrough termination) or a backend # that terminates encryption in this router (edge) frontend public_ssl - bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v4v6 {{ if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }} + {{ if eq "v4v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v4v6 + {{- else if eq "v6" $router_ip_v4_v6_mode }} + bind :::{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} v6only + {{- else }} + bind :{{env "ROUTER_SERVICE_HTTPS_PORT" "443"}} + {{- end }} + {{- if matchPattern "true|TRUE" (env "ROUTER_USE_PROXY_PROTOCOL" "") }} accept-proxy{{ end }} tcp-request inspect-delay 5s tcp-request content accept if { req_ssl_hello_type 1 } @@ -394,7 +410,13 @@ backend be_secure:{{$cfgIdx}} http-request set-header X-Forwarded-Port %[dst_port] http-request set-header X-Forwarded-Proto http if !{ ssl_fc } http-request set-header X-Forwarded-Proto https if { ssl_fc } + {{- if matchPattern "v6" $router_ip_v4_v6_mode }} + # See the quoting rules in https://tools.ietf.org/html/rfc7239 for IPv6 addresses (v4 addresses get translated to v6 when in hybrid mode) + http-request set-header Forwarded for="[%[src]]";host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] + {{- else }} http-request set-header Forwarded for=%[src];host=%[req.hdr(host)];proto=%[req.hdr(X-Forwarded-Proto)] + {{- end }} + {{- if not (matchPattern "true|TRUE" (index $cfg.Annotations "haproxy.router.openshift.io/disable_cookies")) }} cookie {{$cfg.RoutingKeyName}} insert indirect nocache httponly {{- if and (or (eq $cfg.TLSTermination "edge") (eq $cfg.TLSTermination "reencrypt")) (ne $cfg.InsecureEdgeTerminationPolicy "Allow") }} secure diff --git a/test/extended/router/headers.go b/test/extended/router/headers.go new file mode 100644 index 000000000000..183539836493 --- /dev/null +++ b/test/extended/router/headers.go @@ -0,0 +1,132 @@ +package images + +import ( + "bufio" + "fmt" + "net/http" + "strings" + "time" + + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" + + kapierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + e2e "k8s.io/kubernetes/test/e2e/framework" + + exutil "github.com/openshift/origin/test/extended/util" +) + +var _ = g.Describe("[Conformance][networking][router] router headers", func() { + defer g.GinkgoRecover() + var ( + configPath = exutil.FixturePath("testdata", "router-http-echo-server.yaml") + oc = exutil.NewCLI("router-headers", exutil.KubeConfigPath()) + + routerIP, routerNs string + ) + + g.BeforeEach(func() { + svc, err := oc.AdminKubeClient().Core().Services("default").Get("router", metav1.GetOptions{}) + if kapierrs.IsNotFound(err) { + g.Skip("no router installed on the cluster") + return + } + o.Expect(err).NotTo(o.HaveOccurred()) + routerIP = svc.Spec.ClusterIP + routerNs = oc.KubeFramework().Namespace.Name + }) + + g.Describe("The HAProxy router", func() { + g.It("should set Forwarded headers appropriately", func() { + defer func() { + // This should be done if the test fails but + // for now always dump the logs. + // if g.CurrentGinkgoTestDescription().Failed + dumpRouterHeadersLogs(oc, g.CurrentGinkgoTestDescription().FullTestText) + }() + oc.SetOutputDir(exutil.TestContext.OutputDir) + ns := oc.KubeFramework().Namespace.Name + execPodName := exutil.CreateExecPodOrFail(oc.AdminKubeClient().Core(), ns, "execpod") + defer func() { oc.AdminKubeClient().Core().Pods(ns).Delete(execPodName, metav1.NewDeleteOptions(1)) }() + + g.By(fmt.Sprintf("creating an http echo server from a config file %q", configPath)) + + err := oc.Run("create").Args("-f", configPath).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + var clientIP string + err = wait.Poll(time.Second, changeTimeoutSeconds*time.Second, func() (bool, error) { + pod, err := oc.KubeFramework().ClientSet.Core().Pods(ns).Get("execpod", metav1.GetOptions{}) + if err != nil { + return false, err + } + if len(pod.Status.PodIP) == 0 { + return false, nil + } + + clientIP = pod.Status.PodIP + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + // router expected to listen on port 80 + routerURL := fmt.Sprintf("http://%s", routerIP) + + g.By("waiting for the healthz endpoint to respond") + healthzURI := fmt.Sprintf("http://%s:1936/healthz", routerIP) + err = waitForRouterOKResponseExec(ns, execPodName, healthzURI, routerIP, changeTimeoutSeconds) + o.Expect(err).NotTo(o.HaveOccurred()) + + host := "router-headers.example.com" + g.By(fmt.Sprintf("waiting for the route to become active")) + err = waitForRouterOKResponseExec(ns, execPodName, routerURL, host, changeTimeoutSeconds) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By(fmt.Sprintf("making a request and reading back the echoed headers")) + var payload string + payload, err = getRoutePayloadExec(ns, execPodName, routerURL, host) + o.Expect(err).NotTo(o.HaveOccurred()) + + // The trailing \n is being stripped, so add it back + payload = payload + "\n" + + // parse the echoed request + reader := bufio.NewReader(strings.NewReader(payload)) + req, err := http.ReadRequest(reader) + o.Expect(err).NotTo(o.HaveOccurred()) + + // check that the header is what we expect + g.By(fmt.Sprintf("inspecting the echoed headers")) + ffHeader := req.Header.Get("X-Forwarded-For") + if ffHeader != clientIP { + e2e.Failf("Unexpected header: '%s' (expected %s); All headers: %#v", ffHeader, clientIP, req.Header) + } + }) + }) +}) + +func dumpRouterHeadersLogs(oc *exutil.CLI, name string) { + log, _ := e2e.GetPodLogs(oc.AdminKubeClient(), oc.KubeFramework().Namespace.Name, "router-headers", "router") + e2e.Logf("Weighted Router test %s logs:\n %s", name, log) +} + +func getRoutePayloadExec(ns, execPodName, url, host string) (string, error) { + cmd := fmt.Sprintf(` + set -e + payload=$( curl -s --header 'Host: %s' %q ) || rc=$? + if [[ "${rc:-0}" -eq 0 ]]; then + printf "${payload}" + exit 0 + else + echo "error ${rc}" 1>&2 + exit 1 + fi + `, host, url) + output, err := e2e.RunHostCmd(ns, execPodName, cmd) + if err != nil { + return "", fmt.Errorf("host command failed: %v\n%s", err, output) + } + return output, nil +} diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index c97e535828aa..0146de9b1214 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -120,6 +120,7 @@ // test/extended/testdata/roles/empty-role.yaml // test/extended/testdata/roles/policy-clusterroles.yaml // test/extended/testdata/roles/policy-roles.yaml +// test/extended/testdata/router-http-echo-server.yaml // test/extended/testdata/router-metrics.yaml // test/extended/testdata/run_policy/parallel-bc.yaml // test/extended/testdata/run_policy/serial-bc.yaml @@ -6397,6 +6398,79 @@ func testExtendedTestdataRolesPolicyRolesYaml() (*asset, error) { return a, nil } +var _testExtendedTestdataRouterHttpEchoServerYaml = []byte(`apiVersion: v1 +kind: List +metadata: {} +items: +- apiVersion: v1 + kind: DeploymentConfig + metadata: + name: router-http-echo + spec: + replicas: 1 + selector: + app: router-http-echo + deploymentconfig: router-http-echo + strategy: + type: Rolling + template: + metadata: + labels: + app: router-http-echo + deploymentconfig: router-http-echo + spec: + containers: + - image: openshift/origin-base + name: router-http-echo + command: + - /usr/bin/socat + - TCP4-LISTEN:8676,reuseaddr,fork + - EXEC:'perl -e \"print qq(HTTP/1.0 200 OK\r\n\r\n); while (<>) { print; last if /^\r/}\"' + ports: + - containerPort: 8676 + protocol: TCP + dnsPolicy: ClusterFirst + restartPolicy: Always + securityContext: {} +- apiVersion: v1 + kind: Service + metadata: + name: router-http-echo + labels: + app: router-http-echo + spec: + selector: + app: router-http-echo + ports: + - port: 8676 + name: router-http-echo + protocol: TCP +- apiVersion: v1 + kind: Route + metadata: + name: router-http-echo + spec: + host: router-headers.example.com + to: + kind: Service + name: router-http-echo +`) + +func testExtendedTestdataRouterHttpEchoServerYamlBytes() ([]byte, error) { + return _testExtendedTestdataRouterHttpEchoServerYaml, nil +} + +func testExtendedTestdataRouterHttpEchoServerYaml() (*asset, error) { + bytes, err := testExtendedTestdataRouterHttpEchoServerYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/router-http-echo-server.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _testExtendedTestdataRouterMetricsYaml = []byte(`apiVersion: v1 kind: List items: @@ -20612,6 +20686,7 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/roles/empty-role.yaml": testExtendedTestdataRolesEmptyRoleYaml, "test/extended/testdata/roles/policy-clusterroles.yaml": testExtendedTestdataRolesPolicyClusterrolesYaml, "test/extended/testdata/roles/policy-roles.yaml": testExtendedTestdataRolesPolicyRolesYaml, + "test/extended/testdata/router-http-echo-server.yaml": testExtendedTestdataRouterHttpEchoServerYaml, "test/extended/testdata/router-metrics.yaml": testExtendedTestdataRouterMetricsYaml, "test/extended/testdata/run_policy/parallel-bc.yaml": testExtendedTestdataRun_policyParallelBcYaml, "test/extended/testdata/run_policy/serial-bc.yaml": testExtendedTestdataRun_policySerialBcYaml, @@ -21000,6 +21075,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ "policy-clusterroles.yaml": &bintree{testExtendedTestdataRolesPolicyClusterrolesYaml, map[string]*bintree{}}, "policy-roles.yaml": &bintree{testExtendedTestdataRolesPolicyRolesYaml, map[string]*bintree{}}, }}, + "router-http-echo-server.yaml": &bintree{testExtendedTestdataRouterHttpEchoServerYaml, map[string]*bintree{}}, "router-metrics.yaml": &bintree{testExtendedTestdataRouterMetricsYaml, map[string]*bintree{}}, "run_policy": &bintree{nil, map[string]*bintree{ "parallel-bc.yaml": &bintree{testExtendedTestdataRun_policyParallelBcYaml, map[string]*bintree{}}, diff --git a/test/extended/testdata/router-http-echo-server.yaml b/test/extended/testdata/router-http-echo-server.yaml new file mode 100644 index 000000000000..4a17bbd91e54 --- /dev/null +++ b/test/extended/testdata/router-http-echo-server.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: List +metadata: {} +items: +- apiVersion: v1 + kind: DeploymentConfig + metadata: + name: router-http-echo + spec: + replicas: 1 + selector: + app: router-http-echo + deploymentconfig: router-http-echo + strategy: + type: Rolling + template: + metadata: + labels: + app: router-http-echo + deploymentconfig: router-http-echo + spec: + containers: + - image: openshift/origin-base + name: router-http-echo + command: + - /usr/bin/socat + - TCP4-LISTEN:8676,reuseaddr,fork + - EXEC:'perl -e \"print qq(HTTP/1.0 200 OK\r\n\r\n); while (<>) { print; last if /^\r/}\"' + ports: + - containerPort: 8676 + protocol: TCP + dnsPolicy: ClusterFirst + restartPolicy: Always + securityContext: {} +- apiVersion: v1 + kind: Service + metadata: + name: router-http-echo + labels: + app: router-http-echo + spec: + selector: + app: router-http-echo + ports: + - port: 8676 + name: router-http-echo + protocol: TCP +- apiVersion: v1 + kind: Route + metadata: + name: router-http-echo + spec: + host: router-headers.example.com + to: + kind: Service + name: router-http-echo