setup crio networking for build containers

Author: bparees

pkg/build/builder/docker.go

@@ -297,6 +297,11 @@ func (d *DockerBuilder) dockerBuild(dir string, tag string, secrets []buildapi.S
 		return err
 	}
 
+	network, resolvConfHostPath, err := getContainerNetworkMode()
+	if err != nil {
+		return err
+	}
+
 	opts := docker.BuildImageOptions{
 		Name:           tag,
 		RmTmpContainer: true,
@@ -305,7 +310,8 @@ func (d *DockerBuilder) dockerBuild(dir string, tag string, secrets []buildapi.S
 		NoCache:        noCache,
 		Pull:           forcePull,
 		BuildArgs:      buildArgs,
-		NetworkMode:    string(getDockerNetworkMode()),
+		NetworkMode:    network,
+		BuildBinds:     fmt.Sprintf("[\"%s:/etc/resolv.conf\"]", resolvConfHostPath),
 	}
 
 	// Though we are capped on memory and cpu at the cgroup parent level,

pkg/build/builder/sti.go

@@ -166,6 +166,11 @@ func (s *S2IBuilder) Build() error {
 		}
 	}
 
+	networkMode, resolvConfHostPath, err := getContainerNetworkMode()
+	if err != nil {
+		return err
+	}
+
 	config := &s2iapi.Config{
 		// Save some processing time by not cleaning up (the container will go away anyway)
 		PreserveWorkingDir: true,
@@ -182,7 +187,7 @@ func (s *S2IBuilder) Build() error {
 
 		Environment:       buildEnvVars(s.build, sourceInfo),
 		Labels:            s2iBuildLabels(s.build, sourceInfo),
-		DockerNetworkMode: getDockerNetworkMode(),
+		DockerNetworkMode: s2iapi.DockerNetworkMode(networkMode),
 
 		Source:     &s2igit.URL{URL: url.URL{Path: srcDir}, Type: s2igit.URLTypeLocal},
 		ContextDir: contextDir,
@@ -197,6 +202,10 @@ func (s *S2IBuilder) Build() error {
 		BlockOnBuild:              true,
 	}
 
+	if len(resolvConfHostPath) != 0 {
+		config.BuildVolumes = []string{fmt.Sprintf("%s:/etc/resolv.conf", resolvConfHostPath)}
+	}
+
 	if s.build.Spec.Strategy.SourceStrategy.ForcePull {
 		glog.V(4).Infof("With force pull true, setting policies to %s", s2iapi.PullAlways)
 		config.BuilderPullPolicy = s2iapi.PullAlways

pkg/build/builder/util.go

@@ -20,17 +20,28 @@ import (
 
 var (
 	// procCGroupPattern is a regular expression that parses the entries in /proc/self/cgroup
-	procCGroupPattern = regexp.MustCompile(`\d+:([a-z_,]+):/.*/(docker-|)([a-z0-9]+).*`)
+	procCGroupPattern = regexp.MustCompile(`\d+:([a-z_,]+):/.*/(\w+-|)([a-z0-9]+).*`)
+
+	// pidPattern is a regexp to match the pid provided in /proc/1/sched inside a container
+	pidPattern = regexp.MustCompile(`.*?\((\d+)`)
+
+	// resolvConfigPattern is a regexp to match the /etc/resolv.conf mount info in /proc/1/mountinfo
+	resolvConfPattern = regexp.MustCompile(`.*?(/.*?) /etc/resolv\.conf `)
 )
 
 // readNetClsCGroup parses /proc/self/cgroup in order to determine the container id that can be used
-// the network namespace that this process is running on.
-func readNetClsCGroup(reader io.Reader) string {
-	cgroups := make(map[string]string)
+// the network namespace that this process is running on, it returns the cgroup and container type
+// (docker vs crio).
+func readNetClsCGroup(reader io.Reader) (string, string) {
+
+	containerType := "docker"
 
+	cgroups := make(map[string]string)
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
 		if match := procCGroupPattern.FindStringSubmatch(scanner.Text()); match != nil {
+			containerType = match[2]
+
 			list := strings.Split(match[1], ",")
 			containerId := match[3]
 			if len(list) > 0 {
@@ -46,26 +57,89 @@ func readNetClsCGroup(reader io.Reader) string {
 	names := []string{"net_cls", "cpu"}
 	for _, group := range names {
 		if value, ok := cgroups[group]; ok {
-			return value
+			return value, containerType
 		}
 	}
 
-	return ""
+	return "", containerType
+}
+
+// readResolveConfHostPath determines the path to the resolv.conf file for this
+// container, as it exists on the host machine.  (/etc/resolv.conf is mounted
+// into the container from the host path).
+func readResolvConfHostPath() (string, error) {
+	// find the /etc/resolv.conf host path based on what is mounted into this
+	// container.
+	resolvConf, err := os.Open("/proc/1/mountinfo")
+	if err != nil {
+		return "", err
+	}
+	defer resolvConf.Close()
+
+	scanner := bufio.NewScanner(resolvConf)
+	for scanner.Scan() {
+		if match := resolvConfPattern.FindStringSubmatch(scanner.Text()); match != nil {
+			// don't allow the path to contain quote, comma, or colon as these could be used
+			// to escape the bindmount argument we are passing to docker and mount other
+			// filepaths from the host.
+			if strings.ContainsAny(match[1], "\",:") {
+				return "", fmt.Errorf("/etc/resolv.conf path from host contains invalid characters (',', '\"', or ':')")
+			}
+			return match[1], nil
+		}
+	}
+	return "", fmt.Errorf("Unable to determine /etc/resolv.conf hostpath")
 }
 
-// getDockerNetworkMode determines whether the builder is running as a container
+// readPid determines the actual host pid of the pid 1 process in this container
+func readPid() (string, error) {
+	// get pid from /proc/1/sched , e.g.: "java (8151, #threads: 53)"
+	pidFile, err := os.Open("/proc/1/sched")
+	if err != nil {
+		return "", err
+	}
+	defer pidFile.Close()
+
+	pidLine, err := bufio.NewReader(pidFile).ReadString('\n')
+	if err != nil {
+		return "", err
+	}
+	match := pidPattern.FindStringSubmatch(pidLine)
+	if match == nil {
+		return "", fmt.Errorf("Unable to determine pid from %s", pidLine)
+	}
+	return match[1], nil
+}
+
+// getContainerNetworkMode determines whether the builder is running as a container
 // by examining /proc/self/cgroup. This context is then passed to source-to-image.
-func getDockerNetworkMode() s2iapi.DockerNetworkMode {
+// It returns a suitable argument for NetworkMode.  If the container platform is
+// CRI-O, it also returns a path for /etc/resolv.conf, suitable for bindmounting.
+func getContainerNetworkMode() (string, string, error) {
 	file, err := os.Open("/proc/self/cgroup")
 	if err != nil {
-		return ""
+		return "", "", err
 	}
 	defer file.Close()
 
-	if id := readNetClsCGroup(file); id != "" {
-		return s2iapi.NewDockerNetworkModeContainer(id)
+	resolvConfHostPath, err := readResolvConfHostPath()
+	if err != nil {
+		return "", "", err
+	}
+
+	if id, containerType := readNetClsCGroup(file); id != "" {
+		glog.V(5).Infof("container type=%s", containerType)
+		if containerType != "crio-" {
+			return s2iapi.DockerNetworkModeContainerPrefix + id, resolvConfHostPath, nil
+		}
+
+		pid, err := readPid()
+		if err != nil {
+			return "", "", err
+		}
+		return fmt.Sprintf("netns:/proc/%s/ns/net", pid), fmt.Sprintf("/var/run" + resolvConfHostPath), nil
 	}
-	return ""
+	return "", "", nil
 }
 
 // GetCGroupLimits returns a struct populated with cgroup limit values gathered

pkg/build/builder/util_test.go

@@ -19,7 +19,7 @@ func TestCGroups_CentOS7_Docker1_7(t *testing.T) {
 1:name=systemd:/system.slice/docker.service
 `
 	buffer := bytes.NewBufferString(example)
-	containerId := readNetClsCGroup(buffer)
+	containerId, _ := readNetClsCGroup(buffer)
 
 	if containerId != "5617ed7e7e487d2c4dd2e013e361109b4eceabfe3fa8c7aea9e37498b1aed5fa" {
 		t.Errorf("got %s, expected 5617ed7e7e487d2c4dd2e013e361109b4eceabfe3fa8c7aea9e37498b1aed5fa", containerId)
@@ -38,7 +38,7 @@ func TestCGroups_Ubuntu_Docker1_9(t *testing.T) {
 3:cpuset:/docker/bfea6eb2d60179355e370a5d277d496eb0fe75d9a5a47c267221e87dbbbbc93b
 2:name=systemd:/`
 	buffer := bytes.NewBufferString(example)
-	containerId := readNetClsCGroup(buffer)
+	containerId, _ := readNetClsCGroup(buffer)
 
 	if containerId != "bfea6eb2d60179355e370a5d277d496eb0fe75d9a5a47c267221e87dbbbbc93b" {
 		t.Errorf("got %s, expected bfea6eb2d60179355e370a5d277d496eb0fe75d9a5a47c267221e87dbbbbc93b", containerId)