PouchContainer Goroutine Leak Detection Practices

1. Goroutine Leak

func main() {
waitCh := make(chan struct{})
go func() {
fmt.Println("Hi, Pouch. I'm new gopher!")
waitCh <- struct{}{}
}()
<-waitCh
}
func main() {
// /exec?cmd=xx&args=yy runs the shell command in the host
http.HandleFunc("/exec", func(w http.ResponseWriter, r *http.Request) {
defer func() { log.Printf("finish %v\n", r.URL) }()
out, err := genCmd(r).CombinedOutput()
if err != nil {
w.WriteHeader(500)
w.Write([]byte(err.Error()))
return
}
w.Write(out)
})
log.Fatal(http.ListenAndServe(":8080", nil))
}
func genCmd(r *http.Request) (cmd *exec.Cmd) {
var args []string
if got := r.FormValue("args"); got != "" {
args = strings.Split(got, " ")
}
if c := r.FormValue("cmd"); len(args) == 0 {
cmd = exec.Command(c)
} else {
cmd = exec.Command(c, args...)
}
return
}
func main() {
logGoNum()
// without sender and blocking....
var ch chan int
go func(ch chan int) {
<-ch
}(ch)
for range time.Tick(2 * time.Second) {
logGoNum()
}
}
func logGoNum() {
log.Printf("goroutine number: %d\n", runtime.NumGoroutine())
}

2. Pouch Logs API Practices

2.1 Detailed Scenarios

func logsContainer(ctx context.Context, w http.ResponseWriter, r *http.Request) {
...
writeLogStream(ctx, w, msgCh)
return
}
func writeLogStream(ctx context.Context, w http.ResponseWriter, msgCh <-chan Message) {
for {
select {
case <-ctx.Done():
return
case msg, ok := <-msgCh:
if !ok {
return
}
w.Write(msg.Byte())
}
}
}

2.2 How to Detect a goroutine Leak

# step 1: create background job
pouch run -d busybox sh -c "while true; do sleep 1; done"
# step 2: follow the log and stop it after 3 seconds
curl -m 3 {ip}:{port}/v1.24/containers/{container_id}/logs?stdout=1&follow=1
# step 3: after 3 seconds, dump the stack info
curl -s "{ip}:{port}/debug/pprof/goroutine?debug=2" | grep -A 10 logsContainer
github.com/alibaba/pouch/apis/server.(*Server).logsContainer(0xc420330b80, 0x251b3e0, 0xc420d93240, 0x251a1e0, 0xc420432c40, 0xc4203f7a00, 0x3, 0x3)
/tmp/pouchbuild/src/github.com/alibaba/pouch/apis/server/container_bridge.go:339 +0x347
github.com/alibaba/pouch/apis/server.(*Server).(github.com/alibaba/pouch/apis/server.logsContainer)-fm(0x251b3e0, 0xc420d93240, 0x251a1e0, 0xc420432c40, 0xc4203f7a00, 0x3, 0x3)
/tmp/pouchbuild/src/github.com/alibaba/pouch/apis/server/router.go:53 +0x5c
github.com/alibaba/pouch/apis/server.withCancelHandler.func1(0x251b3e0, 0xc420d93240, 0x251a1e0, 0xc420432c40, 0xc4203f7a00, 0xc4203f7a00, 0xc42091dad0)
/tmp/pouchbuild/src/github.com/alibaba/pouch/apis/server/router.go:114 +0x57
github.com/alibaba/pouch/apis/server.filter.func1(0x251a1e0, 0xc420432c40, 0xc4203f7a00)
/tmp/pouchbuild/src/github.com/alibaba/pouch/apis/server/router.go:181 +0x327
net/http.HandlerFunc.ServeHTTP(0xc420a84090, 0x251a1e0, 0xc420432c40, 0xc4203f7a00)
/usr/local/go/src/net/http/server.go:1918 +0x44
github.com/alibaba/pouch/vendor/github.com/gorilla/mux.(*Router).ServeHTTP(0xc4209fad20, 0x251a1e0, 0xc420432c40, 0xc4203f7a00)
/tmp/pouchbuild/src/github.com/alibaba/pouch/vendor/github.com/gorilla/mux/mux.go:133 +0xed
net/http.serverHandler.ServeHTTP(0xc420a18d00, 0x251a1e0, 0xc420432c40, 0xc4203f7800)

2.3 Solutions

// HTTP Handler Interceptors
func withCancelHandler(h handler) handler {
return func(ctx context.Context, rw http.ResponseWriter, req *http.Request) error {
// https://golang.org/pkg/net/http/#CloseNotifier
if notifier, ok := rw.(http.CloseNotifier); ok {
var cancel context.CancelFunc
ctx, cancel = context.WithCancel(ctx)
waitCh := make(chan struct{})
defer close(waitCh)
closeNotify := notifier.CloseNotify()
go func() {
select {
case <-closeNotify:
cancel()
case <-waitCh:
}
}()
}
return h(ctx, rw, req)
}
}

3. Commonly Used Analysis Tools

3.1 net/http/pprof

goroutine 93 [chan receive]:
github.com/alibaba/pouch/daemon/mgr.NewContainerMonitor.func1(0xc4202ce618)
/tmp/pouchbuild/src/github.com/alibaba/pouch/daemon/mgr/container_monitor.go:62 +0x45
created by github.com/alibaba/pouch/daemon/mgr.NewContainerMonitor
/tmp/pouchbuild/src/github.com/alibaba/pouch/daemon/mgr/container_monitor.go:60 +0x8d
goroutine 94 [chan receive]:
github.com/alibaba/pouch/daemon/mgr.(*ContainerManager).execProcessGC(0xc42037e090)
/tmp/pouchbuild/src/github.com/alibaba/pouch/daemon/mgr/container.go:2177 +0x1a5
created by github.com/alibaba/pouch/daemon/mgr.NewContainerManager
/tmp/pouchbuild/src/github.com/alibaba/pouch/daemon/mgr/container.go:179 +0x50b

3.2 runtime.NumGoroutine

func TestXXX(t *testing.T) {
orgNum := runtime.NumGoroutine()
defer func() {
if got := runtime.NumGoroutine(); orgNum != got {
t.Fatalf("xxx", orgNum, got)
}
}()
...
}

3.3 github.com/google/gops

Summary

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store