Analysis on Health Check Logic of a Kubernetes Ingress Controller

Health Check Configuration

Image for post
Image for post
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
curl -XGET http://<NGINX_INGRESS_CONTROLLER_POD_ID>:10254/healthz

Analysis on Health Check Logic

Port 10254 and Path /healthz

// Initialize an HTTP request handler.
mux := http.NewServeMux()
go registerHandlers(conf.EnableProfiling, conf.ListenPorts.Health, ngx, mux)
func registerHandlers(enableProfiling bool, port int, ic *controller.NGINXController, mux *http.ServeMux) {
// Register a health check handler.
healthz.InstallHandler(mux,
healthz.PingHealthz,
ic,
)
// Use Prometheus to obtain metrics.
mux.Handle("/metrics", promhttp.Handler())
// Obtain the Ingress Controller version information.
mux.HandleFunc("/build", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
b, _ := json.Marshal(version.String())
w.Write(b)
})
// Stop the Ingress Controller pod.
mux.HandleFunc("/stop", func(w http.ResponseWriter, r *http.Request) {
err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM)
if err != nil {
glog.Errorf("Unexpected error: %v", err)
}
})
// Obtain the monitored performance information.
if enableProfiling {
mux.HandleFunc("/debug/pprof/", pprof.Index)
mux.HandleFunc("/debug/pprof/heap", pprof.Index)
mux.HandleFunc("/debug/pprof/mutex", pprof.Index)
mux.HandleFunc("/debug/pprof/goroutine", pprof.Index)
mux.HandleFunc("/debug/pprof/threadcreate", pprof.Index)
mux.HandleFunc("/debug/pprof/block", pprof.Index)
mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
}
// Start an HTTP server.
server := &http.Server{
Addr: fmt.Sprintf(":%v", port), // Assign the listening port.
Handler: mux,
ReadTimeout: 10 * time.Second,
ReadHeaderTimeout: 10 * time.Second,
WriteTimeout: 300 * time.Second,
IdleTimeout: 120 * time.Second,
}
glog.Fatal(server.ListenAndServe())
}
httpPort      = flags.Int("http-port", 80, `Port to use for servicing HTTP traffic.`)
httpsPort = flags.Int("https-port", 443, `Port to use for servicing HTTPS traffic.`)
statusPort = flags.Int("status-port", 18080, `Port to use for exposing NGINX status pages.`)
sslProxyPort = flags.Int("ssl-passthrough-proxy-port", 442, `Port to use internally for SSL Passthrough.`)
defServerPort = flags.Int("default-server-port", 8181, `Port to use for exposing the default server (catch-all).`)
healthzPort = flags.Int("healthz-port", 10254, "Port to use for the healthz endpoint.")
func InstallHandler(mux mux, checks ...HealthzChecker) {
// If no health check is specified, only PingHealthz is registered by default.
if len(checks) == 0 {
glog.V(5).Info("No default health checks specified. Installing the ping handler.")
checks = []HealthzChecker{PingHealthz}
}
glog.V(5).Info("Installing healthz checkers:", strings.Join(checkerNames(checks...), ", "))
// Register the root health check handler. The root handler will call other specific handlers in turn.
mux.Handle("/healthz", handleRootHealthz(checks...))
for _, check := range checks {
// Register other specific health check handlers.
mux.Handle(fmt.Sprintf("/healthz/%v", check.Name()), adaptCheckToHandler(check.Check))
}
}

Health Check Mechanism

func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
failed := false
var verboseOut bytes.Buffer
for _, check := range checks {
if err := check.Check(r); err != nil {
// don't include the error since this endpoint is public. If someone wants more detail
// they should have explicit permission to the detailed checks.
glog.V(6).Infof("healthz check %v failed: %v", check.Name(), err)
fmt.Fprintf(&verboseOut, "[-]%v failed: reason withheld\n", check.Name())
failed = true
} else {
fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name())
}
}
// always be verbose on failure
if failed {
http.Error(w, fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)
return
}
if _, found := r.URL.Query()["verbose"]; !found {
fmt.Fprint(w, "ok")
return
}
verboseOut.WriteTo(w)
fmt.Fprint(w, "healthz check passed\n")
})
}

healthz.PingHealthz

// PingHealthz returns true automatically when checked
var PingHealthz HealthzChecker = ping{}
// ping implements the simplest possible healthz checker.
type ping struct{}
func (ping) Name() string {
return "ping"
}
// PingHealthz is a health check that returns true.
func (ping) Check(_ *http.Request) error {
return nil
}

controller.NGINXController

const (
ngxHealthPath = "/healthz"
nginxPID = "/tmp/nginx.pid"
)
func (n NGINXController) Name() string {
return "nginx-ingress-controller"
}
func (n *NGINXController) Check(_ *http.Request) error {
// 1. Check the health of Nginx. The access URL is http://0.0.0.0:18080/healthz.
res, err := http.Get(fmt.Sprintf("http://0.0.0.0:%v%v", n.cfg.ListenPorts.Status, ngxHealthPath))
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("ingress controller is not healthy")
}
// 2. If dynamic-configuration is enabled, check the back-end service information that Nginx maintains in the memory. The access URL is http://0.0.0.0:18080/is-dynamic-lb-initialized.
if n.cfg.DynamicConfigurationEnabled {
res, err := http.Get(fmt.Sprintf("http://0.0.0.0:%v/is-dynamic-lb-initialized", n.cfg.ListenPorts.Status))
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != 200 {
return fmt.Errorf("dynamic load balancer not started")
}
}
// 3. Check whether the main Nginx process is running properly.
fs, err := proc.NewFS("/proc")
if err != nil {
return errors.Wrap(err, "unexpected error reading /proc directory")
}
f, err := n.fileSystem.ReadFile(nginxPID)
if err != nil {
return errors.Wrapf(err, "unexpected error reading %v", nginxPID)
}
pid, err := strconv.Atoi(strings.TrimRight(string(f), "\r\n"))
if err != nil {
return errors.Wrapf(err, "unexpected error reading the nginx PID from %v", nginxPID)
}
_, err = fs.NewProc(pid)
return err
}
# used for NGINX healthcheck and access to nginx stats
server {
listen 18080 default_server backlog=511;
listen [::]:18080 default_server backlog=511;
set $proxy_upstream_name "-";
# Access the path. If 200 is returned, Nginx can receive requests.
location /healthz {
access_log off;
return 200;
}
# Check whether the memory contains back-end service information.
location /is-dynamic-lb-initialized {
access_log off;
content_by_lua_block {
local configuration = require("configuration")
local backend_data = configuration.get_backends_data()
if not backend_data then
ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR)
return
end
ngx.say("OK")
ngx.exit(ngx.HTTP_OK)
}
}
# Obtain the basic monitored statistics.
location /nginx_status {
set $proxy_upstream_name "internal";
access_log off;
stub_status on;
}
# 404 is returned by default.
location / {
set $proxy_upstream_name "upstream-default-backend";
proxy_pass http://upstream-default-backend;
}
}

Summary

Follow me to keep abreast with the latest technology news, industry insights, and developer trends.

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store