Kubernetes中Nginx配置熱加載的全過程

Posted on by

前言

Nginx本身是支持熱更新的，通過nginx -s reload指令，實際通過向進程發送HUB信號實現不停服重新加載配置，然而在Docker或者Kubernetes中，每次都需要進容器執行nginx -s reload指令，單docker容器還好說，可以在外面通過exec指定容器執行該指令進行熱加載，Kubernetes的話，就比較難受瞭

今天介紹一下Kubernetes中Nginx熱加載配置的處理方法——reloader

reloader地址：https://github.com/stakater/Reloader

reloader主要就是用來監測ConfigMap或Secret的變化，然後對相關DeploymentConfig的Deployment、DaemonSet執行滾動升級

reloader需要kubernetes1.9以上的版本才支持

使用方法

首先是安裝部署reloader

# 直接通過官方yaml文件部署
kubectl apply -f https://raw.githubusercontent.com/stakater/Reloader/master/deployments/kubernetes/reloader.yaml

默認情況下reloader是部署在default命名空間，但是它是監控所有命名空間的configmaps和secrets

當然，如果不想監控某個configmap或secret，可以通過–resources-to-ignore=configMaps/secrets來忽略某個資源

部署成功後，就可以直接使用瞭，我提前部署瞭nginx和configmap

這是目前的配置，看一下Nginx目前的配置

接著，我修改Nginx的Deployment，添加reloader，監聽nginx-config這個ConfigMap，執行reload

{
  "kind": "Deployment",
  "apiVersion": "extensions/v1beta1",
  "metadata": {
    "name": "nginx",
    "namespace": "default",
    "selfLink": "/apis/extensions/v1beta1/namespaces/default/deployments/nginx",
    "uid": "7eee5fa8-7514-11ec-a916-0210d5e9ca3b",
    "resourceVersion": "286141",
    "generation": 10,
    "creationTimestamp": "2022-01-14T08:32:23Z",
    "labels": {
      "k8s-app": "nginx"
    },
    "annotations": {
      "deployment.kubernetes.io/revision": "9",
      "description": "nginx應用"
      # 主要是這行
      "reloader.stakater.com/reload": "nginx-config"
    }
  },
  "spec": {
    "replicas": 1,
    "selector": {
      "matchLabels": {
        "k8s-app": "nginx"
      }
    }
    ……

然後apply該Deployment，之後我們去更新ConfigMap，更新nginx配置文件

更新完成，去掉proxy_redirect，然後去看nginx容器是否執行滾動更新

可以看到，nginx執行瞭滾動更新，接著看下nginx配置文件是否更新

這樣很簡單的通過reloader就可以實現Nginx的配置熱加載

除瞭這種方法，常見的方法還有使用sidecar，通過sidecar去做的話，需要自己寫監聽腳本，比較麻煩，但是有時候也相對靈活，這裡也附一個sidecar的python腳本

#!/usr/bin/env python
# -*- encoding: utf8 -*-
"""
需求：nginx配置文件變化，自動更新配置文件，類似nginx -s reload
實現：
    1、用pyinotify實時監控nginx配置文件變化
    2、如果配置文件變化，給系統發送HUP來reload nginx
"""
import os
import re
import pyinotify
import logging
from threading import Timer

# Param
LOG_PATH = "/root/python/log"
CONF_PATHS = [
  "/etc/nginx",
]
DELAY = 5
SUDO = False
RELOAD_COMMAND = "nginx -s reload"
if SUDO:
  RELOAD_COMMAND = "sudo " + RELOAD_COMMAND

# Log
logger = logging.getLogger(__name__)
logger.setLevel(level = logging.INFO)
log_handler = logging.FileHandler(LOG_PATH)
log_handler.setLevel(logging.INFO)
log_formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
log_handler.setFormatter(log_formatter)
logger.addHandler(log_handler)

# Reloader
def reload_nginx():
  os.system(RELOAD_COMMAND)
  logger.info("nginx is reloaded")

t = Timer(DELAY, reload_nginx)

def trigger_reload_nginx(pathname, action):
  logger.info("nginx monitor is triggered because %s is %s" % (pathname, action))
  global t
  if t.is_alive():
    t.cancel()
    t = Timer(DELAY, reload_nginx)
    t.start()
  else:
    t = Timer(DELAY, reload_nginx)
    t.start()

events = pyinotify.IN_MODIFY | pyinotify.IN_CREATE | pyinotify.IN_DELETE

watcher = pyinotify.WatchManager()
watcher.add_watch(CONF_PATHS, events, rec=True, auto_add=True)

class EventHandler(pyinotify.ProcessEvent):
  def process_default(self, event):
    if event.name.endswith(".conf"):
      if event.mask == pyinotify.IN_CREATE:
        action = "created"
      if event.mask == pyinotify.IN_MODIFY:
        action = "modified"
      if event.mask == pyinotify.IN_DELETE:
        action = "deleted"
      trigger_reload_nginx(event.pathname, action)

handler = EventHandler()
notifier = pyinotify.Notifier(watcher, handler)

# Start
logger.info("Start Monitoring")
notifier.loop()

如果喜歡用go的，這裡也提供go腳本

package main

import (
    "log"
    "os"
    "path/filepath"
    "syscall"

    "github.com/fsnotify/fsnotify"
    proc "github.com/shirou/gopsutil/process"
)

const (
    nginxProcessName = "nginx"
    defaultNginxConfPath = "/etc/nginx"
    watchPathEnvVarName = "WATCH_NGINX_CONF_PATH"
)

var stderrLogger = log.New(os.Stderr, "error: ", log.Lshortfile)
var stdoutLogger = log.New(os.Stdout, "", log.Lshortfile)

func getMasterNginxPid() (int, error) {
    processes, processesErr := proc.Processes()
    if processesErr != nil {
        return 0, processesErr
    }

    nginxProcesses := map[int32]int32{}

    for _, process := range processes {
        processName, processNameErr := process.Name()
        if processNameErr != nil {
            return 0, processNameErr
        }

        if processName == nginxProcessName {
            ppid, ppidErr := process.Ppid()

            if ppidErr != nil {
                return 0, ppidErr
            }

            nginxProcesses[process.Pid] = ppid
        }
    }

    var masterNginxPid int32

    for pid, ppid := range nginxProcesses {
        if ppid == 0 {
            masterNginxPid = pid

            break
        }
    }

    stdoutLogger.Println("found master nginx pid:", masterNginxPid)

    return int(masterNginxPid), nil
}

func signalNginxReload(pid int) error {
    stdoutLogger.Printf("signaling master nginx process (pid: %d) -> SIGHUP\n", pid)
    nginxProcess, nginxProcessErr := os.FindProcess(pid)

    if nginxProcessErr != nil {
        return nginxProcessErr
    }

    return nginxProcess.Signal(syscall.SIGHUP)
}

func main() {
    watcher, watcherErr := fsnotify.NewWatcher()
    if watcherErr != nil {
        stderrLogger.Fatal(watcherErr)
    }
    defer watcher.Close()

    done := make(chan bool)
    go func() {
        for {
            select {
            case event, ok := <-watcher.Events:
                if !ok {
                    return
                }

                if event.Op&fsnotify.Create == fsnotify.Create {
                    if filepath.Base(event.Name) == "..data" {
                        stdoutLogger.Println("config map updated")

                        nginxPid, nginxPidErr := getMasterNginxPid()
                        if nginxPidErr != nil {
                            stderrLogger.Printf("getting master nginx pid failed: %s", nginxPidErr.Error())

                            continue
                        }

                        if err := signalNginxReload(nginxPid); err != nil {
                            stderrLogger.Printf("signaling master nginx process failed: %s", err)
                        }
                    }
                }
            case err, ok := <-watcher.Errors:
                if !ok {
                    return
                }
                stderrLogger.Printf("received watcher.Error: %s", err)
            }
        }
    }()

    pathToWatch, ok := os.LookupEnv(watchPathEnvVarName)
    if !ok {
        pathToWatch = defaultNginxConfPath
    }

    stdoutLogger.Printf("adding path: `%s` to watch\n", pathToWatch)

    if err := watcher.Add(pathToWatch); err != nil {
        stderrLogger.Fatal(err)
    }
    <-done
}

ok,今天的內容就到這裡

總結

到此這篇關於Kubernetes中Nginx配置熱加載的文章就介紹到這瞭,更多相關Kubernetes中Nginx配置熱加載內容請搜索WalkonNet以前的文章或繼續瀏覽下面的相關文章希望大傢以後多多支持WalkonNet！

推薦閱讀：