记一次线上GoLang项目OOM排查过程

func DownloadPodLogs(c *gin.Context) {
	host := c.Query("host")
	if host == "" {
		c.JSON(http.StatusBadRequest, gin.H{"error": "没有Host参数"})
		return
	}
	u, _ := url.ParseRequestURI(fmt.Sprintf("http://%s:%s/api/logs/download", host, daemonServicePort))
	data := url.Values{}
	data.Set("namespace", c.Param("namespace"))
	data.Set("app", c.Param("app"))
	data.Set("pod", c.Param("pod"))
	data.Set("path", c.Param("filename"))
	fmt.Println(data.Encode())
	u.RawQuery = data.Encode()
	res, err := http.Get(u.String())
	if err != nil {
		c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
		return
	}

	defer res.Body.Close()

	c.Status(res.StatusCode)
	c.Header("Content-Type", "application/force-download")
	c.Header("Content-Disposition", fmt.Sprintf("attachment;filename*=utf-8''%s", c.Param("filename")))
	_, err = io.Copy(c.Writer, res.Body)

	if err != nil {
		log.Panic(err)
	}
	c.Writer.Flush()
}

func Copy(dst Writer, src Reader) (written int64, err error) {
	return copyBuffer(dst, src, nil)
}

// copyBuffer is the actual implementation of Copy and CopyBuffer.
// if buf is nil, one is allocated.
func copyBuffer(dst Writer, src Reader, buf []byte) (written int64, err error) {
	// If the reader has a WriteTo method, use it to do the copy.
	// Avoids an allocation and a copy.
	if wt, ok := src.(WriterTo); ok {
		return wt.WriteTo(dst)
	}
	// Similarly, if the writer has a ReadFrom method, use it to do the copy.
	if rt, ok := dst.(ReaderFrom); ok {
		return rt.ReadFrom(src)
	}
	if buf == nil {
		size := 32 * 1024
		if l, ok := src.(*LimitedReader); ok && int64(size) > l.N {
			if l.N < 1 {
				size = 1
			} else {
				size = int(l.N)
			}
		}
		buf = make([]byte, size)
	}
	for {
		nr, er := src.Read(buf)
		if nr > 0 {
			nw, ew := dst.Write(buf[0:nr])
			if nw < 0 || nr < nw {
				nw = 0
				if ew == nil {
					ew = errInvalidWrite
				}
			}
			written += int64(nw)
			if ew != nil {
				err = ew
				break
			}
			if nr != nw {
				err = ErrShortWrite
				break
			}
		}
		if er != nil {
			if er != EOF {
				err = er
			}
			break
		}
	}
	return written, err
}

	buf := make([]byte, 32*1024) // 32 KB buffer
	for {
		n, err := res.Body.Read(buf)
		if n > 0 {
			if _, writeErr := c.Writer.Write(buf[:n]); writeErr != nil {
				log.Printf("Write error: %v", writeErr)
				return
			}
			c.Writer.Flush() // 强制立即发送
		}
		if err != nil {
			if err != io.EOF {
				log.Printf("Read error: %v", err)
			}
			break
		}
	}

	buf := make([]byte, 32*1024) // 32 KB buffer
	c.Stream(func(w io.Writer) bool {
		n, err := res.Body.Read(buf)
		if n > 0 {
			_, writeErr := w.Write(buf[:n])
			if writeErr != nil {
				log.Printf("Write error: %v", writeErr)
				return false
			}
		}
		if err != nil {
			if err != io.EOF {
				log.Printf("Read error: %v", err)
			}
			return false
		}
		return n > 0
	})

//go:build pprof
// +build pprof

package main

import (
	"fmt"
	"net/http"
	_ "net/http/pprof"
)

func init() {
	go pprofServer()
}

func pprofServer() {
	ip := "0.0.0.0:6060"
	if err := http.ListenAndServe(ip, nil); err != nil {
		fmt.Printf("start pprof failed on %s\n", ip)
	}
}

/debug/pprof/
profiles:
0    block
5    goroutine
3    heap
0    mutex
9    threadcreate
full goroutine stack dump

$ go tool pprof http://10.5.x.x:30596/debug/pprof/heap?seconds=60
Fetching profile over HTTP from http://10.5.x.x:30596/debug/pprof/heap?seconds=60
Saved profile in /home/ycyin/pprof/pprof.main.alloc_objects.alloc_space.inuse_objects.inuse_space.001.pb.gz
File: main
Type: inuse_space
Time: Dec 4, 2024 at 2:27pm (CST)
Duration: 60s, Total samples = 288.28MB 
Entering interactive mode (type "help" for commands, "o" for options)
(pprof) top
Showing nodes accounting for 280.73MB, 97.38% of 288.28MB total
Dropped 3 nodes (cum <= 1.44MB)
Showing top 10 nodes out of 26
      flat  flat%   sum%        cum   cum%
  284.26MB 98.60% 98.60%   284.26MB 98.60%  github.com/gin-contrib/cache.(*cachedWriter).Write
   -3.53MB  1.22% 97.38%    -3.53MB  1.22%  compress/flate.NewWriter
         0     0% 97.38%    -3.53MB  1.22%  bufio.(*Writer).Flush
         0     0% 97.38%    -3.53MB  1.22%  compress/gzip.(*Writer).Write
         0     0% 97.38%   284.76MB 98.78%  github.com/gin-contrib/cache.CachePage.func1
         0     0% 97.38%   281.23MB 97.55%  github.com/gin-gonic/gin.(*Context).Next (inline)
         0     0% 97.38%   284.26MB 98.60%  github.com/gin-gonic/gin.(*Context).Stream
         0     0% 97.38%   281.23MB 97.55%  github.com/gin-gonic/gin.(*Engine).ServeHTTP
         0     0% 97.38%   281.23MB 97.55%  github.com/gin-gonic/gin.(*Engine).handleHTTPRequest
         0     0% 97.38%   281.23MB 97.55%  github.com/gin-gonic/gin.CustomRecoveryWithWriter.func1

$ curl -s http://172.26.29.58:30596/debug/pprof/heap?seconds=60 > heap.out

$ go tool pprof heap.out
File: main
Type: inuse_space
Time: Dec 4, 2024 at 3:02pm (CST)
Duration: 60s, Total samples = 288.28MB 
Entering interactive mode (type "help" for commands, "o" for options)
(pprof) web

$ go tool pprof -http=:8080 cpu.prof
Serving web UI on http://localhost:8080