mirror of
https://github.com/shtorm-7/sing-box-extended.git
synced 2026-10-26 21:16:56 +00:00
Record memory usage continuously in OOM report
This commit is contained in:
@@ -33,6 +33,7 @@ type Daemon struct {
|
||||
logger log.ContextLogger
|
||||
startedService *daemon.StartedService
|
||||
powerManager *powerreport.Manager
|
||||
oomRecorder *oomkiller.Recorder
|
||||
server *grpc.Server
|
||||
runtimeWorkingDirectory string
|
||||
lifecycleAccess sync.Mutex
|
||||
@@ -63,14 +64,15 @@ func newDaemon() (*Daemon, error) {
|
||||
Context: ctx,
|
||||
LogMaxLines: 3000,
|
||||
})
|
||||
reporter := libbox.NewOOMReporter(d.startedService)
|
||||
service.MustRegister[oomkiller.OOMReporter](ctx, reporter)
|
||||
d.oomRecorder = oomkiller.NewRecorder(libbox.OOMRecorderOptions(d.startedService))
|
||||
service.MustRegister[*oomkiller.Recorder](ctx, d.oomRecorder)
|
||||
d.oomRecorder.Start()
|
||||
d.powerManager = powerreport.NewManager()
|
||||
service.MustRegister[*powerreport.Manager](ctx, d.powerManager)
|
||||
managedService := daemon.NewManagedService(daemon.ManagedServiceOptions{
|
||||
Handler: &managedHandler{d},
|
||||
Debug: debugEnabled,
|
||||
OOMReporter: reporter,
|
||||
OOMRecorder: d.oomRecorder,
|
||||
})
|
||||
authorizer := newAuthorizer(d)
|
||||
serverOptions := []grpc.ServerOption{
|
||||
@@ -289,6 +291,7 @@ func (d *Daemon) Close() {
|
||||
}
|
||||
_ = d.startedService.CloseService()
|
||||
d.startedService.Close()
|
||||
_ = d.oomRecorder.Close()
|
||||
if d.platform != nil {
|
||||
_ = d.platform.Close()
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ type CommandServer struct {
|
||||
platformInterface PlatformInterface
|
||||
platformWrapper *platformInterfaceWrapper
|
||||
powerManager *powerreport.Manager
|
||||
oomRecorder *oomkiller.Recorder
|
||||
grpcServer *grpc.Server
|
||||
listener net.Listener
|
||||
endPauseTimer *time.Timer
|
||||
@@ -83,12 +84,14 @@ func NewCommandServer(handler CommandServerHandler, platformInterface PlatformIn
|
||||
// GroupID: sGroupID,
|
||||
// SystemProxyEnabled: false,
|
||||
})
|
||||
reporter := &oomReporter{startedService: server.StartedService}
|
||||
service.MustRegister[oomkiller.OOMReporter](ctx, reporter)
|
||||
oomRecorder := oomkiller.NewRecorder(OOMRecorderOptions(server.StartedService))
|
||||
service.MustRegister[*oomkiller.Recorder](ctx, oomRecorder)
|
||||
oomRecorder.Start()
|
||||
server.oomRecorder = oomRecorder
|
||||
server.managedService = daemon.NewManagedService(daemon.ManagedServiceOptions{
|
||||
Handler: (*platformHandler)(server),
|
||||
Debug: sDebug,
|
||||
OOMReporter: reporter,
|
||||
OOMRecorder: oomRecorder,
|
||||
})
|
||||
if sPowerReportEnabled {
|
||||
err := powerManager.Start(PowerReportOptions(server.StartedService))
|
||||
@@ -195,6 +198,7 @@ func (s *CommandServer) Close() {
|
||||
}
|
||||
common.Close(s.listener)
|
||||
s.StartedService.Close()
|
||||
s.oomRecorder.Close()
|
||||
s.powerManager.Close()
|
||||
}
|
||||
|
||||
|
||||
@@ -7,7 +7,6 @@ import (
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -52,28 +51,27 @@ type labelMap struct {
|
||||
labelSet
|
||||
}
|
||||
|
||||
func WriteFile(destPath string, name string) (string, error) {
|
||||
func WriteFile(filePath string, name string) error {
|
||||
writer, ok := profileWriters[name]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("unsupported profile %q", name)
|
||||
return fmt.Errorf("unsupported profile %q", name)
|
||||
}
|
||||
|
||||
filePath := filepath.Join(destPath, name+".pb")
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
if err := writer(file); err != nil {
|
||||
err = writer(file)
|
||||
if err != nil {
|
||||
_ = os.Remove(filePath)
|
||||
return "", err
|
||||
return err
|
||||
}
|
||||
if err := file.Close(); err != nil {
|
||||
err = file.Close()
|
||||
if err != nil {
|
||||
_ = os.Remove(filePath)
|
||||
return "", err
|
||||
return err
|
||||
}
|
||||
return filePath, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
var profileWriters = map[string]func(io.Writer) error{
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
//go:build badlinkname
|
||||
|
||||
package runtimeinfo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"cmp"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Mirrors runtime.g of the pinned Go toolchain up to goid; the offsets of gopc and startpc are
|
||||
// discovered by probeGoroutine.
|
||||
|
||||
type runtimeG struct {
|
||||
stackLo uintptr
|
||||
stackHi uintptr
|
||||
_ [2]uintptr
|
||||
_ [3]unsafe.Pointer
|
||||
_ [6]uintptr
|
||||
_ [4]uintptr
|
||||
_ unsafe.Pointer
|
||||
atomicstatus uint32
|
||||
_ uint32
|
||||
goid uint64
|
||||
}
|
||||
|
||||
//go:linkname allgs runtime.allgs
|
||||
var allgs []*runtimeG
|
||||
|
||||
const probeScanBytes = 512
|
||||
|
||||
var (
|
||||
gopcOffset uintptr
|
||||
startpcOffset uintptr
|
||||
probeDone chan bool
|
||||
)
|
||||
|
||||
func (gp *runtimeG) word(offset uintptr) uintptr {
|
||||
return *(*uintptr)(unsafe.Add(unsafe.Pointer(gp), offset))
|
||||
}
|
||||
|
||||
func (gp *runtimeG) gopc() uintptr {
|
||||
return gp.word(gopcOffset)
|
||||
}
|
||||
|
||||
func (gp *runtimeG) startpc() uintptr {
|
||||
return gp.word(startpcOffset)
|
||||
}
|
||||
|
||||
const (
|
||||
statusDead = 6
|
||||
statusScan = 0x1000
|
||||
)
|
||||
|
||||
var statusNames = map[uint32]string{
|
||||
0: "idle",
|
||||
1: "runnable",
|
||||
2: "running",
|
||||
3: "syscall",
|
||||
4: "waiting",
|
||||
6: "dead",
|
||||
8: "copystack",
|
||||
9: "preempted",
|
||||
}
|
||||
|
||||
var (
|
||||
layoutOnce sync.Once
|
||||
layoutVerified bool
|
||||
)
|
||||
|
||||
func probeLayout() {
|
||||
probeDone = make(chan bool, 1)
|
||||
go probeGoroutine()
|
||||
layoutVerified = <-probeDone
|
||||
}
|
||||
|
||||
func probeGoroutine() {
|
||||
id := currentGoroutineID()
|
||||
var self *runtimeG
|
||||
for _, gp := range allgs {
|
||||
if gp.goid == id {
|
||||
self = gp
|
||||
break
|
||||
}
|
||||
}
|
||||
if self == nil {
|
||||
probeDone <- false
|
||||
return
|
||||
}
|
||||
creatorName := functionName(reflect.ValueOf(probeLayout).Pointer())
|
||||
selfName := functionName(reflect.ValueOf(probeGoroutine).Pointer())
|
||||
goidOffset := unsafe.Offsetof(self.goid)
|
||||
for offset := goidOffset + 8; offset < goidOffset+probeScanBytes; offset += 8 {
|
||||
name := functionName(self.word(offset))
|
||||
if gopcOffset == 0 && name == creatorName {
|
||||
gopcOffset = offset
|
||||
} else if startpcOffset == 0 && name == selfName {
|
||||
startpcOffset = offset
|
||||
}
|
||||
}
|
||||
probeDone <- gopcOffset != 0 && startpcOffset != 0
|
||||
}
|
||||
|
||||
func functionName(pc uintptr) string {
|
||||
function := runtime.FuncForPC(pc)
|
||||
if function == nil {
|
||||
return "unknown"
|
||||
}
|
||||
return function.Name()
|
||||
}
|
||||
|
||||
func currentGoroutineID() uint64 {
|
||||
var stackBuffer [64]byte
|
||||
n := runtime.Stack(stackBuffer[:], false)
|
||||
fields := bytes.Fields(stackBuffer[:n])
|
||||
if len(fields) < 2 {
|
||||
return 0
|
||||
}
|
||||
id, _ := strconv.ParseUint(string(fields[1]), 10, 64)
|
||||
return id
|
||||
}
|
||||
|
||||
func collectGoroutines() *GoroutineReport {
|
||||
layoutOnce.Do(probeLayout)
|
||||
if !layoutVerified {
|
||||
return nil
|
||||
}
|
||||
report := &GoroutineReport{ByStatus: make(map[string]int)}
|
||||
groups := make(map[string]*GoroutineGroup)
|
||||
for _, gp := range allgs {
|
||||
status := atomic.LoadUint32(&gp.atomicstatus) &^ statusScan
|
||||
stackSize := uint64(gp.stackHi - gp.stackLo)
|
||||
if status == statusDead {
|
||||
report.Dead++
|
||||
if gp.stackLo != 0 {
|
||||
report.DeadStackBytes += stackSize
|
||||
}
|
||||
continue
|
||||
}
|
||||
report.Total++
|
||||
report.StackBytes += stackSize
|
||||
statusName, known := statusNames[status]
|
||||
if !known {
|
||||
statusName = strconv.Itoa(int(status))
|
||||
}
|
||||
report.ByStatus[statusName]++
|
||||
name := functionName(gp.startpc())
|
||||
group, found := groups[name]
|
||||
if !found {
|
||||
group = &GoroutineGroup{Function: name, CreatedBy: functionName(gp.gopc()), MinStackBytes: stackSize}
|
||||
groups[name] = group
|
||||
}
|
||||
group.Count++
|
||||
group.StackBytes += stackSize
|
||||
group.MaxStackBytes = max(group.MaxStackBytes, stackSize)
|
||||
group.MinStackBytes = min(group.MinStackBytes, stackSize)
|
||||
}
|
||||
report.ByFunction = make([]GoroutineGroup, 0, len(groups))
|
||||
for _, group := range groups {
|
||||
report.ByFunction = append(report.ByFunction, *group)
|
||||
}
|
||||
slices.SortFunc(report.ByFunction, func(a, b GoroutineGroup) int {
|
||||
return cmp.Compare(b.StackBytes, a.StackBytes)
|
||||
})
|
||||
return report
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build !badlinkname
|
||||
|
||||
package runtimeinfo
|
||||
|
||||
func collectGoroutines() *GoroutineReport {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
//go:build badlinkname
|
||||
|
||||
package runtimeinfo
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"unsafe"
|
||||
|
||||
"github.com/sagernet/sing/common/buf"
|
||||
)
|
||||
|
||||
// Mirrors sync.Pool, sync.poolLocal and sync.poolChainElt of the pinned Go toolchain.
|
||||
|
||||
type syncPool struct {
|
||||
local unsafe.Pointer
|
||||
localSize uintptr
|
||||
victim unsafe.Pointer
|
||||
victimSize uintptr
|
||||
_ func() any
|
||||
}
|
||||
|
||||
type poolLocal struct {
|
||||
private [2]unsafe.Pointer
|
||||
sharedHead unsafe.Pointer
|
||||
_ unsafe.Pointer
|
||||
_ [128 - 4*unsafe.Sizeof(uintptr(0))]byte
|
||||
}
|
||||
|
||||
type poolChainElt struct {
|
||||
headTail uint64
|
||||
_ []unsafe.Pointer
|
||||
_ unsafe.Pointer
|
||||
prev unsafe.Pointer
|
||||
}
|
||||
|
||||
func collectBufferPools() []PoolReport {
|
||||
allocator := reflect.ValueOf(buf.DefaultAllocator)
|
||||
if allocator.Kind() != reflect.Pointer {
|
||||
return nil
|
||||
}
|
||||
pools := allocator.Elem().FieldByName("buffers")
|
||||
if !pools.IsValid() || pools.Kind() != reflect.Array || pools.Type().Elem() != reflect.TypeFor[sync.Pool]() {
|
||||
return nil
|
||||
}
|
||||
if unsafe.Sizeof(syncPool{}) != unsafe.Sizeof(sync.Pool{}) || unsafe.Sizeof(poolLocal{}) != 128 {
|
||||
return nil
|
||||
}
|
||||
result := make([]PoolReport, 0, pools.Len())
|
||||
for i := range pools.Len() {
|
||||
pool := (*syncPool)(pools.Index(i).Addr().UnsafePointer())
|
||||
size := min(1<<(6+i), buf.MaxPooledBufferSize)
|
||||
report := PoolReport{
|
||||
Size: size,
|
||||
Cached: countPoolLocals(pool.local, pool.localSize),
|
||||
Victim: countPoolLocals(pool.victim, pool.victimSize),
|
||||
}
|
||||
report.Bytes = uint64(report.Cached+report.Victim) * uint64(size)
|
||||
result = append(result, report)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func countPoolLocals(locals unsafe.Pointer, count uintptr) int {
|
||||
if locals == nil {
|
||||
return 0
|
||||
}
|
||||
var total int
|
||||
for i := range count {
|
||||
local := (*poolLocal)(unsafe.Add(locals, i*unsafe.Sizeof(poolLocal{})))
|
||||
if local.private[0] != nil {
|
||||
total++
|
||||
}
|
||||
for element := (*poolChainElt)(atomic.LoadPointer(&local.sharedHead)); element != nil; element = (*poolChainElt)(atomic.LoadPointer(&element.prev)) {
|
||||
headTail := atomic.LoadUint64(&element.headTail)
|
||||
total += int(uint32(headTail>>32) - uint32(headTail))
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build !badlinkname
|
||||
|
||||
package runtimeinfo
|
||||
|
||||
func collectBufferPools() []PoolReport {
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
package runtimeinfo
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"runtime/metrics"
|
||||
)
|
||||
|
||||
type Report struct {
|
||||
Metrics map[string]uint64 `json:"metrics"`
|
||||
Goroutines *GoroutineReport `json:"goroutines,omitempty"`
|
||||
Pools []PoolReport `json:"bufferPools,omitempty"`
|
||||
}
|
||||
|
||||
type GoroutineReport struct {
|
||||
Total int `json:"total"`
|
||||
StackBytes uint64 `json:"stackBytes"`
|
||||
Dead int `json:"dead"`
|
||||
DeadStackBytes uint64 `json:"deadStackBytes"`
|
||||
ByStatus map[string]int `json:"byStatus"`
|
||||
ByFunction []GoroutineGroup `json:"byFunction"`
|
||||
}
|
||||
|
||||
type GoroutineGroup struct {
|
||||
Function string `json:"function"`
|
||||
CreatedBy string `json:"createdBy,omitempty"`
|
||||
Count int `json:"count"`
|
||||
StackBytes uint64 `json:"stackBytes"`
|
||||
MaxStackBytes uint64 `json:"maxStackBytes"`
|
||||
MinStackBytes uint64 `json:"minStackBytes"`
|
||||
}
|
||||
|
||||
type PoolReport struct {
|
||||
Size int `json:"size"`
|
||||
Cached int `json:"cached"`
|
||||
Victim int `json:"victim"`
|
||||
Bytes uint64 `json:"bytes"`
|
||||
}
|
||||
|
||||
func Collect() Report {
|
||||
return Report{
|
||||
Metrics: collectMetrics(),
|
||||
Goroutines: collectGoroutines(),
|
||||
Pools: collectBufferPools(),
|
||||
}
|
||||
}
|
||||
|
||||
func WriteFile(path string) error {
|
||||
content, err := json.MarshalIndent(Collect(), "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(path, content, 0o666)
|
||||
}
|
||||
|
||||
func collectMetrics() map[string]uint64 {
|
||||
descriptions := metrics.All()
|
||||
samples := make([]metrics.Sample, 0, len(descriptions))
|
||||
for _, description := range descriptions {
|
||||
if description.Kind != metrics.KindUint64 {
|
||||
continue
|
||||
}
|
||||
samples = append(samples, metrics.Sample{Name: description.Name})
|
||||
}
|
||||
metrics.Read(samples)
|
||||
result := make(map[string]uint64, len(samples))
|
||||
for _, sample := range samples {
|
||||
if sample.Value.Kind() != metrics.KindUint64 {
|
||||
continue
|
||||
}
|
||||
result[sample.Name] = sample.Value.Uint64()
|
||||
}
|
||||
return result
|
||||
}
|
||||
@@ -5,21 +5,18 @@ package libbox
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/sagernet/sing-box/common/trafficcontrol"
|
||||
"github.com/sagernet/sing-box/daemon"
|
||||
"github.com/sagernet/sing-box/experimental/libbox/internal/oomprofile"
|
||||
"github.com/sagernet/sing-box/experimental/libbox/internal/runtimeinfo"
|
||||
"github.com/sagernet/sing-box/log"
|
||||
"github.com/sagernet/sing-box/service/oomkiller"
|
||||
"github.com/sagernet/sing/common/byteformats"
|
||||
F "github.com/sagernet/sing/common/format"
|
||||
"github.com/sagernet/sing/common/memory"
|
||||
)
|
||||
|
||||
var oomReportProfiles = []string{
|
||||
@@ -34,157 +31,66 @@ var oomReportProfiles = []string{
|
||||
type oomReportMetadata struct {
|
||||
reportMetadata
|
||||
RecordedAt string `json:"recordedAt"`
|
||||
EndedAt string `json:"endedAt,omitempty"`
|
||||
MemoryLimit string `json:"memoryLimit,omitempty"`
|
||||
MemoryUsage string `json:"memoryUsage"`
|
||||
AvailableMemory string `json:"availableMemory,omitempty"`
|
||||
// Heap
|
||||
HeapAlloc string `json:"heapAlloc,omitempty"`
|
||||
HeapObjects uint64 `json:"heapObjects,omitempty,string"`
|
||||
HeapInuse string `json:"heapInuse,omitempty"`
|
||||
HeapIdle string `json:"heapIdle,omitempty"`
|
||||
HeapReleased string `json:"heapReleased,omitempty"`
|
||||
HeapSys string `json:"heapSys,omitempty"`
|
||||
// Stack
|
||||
StackInuse string `json:"stackInuse,omitempty"`
|
||||
StackSys string `json:"stackSys,omitempty"`
|
||||
// Runtime metadata
|
||||
MSpanInuse string `json:"mSpanInuse,omitempty"`
|
||||
MSpanSys string `json:"mSpanSys,omitempty"`
|
||||
MCacheSys string `json:"mCacheSys,omitempty"`
|
||||
BuckHashSys string `json:"buckHashSys,omitempty"`
|
||||
GCSys string `json:"gcSys,omitempty"`
|
||||
OtherSys string `json:"otherSys,omitempty"`
|
||||
Sys string `json:"sys,omitempty"`
|
||||
// GC & runtime
|
||||
TotalAlloc string `json:"totalAlloc,omitempty"`
|
||||
NumGC uint32 `json:"numGC,omitempty,string"`
|
||||
NumGoroutine int `json:"numGoroutine,omitempty,string"`
|
||||
NextGC string `json:"nextGC,omitempty"`
|
||||
LastGC string `json:"lastGC,omitempty"`
|
||||
Snapshots int `json:"snapshots,omitempty,string"`
|
||||
}
|
||||
|
||||
type oomReporter struct {
|
||||
startedService *daemon.StartedService
|
||||
func OOMRecorderOptions(startedService *daemon.StartedService) oomkiller.RecorderOptions {
|
||||
return oomkiller.RecorderOptions{
|
||||
BasePath: sWorkingPath,
|
||||
Logger: log.StdLogger(),
|
||||
AcceptDraft: acceptOOMDraft,
|
||||
MetadataCallback: func(status oomkiller.ReportStatus) any {
|
||||
metadata := oomReportMetadata{
|
||||
reportMetadata: baseReportMetadata(),
|
||||
RecordedAt: status.RecordedAt.UTC().Format(time.RFC3339),
|
||||
MemoryUsage: byteformats.FormatMemoryBytes(status.PeakMemory),
|
||||
Snapshots: status.Snapshots,
|
||||
}
|
||||
metadata.StartedAt = status.StartedAt.UTC().Format(time.RFC3339)
|
||||
if !status.EndedAt.IsZero() {
|
||||
metadata.EndedAt = status.EndedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if status.MemoryLimit > 0 {
|
||||
metadata.MemoryLimit = byteformats.FormatMemoryBytes(status.MemoryLimit)
|
||||
}
|
||||
if status.AvailableKnown {
|
||||
metadata.AvailableMemory = byteformats.FormatMemoryBytes(status.MinAvailable)
|
||||
}
|
||||
return metadata
|
||||
},
|
||||
OwnerCallback: chownReport,
|
||||
LogCallback: func() []byte {
|
||||
return formatLogEntries(startedService.SavedLog())
|
||||
},
|
||||
SnapshotCallback: func(directory string, prefix string) {
|
||||
for _, name := range oomReportProfiles {
|
||||
writeOOMProfile(filepath.Join(directory, prefix+"."+name+".pb"), name)
|
||||
}
|
||||
runtimeInfoPath := filepath.Join(directory, prefix+".runtime.json")
|
||||
err := runtimeinfo.WriteFile(runtimeInfoPath)
|
||||
if err == nil {
|
||||
chownReport(runtimeInfoPath)
|
||||
}
|
||||
copyConfigSnapshot(directory)
|
||||
content := oomConnectionsContent(startedService)
|
||||
if content != nil {
|
||||
writeReportFile(directory, prefix+".connections.json", content)
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
var _ oomkiller.OOMReporter = (*oomReporter)(nil)
|
||||
|
||||
func NewOOMReporter(startedService *daemon.StartedService) oomkiller.OOMReporter {
|
||||
return &oomReporter{startedService: startedService}
|
||||
}
|
||||
|
||||
func (r *oomReporter) WriteReport(memoryUsage uint64) error {
|
||||
draftPath := filepath.Join(sWorkingPath, "oom_draft")
|
||||
draftInfo, err := os.Stat(draftPath)
|
||||
func acceptOOMDraft(metadataContent []byte) bool {
|
||||
var draftMetadata reportMetadata
|
||||
err := json.Unmarshal(metadataContent, &draftMetadata)
|
||||
if err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
draftInfo = nil
|
||||
return false
|
||||
}
|
||||
reportsDir := filepath.Join(sWorkingPath, "oom_reports")
|
||||
err = os.MkdirAll(reportsDir, 0o777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chownReport(reportsDir)
|
||||
|
||||
destPath, err := nextAvailableReportPath(reportsDir, time.Now().UTC())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = r.writeSnapshot(destPath, memoryUsage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return discardDraftIfCurrent(draftPath, draftInfo)
|
||||
}
|
||||
|
||||
func (r *oomReporter) WriteDraft(memoryUsage uint64) error {
|
||||
draftPath := filepath.Join(sWorkingPath, "oom_draft")
|
||||
os.RemoveAll(draftPath)
|
||||
return r.writeSnapshot(draftPath, memoryUsage)
|
||||
}
|
||||
|
||||
func (r *oomReporter) DiscardDraft() error {
|
||||
draftPath := filepath.Join(sWorkingPath, "oom_draft")
|
||||
return os.RemoveAll(draftPath)
|
||||
}
|
||||
|
||||
func discardDraftIfCurrent(draftPath string, draftInfo os.FileInfo) error {
|
||||
if draftInfo == nil {
|
||||
return nil
|
||||
}
|
||||
currentInfo, err := os.Stat(draftPath)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if !os.SameFile(draftInfo, currentInfo) {
|
||||
return nil
|
||||
}
|
||||
return os.RemoveAll(draftPath)
|
||||
}
|
||||
|
||||
func (r *oomReporter) writeSnapshot(destPath string, memoryUsage uint64) error {
|
||||
now := time.Now().UTC()
|
||||
err := os.MkdirAll(destPath, 0o777)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chownReport(destPath)
|
||||
|
||||
for _, name := range oomReportProfiles {
|
||||
writeOOMProfile(destPath, name)
|
||||
}
|
||||
|
||||
writeReportFile(destPath, "cmdline", []byte(strings.Join(os.Args, "\000")))
|
||||
|
||||
var memStats runtime.MemStats
|
||||
runtime.ReadMemStats(&memStats)
|
||||
|
||||
metadata := oomReportMetadata{
|
||||
reportMetadata: baseReportMetadata(),
|
||||
RecordedAt: now.Format(time.RFC3339),
|
||||
MemoryUsage: byteformats.FormatMemoryBytes(memoryUsage),
|
||||
// Heap
|
||||
HeapAlloc: byteformats.FormatMemoryBytes(memStats.HeapAlloc),
|
||||
HeapObjects: memStats.HeapObjects,
|
||||
HeapInuse: byteformats.FormatMemoryBytes(memStats.HeapInuse),
|
||||
HeapIdle: byteformats.FormatMemoryBytes(memStats.HeapIdle),
|
||||
HeapReleased: byteformats.FormatMemoryBytes(memStats.HeapReleased),
|
||||
HeapSys: byteformats.FormatMemoryBytes(memStats.HeapSys),
|
||||
// Stack
|
||||
StackInuse: byteformats.FormatMemoryBytes(memStats.StackInuse),
|
||||
StackSys: byteformats.FormatMemoryBytes(memStats.StackSys),
|
||||
// Runtime metadata
|
||||
MSpanInuse: byteformats.FormatMemoryBytes(memStats.MSpanInuse),
|
||||
MSpanSys: byteformats.FormatMemoryBytes(memStats.MSpanSys),
|
||||
MCacheSys: byteformats.FormatMemoryBytes(memStats.MCacheSys),
|
||||
BuckHashSys: byteformats.FormatMemoryBytes(memStats.BuckHashSys),
|
||||
GCSys: byteformats.FormatMemoryBytes(memStats.GCSys),
|
||||
OtherSys: byteformats.FormatMemoryBytes(memStats.OtherSys),
|
||||
Sys: byteformats.FormatMemoryBytes(memStats.Sys),
|
||||
// GC & runtime
|
||||
TotalAlloc: byteformats.FormatMemoryBytes(memStats.TotalAlloc),
|
||||
NumGC: memStats.NumGC,
|
||||
NumGoroutine: runtime.NumGoroutine(),
|
||||
NextGC: byteformats.FormatMemoryBytes(memStats.NextGC),
|
||||
}
|
||||
if memStats.LastGC > 0 {
|
||||
metadata.LastGC = time.Unix(0, int64(memStats.LastGC)).UTC().Format(time.RFC3339)
|
||||
}
|
||||
availableMemory := memory.Available()
|
||||
if availableMemory > 0 {
|
||||
metadata.AvailableMemory = byteformats.FormatMemoryBytes(availableMemory)
|
||||
}
|
||||
writeReportMetadata(destPath, metadata)
|
||||
copyConfigSnapshot(destPath)
|
||||
writeOOMLog(destPath, r.startedService.SavedLog())
|
||||
r.writeOOMConnections(destPath)
|
||||
|
||||
return nil
|
||||
return draftMetadata.AppVersion == sAppVersion && draftMetadata.AppMarketingVersion == sAppMarketingVersion
|
||||
}
|
||||
|
||||
type oomConnectionsInfo struct {
|
||||
@@ -213,14 +119,14 @@ type oomConnectionInfo struct {
|
||||
Download string `json:"download,omitempty"`
|
||||
}
|
||||
|
||||
func (r *oomReporter) writeOOMConnections(destPath string) {
|
||||
instance := r.startedService.Instance()
|
||||
func oomConnectionsContent(startedService *daemon.StartedService) []byte {
|
||||
instance := startedService.Instance()
|
||||
if instance == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
trafficManager := instance.TrafficManager()
|
||||
if trafficManager == nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
connections := trafficManager.Connections()
|
||||
sort.Slice(connections, func(i, j int) bool {
|
||||
@@ -235,9 +141,9 @@ func (r *oomReporter) writeOOMConnections(destPath string) {
|
||||
}
|
||||
data, err := json.MarshalIndent(info, "", " ")
|
||||
if err != nil {
|
||||
return
|
||||
return nil
|
||||
}
|
||||
writeReportFile(destPath, "connections.json", data)
|
||||
return data
|
||||
}
|
||||
|
||||
func buildOOMConnections(connections []*trafficcontrol.TrackerMetadata) []oomConnectionInfo {
|
||||
@@ -314,14 +220,6 @@ func formatLogEntries(entries []*log.Entry) []byte {
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
func writeOOMLog(destPath string, entries []*log.Entry) {
|
||||
content := formatLogEntries(entries)
|
||||
if content == nil {
|
||||
return
|
||||
}
|
||||
writeReportFile(destPath, "go.log", content)
|
||||
}
|
||||
|
||||
func writeWithoutColors(buffer *bytes.Buffer, message string) {
|
||||
start := 0
|
||||
for index := 0; index < len(message); {
|
||||
@@ -343,54 +241,18 @@ func writeWithoutColors(buffer *bytes.Buffer, message string) {
|
||||
buffer.WriteString(message[start:])
|
||||
}
|
||||
|
||||
func writeOOMProfile(destPath string, name string) {
|
||||
filePath, err := oomprofile.WriteFile(destPath, name)
|
||||
func writeOOMProfile(filePath string, name string) {
|
||||
err := oomprofile.WriteFile(filePath, name)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
chownReport(filePath)
|
||||
}
|
||||
|
||||
func promoteOOMDraftAt(workingPath string) {
|
||||
draftPath := filepath.Join(workingPath, "oom_draft")
|
||||
info, err := os.Stat(draftPath)
|
||||
if err != nil || !info.IsDir() {
|
||||
return
|
||||
}
|
||||
metadataContent, err := os.ReadFile(filepath.Join(draftPath, "metadata.json"))
|
||||
if err != nil {
|
||||
os.RemoveAll(draftPath)
|
||||
return
|
||||
}
|
||||
var draftMetadata reportMetadata
|
||||
err = json.Unmarshal(metadataContent, &draftMetadata)
|
||||
if err != nil || draftMetadata.AppVersion != sAppVersion || draftMetadata.AppMarketingVersion != sAppMarketingVersion {
|
||||
os.RemoveAll(draftPath)
|
||||
return
|
||||
}
|
||||
reportsDir := filepath.Join(workingPath, "oom_reports")
|
||||
initReportDir(reportsDir)
|
||||
destPath, err := nextAvailableReportPath(reportsDir, info.ModTime().UTC())
|
||||
if err != nil {
|
||||
os.RemoveAll(draftPath)
|
||||
return
|
||||
}
|
||||
err = os.Rename(draftPath, destPath)
|
||||
if err != nil {
|
||||
os.RemoveAll(draftPath)
|
||||
return
|
||||
}
|
||||
chownReport(destPath)
|
||||
}
|
||||
|
||||
func promoteOOMDraft() {
|
||||
promoteOOMDraftAt(sWorkingPath)
|
||||
}
|
||||
|
||||
func PromoteOOMDraft() {
|
||||
promoteOOMDraft()
|
||||
oomkiller.PromoteDraft(sWorkingPath, acceptOOMDraft)
|
||||
}
|
||||
|
||||
func PromoteOOMDraftAt(workingPath string) {
|
||||
promoteOOMDraftAt(workingPath)
|
||||
oomkiller.PromoteDraft(workingPath, acceptOOMDraft)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
package libbox
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/sagernet/sing-box/daemon"
|
||||
@@ -29,7 +30,7 @@ func PowerReportOptions(startedService *daemon.StartedService) powerreport.Optio
|
||||
},
|
||||
ProfileCallback: func(path string) {
|
||||
for _, name := range oomReportProfiles {
|
||||
writeOOMProfile(path, name)
|
||||
writeOOMProfile(filepath.Join(path, name+".pb"), name)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@@ -92,6 +92,7 @@ func ReloadSetupOptions(options *SetupOptions) {
|
||||
if sOOMKillerEnabled {
|
||||
if sOOMMemoryLimit == 0 && C.IsIos {
|
||||
sOOMMemoryLimit = oomkiller.DefaultAppleNetworkExtensionMemoryLimit
|
||||
debug.SetGCPercent(oomkiller.DefaultAppleNetworkExtensionGCPercent)
|
||||
}
|
||||
if sOOMMemoryLimit > 0 {
|
||||
debug.SetMemoryLimit(sOOMMemoryLimit * 4 / 5)
|
||||
|
||||
Reference in New Issue
Block a user