From 351ce12f7398a7995af5f6451e65039ff4902169 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=96=E7=95=8C?= Date: Sat, 5 Sep 2026 16:53:14 +0800 Subject: [PATCH] Record reloads in reports and dump heap on reset --- daemon/instance.go | 8 +-- daemon/started_service.go | 12 ++++- service/oomkiller/recorder.go | 77 +++++++++++++++++++++++++++-- service/oomkiller/service.go | 10 ++-- service/oomkiller/service_darwin.go | 2 +- service/oomkiller/timer.go | 49 +++++++++++++----- service/powerreport/record.go | 1 + service/powerreport/recorder.go | 3 +- 8 files changed, 133 insertions(+), 29 deletions(-) diff --git a/daemon/instance.go b/daemon/instance.go index e60701dc..bb6a3403 100644 --- a/daemon/instance.go +++ b/daemon/instance.go @@ -82,7 +82,7 @@ type OverrideOptions struct { ExcludePackage []string } -func (s *StartedService) newInstance(ctx context.Context, profileContent string, overrideOptions *OverrideOptions) (*Instance, error) { +func (s *StartedService) newInstance(ctx context.Context, profileContent string, overrideOptions *OverrideOptions, reloading bool) (*Instance, error) { selectedLocale := locale.FromContext(ctx) ctx, _ = locale.ContextWithLocale(s.ctx, selectedLocale.Locale) ctx = service.ExtendContext(ctx) @@ -138,7 +138,7 @@ func (s *StartedService) newInstance(ctx context.Context, profileContent string, i.clashMode = service.PtrFromContext[clashmode.Manager](ctx) i.trafficManager = service.PtrFromContext[trafficcontrol.Manager](ctx) i.pauseManager = service.FromContext[pause.Manager](ctx) - i.registerPowerReport(ctx) + i.registerPowerReport(ctx, reloading) i.cacheFile = service.FromContext[adapter.CacheFile](ctx) i.outboundManager = service.FromContext[adapter.OutboundManager](ctx) i.endpointManager = service.FromContext[adapter.EndpointManager](ctx) @@ -176,14 +176,14 @@ func (i *Instance) Close() error { return i.instance.Close() } -func (i *Instance) registerPowerReport(ctx context.Context) { +func (i *Instance) registerPowerReport(ctx context.Context, reloading bool) { powerManager := service.FromContext[*powerreport.Manager](ctx) if powerManager == nil { return } recorder := powerManager.Recorder() if recorder != nil { - recorder.RecordServiceStart(i.instance.CreatedAt()) + recorder.RecordServiceStart(i.instance.CreatedAt(), reloading) } if i.pauseManager == nil { return diff --git a/daemon/started_service.go b/daemon/started_service.go index 5e8c7998..f1fb0f0d 100644 --- a/daemon/started_service.go +++ b/daemon/started_service.go @@ -20,6 +20,7 @@ import ( "github.com/sagernet/sing-box/experimental/locale" "github.com/sagernet/sing-box/log" "github.com/sagernet/sing-box/protocol/group" + "github.com/sagernet/sing-box/service/oomkiller" "github.com/sagernet/sing/common" "github.com/sagernet/sing/common/memory" "github.com/sagernet/sing/common/observable" @@ -259,15 +260,22 @@ func (s *StartedService) StartOrReloadService(ctx context.Context, profileConten s.instance = nil s.updateStatus(ServiceStatus_STOPPING) s.serviceAccess.Unlock() + oomRecorder := service.FromContext[*oomkiller.Recorder](s.ctx) + if oomRecorder != nil { + oomRecorder.BeginReload() + defer oomRecorder.EndReload() + } _ = oldInstance.Close() runtimeDebug.FreeOSMemory() s.serviceAccess.Lock() } s.startInterrupted = false s.updateStatus(ServiceStatus_STARTING) - s.resetLogs() + if oldInstance == nil { + s.resetLogs() + } s.serviceAccess.Unlock() - instance, err := s.newInstance(ctx, profileContent, options) + instance, err := s.newInstance(ctx, profileContent, options, oldInstance != nil) if err != nil { s.serviceAccess.Lock() s.updateStatusError(err) diff --git a/service/oomkiller/recorder.go b/service/oomkiller/recorder.go index 1ec6b4fa..23d0494f 100644 --- a/service/oomkiller/recorder.go +++ b/service/oomkiller/recorder.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "runtime" + runtimeDebug "runtime/debug" "runtime/metrics" "strconv" "sync" @@ -23,6 +24,7 @@ const ( eventsFileName = "events.jsonl" metadataFileName = "metadata.json" logFileName = "go.log" + heapDumpFileName = "heap.dump" lockFileName = ".lock" normalSampleInterval = time.Minute @@ -44,6 +46,7 @@ const ( eventTypeStart = "start" eventTypeStop = "stop" + eventTypeReload = "reload" eventTypePressure = "pressure" eventTypeState = "state" eventTypeReset = "reset" @@ -86,6 +89,8 @@ type Recorder struct { closed bool draftCreated bool draftLock *os.File + reloading bool + carriedState *timerState status ReportStatus lastRowAt time.Time lastRowState pressureState @@ -131,6 +136,7 @@ type eventRecord struct { Sequence int `json:"seq,omitempty"` Prefix string `json:"prefix,omitempty"` Runtime *runtimeStats `json:"runtime,omitempty"` + HeapDump bool `json:"heapDump,omitempty"` } type runtimeStats struct { @@ -217,7 +223,7 @@ func (r *Recorder) WriteReport() error { sample.availableKnown = true sample.available = memory.Available() } - err := r.snapshot(SnapshotReasonManual, sample, true) + err := r.snapshot(SnapshotReasonManual, sample, false, true) if err != nil { return E.Cause(err, "write snapshot") } @@ -243,12 +249,41 @@ func (r *Recorder) WriteReport() error { return nil } -func (r *Recorder) instanceStarted(config timerConfig, thresholds pressureThresholds) { +func (r *Recorder) BeginReload() { + r.access.Lock() + defer r.access.Unlock() + r.reloading = true +} + +func (r *Recorder) EndReload() { + r.access.Lock() + defer r.access.Unlock() + if !r.reloading { + return + } + r.reloading = false + if r.carriedState != nil { + r.carriedState = nil + r.appendEventLocked(eventRecord{Type: eventTypeStop}) + } +} + +func (r *Recorder) instanceStarted(config timerConfig, thresholds pressureThresholds) *timerState { r.access.Lock() defer r.access.Unlock() r.status.MemoryLimit = config.memoryLimit + eventType := eventTypeStart + var carriedState *timerState + if r.reloading { + r.reloading = false + carriedState = r.carriedState + r.carriedState = nil + if carriedState != nil { + eventType = eventTypeReload + } + } r.appendEventLocked(eventRecord{ - Type: eventTypeStart, + Type: eventType, Policy: config.policyMode.String(), MemoryLimit: config.memoryLimit, TriggerBytes: thresholds.trigger, @@ -256,11 +291,16 @@ func (r *Recorder) instanceStarted(config timerConfig, thresholds pressureThresh ResumeBytes: thresholds.resume, ReportOnly: config.killerDisabled, }) + return carriedState } -func (r *Recorder) instanceStopped() { +func (r *Recorder) instanceStopped(state timerState) { r.access.Lock() defer r.access.Unlock() + if r.reloading { + r.carriedState = &state + return + } r.appendEventLocked(eventRecord{Type: eventTypeStop}) } @@ -347,7 +387,7 @@ func (r *Recorder) recordReset(reason string, before memorySample, after memoryS }) } -func (r *Recorder) snapshot(reason string, sample memorySample, force bool) error { +func (r *Recorder) snapshot(reason string, sample memorySample, heapDump bool, force bool) error { r.snapshotAccess.Lock() defer r.snapshotAccess.Unlock() now := time.Now() @@ -387,6 +427,10 @@ func (r *Recorder) snapshot(reason string, sample memorySample, force bool) erro if r.snapshotCallback != nil { r.snapshotCallback(r.draftPath, prefix) } + var heapDumped bool + if heapDump { + heapDumped = r.writeHeapDump() + } var memStats runtime.MemStats runtime.ReadMemStats(&memStats) stats := &runtimeStats{ @@ -420,10 +464,33 @@ func (r *Recorder) snapshot(reason string, sample memorySample, force bool) erro MemoryBytes: sample.usage, AvailableBytes: sample.available, Runtime: stats, + HeapDump: heapDumped, }) return nil } +func (r *Recorder) writeHeapDump() bool { + dumpPath := filepath.Join(r.draftPath, heapDumpFileName) + tempPath := dumpPath + ".tmp" + file, err := os.Create(tempPath) + if err != nil { + r.logger.Error(E.Cause(err, "OOM report: create heap dump")) + return false + } + runtimeDebug.WriteHeapDump(file.Fd()) + err = file.Close() + if err == nil { + err = os.Rename(tempPath, dumpPath) + } + if err != nil { + os.Remove(tempPath) + r.logger.Error(E.Cause(err, "OOM report: write heap dump")) + return false + } + r.chown(dumpPath) + return true +} + func (r *Recorder) observeLocked(sample memorySample) { if sample.usage > r.status.PeakMemory { r.status.PeakMemory = sample.usage diff --git a/service/oomkiller/service.go b/service/oomkiller/service.go index 7ce51616..b46d7dfa 100644 --- a/service/oomkiller/service.go +++ b/service/oomkiller/service.go @@ -49,18 +49,20 @@ func (s *Service) startTimer() error { return E.New("memory pressure monitoring is not available on this platform without memory_limit") } s.adaptiveTimer = newAdaptiveTimer(s.logger, s.network, s.connections, s.recorder, s.timerConfig) + var carriedState *timerState if s.recorder != nil { - s.recorder.instanceStarted(s.timerConfig, s.adaptiveTimer.limitThresholds) + carriedState = s.recorder.instanceStarted(s.timerConfig, s.adaptiveTimer.limitThresholds) } - s.adaptiveTimer.start() + s.adaptiveTimer.start(carriedState) return nil } func (s *Service) stopTimer() { + var state timerState if s.adaptiveTimer != nil { - s.adaptiveTimer.stop() + state = s.adaptiveTimer.stop() } if s.recorder != nil { - s.recorder.instanceStopped() + s.recorder.instanceStopped(state) } } diff --git a/service/oomkiller/service_darwin.go b/service/oomkiller/service_darwin.go index a3814cde..a0a07486 100644 --- a/service/oomkiller/service_darwin.go +++ b/service/oomkiller/service_darwin.go @@ -100,7 +100,7 @@ func goMemoryPressureCallback(status C.ulong) { } s.adaptiveTimer.notifyPressure() if s.recorder != nil { - s.recorder.snapshot(SnapshotReasonPressure, sample, false) + s.recorder.snapshot(SnapshotReasonPressure, sample, false, false) } } } diff --git a/service/oomkiller/timer.go b/service/oomkiller/timer.go index 5dfdd7b3..a49bc4b9 100644 --- a/service/oomkiller/timer.go +++ b/service/oomkiller/timer.go @@ -98,6 +98,15 @@ func buildTimerConfig(options option.OOMKillerServiceOptions, memoryLimit uint64 }, nil } +type timerState struct { + state pressureState + currentInterval time.Duration + forceMinInterval bool + pendingPressureBaseline bool + pressureBaseline memorySample + pressureBaselineTime time.Time +} + type adaptiveTimer struct { timerConfig logger log.ContextLogger @@ -106,14 +115,9 @@ type adaptiveTimer struct { recorder *Recorder limitThresholds pressureThresholds - access sync.Mutex - timer *time.Timer - state pressureState - currentInterval time.Duration - forceMinInterval bool - pendingPressureBaseline bool - pressureBaseline memorySample - pressureBaselineTime time.Time + access sync.Mutex + timer *time.Timer + timerState } func newAdaptiveTimer(logger log.ContextLogger, network adapter.NetworkManager, connections adapter.ConnectionManager, recorder *Recorder, config timerConfig) *adaptiveTimer { @@ -130,9 +134,17 @@ func newAdaptiveTimer(logger log.ContextLogger, network adapter.NetworkManager, return t } -func (t *adaptiveTimer) start() { +func (t *adaptiveTimer) start(carriedState *timerState) { t.access.Lock() defer t.access.Unlock() + if t.timer != nil { + return + } + if carriedState != nil { + t.timerState = *carriedState + t.timer = time.AfterFunc(t.minInterval, t.poll) + return + } t.startLocked() } @@ -145,13 +157,14 @@ func (t *adaptiveTimer) startLocked() { t.timer = time.AfterFunc(t.minInterval, t.poll) } -func (t *adaptiveTimer) stop() { +func (t *adaptiveTimer) stop() timerState { t.access.Lock() defer t.access.Unlock() if t.timer != nil { t.timer.Stop() t.timer = nil } + return t.timerState } func (t *adaptiveTimer) poll() { @@ -230,8 +243,20 @@ func (t *adaptiveTimer) poll() { badCleanup() runtimeDebug.FreeOSMemory() if t.recorder != nil { - t.recorder.recordReset(reason, sample, readMemorySample(t.policyMode), connections, t.killerDisabled) - t.recorder.snapshot(SnapshotReasonReset, sample, false) + after := readMemorySample(t.policyMode) + t.recorder.recordReset(reason, sample, after, connections, t.killerDisabled) + t.recorder.snapshot(SnapshotReasonReset, sample, t.belowTrigger(after), false) + } +} + +func (t *adaptiveTimer) belowTrigger(sample memorySample) bool { + switch t.policyMode { + case policyModeMemoryLimit, policyModeNetworkExtension: + return sample.usage < t.limitThresholds.trigger + case policyModeAvailable: + return !sample.availableKnown || sample.available > t.availableThresholds(sample).trigger + default: + return true } } diff --git a/service/powerreport/record.go b/service/powerreport/record.go index d8e555ef..4dd5f939 100644 --- a/service/powerreport/record.go +++ b/service/powerreport/record.go @@ -132,6 +132,7 @@ type eventRecord struct { Device *deviceState `json:"device,omitempty"` Window *wakeWindow `json:"window,omitempty"` LogBaseMonoMS int64 `json:"logBaseMonoMS,omitempty"` + Reload bool `json:"reload,omitempty"` } type wakeWindow struct { diff --git a/service/powerreport/recorder.go b/service/powerreport/recorder.go index 782fbcf1..06bbdaac 100644 --- a/service/powerreport/recorder.go +++ b/service/powerreport/recorder.go @@ -348,7 +348,7 @@ func (r *Recorder) CountConnectionOpened() { // The service log prefixes each line with whole monotonic seconds since createdAt, so a // line at [N] sits at logBaseMonoMS + N*1000 on the monoMS axis of events and timeline rows. -func (r *Recorder) RecordServiceStart(createdAt time.Time) { +func (r *Recorder) RecordServiceStart(createdAt time.Time, reload bool) { now := time.Now() r.access.Lock() if !r.started || r.closed { @@ -357,6 +357,7 @@ func (r *Recorder) RecordServiceStart(createdAt time.Time) { } event := r.newEvent(eventTypeService, now) event.LogBaseMonoMS = int64(createdAt.Sub(r.baseTime) / time.Millisecond) + event.Reload = reload r.events = append(r.events, event) r.access.Unlock() r.notifyWorker()