Record reloads in reports and dump heap on reset

daemon/instance.go

@@ -82,7 +82,7 @@ type OverrideOptions struct {
 	ExcludePackage []string
 }
 
-func (s *StartedService) newInstance(ctx context.Context, profileContent string, overrideOptions *OverrideOptions) (*Instance, error) {
+func (s *StartedService) newInstance(ctx context.Context, profileContent string, overrideOptions *OverrideOptions, reloading bool) (*Instance, error) {
 	selectedLocale := locale.FromContext(ctx)
 	ctx, _ = locale.ContextWithLocale(s.ctx, selectedLocale.Locale)
 	ctx = service.ExtendContext(ctx)
@@ -138,7 +138,7 @@ func (s *StartedService) newInstance(ctx context.Context, profileContent string,
 	i.clashMode = service.PtrFromContext[clashmode.Manager](ctx)
 	i.trafficManager = service.PtrFromContext[trafficcontrol.Manager](ctx)
 	i.pauseManager = service.FromContext[pause.Manager](ctx)
-	i.registerPowerReport(ctx)
+	i.registerPowerReport(ctx, reloading)
 	i.cacheFile = service.FromContext[adapter.CacheFile](ctx)
 	i.outboundManager = service.FromContext[adapter.OutboundManager](ctx)
 	i.endpointManager = service.FromContext[adapter.EndpointManager](ctx)
@@ -176,14 +176,14 @@ func (i *Instance) Close() error {
 	return i.instance.Close()
 }
 
-func (i *Instance) registerPowerReport(ctx context.Context) {
+func (i *Instance) registerPowerReport(ctx context.Context, reloading bool) {
 	powerManager := service.FromContext[*powerreport.Manager](ctx)
 	if powerManager == nil {
 		return
 	}
 	recorder := powerManager.Recorder()
 	if recorder != nil {
-		recorder.RecordServiceStart(i.instance.CreatedAt())
+		recorder.RecordServiceStart(i.instance.CreatedAt(), reloading)
 	}
 	if i.pauseManager == nil {
 		return

daemon/started_service.go

@@ -20,6 +20,7 @@ import (
 	"github.com/sagernet/sing-box/experimental/locale"
 	"github.com/sagernet/sing-box/log"
 	"github.com/sagernet/sing-box/protocol/group"
+	"github.com/sagernet/sing-box/service/oomkiller"
 	"github.com/sagernet/sing/common"
 	"github.com/sagernet/sing/common/memory"
 	"github.com/sagernet/sing/common/observable"
@@ -259,15 +260,22 @@ func (s *StartedService) StartOrReloadService(ctx context.Context, profileConten
 		s.instance = nil
 		s.updateStatus(ServiceStatus_STOPPING)
 		s.serviceAccess.Unlock()
+		oomRecorder := service.FromContext[*oomkiller.Recorder](s.ctx)
+		if oomRecorder != nil {
+			oomRecorder.BeginReload()
+			defer oomRecorder.EndReload()
+		}
 		_ = oldInstance.Close()
 		runtimeDebug.FreeOSMemory()
 		s.serviceAccess.Lock()
 	}
 	s.startInterrupted = false
 	s.updateStatus(ServiceStatus_STARTING)
-	s.resetLogs()
+	if oldInstance == nil {
+		s.resetLogs()
+	}
 	s.serviceAccess.Unlock()
-	instance, err := s.newInstance(ctx, profileContent, options)
+	instance, err := s.newInstance(ctx, profileContent, options, oldInstance != nil)
 	if err != nil {
 		s.serviceAccess.Lock()
 		s.updateStatusError(err)

service/oomkiller/recorder.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
+	runtimeDebug "runtime/debug"
 	"runtime/metrics"
 	"strconv"
 	"sync"
@@ -23,6 +24,7 @@ const (
 	eventsFileName   = "events.jsonl"
 	metadataFileName = "metadata.json"
 	logFileName      = "go.log"
+	heapDumpFileName = "heap.dump"
 	lockFileName     = ".lock"
 
 	normalSampleInterval   = time.Minute
@@ -44,6 +46,7 @@ const (
 
 	eventTypeStart    = "start"
 	eventTypeStop     = "stop"
+	eventTypeReload   = "reload"
 	eventTypePressure = "pressure"
 	eventTypeState    = "state"
 	eventTypeReset    = "reset"
@@ -86,6 +89,8 @@ type Recorder struct {
 	closed           bool
 	draftCreated     bool
 	draftLock        *os.File
+	reloading        bool
+	carriedState     *timerState
 	status           ReportStatus
 	lastRowAt        time.Time
 	lastRowState     pressureState
@@ -131,6 +136,7 @@ type eventRecord struct {
 	Sequence         int           `json:"seq,omitempty"`
 	Prefix           string        `json:"prefix,omitempty"`
 	Runtime          *runtimeStats `json:"runtime,omitempty"`
+	HeapDump         bool          `json:"heapDump,omitempty"`
 }
 
 type runtimeStats struct {
@@ -217,7 +223,7 @@ func (r *Recorder) WriteReport() error {
 		sample.availableKnown = true
 		sample.available = memory.Available()
 	}
-	err := r.snapshot(SnapshotReasonManual, sample, true)
+	err := r.snapshot(SnapshotReasonManual, sample, false, true)
 	if err != nil {
 		return E.Cause(err, "write snapshot")
 	}
@@ -243,12 +249,41 @@ func (r *Recorder) WriteReport() error {
 	return nil
 }
 
-func (r *Recorder) instanceStarted(config timerConfig, thresholds pressureThresholds) {
+func (r *Recorder) BeginReload() {
+	r.access.Lock()
+	defer r.access.Unlock()
+	r.reloading = true
+}
+
+func (r *Recorder) EndReload() {
+	r.access.Lock()
+	defer r.access.Unlock()
+	if !r.reloading {
+		return
+	}
+	r.reloading = false
+	if r.carriedState != nil {
+		r.carriedState = nil
+		r.appendEventLocked(eventRecord{Type: eventTypeStop})
+	}
+}
+
+func (r *Recorder) instanceStarted(config timerConfig, thresholds pressureThresholds) *timerState {
 	r.access.Lock()
 	defer r.access.Unlock()
 	r.status.MemoryLimit = config.memoryLimit
+	eventType := eventTypeStart
+	var carriedState *timerState
+	if r.reloading {
+		r.reloading = false
+		carriedState = r.carriedState
+		r.carriedState = nil
+		if carriedState != nil {
+			eventType = eventTypeReload
+		}
+	}
 	r.appendEventLocked(eventRecord{
-		Type:         eventTypeStart,
+		Type:         eventType,
 		Policy:       config.policyMode.String(),
 		MemoryLimit:  config.memoryLimit,
 		TriggerBytes: thresholds.trigger,
@@ -256,11 +291,16 @@ func (r *Recorder) instanceStarted(config timerConfig, thresholds pressureThresh
 		ResumeBytes:  thresholds.resume,
 		ReportOnly:   config.killerDisabled,
 	})
+	return carriedState
 }
 
-func (r *Recorder) instanceStopped() {
+func (r *Recorder) instanceStopped(state timerState) {
 	r.access.Lock()
 	defer r.access.Unlock()
+	if r.reloading {
+		r.carriedState = &state
+		return
+	}
 	r.appendEventLocked(eventRecord{Type: eventTypeStop})
 }
 
@@ -347,7 +387,7 @@ func (r *Recorder) recordReset(reason string, before memorySample, after memoryS
 	})
 }
 
-func (r *Recorder) snapshot(reason string, sample memorySample, force bool) error {
+func (r *Recorder) snapshot(reason string, sample memorySample, heapDump bool, force bool) error {
 	r.snapshotAccess.Lock()
 	defer r.snapshotAccess.Unlock()
 	now := time.Now()
@@ -387,6 +427,10 @@ func (r *Recorder) snapshot(reason string, sample memorySample, force bool) erro
 	if r.snapshotCallback != nil {
 		r.snapshotCallback(r.draftPath, prefix)
 	}
+	var heapDumped bool
+	if heapDump {
+		heapDumped = r.writeHeapDump()
+	}
 	var memStats runtime.MemStats
 	runtime.ReadMemStats(&memStats)
 	stats := &runtimeStats{
@@ -420,10 +464,33 @@ func (r *Recorder) snapshot(reason string, sample memorySample, force bool) erro
 		MemoryBytes:    sample.usage,
 		AvailableBytes: sample.available,
 		Runtime:        stats,
+		HeapDump:       heapDumped,
 	})
 	return nil
 }
 
+func (r *Recorder) writeHeapDump() bool {
+	dumpPath := filepath.Join(r.draftPath, heapDumpFileName)
+	tempPath := dumpPath + ".tmp"
+	file, err := os.Create(tempPath)
+	if err != nil {
+		r.logger.Error(E.Cause(err, "OOM report: create heap dump"))
+		return false
+	}
+	runtimeDebug.WriteHeapDump(file.Fd())
+	err = file.Close()
+	if err == nil {
+		err = os.Rename(tempPath, dumpPath)
+	}
+	if err != nil {
+		os.Remove(tempPath)
+		r.logger.Error(E.Cause(err, "OOM report: write heap dump"))
+		return false
+	}
+	r.chown(dumpPath)
+	return true
+}
+
 func (r *Recorder) observeLocked(sample memorySample) {
 	if sample.usage > r.status.PeakMemory {
 		r.status.PeakMemory = sample.usage

service/oomkiller/service.go

@@ -49,18 +49,20 @@ func (s *Service) startTimer() error {
 		return E.New("memory pressure monitoring is not available on this platform without memory_limit")
 	}
 	s.adaptiveTimer = newAdaptiveTimer(s.logger, s.network, s.connections, s.recorder, s.timerConfig)
+	var carriedState *timerState
 	if s.recorder != nil {
-		s.recorder.instanceStarted(s.timerConfig, s.adaptiveTimer.limitThresholds)
+		carriedState = s.recorder.instanceStarted(s.timerConfig, s.adaptiveTimer.limitThresholds)
 	}
-	s.adaptiveTimer.start()
+	s.adaptiveTimer.start(carriedState)
 	return nil
 }
 
 func (s *Service) stopTimer() {
+	var state timerState
 	if s.adaptiveTimer != nil {
-		s.adaptiveTimer.stop()
+		state = s.adaptiveTimer.stop()
 	}
 	if s.recorder != nil {
-		s.recorder.instanceStopped()
+		s.recorder.instanceStopped(state)
 	}
 }

service/oomkiller/service_darwin.go

@@ -100,7 +100,7 @@ func goMemoryPressureCallback(status C.ulong) {
 		}
 		s.adaptiveTimer.notifyPressure()
 		if s.recorder != nil {
-			s.recorder.snapshot(SnapshotReasonPressure, sample, false)
+			s.recorder.snapshot(SnapshotReasonPressure, sample, false, false)
 		}
 	}
 }

service/oomkiller/timer.go

@@ -98,6 +98,15 @@ func buildTimerConfig(options option.OOMKillerServiceOptions, memoryLimit uint64
 	}, nil
 }
 
+type timerState struct {
+	state                   pressureState
+	currentInterval         time.Duration
+	forceMinInterval        bool
+	pendingPressureBaseline bool
+	pressureBaseline        memorySample
+	pressureBaselineTime    time.Time
+}
+
 type adaptiveTimer struct {
 	timerConfig
 	logger          log.ContextLogger
@@ -106,14 +115,9 @@ type adaptiveTimer struct {
 	recorder        *Recorder
 	limitThresholds pressureThresholds
 
-	access                  sync.Mutex
-	timer                   *time.Timer
-	state                   pressureState
-	currentInterval         time.Duration
-	forceMinInterval        bool
-	pendingPressureBaseline bool
-	pressureBaseline        memorySample
-	pressureBaselineTime    time.Time
+	access sync.Mutex
+	timer  *time.Timer
+	timerState
 }
 
 func newAdaptiveTimer(logger log.ContextLogger, network adapter.NetworkManager, connections adapter.ConnectionManager, recorder *Recorder, config timerConfig) *adaptiveTimer {
@@ -130,9 +134,17 @@ func newAdaptiveTimer(logger log.ContextLogger, network adapter.NetworkManager,
 	return t
 }
 
-func (t *adaptiveTimer) start() {
+func (t *adaptiveTimer) start(carriedState *timerState) {
 	t.access.Lock()
 	defer t.access.Unlock()
+	if t.timer != nil {
+		return
+	}
+	if carriedState != nil {
+		t.timerState = *carriedState
+		t.timer = time.AfterFunc(t.minInterval, t.poll)
+		return
+	}
 	t.startLocked()
 }
 
@@ -145,13 +157,14 @@ func (t *adaptiveTimer) startLocked() {
 	t.timer = time.AfterFunc(t.minInterval, t.poll)
 }
 
-func (t *adaptiveTimer) stop() {
+func (t *adaptiveTimer) stop() timerState {
 	t.access.Lock()
 	defer t.access.Unlock()
 	if t.timer != nil {
 		t.timer.Stop()
 		t.timer = nil
 	}
+	return t.timerState
 }
 
 func (t *adaptiveTimer) poll() {
@@ -230,8 +243,20 @@ func (t *adaptiveTimer) poll() {
 	badCleanup()
 	runtimeDebug.FreeOSMemory()
 	if t.recorder != nil {
-		t.recorder.recordReset(reason, sample, readMemorySample(t.policyMode), connections, t.killerDisabled)
-		t.recorder.snapshot(SnapshotReasonReset, sample, false)
+		after := readMemorySample(t.policyMode)
+		t.recorder.recordReset(reason, sample, after, connections, t.killerDisabled)
+		t.recorder.snapshot(SnapshotReasonReset, sample, t.belowTrigger(after), false)
+	}
+}
+
+func (t *adaptiveTimer) belowTrigger(sample memorySample) bool {
+	switch t.policyMode {
+	case policyModeMemoryLimit, policyModeNetworkExtension:
+		return sample.usage < t.limitThresholds.trigger
+	case policyModeAvailable:
+		return !sample.availableKnown || sample.available > t.availableThresholds(sample).trigger
+	default:
+		return true
 	}
 }
 

service/powerreport/record.go

@@ -132,6 +132,7 @@ type eventRecord struct {
 	Device        *deviceState `json:"device,omitempty"`
 	Window        *wakeWindow  `json:"window,omitempty"`
 	LogBaseMonoMS int64        `json:"logBaseMonoMS,omitempty"`
+	Reload        bool         `json:"reload,omitempty"`
 }
 
 type wakeWindow struct {

service/powerreport/recorder.go

@@ -348,7 +348,7 @@ func (r *Recorder) CountConnectionOpened() {
 
 // The service log prefixes each line with whole monotonic seconds since createdAt, so a
 // line at [N] sits at logBaseMonoMS + N*1000 on the monoMS axis of events and timeline rows.
-func (r *Recorder) RecordServiceStart(createdAt time.Time) {
+func (r *Recorder) RecordServiceStart(createdAt time.Time, reload bool) {
 	now := time.Now()
 	r.access.Lock()
 	if !r.started || r.closed {
@@ -357,6 +357,7 @@ func (r *Recorder) RecordServiceStart(createdAt time.Time) {
 	}
 	event := r.newEvent(eventTypeService, now)
 	event.LogBaseMonoMS = int64(createdAt.Sub(r.baseTime) / time.Millisecond)
+	event.Reload = reload
 	r.events = append(r.events, event)
 	r.access.Unlock()
 	r.notifyWorker()