Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions src/Wisp.App/App.xaml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,9 @@ protected override async void OnStartup(StartupEventArgs e)
}

_runExitMarker = RunExitMarkerStore.BeginCurrent(_runId);
// Windows delivers its shutdown notice to this window on its own thread, so the close request
// is recorded even while the dashboard thread is busy.
_sessionEndSentinel = SessionEndSentinel.TryStart(() =>
{
if (Volatile.Read(ref _fatalExit) == 0) _runExitMarker?.MarkShutdownStarted(ShutdownIntent.OperatingSystem);
}, () => _runExitMarker?.ClearShutdownIntent(ShutdownIntent.OperatingSystem));
// Windows delivers its shutdown, sleep and lock notices to this window on its own thread, so they
// are recorded even while the dashboard thread is busy.
_sessionEndSentinel = SessionEndSentinel.TryStart(RecordSessionNotice);
// The native handler records the faulting address and call stack of a hardware fault
// for this run; the next run reports it with the crash.
NativeCrashCapture.TryInstall(_crashReports.StoreDirectory, _runId);
Expand Down Expand Up @@ -350,6 +347,23 @@ private async Task RecoverPreviousRunAsync(RunExitMarker previous, DateTimeOffse
catch (Exception) { } // Diagnostics and shutdown races must not interrupt startup.
}

// The run marker keeps what Windows said last, so the next start can tell a run that ended on a
// sleeping or locked PC from one that stopped without explanation.
private void RecordSessionNotice(SessionNotice notice)
{
switch (notice)
{
case SessionNotice.Ending:
if (Volatile.Read(ref _fatalExit) == 0) _runExitMarker?.MarkShutdownStarted(ShutdownIntent.OperatingSystem);
break;
case SessionNotice.Continuing: _runExitMarker?.ClearShutdownIntent(ShutdownIntent.OperatingSystem); break;
case SessionNotice.Suspending: _runExitMarker?.MarkSuspended(DateTimeOffset.UtcNow); break;
case SessionNotice.Resumed: _runExitMarker?.MarkResumed(); break;
case SessionNotice.Locked: _runExitMarker?.MarkSessionLocked(DateTimeOffset.UtcNow); break;
case SessionNotice.Unlocked: _runExitMarker?.MarkSessionUnlocked(); break;
}
}

protected override void OnSessionEnding(SessionEndingCancelEventArgs e)
{
base.OnSessionEnding(e);
Expand Down
2 changes: 2 additions & 0 deletions src/Wisp.App/CrashDiagnostics/CrashReport.cs
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ internal string Format()
}
else if (exit.WindowsStopped) report.AppendLine("Windows stopped without shutting down (Kernel-Power 41 or EventLog 6008) and restarted within 10 minutes of the previous run's last checkpoint.");
else if (exit.WindowsShutDown) report.AppendLine("Windows shut down or signed out (EventLog 6006 or User32 1074) within minutes of the run's last checkpoint while Wisp was still running.");
else if (exit.Suspended) report.AppendLine("Windows was about to sleep or hibernate when the run was last seen, and the run did not resume. No shutdown, stop or crash event was found.");
else if (exit.SessionLocked) report.AppendLine("The Windows session was locked when the run was last seen. No shutdown, stop or crash event was found.");
else report.AppendLine("The previous run did not finish normal shutdown. The cause is not established; termination or power loss can also leave this marker.");
if (exit.Native is { } native)
{
Expand Down
4 changes: 4 additions & 0 deletions src/Wisp.App/CrashDiagnostics/PreviousRunRecovery.cs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ internal sealed class PreviousRunRecovery(CrashReportStore reports, IWindowsFaul
exit = exit with { WindowsShutDown = shutDown };
}
}
// Nobody closed Wisp on a PC that was asleep, or whose session was locked, when the run was last
// seen, and Windows recorded no stop. Markers from earlier versions carry neither state.
if (exit.Fault is null && exit.Native is null && !exit.WindowsStopped && !exit.WindowsShutDown)
exit = exit with { Suspended = safe.SuspendedAtUtc is not null, SessionLocked = safe.SessionLockedAtUtc is not null };
var report = safe.ToReport(exit.Fault?.TimeUtc ?? nextRunStartedAtUtc) with
{
Origin = exit.Fault is null ? CrashOrigin.UnexpectedExit : CrashOrigin.WindowsApplicationFault,
Expand Down
45 changes: 43 additions & 2 deletions src/Wisp.App/CrashDiagnostics/RunExitMarkerStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ internal sealed record RunExitMarker(Guid RunId, int ProcessId, long ProcessCrea
{
public HealthContextSnapshot? Context { get; init; }
public ShutdownIntent ShutdownIntent { get; init; }
// Windows reported that it was about to sleep or hibernate, and has not reported resuming since.
public DateTimeOffset? SuspendedAtUtc { get; init; }
// The Windows session was locked (Win+L, the lock screen or a user switch) and has not been unlocked since.
public DateTimeOffset? SessionLockedAtUtc { get; init; }

internal RunExitMarker? Sanitize()
{
Expand All @@ -32,10 +36,15 @@ internal sealed record RunExitMarker(Guid RunId, int ProcessId, long ProcessCrea
{
Build = new(report.WispVersion, report.PrivateBuildId, module,
report.RuntimeVersion, report.WindowsVersion, architecture),
Context = ValidContext(Context) ? Context : null
Context = ValidContext(Context) ? Context : null,
SuspendedAtUtc = AfterStart(SuspendedAtUtc) ? SuspendedAtUtc : null,
SessionLockedAtUtc = AfterStart(SessionLockedAtUtc) ? SessionLockedAtUtc : null
};
}

// A moment this run can have observed: UTC, at or after its start.
internal bool AfterStart(DateTimeOffset? value) => value is { } moment && moment.Offset == TimeSpan.Zero && moment >= StartedAtUtc;

internal CrashReport ToReport(DateTimeOffset detectedAtUtc) => new(RunId, detectedAtUtc,
Build.WispVersion, Build.PrivateBuildId, Build.RuntimeVersion, CrashOrigin.UnexpectedExit, true, [])
{
Expand All @@ -60,6 +69,9 @@ private static bool SafeVersion(string? value) => value is { Length: > 0 and <=
internal sealed class RunExitMarkerStore : IDisposable
{
private const int MaximumMarkerBytes = 8192;
// The checkpoint timer does not run while Windows sleeps, so a checkpoint this long after the sleep
// notice means Windows is running again, whether or not the resume notice arrived.
internal static readonly TimeSpan ResumeGrace = TimeSpan.FromMinutes(1);
private readonly string _directory;
private readonly SafeFileHandle _parent, _lease;
private RunExitMarker _current;
Expand Down Expand Up @@ -165,6 +177,34 @@ internal bool ClearShutdownIntent(ShutdownIntent intent)
}
}

// Windows is about to sleep or hibernate. Checkpoints stop with it, so a run that never resumes keeps
// this state for the next start to read; resuming withdraws it.
internal bool MarkSuspended(DateTimeOffset nowUtc) =>
Mark(current => current.AfterStart(nowUtc) ? current with { SuspendedAtUtc = nowUtc } : null);

internal bool MarkResumed() => Mark(current => current.SuspendedAtUtc is null ? null : current with { SuspendedAtUtc = null });

// The Windows session was locked: a run that ends while it stays locked was not closed by the user.
internal bool MarkSessionLocked(DateTimeOffset nowUtc) =>
Mark(current => current.AfterStart(nowUtc) ? current with { SessionLockedAtUtc = nowUtc } : null);

internal bool MarkSessionUnlocked() => Mark(current => current.SessionLockedAtUtc is null ? null : current with { SessionLockedAtUtc = null });

private bool Mark(Func<RunExitMarker, RunExitMarker?> change)
{
lock (_gate)
{
if (_disposed || _current.CleanExit) return false;
try
{
if (change(_current) is not { } next || Read(_parent)?.RunId != _current.RunId) return false;
Write(next); _current = next;
return true;
}
catch (Exception) { return false; }
}
}

internal bool Checkpoint(HealthContextSnapshot snapshot)
{
lock (_gate)
Expand All @@ -174,7 +214,8 @@ internal bool Checkpoint(HealthContextSnapshot snapshot)
sample.TimestampUtc <= snapshot.CapturedAtUtc && snapshot.CapturedAtUtc - sample.TimestampUtc <= TimeSpan.FromSeconds(30))
.OrderBy(sample => sample.TimestampUtc).TakeLast(3).ToImmutableArray();
if (samples.Length == 0 || _current.Context?.CapturedAtUtc >= snapshot.CapturedAtUtc) return false;
var next = _current with { Context = new(snapshot.CapturedAtUtc, samples, []) };
var resumed = _current.SuspendedAtUtc is { } suspended && snapshot.CapturedAtUtc - suspended >= ResumeGrace;
var next = _current with { Context = new(snapshot.CapturedAtUtc, samples, []), SuspendedAtUtc = resumed ? null : _current.SuspendedAtUtc };
try
{
if (Read(_parent)?.RunId != _current.RunId) return false;
Expand Down
62 changes: 50 additions & 12 deletions src/Wisp.App/CrashDiagnostics/SessionEndSentinel.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,36 +2,44 @@

namespace Wisp.App.CrashDiagnostics;

// What Windows told the sentinel window: the session is ending or going on after all, the PC is
// about to sleep or has resumed, and the session was locked or unlocked.
internal enum SessionNotice { Ending, Continuing, Suspending, Resumed, Locked, Unlocked }

// A hidden top-level window on its own thread receives WM_QUERYENDSESSION and WM_ENDSESSION even
// while the dashboard thread is busy, so a Windows shutdown or sign-out is recorded as a close
// request before Windows ends the process. Without it, a run that Windows ended was reported as an
// unexplained exit.
// unexplained exit. The same window receives the sleep, resume, lock and unlock notices, which the
// run marker keeps so the next start can tell a run that ended on a sleeping or locked PC apart.
internal sealed class SessionEndSentinel : IDisposable
{
private const uint WM_DESTROY = 0x0002, WM_CLOSE = 0x0010, WM_QUERYENDSESSION = 0x0011, WM_ENDSESSION = 0x0016;
private const uint WM_DESTROY = 0x0002, WM_CLOSE = 0x0010, WM_QUERYENDSESSION = 0x0011, WM_ENDSESSION = 0x0016,
WM_POWERBROADCAST = 0x0218, WM_WTSSESSION_CHANGE = 0x02B1;
private const long PBT_APMSUSPEND = 0x0004, PBT_APMRESUMESUSPEND = 0x0007, PBT_APMRESUMEAUTOMATIC = 0x0012;
private const long WTS_SESSION_LOCK = 0x7, WTS_SESSION_UNLOCK = 0x8;
private const uint NOTIFY_FOR_THIS_SESSION = 0;
private const int WS_OVERLAPPED = 0x00000000;
private const int ERROR_CLASS_ALREADY_EXISTS = 1410;
private const string ClassName = "WispSessionEndSentinel";
private readonly Action _sessionEnding;
private readonly Action? _sessionContinuing;
private readonly Action<SessionNotice> _notice;
private readonly WindowProcedure _procedure; // Keeps the delegate alive for the window's lifetime.
private readonly ManualResetEventSlim _started = new(false);
private Thread? _thread;
private IntPtr _window;
private int _signalled;
private bool _sessionNotifications;

private delegate IntPtr WindowProcedure(IntPtr window, uint message, IntPtr wParam, IntPtr lParam);

private SessionEndSentinel(Action sessionEnding, Action? sessionContinuing)
private SessionEndSentinel(Action<SessionNotice> notice)
{
_sessionEnding = sessionEnding;
_sessionContinuing = sessionContinuing;
_notice = notice;
_procedure = Procedure;
}

internal static SessionEndSentinel? TryStart(Action sessionEnding, Action? sessionContinuing = null)
internal static SessionEndSentinel? TryStart(Action<SessionNotice> notice)
{
var sentinel = new SessionEndSentinel(sessionEnding, sessionContinuing);
var sentinel = new SessionEndSentinel(notice);
try
{
sentinel._thread = new Thread(sentinel.Run) { IsBackground = true, Name = "Wisp session sentinel" };
Expand Down Expand Up @@ -67,6 +75,10 @@ private void Run()
catch (Exception) { }
finally { _started.Set(); }
if (_window == IntPtr.Zero) return;
// Lock and unlock notices need a registration, which can wait for the Terminal Services service;
// without one the sentinel still records shutdowns and sleep.
try { _sessionNotifications = WTSRegisterSessionNotification(_window, NOTIFY_FOR_THIS_SESSION); }
catch (Exception) { }
while (GetMessageW(out var message, IntPtr.Zero, 0, 0) > 0)
{
TranslateMessage(ref message);
Expand All @@ -85,7 +97,21 @@ private IntPtr Procedure(IntPtr window, uint message, IntPtr wParam, IntPtr lPar
if (wParam != IntPtr.Zero) Signal();
else Continue();
return IntPtr.Zero;
case WM_POWERBROADCAST:
if ((long)wParam == PBT_APMSUSPEND) Notify(SessionNotice.Suspending);
else if ((long)wParam is PBT_APMRESUMEAUTOMATIC or PBT_APMRESUMESUSPEND) Notify(SessionNotice.Resumed);
return 1;
case WM_WTSSESSION_CHANGE:
if ((long)wParam == WTS_SESSION_LOCK) Notify(SessionNotice.Locked);
else if ((long)wParam == WTS_SESSION_UNLOCK) Notify(SessionNotice.Unlocked);
return IntPtr.Zero;
case WM_CLOSE:
if (_sessionNotifications)
{
_sessionNotifications = false;
try { WTSUnRegisterSessionNotification(window); }
catch (Exception) { }
}
DestroyWindow(window);
return IntPtr.Zero;
case WM_DESTROY:
Expand All @@ -99,16 +125,20 @@ private IntPtr Procedure(IntPtr window, uint message, IntPtr wParam, IntPtr lPar
private void Signal()
{
if (Interlocked.Exchange(ref _signalled, 1) != 0) return;
try { _sessionEnding(); }
catch (Exception) { }
Notify(SessionNotice.Ending);
}

// Windows asked every window whether the session may end and a program vetoed it: the close
// request recorded on the query is withdrawn, and a later shutdown is recorded again.
private void Continue()
{
if (Interlocked.Exchange(ref _signalled, 0) == 0) return;
try { _sessionContinuing?.Invoke(); }
Notify(SessionNotice.Continuing);
}

private void Notify(SessionNotice notice)
{
try { _notice(notice); }
catch (Exception) { }
}

Expand Down Expand Up @@ -182,4 +212,12 @@ private static extern IntPtr CreateWindowExW(int exStyle, string className, stri

[DllImport("kernel32.dll", CharSet = CharSet.Unicode)]
private static extern IntPtr GetModuleHandleW(string? moduleName);

[DllImport("wtsapi32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool WTSRegisterSessionNotification(IntPtr window, uint flags);

[DllImport("wtsapi32.dll", SetLastError = true)]
[return: MarshalAs(UnmanagedType.Bool)]
private static extern bool WTSUnRegisterSessionNotification(IntPtr window);
}
8 changes: 7 additions & 1 deletion src/Wisp.App/CrashDiagnostics/WindowsFaultEventReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,10 @@ internal sealed record RecoveredExitInfo(DateTimeOffset RunStartedAtUtc, DateTim
public bool WindowsStopped { get; init; }
// Windows shut down or signed the user out within minutes of the run's last checkpoint while Wisp was still running.
public bool WindowsShutDown { get; init; }
// Windows was about to sleep or hibernate when the run was last seen, and the run never resumed.
public bool Suspended { get; init; }
// The Windows session was locked when the run was last seen.
public bool SessionLocked { get; init; }

internal RecoveredExitInfo? Sanitize()
{
Expand All @@ -38,7 +42,9 @@ internal sealed record RecoveredExitInfo(DateTimeOffset RunStartedAtUtc, DateTim
Fault = fault with { Module = WindowsFaultEventReader.SafeModule(fault.Module), ModuleVersion = WindowsFaultEventReader.SafeVersion(fault.ModuleVersion) },
Native = Native?.Sanitize(),
WindowsStopped = false,
WindowsShutDown = false
WindowsShutDown = false,
Suspended = false,
SessionLocked = false
};
}
return this with { Fault = null, Native = Native?.Sanitize() };
Expand Down
4 changes: 3 additions & 1 deletion src/Wisp.App/DebugLogging/SupplementaryIncidentCapture.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,9 @@ internal static SupplementaryIncident Crash(CrashReport report)
return SupplementaryIncidentSchema.Create(category,
reason: category == "windows-fault" ? "windows-application-fault" : category != "unexpected-exit" ? "exception-raised" :
safe?.RecoveredExit?.WindowsStopped == true ? "system-stopped" :
safe?.RecoveredExit?.WindowsShutDown == true ? "system-shutdown" : "shutdown-incomplete",
safe?.RecoveredExit?.WindowsShutDown == true ? "system-shutdown" :
safe?.RecoveredExit?.Suspended == true ? "system-sleep" :
safe?.RecoveredExit?.SessionLocked == true ? "session-locked" : "shutdown-incomplete",
failureStage: "crash-" + origin,
code: safe?.RecoveredExit?.Fault?.ExceptionCode ?? (safe?.Exceptions.FirstOrDefault() is { } first ? unchecked((uint)first.HResult) : null),
exceptions: Exceptions(safe?.Exceptions), context: Context(safe?.Context, report.TimeUtc,
Expand Down
10 changes: 6 additions & 4 deletions src/Wisp.App/DiagnosticsViewModel.CrashReports.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,22 +28,24 @@ internal void InitializeCrashReport(CrashReport? report, Func<Guid, bool> acknow
{
CrashReportNoticeGeneration++;
var safe = report?.Sanitize();
_crashReport = EndedByWindowsShutdown(safe) ? null : safe;
_crashReport = EndedWithoutTheUser(safe) ? null : safe;
_acknowledgeCrashReport = acknowledge;
_crashReportCopyStatus = "";
NotifyCrashReport();
}

internal bool TryInitializeRecoveredCrashReport(CrashReport report, long expectedGeneration, Func<Guid, bool> acknowledge)
{
if (CrashReportNoticeGeneration != expectedGeneration || EndedByWindowsShutdown(report.Sanitize())) return false;
if (CrashReportNoticeGeneration != expectedGeneration || EndedWithoutTheUser(report.Sanitize())) return false;
InitializeCrashReport(report, acknowledge);
return true;
}

// Windows shutting down or signing out while Wisp ran is not an unexpected end, so that run gets no card.
// Windows shutting down or signing out while Wisp ran is not an unexpected end, and neither is a run that
// ended while the PC was asleep or its session locked, so those runs get no card.
// The report stays in the store, where the diagnostics bundle and Tools reporting read it.
private static bool EndedByWindowsShutdown(CrashReport? report) => report?.RecoveredExit?.WindowsShutDown == true;
private static bool EndedWithoutTheUser(CrashReport? report) =>
report?.RecoveredExit is { } exit && (exit.WindowsShutDown || exit.Suspended || exit.SessionLocked);

public void ReportCrashDetailsCopied(bool copied)
{
Expand Down
2 changes: 1 addition & 1 deletion src/Wisp.App/Supplementary/SupplementaryIncident.cs
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ internal static class SupplementaryIncidentSchema
{ "unknown", "application-still-running", "restarted", "deferred", "no-verified-installation", "restart-failed" };
internal static readonly HashSet<string> Reasons = new(StringComparer.Ordinal)
{
"exception-raised", "windows-application-fault", "shutdown-incomplete", "system-stopped", "system-shutdown",
"exception-raised", "windows-application-fault", "shutdown-incomplete", "system-stopped", "system-shutdown", "system-sleep", "session-locked",
"classification-missing", "policy-rejected", "policy-cache-unavailable", "unknown", "access-denied", "unsupported-build", "unsupported_os", "unsupported_gpu", "unsupported_format",
"capture_failed", "encoder_failed", "audio_failed", "audio_capture_failed", "audio_unavailable", "buffer_full",
"no_keyframe", "not_ready", "save_in_progress", "storage_failed", "mux_failed", "protocol_error",
Expand Down
Loading
Loading