| | | 1 | | using AmbientServices.Extensions; |
| | | 2 | | using System; |
| | | 3 | | using System.Collections.Generic; |
| | | 4 | | using System.Linq; |
| | | 5 | | using System.Reflection; |
| | | 6 | | using System.Threading; |
| | | 7 | | using System.Threading.Tasks; |
| | | 8 | | |
| | | 9 | | namespace AmbientServices; |
| | | 10 | | |
| | | 11 | | /// <summary> |
| | | 12 | | /// A single-instance class that holds status for the entire system. |
| | | 13 | | /// </summary> |
| | | 14 | | /// <remarks> |
| | | 15 | | /// <pitch> |
| | | 16 | | /// The front door of the status subsystem: one instance (usually <see cref="DefaultInstance"/>) discovers every status |
| | | 17 | | /// It coordinates and summarizes; the actual testing logic lives in <see cref="StatusChecker"/> and <see cref="StatusAu |
| | | 18 | | /// </pitch> |
| | | 19 | | /// <pledge> |
| | | 20 | | /// Discovery: when constructed with automatic loading, every non-abstract <see cref="StatusChecker"/> derivative with a |
| | | 21 | | /// <see cref="Stop"/> stops scheduled audits, disposes every registered checker, and resets the instance so Start may b |
| | | 22 | | /// The summary properties (<see cref="Results"/>, <see cref="Summary"/>, <see cref="SummaryAlertsAndFailures"/>, <see c |
| | | 23 | | /// Aggregation treats the registered checkers as heterogeneous children: the overall rating is the worst of their ratin |
| | | 24 | | /// </pledge> |
| | | 25 | | /// <plan> |
| | | 26 | | /// Checkers live in a <see cref="ConcurrentHashSet{T}"/>. Automatic discovery hooks <see cref="AppDomain.AssemblyLoad" |
| | | 27 | | /// The summary properties sort the checkers' latest results by rating, wrap them in a root "/" heterogeneous <see cref= |
| | | 28 | | /// </plan> |
| | | 29 | | /// <priority> |
| | | 30 | | /// 1. Finding problems at startup over discovering them lazily: checkers and auditors are discovered and constructed at |
| | | 31 | | /// 2. Explicit start over running on discovery: nothing runs until <see cref="Start"/> is called, so merely referencing |
| | | 32 | | /// 3. Answering from the last recorded results over auditing on read: the summary properties snapshot what was already |
| | | 33 | | /// 4. Reporting the checkers that could not finish over failing the whole refresh: a refresh returns the incomplete one |
| | | 34 | | /// </priority> |
| | | 35 | | /// </remarks> |
| | | 36 | | public class Status |
| | | 37 | | { |
| | | 38 | | internal const string DefaultSource = "LOCALHOST"; |
| | | 39 | | internal const string DefaultTarget = "Unknown Target"; |
| | | 40 | | internal static readonly AmbientLogger<Status> Logger = new(); |
| | | 41 | | /// <summary> |
| | | 42 | | /// Gets the base instance that contains the overall status and is initialized with all checkers and auditors with p |
| | | 43 | | /// Note that even the default instance must be started by calling <see cref="Start"/> before checks and audits will |
| | | 44 | | /// </summary> |
| | | 45 | | public static Status DefaultInstance { get; } = new(true); |
| | | 46 | | |
| | | 47 | | private readonly bool _loadAllCheckers; |
| | | 48 | | private readonly ConcurrentHashSet<StatusChecker> _checkers = new(); |
| | | 49 | | private int _shuttingDown; // interlocked |
| | | 50 | | private int _started; // interlocked |
| | | 51 | | |
| | | 52 | | /// <summary> |
| | | 53 | | /// Constructs a new Status instance which will keep track of status checkers and auditors and shut them down when i |
| | | 54 | | /// If <paramref name="loadAllCheckers"/> is true, constructs and registers all <see cref="StatusChecker"/> classes |
| | | 55 | | /// If <paramref name="loadAllCheckers"/> is false, constructs an empty collection of checkers which may be added to |
| | | 56 | | /// Note that checkers with <see cref="StatusIgnoreCheckerAttribute"/> applied will never be included automatically. |
| | | 57 | | /// </summary> |
| | | 58 | | /// <param name="loadAllCheckers">Whether or not to load all checkers (and auditors) in all loaded assemblies and an |
| | | 59 | | public Status(bool loadAllCheckers) |
| | | 60 | | { |
| | | 61 | | _loadAllCheckers = loadAllCheckers; |
| | | 62 | | } |
| | | 63 | | |
| | | 64 | | /// <summary> |
| | | 65 | | /// Checks to see whether or not we're started shutting down the status system. |
| | | 66 | | /// </summary> |
| | | 67 | | internal bool ShuttingDown => _shuttingDown != 0; |
| | | 68 | | |
| | | 69 | | /// <summary> |
| | | 70 | | /// Starts the status system by searching the system for checkers and auditors (unless the constructor parameter say |
| | | 71 | | /// A call to Start must be matched by a call to <see cref="Stop"/> or else disposable items will not be disposed an |
| | | 72 | | /// Start may only be called once. |
| | | 73 | | /// </summary> |
| | | 74 | | /// <param name="cancel">A <see cref="CancellationToken"/> the caller can use to stop the operation before it comple |
| | | 75 | | public ValueTask Start(CancellationToken cancel = default) |
| | | 76 | | { |
| | | 77 | | Logger.Filter("StartStop")?.Log(new { Action = "StatusStarting" }); |
| | | 78 | | if (Interlocked.Exchange(ref _started, 1) != 0) throw new InvalidOperationException("The Status system has alrea |
| | | 79 | | if (_loadAllCheckers) |
| | | 80 | | { |
| | | 81 | | // add checkers and auditors from all assemblies subsequently loaded |
| | | 82 | | AppDomain.CurrentDomain.AssemblyLoad += CurrentDomain_AssemblyLoad; |
| | | 83 | | // add checkers and auditors from all assemblies currently loaded |
| | | 84 | | foreach (System.Reflection.Assembly assembly in AppDomain.CurrentDomain.GetAssemblies()) |
| | | 85 | | { |
| | | 86 | | cancel.ThrowIfCancellationRequested(); |
| | | 87 | | // add checkers and auditors from this assembly |
| | | 88 | | AddCheckersAndAuditors(assembly); |
| | | 89 | | } |
| | | 90 | | } |
| | | 91 | | Logger.Filter("StartStop")?.Log(new { Action = "StatusStarted" }); |
| | | 92 | | return default; |
| | | 93 | | } |
| | | 94 | | /// <summary> |
| | | 95 | | /// Stops the status system by disposing of all the status nodes. |
| | | 96 | | /// </summary> |
| | | 97 | | public async ValueTask Stop() |
| | | 98 | | { |
| | | 99 | | Logger.Filter("StartStop")?.Log(new { Action = "StatusStopping" }); |
| | | 100 | | // make sure everyone can tell we're shutting down |
| | | 101 | | Interlocked.Exchange(ref _shuttingDown, 1); |
| | | 102 | | // stop the timers on each node |
| | | 103 | | foreach (StatusChecker checker in _checkers) |
| | | 104 | | { |
| | | 105 | | await checker.BeginStop(); |
| | | 106 | | } |
| | | 107 | | // wait for each one to stop |
| | | 108 | | foreach (StatusChecker checker in _checkers) |
| | | 109 | | { |
| | | 110 | | await checker.FinishStop(); |
| | | 111 | | } |
| | | 112 | | // dispose each one |
| | | 113 | | foreach (StatusChecker checker in _checkers) |
| | | 114 | | { |
| | | 115 | | checker.Dispose(); |
| | | 116 | | } |
| | | 117 | | Logger.Filter("StartStop")?.Log( new { Action = "StatusStopped" }); |
| | | 118 | | // now that we're done, reset everything back to where we were before we started |
| | | 119 | | _checkers.Clear(); |
| | | 120 | | Interlocked.Exchange(ref _started, 0); |
| | | 121 | | Interlocked.Exchange(ref _shuttingDown, 0); |
| | | 122 | | } |
| | | 123 | | |
| | | 124 | | private void CurrentDomain_AssemblyLoad(object? sender, AssemblyLoadEventArgs args) |
| | | 125 | | { |
| | | 126 | | AddCheckersAndAuditors(args.LoadedAssembly); |
| | | 127 | | } |
| | | 128 | | |
| | | 129 | | /// <summary> |
| | | 130 | | /// Adds checkers and auditors defined in the specified assembly. |
| | | 131 | | /// </summary> |
| | | 132 | | /// <param name="assembly">The <see cref="Assembly"/> to look in.</param> |
| | | 133 | | private void AddCheckersAndAuditors(Assembly assembly) |
| | | 134 | | { |
| | | 135 | | // does the loaded assembly refer to this one? if it doesn't, there can't possibly be any of the classes we're |
| | | 136 | | if (assembly.DoesAssemblyReferDirectlyToAssembly(Assembly.GetExecutingAssembly())) |
| | | 137 | | { |
| | | 138 | | // loop through all the types looking for types that are not abstract, inherit from StatusNode (directly or |
| | | 139 | | foreach (Type type in assembly.GetLoadableTypes()) |
| | | 140 | | { |
| | | 141 | | // does this checker have the IgnoreCheckerAttribute? skip this one |
| | | 142 | | if (type.GetCustomAttribute<StatusIgnoreCheckerAttribute>() != null) continue; |
| | | 143 | | if (IsTestableStatusCheckerClass(type)) |
| | | 144 | | { |
| | | 145 | | // construct an instance (it will be added to the list by the constructor) |
| | | 146 | | StatusChecker checker = (StatusChecker)Activator.CreateInstance(type)!; |
| | | 147 | | AddCheckerOrAuditor(checker); |
| | | 148 | | } |
| | | 149 | | } |
| | | 150 | | } |
| | | 151 | | } |
| | | 152 | | /// <summary> |
| | | 153 | | /// Adds the specified checker or auditor to the list of checkers and auditors and for auditors, |
| | | 154 | | /// schedules the initial audit for 10ms afterwards (using an <see cref="AmbientEventTimer"/> so that the timing of |
| | | 155 | | /// </summary> |
| | | 156 | | /// <param name="checker">The <see cref="StatusChecker"/> to add.</param> |
| | | 157 | | public void AddCheckerOrAuditor(StatusChecker checker) |
| | | 158 | | { |
| | | 159 | | #if NET5_0_OR_GREATER |
| | | 160 | | ArgumentNullException.ThrowIfNull(checker); |
| | | 161 | | #else |
| | | 162 | | if (checker is null) throw new ArgumentNullException(nameof(checker)); |
| | | 163 | | #endif |
| | | 164 | | Logger.Filter("Registration")?.Log(new { Action = $"AddingStatusChecker", CheckerName = checker.GetType().Name } |
| | | 165 | | _checkers.Add(checker); |
| | | 166 | | // is this checker an auditor? |
| | | 167 | | StatusAuditor? auditor = checker as StatusAuditor; |
| | | 168 | | // kick off the initial audit (note that this cannot be done in the StatusAuditor constructor because it might r |
| | | 169 | | auditor?.ScheduleInitialAudit(); |
| | | 170 | | } |
| | | 171 | | /// <summary> |
| | | 172 | | /// Removes the specified checker or auditor from the global list. |
| | | 173 | | /// Subsequent audits may still occur, as they are controlled by the <see cref="StatusAuditor"/> class. |
| | | 174 | | /// No further audits will be scheduled, but no blocking wil occur if one is already in progress. |
| | | 175 | | /// </summary> |
| | | 176 | | /// <param name="checker">The <see cref="StatusChecker"/> to remove.</param> |
| | | 177 | | public void RemoveCheckerOrAuditor(StatusChecker checker) |
| | | 178 | | { |
| | | 179 | | #if NET5_0_OR_GREATER |
| | | 180 | | ArgumentNullException.ThrowIfNull(checker); |
| | | 181 | | #else |
| | | 182 | | if (checker is null) throw new ArgumentNullException(nameof(checker)); |
| | | 183 | | #endif |
| | | 184 | | _checkers.Remove(checker); |
| | | 185 | | Logger.Filter("Registration")?.Log(new { Action = $"RemovedStatusChecker", CheckerName = checker.GetType().Name |
| | | 186 | | } |
| | | 187 | | |
| | | 188 | | private static float? Rating(StatusResults results) |
| | | 189 | | { |
| | | 190 | | if (results == null || results.Report == null || results.Report.Alert == null) return null; |
| | | 191 | | return results.Report.Alert.Rating; |
| | | 192 | | } |
| | | 193 | | internal static int RatingCompare(StatusResults a, StatusResults b) |
| | | 194 | | { |
| | | 195 | | float? fa = Rating(a); |
| | | 196 | | float? fb = Rating(b); |
| | | 197 | | if (fa == null) return (fb == null) ? 0 : -1; |
| | | 198 | | return (fb == null) ? 1 : fa.Value.CompareTo(fb.Value); |
| | | 199 | | } |
| | | 200 | | /// <summary> |
| | | 201 | | /// Refreshes the status audits immediately, returning an enumeration of status checkers that did not complete befor |
| | | 202 | | /// Normally audits will be refreshed automatically in the background, but in some circumstances, users may want to |
| | | 203 | | /// </summary> |
| | | 204 | | /// <returns>An enumeration of <see cref="StatusChecker"/>s that did not complete refreshing before being cancelled. |
| | | 205 | | public async ValueTask<IEnumerable<StatusChecker>> RefreshAsync(CancellationToken cancel = default) |
| | | 206 | | { |
| | | 207 | | Logger.Filter("Check")?.Log(new { Action = "StatusExplicitRefresh" }); |
| | | 208 | | // asynchronously get the status of each system |
| | | 209 | | Dictionary<StatusChecker, Task<StatusResults>> checkerTasks = new(_checkers.Count); |
| | | 210 | | foreach (StatusChecker checker in _checkers) |
| | | 211 | | { |
| | | 212 | | Task<StatusResults> task = Task.Run(() => checker.GetStatus(cancel).AsTask(), cancel); |
| | | 213 | | checkerTasks.Add(checker, task); |
| | | 214 | | } |
| | | 215 | | // wait for either all the checker tasks to complete, or for the cancellation token to be canceled |
| | | 216 | | Task allCheckers = Task.WhenAll(checkerTasks.Values); |
| | | 217 | | if (cancel.CanBeCanceled) |
| | | 218 | | { |
| | | 219 | | // signal completion from a token registration we dispose as soon as the wait ends, so we never leave a call |
| | | 220 | | TaskCompletionSource<bool> cancellationSignal = new(TaskCreationOptions.RunContinuationsAsynchronously); |
| | | 221 | | using (cancel.Register(() => cancellationSignal.TrySetResult(true))) |
| | | 222 | | { |
| | | 223 | | await Task.WhenAny(allCheckers, cancellationSignal.Task); |
| | | 224 | | } |
| | | 225 | | } |
| | | 226 | | else |
| | | 227 | | { |
| | | 228 | | // no cancellation is possible; wait via WhenAny (not a direct await) so a faulted checker is inspected belo |
| | | 229 | | await Task.WhenAny(allCheckers); |
| | | 230 | | } |
| | | 231 | | // make a list of those that got canceled or catastrophically failed (GetStatus should never throw an exception, |
| | | 232 | | List<StatusChecker> canceledOrFailedCheckers = new(); |
| | | 233 | | foreach (KeyValuePair<StatusChecker, Task<StatusResults>> kvp in checkerTasks) |
| | | 234 | | { |
| | | 235 | | StatusChecker checker = kvp.Key; |
| | | 236 | | Task<StatusResults> resultsTask = kvp.Value; |
| | | 237 | | if (resultsTask.IsFaulted) // this means that GetStatus threw an exception--this should have been caught int |
| | | 238 | | { |
| | | 239 | | StatusResultsBuilder builder = new(checker); |
| | | 240 | | builder.AddException(resultsTask.Exception!); // if IsFaulted, there should be a non-null Exception! |
| | | 241 | | checker.SetLatestResults(builder.FinalResults); |
| | | 242 | | // in this case the checker failed, but we put the results back into the checker, so we've made it appea |
| | | 243 | | } |
| | | 244 | | else if (resultsTask.Status != TaskStatus.RanToCompletion || resultsTask.IsCanceled) // cancelled (or someho |
| | | 245 | | { |
| | | 246 | | // in this case, we won't update the checker itself, because the caller could have just specified a very |
| | | 247 | | canceledOrFailedCheckers.Add(checker); |
| | | 248 | | } |
| | | 249 | | // else the task completed successfully so the results are in the latest results |
| | | 250 | | } |
| | | 251 | | return canceledOrFailedCheckers; |
| | | 252 | | } |
| | | 253 | | /// <summary> |
| | | 254 | | /// Gets the <see cref="StatusResults"/> for the entire system. |
| | | 255 | | /// </summary> |
| | | 256 | | public StatusResults Results |
| | | 257 | | { |
| | | 258 | | get |
| | | 259 | | { |
| | | 260 | | DateTime now = AmbientClock.UtcNow; |
| | | 261 | | List<StatusResults> results = new(_checkers.Select(checker => checker.LatestResults)); |
| | | 262 | | results.Sort((a, b) => RatingCompare(a, b)); |
| | | 263 | | StatusResults overallResults = new(null, "/", now, 0, Array.Empty<StatusProperty>(), StatusNatureOfSystem.Ch |
| | | 264 | | return overallResults; |
| | | 265 | | } |
| | | 266 | | } |
| | | 267 | | /// <summary> |
| | | 268 | | /// Gets the <see cref="StatusAuditAlert"/> containing the full summarized results for the entire system, including |
| | | 269 | | /// </summary> |
| | | 270 | | public StatusAuditAlert Summary |
| | | 271 | | { |
| | | 272 | | get |
| | | 273 | | { |
| | | 274 | | DateTime now = AmbientClock.UtcNow; |
| | | 275 | | List<StatusResults> results = new(_checkers.Select(checker => checker.LatestResults)); |
| | | 276 | | results.Sort((a, b) => RatingCompare(a, b)); |
| | | 277 | | StatusResults overallResults = new(null, "/", now, 0, Array.Empty<StatusProperty>(), StatusNatureOfSystem.Ch |
| | | 278 | | StatusAuditAlert alerts = overallResults.GetSummaryAlerts(true, float.MaxValue, false); |
| | | 279 | | return alerts; |
| | | 280 | | } |
| | | 281 | | } |
| | | 282 | | /// <summary> |
| | | 283 | | /// Gets the <see cref="StatusAuditAlert"/> containing the summarized alerts and failures for the entire system. |
| | | 284 | | /// </summary> |
| | | 285 | | public StatusAuditAlert SummaryAlertsAndFailures |
| | | 286 | | { |
| | | 287 | | get |
| | | 288 | | { |
| | | 289 | | DateTime now = AmbientClock.UtcNow; |
| | | 290 | | List<StatusResults> results = new(_checkers.Select(checker => checker.LatestResults)); |
| | | 291 | | results.Sort((a, b) => RatingCompare(a, b)); |
| | | 292 | | StatusResults overallResults = new(null, "/", now, 0, Array.Empty<StatusProperty>(), StatusNatureOfSystem.Ch |
| | | 293 | | StatusAuditAlert alerts = overallResults.GetSummaryAlerts(true, StatusRating.Alert, false); |
| | | 294 | | return alerts; |
| | | 295 | | } |
| | | 296 | | } |
| | | 297 | | /// <summary> |
| | | 298 | | /// Gets the <see cref="StatusAuditAlert"/> containing the summarized failures for the entire system. |
| | | 299 | | /// </summary> |
| | | 300 | | public StatusAuditAlert SummaryFailures |
| | | 301 | | { |
| | | 302 | | get |
| | | 303 | | { |
| | | 304 | | DateTime now = AmbientClock.UtcNow; |
| | | 305 | | List<StatusResults> results = new(_checkers.Select(checker => checker.LatestResults)); |
| | | 306 | | results.Sort((a, b) => RatingCompare(a, b)); |
| | | 307 | | StatusResults overallResults = new(null, "/", now, 0, Array.Empty<StatusProperty>(), StatusNatureOfSystem.Ch |
| | | 308 | | StatusAuditAlert alerts = overallResults.GetSummaryAlerts(true, StatusRating.Fail, false); |
| | | 309 | | return alerts; |
| | | 310 | | } |
| | | 311 | | } |
| | | 312 | | /// <summary> |
| | | 313 | | /// Gets all the historical <see cref="StatusResults"/> for the entire system. |
| | | 314 | | /// </summary> |
| | | 315 | | public IEnumerable<StatusResults> History |
| | | 316 | | { |
| | | 317 | | get |
| | | 318 | | { |
| | | 319 | | DateTime now = AmbientClock.UtcNow; |
| | | 320 | | List<StatusResults> historicalResults = new(_checkers.SelectMany(checker => checker.History).OrderByDescendi |
| | | 321 | | return historicalResults; |
| | | 322 | | } |
| | | 323 | | } |
| | | 324 | | /// <summary> |
| | | 325 | | /// Checks to see if the specified type represents a testable status checker class (ie. one with a public constructo |
| | | 326 | | /// </summary> |
| | | 327 | | /// <param name="type">The type to check.</param> |
| | | 328 | | /// <returns>true if the specified type is a testable status checker class.</returns> |
| | | 329 | | public static bool IsTestableStatusCheckerClass(Type type) |
| | | 330 | | { |
| | | 331 | | #if NET5_0_OR_GREATER |
| | | 332 | | ArgumentNullException.ThrowIfNull(type); |
| | | 333 | | #else |
| | | 334 | | if (type is null) throw new ArgumentNullException(nameof(type)); |
| | | 335 | | #endif |
| | | 336 | | return !type.IsAbstract && typeof(StatusChecker).IsAssignableFrom(type) && type.GetConstructor(Array.Empty<Type> |
| | | 337 | | } |
| | | 338 | | } |
| | | 339 | | /// <summary> |
| | | 340 | | /// A class attribute used mostly for testing that causes a <see cref="StatusChecker"/> or <see cref="StatusAuditor"/> c |
| | | 341 | | /// </summary> |
| | | 342 | | [AttributeUsage(AttributeTargets.Class)] |
| | | 343 | | public sealed class StatusIgnoreCheckerAttribute : Attribute |
| | | 344 | | { |
| | | 345 | | /// <summary> |
| | | 346 | | /// Constructs the IgnoreCheckerAttribute. |
| | | 347 | | /// </summary> |
| | 2 | 348 | | public StatusIgnoreCheckerAttribute() |
| | | 349 | | { |
| | 2 | 350 | | } |
| | | 351 | | } |