| | | 1 | | using AmbientServices.Extensions; |
| | | 2 | | using System; |
| | | 3 | | using System.Text; |
| | | 4 | | using System.Threading; |
| | | 5 | | using System.Threading.Tasks; |
| | | 6 | | |
| | | 7 | | namespace AmbientServices; |
| | | 8 | | |
| | | 9 | | /// <summary> |
| | | 10 | | /// An immutable class that contains information about a status audit alert. |
| | | 11 | | /// Alerts with the same <see cref="StatusAuditAlert.Rating"/>, and <see cref="StatusAuditAlert.AuditAlertCode"/> are co |
| | | 12 | | /// </summary> |
| | | 13 | | /// <remarks> |
| | | 14 | | /// <pitch>The atomic unit of alerting: one rating plus a stable alert code and paired terse (SMS-safe) and detailed (HT |
| | | 15 | | /// <pledge> |
| | | 16 | | /// Equality — and therefore aggregatability — is defined by <see cref="Rating"/> and a case-insensitive <see cref="Audi |
| | | 17 | | /// Neither the code nor the terse message may contain sensitive details, and the terse message must be free of line bre |
| | | 18 | | /// </pledge> |
| | | 19 | | /// <priority> |
| | | 20 | | /// 1. Collapsing across servers over expressing per-server detail: equality is the rating and a case-insensitive alert |
| | | 21 | | /// 2. A notification safe to send anywhere over one that says everything: neither the code nor the terse message may ca |
| | | 22 | | /// </priority> |
| | | 23 | | /// </remarks> |
| | | 24 | | public sealed class StatusAuditAlert : IEquatable<StatusAuditAlert> |
| | | 25 | | { |
| | | 26 | | /// <summary> |
| | | 27 | | /// An empty <see cref="StatusAuditAlert"/> in case they need to be compared. |
| | | 28 | | /// </summary> |
| | | 29 | | public static StatusAuditAlert Empty { get; } = new(); |
| | | 30 | | /// <summary> |
| | | 31 | | /// A <see cref="StatusAuditAlert"/> for when no alert was reported. |
| | | 32 | | /// </summary> |
| | | 33 | | public static StatusAuditAlert None { get; } = new(StatusRating.Okay, "NoAlert", "No Alerts", "There are no alerts." |
| | | 34 | | |
| | | 35 | | /// <summary> |
| | | 36 | | /// Gets a short string containing a code for this condition, or empty string if the report should not be collated b |
| | | 37 | | /// Audit alert codes should not contain numbers or strings that might vary for the same type of error, and the mess |
| | | 38 | | /// Some examples might be, Timeout, Configuration, BadRequest, AccessDenied, ProgramError, NotFound, or MirrorBroke |
| | | 39 | | /// This string should never contain any sensitive details. |
| | | 40 | | /// </summary> |
| | | 41 | | public string AuditAlertCode { get; } |
| | | 42 | | /// <summary> |
| | | 43 | | /// A status rating indicating the overall state of the system represented by the report. |
| | | 44 | | /// </summary> |
| | | 45 | | public float Rating { get; } |
| | | 46 | | /// <summary> |
| | | 47 | | /// Gets a short message indicating that there is a suboptimal status (one that would be appropriate for SMS or a mo |
| | | 48 | | /// The message must not contain any line breaks or markup so that it can be sent in a text message. |
| | | 49 | | /// This message must not contain any sensitive details. |
| | | 50 | | /// </summary> |
| | | 51 | | public string Terse { get; } |
| | | 52 | | /// <summary> |
| | | 53 | | /// Gets a detailed message indicating the cause of a suboptimal status (one that would be appropriate for email, we |
| | | 54 | | /// The message should use the same format as documentation details. |
| | | 55 | | /// If authorization was not granted, and the error might contain sensitive details, the message may have been repla |
| | | 56 | | /// </summary> |
| | | 57 | | public string Details { get; } |
| | | 58 | | |
| | | 59 | | /// <summary> |
| | | 60 | | /// Constructs an empty <see cref="StatusAuditAlert"/>. |
| | | 61 | | /// </summary> |
| | | 62 | | private StatusAuditAlert() |
| | | 63 | | { |
| | | 64 | | Rating = StatusRating.Okay; |
| | | 65 | | AuditAlertCode = "OkayCode"; |
| | | 66 | | Terse = "ok"; |
| | | 67 | | Details = "The system is functioning normally."; |
| | | 68 | | } |
| | | 69 | | /// <summary> |
| | | 70 | | /// Constructs a <see cref="StatusAuditAlert"/> with the specified values. |
| | | 71 | | /// </summary> |
| | | 72 | | /// <param name="rating">A status rating indicating whether or not the system is working.</param> |
| | | 73 | | /// <param name="auditAlertCode">The code for this audit result, or empty string if the alert is a composite alert.< |
| | | 74 | | /// <param name="terse">The terse alert message.</param> |
| | | 75 | | /// <param name="details">The details alert message.</param> |
| | | 76 | | public StatusAuditAlert(float rating, string auditAlertCode, string terse, string details) |
| | | 77 | | { |
| | | 78 | | Rating = rating; |
| | | 79 | | AuditAlertCode = auditAlertCode ?? throw new ArgumentNullException(nameof(auditAlertCode)); |
| | | 80 | | Terse = terse ?? throw new ArgumentNullException(nameof(terse)); |
| | | 81 | | Details = details ?? throw new ArgumentNullException(nameof(details)); |
| | | 82 | | } |
| | | 83 | | |
| | | 84 | | /// <summary> |
| | | 85 | | /// Gets the 32-bit hash code for this object. |
| | | 86 | | /// </summary> |
| | | 87 | | /// <returns>The 32-bit hash code for this object.</returns> |
| | | 88 | | public override int GetHashCode() |
| | | 89 | | { |
| | | 90 | | return Rating.GetHashCode() ^ AuditAlertCode.GetHashCode(StringComparison.Ordinal); |
| | | 91 | | } |
| | | 92 | | /// <summary> |
| | | 93 | | /// Checks to see if this <see cref="StatusAuditAlert"/> is logically equal to another one. |
| | | 94 | | /// </summary> |
| | | 95 | | /// <param name="obj">The other object to compare to.</param> |
| | | 96 | | /// <returns>true if the objects are logically equivalent, false if they are not.</returns> |
| | | 97 | | public override bool Equals(object? obj) |
| | | 98 | | { |
| | | 99 | | if (obj is not StatusAuditAlert alert) return false; |
| | | 100 | | return Equals(alert); |
| | | 101 | | } |
| | | 102 | | |
| | | 103 | | /// <summary> |
| | | 104 | | /// Checks to see if this <see cref="StatusAuditAlert"/> is logically equal to another one. |
| | | 105 | | /// </summary> |
| | | 106 | | /// <param name="other">The other <see cref="StatusAuditAlert"/> to compare to.</param> |
| | | 107 | | /// <returns>true if the objects are logically equivalent, false if they are not.</returns> |
| | | 108 | | public bool Equals(StatusAuditAlert? other) |
| | | 109 | | { |
| | | 110 | | if (other is null) return false; |
| | | 111 | | return Rating.Equals(other.Rating) && string.Equals(AuditAlertCode, other.AuditAlertCode, StringComparison.Ordin |
| | | 112 | | } |
| | | 113 | | |
| | | 114 | | /// <summary> |
| | | 115 | | /// Compares <paramref name="a"/> and <paramref name="b"/> and returns whether or not <paramref name="a"/> is equal |
| | | 116 | | /// </summary> |
| | | 117 | | /// <param name="a">The first <see cref="StatusAuditAlert"/> to compare.</param> |
| | | 118 | | /// <param name="b">The second <see cref="StatusAuditAlert"/> to compare.</param> |
| | | 119 | | /// <returns><b>true</b> if <paramref name="a"/> is equal to <paramref name="b"/>.</returns> |
| | | 120 | | public static bool operator ==(StatusAuditAlert? a, StatusAuditAlert? b) |
| | | 121 | | { |
| | | 122 | | if (a is null) return b is null; |
| | | 123 | | return a.Equals(b); |
| | | 124 | | } |
| | | 125 | | /// <summary> |
| | | 126 | | /// Compares <paramref name="a"/> and <paramref name="b"/> and returns whether or not <paramref name="a"/> is NOT eq |
| | | 127 | | /// </summary> |
| | | 128 | | /// <param name="a">The first <see cref="StatusAuditAlert"/> to compare.</param> |
| | | 129 | | /// <param name="b">The second <see cref="StatusAuditAlert"/> to compare.</param> |
| | | 130 | | /// <returns><b>true</b> if <paramref name="a"/> is NOT equal to <paramref name="b"/>.</returns> |
| | | 131 | | public static bool operator !=(StatusAuditAlert? a, StatusAuditAlert? b) |
| | | 132 | | { |
| | | 133 | | if (a is null) return b is not null; |
| | | 134 | | return !a.Equals(b); |
| | | 135 | | } |
| | | 136 | | /// <summary> |
| | | 137 | | /// Gets a string representation of the instance. |
| | | 138 | | /// </summary> |
| | | 139 | | /// <returns>A string representation of the instance.</returns> |
| | | 140 | | public override string ToString() |
| | | 141 | | { |
| | | 142 | | return $"{Rating}({AuditAlertCode}): {Terse}({Details})"; |
| | | 143 | | } |
| | | 144 | | } |
| | | 145 | | /// <summary> |
| | | 146 | | /// An immutable class that contains a report generated by an audit of specific target system. |
| | | 147 | | /// Separate from <see cref="StatusAuditAlert"/> because not all audits generate alerts. |
| | | 148 | | /// Reports with the same <see cref="StatusAuditReport.Alert"/> are considered equivalent and may be combined during rep |
| | | 149 | | /// </summary> |
| | | 150 | | /// <remarks> |
| | | 151 | | /// <pitch>The record of one audit run: when it started, how long it took, when the next one is expected, and the worst |
| | | 152 | | /// <pledge> |
| | | 153 | | /// Equality is defined solely by <see cref="Alert"/> — the timing fields never prevent equivalent reports from being ag |
| | | 154 | | /// The shared <see cref="Pending"/> sentinel represents "the first audit has not run yet" and carries a <see cref="Stat |
| | | 155 | | /// </pledge> |
| | | 156 | | /// </remarks> |
| | | 157 | | public sealed class StatusAuditReport : IEquatable<StatusAuditReport> |
| | | 158 | | { |
| | | 159 | | /// <summary> |
| | | 160 | | /// A <see cref="StatusAuditReport"/> indicating that the first audit is pending. |
| | | 161 | | /// </summary> |
| | 2 | 162 | | public static readonly StatusAuditReport Pending = new(); |
| | | 163 | | |
| | | 164 | | /// <summary> |
| | | 165 | | /// Gets the time when the audit of the target system started. |
| | | 166 | | /// </summary> |
| | | 167 | | public DateTime AuditStartTime { get; } |
| | | 168 | | /// <summary> |
| | | 169 | | /// Gets the time when the audit occurred. |
| | | 170 | | /// </summary> |
| | | 171 | | public TimeSpan AuditDuration { get; } |
| | | 172 | | /// <summary> |
| | | 173 | | /// Gets the time when the system should be reaudited (if any). |
| | | 174 | | /// </summary> |
| | | 175 | | public DateTime? NextAuditTime { get; } |
| | | 176 | | /// <summary> |
| | | 177 | | /// Gets the <see cref="StatusAuditAlert"/> for this audit, if any. |
| | | 178 | | /// No <see cref="StatusAuditAlert"/> implies that there were no issues, which should default to <see cref="StatusRa |
| | | 179 | | /// </summary> |
| | | 180 | | public StatusAuditAlert? Alert { get; } |
| | | 181 | | |
| | | 182 | | /// <summary> |
| | | 183 | | /// Constructs the "Pending" <see cref="StatusAuditReport"/>. |
| | | 184 | | /// </summary> |
| | 2 | 185 | | private StatusAuditReport() |
| | | 186 | | { |
| | 2 | 187 | | AuditStartTime = AmbientClock.UtcNow; |
| | 2 | 188 | | AuditDuration = TimeSpan.FromTicks(0); |
| | 2 | 189 | | NextAuditTime = AmbientClock.UtcNow; |
| | 2 | 190 | | Alert = new StatusAuditAlert(StatusRating.Pending, "Pending", "Pending", "The first audit has not run yet!"); |
| | 2 | 191 | | } |
| | | 192 | | /// <summary> |
| | | 193 | | /// Constructs a <see cref="StatusAuditReport"/> with the specified values. |
| | | 194 | | /// </summary> |
| | | 195 | | /// <param name="auditStartTime">The <see cref="DateTime"/> when the status was assessed.</param> |
| | | 196 | | /// <param name="auditDuration">A <see cref="TimeSpan"/> indicating how long the audit took.</param> |
| | | 197 | | /// <param name="nextAuditTime">The next time the audit should happen (if any).</param> |
| | | 198 | | /// <param name="alert">A <see cref="StatusAuditAlert"/> if the audit includes an alert, if any</param> |
| | 2 | 199 | | public StatusAuditReport(DateTime auditStartTime, TimeSpan auditDuration, DateTime? nextAuditTime = null, StatusAudi |
| | | 200 | | { |
| | 2 | 201 | | AuditStartTime = auditStartTime; |
| | 2 | 202 | | AuditDuration = auditDuration; |
| | 2 | 203 | | NextAuditTime = nextAuditTime; |
| | 2 | 204 | | Alert = alert; |
| | 2 | 205 | | } |
| | | 206 | | |
| | | 207 | | /// <summary> |
| | | 208 | | /// Gets a string representation of this object. |
| | | 209 | | /// </summary> |
| | | 210 | | /// <returns>A string representation of this object.</returns> |
| | | 211 | | public override string ToString() |
| | | 212 | | { |
| | 2 | 213 | | StringBuilder output = new(); |
| | 2 | 214 | | output.Append('@'); |
| | 2 | 215 | | output.Append(AuditStartTime.ToShortTimeString().Replace(" ", "", StringComparison.Ordinal)); |
| | 2 | 216 | | if (Alert != null) |
| | | 217 | | { |
| | 2 | 218 | | output.Append(':'); |
| | 2 | 219 | | output.Append(StatusRating.GetRangeName(Alert.Rating)); |
| | 2 | 220 | | if (!string.IsNullOrEmpty(Alert.Terse)) |
| | | 221 | | { |
| | 2 | 222 | | output.Append('('); |
| | 2 | 223 | | output.Append(Alert.Terse); |
| | 2 | 224 | | output.Append(')'); |
| | | 225 | | } |
| | | 226 | | } |
| | 2 | 227 | | return output.ToString(); |
| | | 228 | | } |
| | | 229 | | /// <summary> |
| | | 230 | | /// Gets the 32-bit hash code for this object. |
| | | 231 | | /// </summary> |
| | | 232 | | /// <returns>The 32-bit hash code for this object.</returns> |
| | | 233 | | public override int GetHashCode() |
| | | 234 | | { |
| | 2 | 235 | | return Alert?.GetHashCode() ?? 0; |
| | | 236 | | } |
| | | 237 | | /// <summary> |
| | | 238 | | /// Checks to see if this <see cref="StatusAuditReport"/> is logically equal to another one. |
| | | 239 | | /// </summary> |
| | | 240 | | /// <param name="obj">The other object to compare to.</param> |
| | | 241 | | /// <returns>true if the objects are logically equivalent, false if they are not.</returns> |
| | | 242 | | public override bool Equals(object? obj) |
| | | 243 | | { |
| | 2 | 244 | | if (obj is not StatusAuditReport alert) return false; |
| | 2 | 245 | | return Equals(alert); |
| | | 246 | | } |
| | | 247 | | |
| | | 248 | | /// <summary> |
| | | 249 | | /// Checks to see if this <see cref="StatusAuditReport"/> is logically equal to another one. |
| | | 250 | | /// </summary> |
| | | 251 | | /// <param name="other">The other <see cref="StatusAuditReport"/> to compare to.</param> |
| | | 252 | | /// <returns>true if the objects are logically equivalent, false if they are not.</returns> |
| | | 253 | | public bool Equals(StatusAuditReport? other) |
| | | 254 | | { |
| | 2 | 255 | | if (other is null) return false; |
| | 2 | 256 | | return Equals(Alert, other.Alert); |
| | | 257 | | } |
| | | 258 | | |
| | | 259 | | /// <summary> |
| | | 260 | | /// Compares <paramref name="a"/> and <paramref name="b"/> and returns whether or not <paramref name="a"/> is equal |
| | | 261 | | /// </summary> |
| | | 262 | | /// <param name="a">The first <see cref="StatusAuditReport"/> to compare.</param> |
| | | 263 | | /// <param name="b">The second <see cref="StatusAuditReport"/> to compare.</param> |
| | | 264 | | /// <returns><b>true</b> if <paramref name="a"/> is equal to <paramref name="b"/>.</returns> |
| | | 265 | | public static bool operator ==(StatusAuditReport? a, StatusAuditReport? b) |
| | | 266 | | { |
| | 2 | 267 | | if (a is null) return b is null; |
| | 2 | 268 | | return a.Equals(b); |
| | | 269 | | } |
| | | 270 | | /// <summary> |
| | | 271 | | /// Compares <paramref name="a"/> and <paramref name="b"/> and returns whether or not <paramref name="a"/> is NOT eq |
| | | 272 | | /// </summary> |
| | | 273 | | /// <param name="a">The first <see cref="StatusAuditReport"/> to compare.</param> |
| | | 274 | | /// <param name="b">The second <see cref="StatusAuditReport"/> to compare.</param> |
| | | 275 | | /// <returns><b>true</b> if <paramref name="a"/> is NOT equal to <paramref name="b"/>.</returns> |
| | | 276 | | public static bool operator !=(StatusAuditReport? a, StatusAuditReport? b) |
| | | 277 | | { |
| | 2 | 278 | | if (a is null) return b is not null; |
| | 2 | 279 | | return !a.Equals(b); |
| | | 280 | | } |
| | | 281 | | } |
| | | 282 | | /// <summary> |
| | | 283 | | /// An abstract class that manages periodic status auditing of a system. |
| | | 284 | | /// Any derivative of this class will be automatically instantiated by the system retained in a system-wide list to trac |
| | | 285 | | /// </summary> |
| | | 286 | | /// <remarks> |
| | | 287 | | /// <pitch> |
| | | 288 | | /// Derive from this instead of <see cref="StatusChecker"/> when a system's health must be actively tested: implement <s |
| | | 289 | | /// </pitch> |
| | | 290 | | /// <pledge><see cref="StatusChecker"/></pledge> |
| | | 291 | | /// <pledge> |
| | | 292 | | /// <see cref="Audit"/> is invoked once shortly after registration and then periodically; the audit frequency self-tunes |
| | | 293 | | /// <see cref="GetStatus"/> always performs a fresh audit rather than returning cached results, and results are recorded |
| | | 294 | | /// </pledge> |
| | | 295 | | /// <plan> |
| | | 296 | | /// Two <see cref="AmbientEventTimer"/>s drive scheduling: a one-shot ~10ms timer for the initial audit (disposed after |
| | | 297 | | /// All mutable scheduling state is interlocked; background audits observe an <see cref="AmbientCancellationTokenSource" |
| | | 298 | | /// </plan> |
| | | 299 | | /// <priority> |
| | | 300 | | /// <see cref="StatusChecker"/> |
| | | 301 | | /// 1. Noticing trouble and recovery quickly over an even sampling rate: a worse rating shortens the interval, so a fail |
| | | 302 | | /// 2. Never letting the tests congest what they test over holding that pace: audit duration damps the schedule the othe |
| | | 303 | | /// 3. Capturing an audit failure as a result over letting it propagate: exceptions escaping <see cref="Audit"/> become |
| | | 304 | | /// </priority> |
| | | 305 | | /// </remarks> |
| | | 306 | | public abstract class StatusAuditor : StatusChecker |
| | | 307 | | { |
| | | 308 | | private readonly StatusResults _shutdownInProgress; // may be returned if results are requested during shutdown |
| | | 309 | | private readonly TimeSpan _baselineAuditFrequency; |
| | | 310 | | private readonly AmbientEventTimer _initialAuditTimer; // only used until the initial audit happens, then dispose |
| | | 311 | | private readonly AmbientEventTimer _auditTimer; |
| | | 312 | | |
| | | 313 | | private AmbientCancellationTokenSource? _backgroundCancelSource = new(); // interlocked |
| | | 314 | | private int _backgroundAuditCount; // interlocked |
| | | 315 | | private int _foregroundAuditCount; // interlocked |
| | | 316 | | private long _nextAuditTime; // interlocked |
| | | 317 | | private long _frequencyTicks; // interlocked -- the current audit frequency, adjusted based on how long th |
| | | 318 | | |
| | | 319 | | /// <summary> |
| | | 320 | | /// Constructs an <see cref="StatusAuditor"/> with the specified values. |
| | | 321 | | /// </summary> |
| | | 322 | | /// <param name="targetSystem">The name of the target system (if any).</param> |
| | | 323 | | /// <param name="baselineAuditFrequency"> |
| | | 324 | | /// The baseline frequency with which audits should be run. |
| | | 325 | | /// The system will start running tests this frequently but will automatically tune the frequency to between one qua |
| | | 326 | | /// This means that systems that process status audits faster will automatically be more responsive in reporting iss |
| | | 327 | | /// If audits start to timeout, audits will happen less frequently even if they are failing so that the status tests |
| | | 328 | | /// <see cref="TimeSpan.Zero"/> and negative time spans are treated as if they were <see cref="TimeSpan.MaxValue"/>. |
| | | 329 | | /// An initial audit will be scheduled to begin ten milliseconds after the start of construction, using an <see cref |
| | | 330 | | /// </param> |
| | | 331 | | /// <param name="status">The <see cref="Status"/> this auditor belongs to, or null if this should be a standalone au |
| | | 332 | | internal protected StatusAuditor(string targetSystem, TimeSpan baselineAuditFrequency, Status? status) |
| | | 333 | | : base(targetSystem) |
| | | 334 | | { |
| | | 335 | | Owner = status; |
| | | 336 | | _baselineAuditFrequency = baselineAuditFrequency; |
| | | 337 | | |
| | | 338 | | _frequencyTicks = baselineAuditFrequency.Ticks; |
| | | 339 | | _nextAuditTime = AmbientClock.UtcNow.AddMilliseconds(10).Ticks; |
| | | 340 | | // create a timer for the initial audit (we'll dispose of this one immediately as soon as that audit finishes) |
| | | 341 | | _initialAuditTimer = new AmbientEventTimer(10); |
| | | 342 | | _initialAuditTimer.Elapsed += InitialAuditTimer_Elapsed; |
| | | 343 | | _initialAuditTimer.AutoReset = false; |
| | | 344 | | // should we update periodically? |
| | | 345 | | if (baselineAuditFrequency < TimeSpan.MaxValue && baselineAuditFrequency > TimeSpan.FromTicks(0)) |
| | | 346 | | { |
| | | 347 | | _auditTimer = new AmbientEventTimer(TimeSpan.FromTicks(_frequencyTicks).TotalMilliseconds); |
| | | 348 | | _auditTimer.Elapsed += AuditTimer_Elapsed; |
| | | 349 | | } |
| | | 350 | | else // other parts of the code assume that _auditTimer is not null, so we will create one here that we don't ho |
| | | 351 | | { |
| | | 352 | | _auditTimer = new AmbientEventTimer(int.MaxValue - 1); |
| | | 353 | | } |
| | | 354 | | _auditTimer.AutoReset = false; |
| | | 355 | | // note that the audit timer should remain stopped until we start it after the first audit happens |
| | | 356 | | _shutdownInProgress = StatusResults.GetPendingResults(null, targetSystem); |
| | | 357 | | } |
| | | 358 | | |
| | | 359 | | /// <summary> |
| | | 360 | | /// Constructs an <see cref="StatusAuditor"/> associated with the default status instance and with the specified pro |
| | | 361 | | /// </summary> |
| | | 362 | | /// <param name="targetSystem">The name of the target system (if any).</param> |
| | | 363 | | /// <param name="baselineAuditFrequency"> |
| | | 364 | | /// The baseline frequency with which audits should be run. |
| | | 365 | | /// The system will start running tests this frequently but will automatically tune the frequency to between one qua |
| | | 366 | | /// This means that systems that process status audits faster will automatically be more responsive in reporting iss |
| | | 367 | | /// If audits start to timeout, audits will happen less frequently even if they are failing so that the status tests |
| | | 368 | | /// <see cref="TimeSpan.Zero"/> and negative time spans are treated as if they were <see cref="TimeSpan.MaxValue"/>. |
| | | 369 | | /// </param> |
| | | 370 | | internal protected StatusAuditor(string targetSystem, TimeSpan baselineAuditFrequency) |
| | | 371 | | : this(targetSystem, baselineAuditFrequency, Status.DefaultInstance) |
| | | 372 | | { |
| | | 373 | | } |
| | | 374 | | /// <summary> |
| | | 375 | | /// Gets the <see cref="Status"/> that owns this auditor, if there is one. |
| | | 376 | | /// </summary> |
| | | 377 | | public Status? Owner { get; } |
| | | 378 | | |
| | | 379 | | internal void ScheduleInitialAudit() |
| | | 380 | | { |
| | | 381 | | _initialAuditTimer.Enabled = true; |
| | | 382 | | } |
| | | 383 | | internal async void InitialAuditTimer_Elapsed(object? sender, System.Timers.ElapsedEventArgs? e) |
| | | 384 | | { |
| | | 385 | | CancellationToken cancel = _backgroundCancelSource?.Token ?? default; |
| | | 386 | | _initialAuditTimer.Enabled = false; |
| | | 387 | | await InternalAuditAsync(false, cancel); |
| | | 388 | | _initialAuditTimer.Close(); |
| | | 389 | | } |
| | | 390 | | internal async void AuditTimer_Elapsed(object? sender, System.Timers.ElapsedEventArgs? e) |
| | | 391 | | { |
| | | 392 | | CancellationToken cancel = _backgroundCancelSource?.Token ?? default; |
| | | 393 | | await InternalAuditAsync(false, cancel); |
| | | 394 | | } |
| | | 395 | | /// <summary> |
| | | 396 | | /// Computes the current status, building a <see cref="StatusResults"/> to hold information about the status. |
| | | 397 | | /// </summary> |
| | | 398 | | /// <param name="cancel">A <see cref="CancellationToken"/> to cancel the operation before it finishes.</param> |
| | | 399 | | public async override sealed ValueTask<StatusResults> GetStatus(CancellationToken cancel = default) |
| | | 400 | | { |
| | | 401 | | return await InternalAuditAsync(true, cancel); |
| | | 402 | | } |
| | | 403 | | private async ValueTask<StatusResults> InternalAuditAsync(bool foreground = false, CancellationToken cancel = defaul |
| | | 404 | | { |
| | | 405 | | StatusResultsBuilder builder = new(this); |
| | | 406 | | try |
| | | 407 | | { |
| | | 408 | | try |
| | | 409 | | { |
| | | 410 | | // in case the timer went off more than once due to test (or overall system) slowness, disable the timer |
| | | 411 | | _auditTimer.Stop(); |
| | | 412 | | // have we already shut down? bail out now! |
| | | 413 | | if (foreground) Interlocked.Increment(ref _foregroundAuditCount); else Interlocked.Increment(ref _backgr |
| | | 414 | | // call the derived object to get the status |
| | | 415 | | await Audit(builder, cancel); |
| | | 416 | | // schedule the next audit |
| | | 417 | | builder.NextAuditTime = ScheduleNextAudit(builder.WorstAlert?.Rating, builder.Elapsed); |
| | | 418 | | } |
| | | 419 | | #pragma warning disable CA1031 // we really DO want to catch ALL exceptions here--this is a status test, and the except |
| | | 420 | | catch (Exception ex) |
| | | 421 | | #pragma warning restore CA1031 |
| | | 422 | | { |
| | | 423 | | builder.AddException(ex); |
| | | 424 | | } |
| | | 425 | | finally |
| | | 426 | | { |
| | | 427 | | _auditTimer.Start(); |
| | | 428 | | } |
| | | 429 | | } |
| | | 430 | | catch (ObjectDisposedException) |
| | | 431 | | { |
| | | 432 | | // ignore this exception--given the design of System.Timers.Timer, it's impossible to prevent |
| | | 433 | | // it happens when an audit happens to get triggered just before shutdown/disposal |
| | | 434 | | return _shutdownInProgress; |
| | | 435 | | } |
| | | 436 | | // get the results |
| | | 437 | | StatusResults newStatusResults = builder.FinalResults; |
| | | 438 | | // save the results AND return them |
| | | 439 | | SetLatestResults(newStatusResults); |
| | | 440 | | return newStatusResults; |
| | | 441 | | } |
| | | 442 | | private DateTime ScheduleNextAudit(float? rating, TimeSpan auditDuration) |
| | | 443 | | { |
| | | 444 | | // set the next audit time |
| | | 445 | | TimeSpan nextInterval = AdjustedAuditInterval(rating, auditDuration); |
| | | 446 | | System.Diagnostics.Debug.Assert(nextInterval.Ticks > 0); |
| | | 447 | | DateTime nextAudit = (nextInterval == TimeSpan.MaxValue) ? DateTime.MaxValue : AmbientClock.UtcNow + nextInterva |
| | | 448 | | Interlocked.Exchange(ref _nextAuditTime, nextAudit.Ticks); |
| | | 449 | | _auditTimer.Interval = nextInterval.TotalMilliseconds; |
| | | 450 | | return nextAudit; |
| | | 451 | | } |
| | | 452 | | private TimeSpan AdjustedAuditInterval(float? rating, TimeSpan auditDuration) |
| | | 453 | | { |
| | | 454 | | /* |
| | | 455 | | * The audit interval is automatically adjusted based on two competing factors: |
| | | 456 | | * 1. The status of the system, whether it is failing, alerting, okay, or superlative |
| | | 457 | | * 2. The duration required to perform the audit |
| | | 458 | | * As the status gets worse, the frequency goes up so that the system can more quickly determine when the failin |
| | | 459 | | * As the duration gets longer, the frequency goes down to ensure that we don't consume too many resources eithe |
| | | 460 | | * */ |
| | | 461 | | if (_baselineAuditFrequency.Ticks <= 0 || _baselineAuditFrequency == TimeSpan.MaxValue) return TimeSpan.MaxValue |
| | | 462 | | float ratingAdjustment; |
| | | 463 | | if (rating == null) |
| | | 464 | | { |
| | | 465 | | Interlocked.Exchange(ref _frequencyTicks, _baselineAuditFrequency.Ticks); |
| | | 466 | | return TimeSpan.FromTicks(_frequencyTicks); |
| | | 467 | | } |
| | | 468 | | else // we have a rating, so we'll adjust based on both the rating and the test duration |
| | | 469 | | { |
| | | 470 | | if (rating <= StatusRating.Fail) |
| | | 471 | | { |
| | | 472 | | ratingAdjustment = 0.75f; // do the test a lot more frequently because the system is failing and we wa |
| | | 473 | | } |
| | | 474 | | else if (rating <= StatusRating.Alert) |
| | | 475 | | { |
| | | 476 | | ratingAdjustment = 0.9f; // do the tests a little more frequently because the system may be in a bad |
| | | 477 | | } |
| | | 478 | | else if (rating <= StatusRating.Okay) |
| | | 479 | | { |
| | | 480 | | ratingAdjustment = 1.1f; // do the tests less frequently because the system is working just fine |
| | | 481 | | } |
| | | 482 | | else |
| | | 483 | | { |
| | | 484 | | ratingAdjustment = 1.5f; // do the tests much less frequently because the system is working superlati |
| | | 485 | | } |
| | | 486 | | } |
| | | 487 | | TimeSpan oldFrequency = TimeSpan.FromTicks(_frequencyTicks); |
| | | 488 | | // if the test took more than 1/1000th of the current frequency, slow it down--status tests shouldn't be taking |
| | | 489 | | float durationAdjustment = (float)Math.Pow((1000 * auditDuration.TotalMilliseconds + 1) / (oldFrequency.TotalMil |
| | | 490 | | // adjust the frequency based on the duration of the audit and the rating--the result should always be between o |
| | | 491 | | Interlocked.Exchange(ref _frequencyTicks, (long)Math.Max(_baselineAuditFrequency.Ticks / 10, Math.Min(_baselineA |
| | | 492 | | System.Diagnostics.Debug.Assert(_frequencyTicks > 0); |
| | | 493 | | return TimeSpan.FromTicks(_frequencyTicks); |
| | | 494 | | } |
| | | 495 | | /// <summary> |
| | | 496 | | /// Computes the current status, filling in <paramref name="statusBuilder"/> with information about the status. |
| | | 497 | | /// Note that this function is only public instead of protected so that it can be unit tested more easily. |
| | | 498 | | /// The status system calls this function internally. |
| | | 499 | | /// Due to race conditions, audits may occur even after the status system shuts down, but should never happen more t |
| | | 500 | | /// </summary> |
| | | 501 | | /// <param name="statusBuilder">A <see cref="StatusResultsBuilder"/> to put the audit results into.</param> |
| | | 502 | | /// <param name="cancel">A <see cref="CancellationToken"/> to cancel the operation before it finishes.</param> |
| | | 503 | | public abstract ValueTask Audit(StatusResultsBuilder statusBuilder, CancellationToken cancel = default); |
| | | 504 | | |
| | | 505 | | /// <summary> |
| | | 506 | | /// Starts stopping any asynchronous activity (such as periodic audits). |
| | | 507 | | /// Due to race conditions, occasionally one more audit may occur after this function returns. |
| | | 508 | | /// </summary> |
| | | 509 | | internal protected override sealed ValueTask BeginStop() |
| | | 510 | | { |
| | | 511 | | _initialAuditTimer.Close(); // just in case--we must have shut down pretty quickly to get here without this tim |
| | | 512 | | _auditTimer.Stop(); |
| | | 513 | | _backgroundCancelSource?.Cancel(); |
| | | 514 | | return default; |
| | | 515 | | } |
| | | 516 | | /// <summary> |
| | | 517 | | /// Finishes stopping any asynchronous activity; |
| | | 518 | | /// </summary> |
| | | 519 | | internal protected override sealed ValueTask FinishStop() |
| | | 520 | | { |
| | | 521 | | return default; |
| | | 522 | | } |
| | | 523 | | /// <summary> |
| | | 524 | | /// Dispose the instance (only used by derived classes). |
| | | 525 | | /// </summary> |
| | | 526 | | /// <param name="disposing">Whether or not we are disposing (as opposed to finalizing).</param> |
| | | 527 | | protected override void Dispose(bool disposing) |
| | | 528 | | { |
| | | 529 | | base.Dispose(disposing); |
| | | 530 | | if (disposing) |
| | | 531 | | { |
| | | 532 | | _initialAuditTimer.Dispose(); // we've usually disposed of this timer, but not if we didn't call ScheduleI |
| | | 533 | | _auditTimer.Dispose(); |
| | | 534 | | if (_backgroundCancelSource != null) |
| | | 535 | | { |
| | | 536 | | _backgroundCancelSource.Dispose(); |
| | | 537 | | _backgroundCancelSource = null; |
| | | 538 | | } |
| | | 539 | | } |
| | | 540 | | } |
| | | 541 | | } |