Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
c0dccab
wip: stub MetricProcessor.cs
mykeelium Oct 1, 2025
2f38da1
wip: cleanup MetricProcessor and Writer
mykeelium Oct 2, 2025
ee826ce
chore: alter histogram value type
mykeelium Dec 29, 2025
1336653
fix: create Dictionary Directly
mykeelium Dec 29, 2025
b4e61cb
wip: create platform for defining and observing metrics
mykeelium Dec 30, 2025
78e9041
wip: adding some ldap metrics to LdapUtils and LdapConnectionPool
mykeelium Dec 31, 2025
6d422ce
feat: add FileMetricSink, MetricsFlushTimer, and MetricWriter. Update…
mykeelium Dec 31, 2025
3c6d140
feat: refine metric logic
mykeelium Jan 5, 2026
2be1793
test: fix AdaptiveTimeout LatencyObservation, Add Tests
mykeelium Jan 5, 2026
f01620f
test: fix AdaptiveTimeout LatencyObservation, Add MetricDefinitionTes…
mykeelium Jan 5, 2026
cf833bc
test: add FileMetricSinkTests
mykeelium Jan 6, 2026
9593443
chore: add notes for IsExternalInit.cs
mykeelium Jan 6, 2026
6425efb
tests: Add MetricAggregatorTests.cs, MetricRegistryTests.cs, and Metr…
mykeelium Jan 6, 2026
73b7e12
tests: adjusting to relax ranges on AdaptiveTimeoutTests
mykeelium Jan 6, 2026
4d7e12a
chore: coderabbit suggestions
mykeelium Jan 6, 2026
c0765c1
chore: more coderabbit suggestions
mykeelium Jan 6, 2026
cdedb02
chore: more coderabbit suggestions
mykeelium Jan 6, 2026
87504a9
feat: update generics for aggregator to send values
mykeelium Feb 25, 2026
cbe08dd
feat: add adaptive timeout metrics
mykeelium Mar 16, 2026
e930501
chore: coderabbit suggested changes
mykeelium Mar 16, 2026
afbc3c6
chore: coderabbit suggested changes on FileMetricSink
mykeelium Mar 16, 2026
77b7b03
chore: coderabbit suggested changes
mykeelium Mar 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 38 additions & 8 deletions src/CommonLib/AdaptiveTimeout.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,9 @@
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using SharpHoundCommonLib.Exceptions;
using SharpHoundCommonLib.Interfaces;
using SharpHoundCommonLib.Models;
using SharpHoundCommonLib.Static;
using SharpHoundRPC.NetAPINative;

namespace SharpHoundCommonLib;
Expand All @@ -26,6 +29,7 @@ public sealed class AdaptiveTimeout : IDisposable {
private const int ExcessiveTimeoutsThreshold = 7;
private const int StdDevMultiplier = 7; // 7 standard deviations should be a very conservative upper bound
private const int CountOfLatestSuccessToKeep = 3;
private readonly IMetricRouter _metrics;

public AdaptiveTimeout(TimeSpan maxTimeout, ILogger log, int sampleCount = 100, int logFrequency = 1000, int minSamplesForAdaptiveTimeout = 30, bool useAdaptiveTimeout = true, bool throwIfExcessiveTimeouts = false) {
if (maxTimeout <= TimeSpan.Zero)
Expand All @@ -47,6 +51,7 @@ public AdaptiveTimeout(TimeSpan maxTimeout, ILogger log, int sampleCount = 100,
_minSamplesForAdaptiveTimeout = minSamplesForAdaptiveTimeout;
_useAdaptiveTimeout = useAdaptiveTimeout;
_throwIfExcessiveTimeouts = throwIfExcessiveTimeouts;
_metrics = Metrics.Factory.CreateMetricRouter();
}

public AdaptiveTimeout(TimeSpan maxTimeout, TimeSpan minTimeout, ILogger log, int sampleCount = 100, int logFrequency = 1000, int minSamplesForAdaptiveTimeout = 30, bool useAdaptiveTimeout = true, bool throwIfExcessiveTimeouts = false)
Expand Down Expand Up @@ -75,15 +80,19 @@ public void ClearSamples() {
/// <typeparam name="T"></typeparam>
/// <param name="func"></param>
/// <param name="parentToken"></param>
/// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
/// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, T> func, CancellationToken parentToken = default) {
public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, T> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
DateTime startTime = default;
var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
_sampler.SampleExecutionTime(() => {
startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
return func(timeoutToken);
}), parentToken);
}, latencyObservation), parentToken);
TimeSpikeSafetyValve(result.IsSuccess, startTime);
if (!result.IsSuccess) {
_metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
}
return result;
}

Expand All @@ -97,15 +106,19 @@ public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, T> fu
/// </summary>
/// <param name="func"></param>
/// <param name="parentToken"></param>
/// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
/// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
public async Task<Result> ExecuteWithTimeout(Action<CancellationToken> func, CancellationToken parentToken = default) {
public async Task<Result> ExecuteWithTimeout(Action<CancellationToken> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
DateTime startTime = default;
var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
_sampler.SampleExecutionTime(() => {
startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
func(timeoutToken);
}), parentToken);
}, latencyObservation), parentToken);
TimeSpikeSafetyValve(result.IsSuccess, startTime);
if (!result.IsSuccess) {
_metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
}
return result;
}

Expand All @@ -120,15 +133,19 @@ public async Task<Result> ExecuteWithTimeout(Action<CancellationToken> func, Can
/// <typeparam name="T"></typeparam>
/// <param name="func"></param>
/// <param name="parentToken"></param>
/// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
/// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, Task<T>> func, CancellationToken parentToken = default) {
public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, Task<T>> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
DateTime startTime = default;
var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
_sampler.SampleExecutionTime(() => {
startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
return func(timeoutToken);
}), parentToken);
}, latencyObservation), parentToken);
TimeSpikeSafetyValve(result.IsSuccess, startTime);
if (!result.IsSuccess) {
_metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
}
return result;
}

Expand All @@ -142,15 +159,19 @@ public async Task<Result<T>> ExecuteWithTimeout<T>(Func<CancellationToken, Task<
/// </summary>
/// <param name="func"></param>
/// <param name="parentToken"></param>
/// <param name="latencyObservation">A method that is used to observe the latency of the request.</param>
/// <returns>Returns a Fail result if a task runs longer than its budgeted time.</returns>
public async Task<Result> ExecuteWithTimeout(Func<CancellationToken, Task> func, CancellationToken parentToken = default) {
public async Task<Result> ExecuteWithTimeout(Func<CancellationToken, Task> func, CancellationToken parentToken = default, Action<double> latencyObservation = null) {
DateTime startTime = default;
var result = await Timeout.ExecuteWithTimeout(GetAdaptiveTimeout(), (timeoutToken) =>
_sampler.SampleExecutionTime(() => {
startTime = DateTime.Now; // for ordinal tracking; see use in TimeSpikeSafetyValve
return func(timeoutToken);
}), parentToken);
}, latencyObservation), parentToken);
TimeSpikeSafetyValve(result.IsSuccess, startTime);
if (!result.IsSuccess) {
_metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
}
return result;
}

Expand All @@ -174,6 +195,9 @@ public async Task<NetAPIResult<T>> ExecuteNetAPIWithTimeout<T>(Func<Cancellation
return func(timeoutToken);
}), parentToken);
TimeSpikeSafetyValve(result.IsSuccess, startTime);
if (!result.IsSuccess) {
_metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
}
return result;
}

Expand All @@ -197,6 +221,9 @@ public async Task<NetAPIResult<T>> ExecuteNetAPIWithTimeout<T>(Func<Cancellation
return func(timeoutToken);
}), parentToken);
TimeSpikeSafetyValve(result.IsSuccess, startTime);
if (!result.IsSuccess) {
_metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
}
return result;
}

Expand All @@ -220,6 +247,9 @@ public async Task<NetAPIResult<T>> ExecuteNetAPIWithTimeout<T>(Func<Cancellation
return func(timeoutToken);
}), parentToken);
TimeSpikeSafetyValve(result.IsSuccess, startTime);
if (!result.IsSuccess) {
_metrics.Observe(AdaptiveTimeoutDefinitions.TimeoutsTotal, 1, new LabelValues());
}
return result;
}

Expand Down
12 changes: 8 additions & 4 deletions src/CommonLib/ExecutionTimeSampler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,35 +43,39 @@ public double StandardDeviation() {

public double Average() => _samples.Average();

public async Task<T> SampleExecutionTime<T>(Func<Task<T>> func) {
public async Task<T> SampleExecutionTime<T>(Func<Task<T>> func, Action<double> latencyObservation = null) {
var stopwatch = Stopwatch.StartNew();
var result = await func.Invoke();
stopwatch.Stop();
latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
AddTimeSample(stopwatch.Elapsed);

return result;
}

public async Task SampleExecutionTime(Func<Task> func) {
public async Task SampleExecutionTime(Func<Task> func, Action<double> latencyObservation = null) {
var stopwatch = Stopwatch.StartNew();
await func.Invoke();
stopwatch.Stop();
latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
AddTimeSample(stopwatch.Elapsed);
}

public T SampleExecutionTime<T>(Func<T> func) {
public T SampleExecutionTime<T>(Func<T> func, Action<double> latencyObservation = null) {
var stopwatch = Stopwatch.StartNew();
var result = func.Invoke();
stopwatch.Stop();
latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
AddTimeSample(stopwatch.Elapsed);

return result;
}

public void SampleExecutionTime(Action func) {
public void SampleExecutionTime(Action func, Action<double> latencyObservation = null) {
var stopwatch = Stopwatch.StartNew();
func.Invoke();
stopwatch.Stop();
latencyObservation?.Invoke(stopwatch.ElapsedMilliseconds);
AddTimeSample(stopwatch.Elapsed);
}

Expand Down
5 changes: 5 additions & 0 deletions src/CommonLib/Interfaces/ILabelValuesCache.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
namespace SharpHoundCommonLib.Interfaces;

public interface ILabelValuesCache {
string[] Intern(string[] values);
}
5 changes: 5 additions & 0 deletions src/CommonLib/Interfaces/IMetricFactory.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
namespace SharpHoundCommonLib.Interfaces;

public interface IMetricFactory {
IMetricRouter CreateMetricRouter();
}
9 changes: 9 additions & 0 deletions src/CommonLib/Interfaces/IMetricRegistry.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
using System.Collections.Generic;
using SharpHoundCommonLib.Models;

namespace SharpHoundCommonLib.Interfaces;

public interface IMetricRegistry {
bool TryRegister(MetricDefinition definition, out int definitionId);
IReadOnlyList<MetricDefinition> Definitions { get; }
}
8 changes: 8 additions & 0 deletions src/CommonLib/Interfaces/IMetricRouter.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
using SharpHoundCommonLib.Models;

namespace SharpHoundCommonLib.Interfaces;

public interface IMetricRouter {
void Observe(int definitionId, double value, LabelValues labelValues);
void Flush();
}
8 changes: 8 additions & 0 deletions src/CommonLib/Interfaces/IMetricSink.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
using SharpHoundCommonLib.Models;

namespace SharpHoundCommonLib.Interfaces;

public interface IMetricSink {
void Observe(in MetricObservation.DoubleMetricObservation observation);
void Flush();
}
17 changes: 17 additions & 0 deletions src/CommonLib/Interfaces/IMetricWriter.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
using System;
using System.Text;
using SharpHoundCommonLib.Models;
using SharpHoundCommonLib.Services;

namespace SharpHoundCommonLib.Interfaces;

public interface IMetricWriter {
void StringBuilderAppendMetric(
StringBuilder builder,
MetricDefinition definition,
LabelValues labelValues,
MetricAggregator aggregator,
DateTimeOffset timestamp,
string timestampOutputString = "yyyy-MM-dd HH:mm:ss.fff"
);
}
Loading
Loading