diff --git a/API.IntegrationTests/GeoLocatedWebApplicationFactory.cs b/API.IntegrationTests/GeoLocatedWebApplicationFactory.cs index 887655d6..c3f1bea6 100644 --- a/API.IntegrationTests/GeoLocatedWebApplicationFactory.cs +++ b/API.IntegrationTests/GeoLocatedWebApplicationFactory.cs @@ -37,6 +37,6 @@ private sealed class FakeIpEnrichmentService : IIpEnrichmentService public GeoPoint? Location { get; set; } public IpEnrichmentData? Enrich(IPAddress ip) => - Location is { } location ? new IpEnrichmentData(null, null, null, null, location, 20) : null; + Location is { } location ? new IpEnrichmentData(null, null, null, null, null, location, 20) : null; } } \ No newline at end of file diff --git a/API/Controller/Admin/GetOnlineDevices.cs b/API/Controller/Admin/GetOnlineDevices.cs index fe59ea65..42817119 100644 --- a/API/Controller/Admin/GetOnlineDevices.cs +++ b/API/Controller/Admin/GetOnlineDevices.cs @@ -60,7 +60,9 @@ public async Task GetOnlineDevices() LatencyMs = x.LatencyMs, Rssi = x.Rssi, Country = x.Country, - Ip = IpAddressUtils.ParseStoredOrNull(x.Ip, _logger) + Ip = IpAddressUtils.ParseStoredOrNull(x.Ip, _logger), + Asn = x.Asn, + AsnOrg = x.AsnOrg }; }) ); @@ -84,5 +86,7 @@ public sealed class AdminOnlineDeviceResponse public required int? Rssi { get; init; } public required string? Country { get; set; } public required IPAddress? Ip { get; set; } + public required long? Asn { get; set; } + public required string? AsnOrg { get; set; } } } \ No newline at end of file diff --git a/Common/OpenShockServiceHelper.cs b/Common/OpenShockServiceHelper.cs index 6ffef408..0bef97f4 100644 --- a/Common/OpenShockServiceHelper.cs +++ b/Common/OpenShockServiceHelper.cs @@ -347,7 +347,7 @@ public static IServiceCollection AddOpenShockServices(this IServiceCollection se services.AddScoped(); // Ensure GeoOptions is always resolvable so IpEnrichmentService can activate even in hosts - // (Cron, LiveControlGateway, SeedE2E) that don't call RegisterGeoOptions(). TryAdd leaves the + // (Cron, SeedE2E) that don't call RegisterGeoOptions(). TryAdd leaves the // API's config-bound instance untouched; other hosts get a disabled default (no DB paths). services.TryAddSingleton(new GeoOptions()); services.AddSingleton(); diff --git a/Common/Redis/DeviceOnline.cs b/Common/Redis/DeviceOnline.cs index 73a52045..27f804e9 100644 --- a/Common/Redis/DeviceOnline.cs +++ b/Common/Redis/DeviceOnline.cs @@ -23,4 +23,6 @@ public sealed class DeviceOnline public int? Rssi { get; set; } public string? Country { get; set; } public string? Ip { get; set; } + public long? Asn { get; set; } + public string? AsnOrg { get; set; } } \ No newline at end of file diff --git a/Common/Services/Geo/IpEnrichmentData.cs b/Common/Services/Geo/IpEnrichmentData.cs index 285d55cd..3377c584 100644 --- a/Common/Services/Geo/IpEnrichmentData.cs +++ b/Common/Services/Geo/IpEnrichmentData.cs @@ -1,6 +1,7 @@ namespace OpenShock.Common.Services.Geo; public sealed record IpEnrichmentData( + long? Asn, string? AsnOrg, bool? IsVpn, string? CountryCode, diff --git a/Common/Services/Geo/IpEnrichmentService.cs b/Common/Services/Geo/IpEnrichmentService.cs index eac262fb..b326a16a 100644 --- a/Common/Services/Geo/IpEnrichmentService.cs +++ b/Common/Services/Geo/IpEnrichmentService.cs @@ -86,6 +86,7 @@ public static bool MatchesVpnProvider(string asnOrg) { if (_asnReader is null && _cityReader is null) return null; + long? asnNumber = null; string? asnOrg = null; // Null means "unknown" (no ASN DB, lookup miss, or failure); only a resolved ASN org yields a verdict. bool? isVpn = null; @@ -96,6 +97,7 @@ public static bool MatchesVpnProvider(string asnOrg) { if (_asnReader.TryAsn(ip, out var asn) && asn is not null) { + asnNumber = asn.AutonomousSystemNumber; asnOrg = asn.AutonomousSystemOrganization; if (asnOrg is not null) { @@ -132,7 +134,7 @@ public static bool MatchesVpnProvider(string asnOrg) } } - return new IpEnrichmentData(asnOrg, isVpn, countryCode, city, location, accuracyRadiusKm); + return new IpEnrichmentData(asnNumber, asnOrg, isVpn, countryCode, city, location, accuracyRadiusKm); } public void Dispose() diff --git a/LiveControlGateway/Controllers/HubControllerBase.cs b/LiveControlGateway/Controllers/HubControllerBase.cs index 159fe575..f38be002 100644 --- a/LiveControlGateway/Controllers/HubControllerBase.cs +++ b/LiveControlGateway/Controllers/HubControllerBase.cs @@ -17,6 +17,7 @@ using System.Security.Claims; using OpenShock.Common.Authentication; using OpenShock.Common.Extensions; +using OpenShock.Common.Services.Geo; using SemVersion = OpenShock.Common.Models.SemVersion; using Timer = System.Timers.Timer; @@ -65,6 +66,9 @@ protected HubLifetime HubLifetime /// public override Guid Id => CurrentHubId; + + /// + public IpEnrichmentData? IpInfo { get; private set; } /// /// Authentication context @@ -143,6 +147,10 @@ protected override async Task ConnectionPrecondition() } _userAgent = HttpContext.Request.Headers.UserAgent.ToString().Truncate(256); + + // Resolved before the lifetime is added, so the connect attempt metric already carries it. + IpInfo = ServiceProvider.GetRequiredService().Enrich(HttpContext.GetRemoteIP()); + var hubLifetimeResult = await _hubLifetimeManager.TryAddDeviceConnection(5, this, LinkedToken); switch (hubLifetimeResult) @@ -250,7 +258,9 @@ protected async Task SelfOnline(ulong uptimeMs, ushort? latency = null, in LatencyMs = latency, Rssi = rssi, Country = HttpContext.GetCFIPCountry(), - Ip = HttpContext.GetRemoteIP() + Ip = HttpContext.GetRemoteIP(), + Asn = IpInfo?.Asn, + AsnOrg = IpInfo?.AsnOrg }); return true; diff --git a/LiveControlGateway/Controllers/IHubController.cs b/LiveControlGateway/Controllers/IHubController.cs index e710bc5e..ad98ba30 100644 --- a/LiveControlGateway/Controllers/IHubController.cs +++ b/LiveControlGateway/Controllers/IHubController.cs @@ -1,4 +1,5 @@ using OpenShock.Common.Models; +using OpenShock.Common.Services.Geo; using OpenShock.Serialization.Gateway; namespace OpenShock.LiveControlGateway.Controllers; @@ -13,6 +14,12 @@ public interface IHubController : IAsyncDisposable /// public Guid Id { get; } + /// + /// GeoIP data for the address this connection came from, resolved once when it connected. + /// Null when no GeoIP database is configured. + /// + public IpEnrichmentData? IpInfo { get; } + /// /// Control shockers /// diff --git a/LiveControlGateway/LifetimeManager/HubLifetime.cs b/LiveControlGateway/LifetimeManager/HubLifetime.cs index 9d1f4820..576d0f1e 100644 --- a/LiveControlGateway/LifetimeManager/HubLifetime.cs +++ b/LiveControlGateway/LifetimeManager/HubLifetime.cs @@ -504,9 +504,12 @@ public async Task> Online(Guid device, SelfO // as we don't want to send a device online status every time, we will do it here online.BootedAt = data.BootedAt; online.LatencyMs = data.LatencyMs; + online.Rssi = data.Rssi; online.Country = data.Country; online.Ip = data.Ip?.ToString(); + online.Asn = data.Asn; + online.AsnOrg = data.AsnOrg; var sendOnlineStatusUpdate = false; @@ -562,6 +565,8 @@ await deviceOnline.InsertAsync(new DeviceOnline Rssi = data.Rssi, Country = data.Country, Ip = data.Ip?.ToString(), + Asn = data.Asn, + AsnOrg = data.AsnOrg, }, Duration.DeviceKeepAliveTimeout); } } @@ -674,4 +679,14 @@ public SelfOnlineData( /// Remote ip address /// public IPAddress? Ip { get; init; } = null; + + /// + /// Autonomous system number of the remote ip, if GeoIP is available + /// + public long? Asn { get; init; } = null; + + /// + /// Organization owning + /// + public string? AsnOrg { get; init; } = null; } \ No newline at end of file diff --git a/LiveControlGateway/LifetimeManager/HubLifetimeManager.cs b/LiveControlGateway/LifetimeManager/HubLifetimeManager.cs index 0abec7fc..614d6e53 100644 --- a/LiveControlGateway/LifetimeManager/HubLifetimeManager.cs +++ b/LiveControlGateway/LifetimeManager/HubLifetimeManager.cs @@ -75,11 +75,32 @@ public HubLifetimeManager( meter.CreateObservableUpDownCounter("openshock_hub_connections", () => { - return new[] + var lifetimes = _lifetimes; + + // Grouped on the org as well as the number: a lookup can resolve the ASN but not its org, + // and the series here must match the tags the per-hub counters were recorded with. + var perAsn = new Dictionary<(string Asn, string Org), int>(); + foreach (var (_, lifetime) in lifetimes) { - new Measurement(_lifetimes.Count, gatewayFqdn) - }; - }, "connections", "Current number of connected hubs"); + var ipInfo = lifetime.HubController.IpInfo; + var key = ((string)GatewayMetrics.AsnTag(ipInfo).Value!, (string)GatewayMetrics.AsnOrgTag(ipInfo).Value!); + perAsn[key] = perAsn.GetValueOrDefault(key) + 1; + } + + // An idle gateway still reports a zero, so a sum over gateways does not go empty. + if (perAsn.Count == 0) perAsn[(GatewayMetrics.Unknown, GatewayMetrics.Unknown)] = 0; + + var measurements = new Measurement[perAsn.Count]; + var i = 0; + foreach (var ((asn, org), count) in perAsn) + { + measurements[i++] = new Measurement(count, gatewayFqdn, + new KeyValuePair("asn", asn), + new KeyValuePair("asn_org", org)); + } + + return measurements; + }, "connections", "Current number of connected hubs by network (ASN)"); // Derived from the hubs themselves rather than tracked alongside them: a separately // maintained counter drifts from the truth the moment a teardown path misses a decrement. @@ -130,7 +151,7 @@ public async Task> TryAddDeviceConnection(byte // There already is a hub lifetime, lets swap! if (!hubLifetime.TryMarkSwapping()) { - _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.Busy); + _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.Busy, hubController); return new Busy(); } @@ -149,7 +170,7 @@ public async Task> TryAddDeviceConnection(byte { _logger.LogTrace("Swapping hub lifetime [{HubId}]", hubController.Id); await hubLifetime.Swap(hubController); - _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.Swapped); + _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.Swapped, hubController); } else { @@ -159,11 +180,11 @@ public async Task> TryAddDeviceConnection(byte // If we fail to initialize, the hub must be removed await RemoveDeviceConnection(hubController); // Here be dragons? _logger.LogError("Failed to initialize hub lifetime [{HubId}]", hubController.Id); - _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.InitFailed); + _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.InitFailed, hubController); return new Error(); } - _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.Connected); + _metrics.HubConnectAttempt(GatewayMetrics.HubConnectOutcome.Connected, hubController); } return hubLifetime; @@ -237,7 +258,7 @@ public async Task RemoveDeviceConnection(IHubController hubController) else { _lifetimes = withoutHub; - _metrics.HubDisconnected(); + _metrics.HubDisconnected(hubController); } } } diff --git a/LiveControlGateway/Metrics/GatewayMetrics.cs b/LiveControlGateway/Metrics/GatewayMetrics.cs index 27348d77..63b0d9eb 100644 --- a/LiveControlGateway/Metrics/GatewayMetrics.cs +++ b/LiveControlGateway/Metrics/GatewayMetrics.cs @@ -1,6 +1,8 @@ using System.Diagnostics.Metrics; using OpenShock.Common.Metrics; using OpenShock.Common.OpenShockDb; +using OpenShock.Common.Services.Geo; +using OpenShock.LiveControlGateway.Controllers; using OpenShock.LiveControlGateway.Options; namespace OpenShock.LiveControlGateway.Metrics; @@ -14,6 +16,11 @@ namespace OpenShock.LiveControlGateway.Metrics; /// Every measurement carries gateway_fqdn as a normal tag rather than a tag on the /// itself - a meter-level tag is a *scope* attribute, which the Prometheus /// exporter emits prefixed as otel_scope_gateway_fqdn. +/// +/// Hub instruments also carry the hub's network as asn and asn_org, from GeoIP. The +/// org is redundant with the number but saves every dashboard a lookup table; it adds no series +/// of its own. Both are when GeoIP is not configured or the lookup missed. +/// /// public sealed class GatewayMetrics { @@ -71,6 +78,9 @@ public static class FrameOutcome public const string ShockerExclusive = "shocker_exclusive"; } + /// Tag value for a hub whose network could not be resolved. + public const string Unknown = "unknown"; + private readonly KeyValuePair _gatewayFqdn; private readonly Counter _hubConnectAttempts; @@ -117,13 +127,29 @@ public GatewayMetrics(LcgOptions lcgOptions, [FromKeyedServices("OpenShock.Gatew /// Record the outcome of a hub connection attempt. /// /// One of - public void HubConnectAttempt(string outcome) => - _hubConnectAttempts.Add(1, _gatewayFqdn, new KeyValuePair("outcome", outcome)); + /// The connecting hub + public void HubConnectAttempt(string outcome, IHubController hub) => + _hubConnectAttempts.Add(1, _gatewayFqdn, new KeyValuePair("outcome", outcome), + AsnTag(hub.IpInfo), AsnOrgTag(hub.IpInfo)); /// /// Record a hub lifetime being torn down. /// - public void HubDisconnected() => _hubDisconnections.Add(1, _gatewayFqdn); + /// The hub that disconnected + public void HubDisconnected(IHubController hub) => + _hubDisconnections.Add(1, _gatewayFqdn, AsnTag(hub.IpInfo), AsnOrgTag(hub.IpInfo)); + + /// + /// The asn tag for a hub's network. + /// + public static KeyValuePair AsnTag(IpEnrichmentData? ipInfo) => + new("asn", ipInfo?.Asn?.ToString() ?? Unknown); + + /// + /// The asn_org tag for a hub's network. + /// + public static KeyValuePair AsnOrgTag(IpEnrichmentData? ipInfo) => + new("asn_org", ipInfo?.AsnOrg ?? Unknown); /// /// Record the outcome of a live control session attempt. This is churn only - the number of diff --git a/LiveControlGateway/Program.cs b/LiveControlGateway/Program.cs index aa7403f3..94e5a187 100644 --- a/LiveControlGateway/Program.cs +++ b/LiveControlGateway/Program.cs @@ -17,6 +17,7 @@ var redisOptions = builder.RegisterRedisOptions(); var databaseOptions = builder.RegisterDatabaseOptions(); builder.RegisterMetricsOptions(); +builder.RegisterGeoOptions(); var lcgOptions = builder.Configuration.GetRequiredSection(LcgOptions.SectionName).Get(); if (lcgOptions is null)