From 3abd5d5d9f79002d9852360b70d33a4103740c6d Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 12 Oct 2026 13:01:53 +0100 Subject: [PATCH 1/2] Add read-only smoke tests for the modules written from the spec Calls 43 read-only endpoints across the new modules against a real organization. A refusal (401/402/403/404/409) is reported, but a 400/422, an exception, or a 2xx whose body cannot be read fails the test. Logging is off and only outcomes are written, never response bodies. CODACY_SMOKE_REPORT writes the outcome lines to a file. Category=Integration, so CI skips it. Co-Authored-By: Claude Sonnet 5.5 --- .../Integration/NewModulesSmokeTests.cs | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 Codacy.Api.Test/Integration/NewModulesSmokeTests.cs diff --git a/Codacy.Api.Test/Integration/NewModulesSmokeTests.cs b/Codacy.Api.Test/Integration/NewModulesSmokeTests.cs new file mode 100644 index 0000000..f95ad90 --- /dev/null +++ b/Codacy.Api.Test/Integration/NewModulesSmokeTests.cs @@ -0,0 +1,146 @@ +using Refit; + +namespace Codacy.Api.Test.Integration; + +/// +/// Read-only smoke tests for the modules written from the OpenAPI specification (4.0.13), run +/// against a real organization. They exist to catch models that cannot deserialize a real +/// response, the failure that went unnoticed for the original security models. +/// +/// +/// A response Codacy refuses (401, 402, 403, 404, 409) is reported but does not fail the test: the +/// organization may not have the feature, plan or permission. A 400 or 422 means this client sent a +/// malformed request, and any other exception, including a 2xx whose body could not be read, means a +/// model does not match the real response. Both fail. Only outcomes are logged, never response +/// bodies, because some of these endpoints return secrets or capability URLs. Endpoints that mutate +/// state, return credentials, or need ids this organization may not have are left out. +/// +[Trait("Category", "Integration")] +public class NewModulesSmokeTests(ITestOutputHelper output) : TestBase(output) +{ + private readonly List _outcomes = []; + private readonly List _failures = []; + + private string TestBranch => Configuration["CodacyApi:TestBranch"] is { Length: > 0 } branch ? branch : "main"; + + [Fact] + public async Task ReadOnlyEndpoints_DeserializeRealResponses() + { + var p = TestProvider; + var o = TestOrganization; + var r = TestRepository; + var ct = CancellationToken; + + // Logging off: the shared client logs response bodies, and some of these hold secrets or capability URLs. + var baseOptions = GetClientOptions(); + using var client = new CodacyClient(new CodacyClientOptions { ApiToken = baseOptions.ApiToken, BaseUrl = baseOptions.BaseUrl }); + + // Repository settings, files, coverage and analysis + await Probe("RepositorySettings.GetQualitySettingsForRepository", () => client.RepositorySettings.GetQualitySettingsForRepositoryAsync(p, o, r, ct)); + await Probe("RepositorySettings.GetBuildServerAnalysisSetting", () => client.RepositorySettings.GetBuildServerAnalysisSettingAsync(p, o, r, ct)); + await Probe("RepositorySettings.GetRepositoryLanguages", () => client.RepositorySettings.GetRepositoryLanguagesAsync(p, o, r, ct)); + await Probe("RepositorySettings.GetRepositoryIntegrationsSettings", () => client.RepositorySettings.GetRepositoryIntegrationsSettingsAsync(p, o, r, ct)); + await Probe("RepositorySettings.GetBranchRequiredChecks", () => client.RepositorySettings.GetBranchRequiredChecksAsync(p, o, r, TestBranch, ct)); + await Probe("RepositoryFiles.ListDirectories", () => client.RepositoryFiles.ListDirectoriesAsync(p, o, r, null, null, null, null, null, 5, ct)); + await Probe("RepositoryFiles.ListIgnoredFiles", () => client.RepositoryFiles.ListIgnoredFilesAsync(p, o, r, null, null, null, 5, ct)); + await Probe("RepositoryCoverageReports.ListCoverageReports", () => client.RepositoryCoverageReports.ListCoverageReportsAsync(p, o, r, 5, ct)); + await Probe("AnalysisActions.GetAutoconfigStatus", () => client.AnalysisActions.GetAutoconfigStatusAsync(p, o, r, ct)); + await Probe("AnalysisActions.GetLatestAutoconfigRunSummary", () => client.AnalysisActions.GetLatestAutoconfigRunSummaryAsync(p, o, r, ct)); + await Probe("AnalysisActions.HasQuickfixSuggestions", () => client.AnalysisActions.HasQuickfixSuggestionsAsync(p, o, r, TestBranch, ct)); + await Probe("Sbom.GetRepositorySbomPresignedUrl", () => client.Sbom.GetRepositorySbomPresignedUrlAsync(p, o, r, ct)); + + // Gate policies, segments, integrations + var policies = await Probe("GatePolicies.ListGatePolicies", () => client.GatePolicies.ListGatePoliciesAsync(p, o, null, 5, ct)); + if (policies?.Data.FirstOrDefault() is { } policy) + { + await Probe("GatePolicies.GetGatePolicy", () => client.GatePolicies.GetGatePolicyAsync(p, o, policy.Id, ct)); + await Probe("GatePolicies.ListRepositoriesFollowingGatePolicy", () => client.GatePolicies.ListRepositoriesFollowingGatePolicyAsync(p, o, policy.Id, null, 5, ct)); + } + + await Probe("Segments.GetSegmentsSyncStatus", () => client.Segments.GetSegmentsSyncStatusAsync(p, o, ct)); + await Probe("Segments.GetSegmentsKeys", () => client.Segments.GetSegmentsKeysAsync(p, o, null, 5, null, ct)); + await Probe("Segments.GetSegmentsKeysWithIds", () => client.Segments.GetSegmentsKeysWithIdsAsync(p, o, null, 5, null, ct)); + await Probe("Jira.GetJiraIntegration", () => client.Jira.GetJiraIntegrationAsync(p, o, ct)); + await Probe("Slack.GetSlackIntegration", () => client.Slack.GetSlackIntegrationAsync(p, o, ct)); + await Probe("Dast.GetDastTargets", () => client.Dast.GetDastTargetsAsync(p, o, null, 5, ct)); + await Probe("Images.ListOrganizationImages", () => client.Images.ListOrganizationImagesAsync(p, o, null, 5, ct)); + + // Organization settings, billing and platform + await Probe("OrganizationSettings.GetProviderSettings", () => client.OrganizationSettings.GetProviderSettingsAsync(p, o, ct)); + await Probe("OrganizationSettings.RetrieveOrganizationOnboardingProgress", () => client.OrganizationSettings.RetrieveOrganizationOnboardingProgressAsync(p, o, ct)); + await Probe("OrganizationSettings.GetGitProviderAppPermissions", () => client.OrganizationSettings.GetGitProviderAppPermissionsAsync(p, o, ct)); + await Probe("OrganizationSettings.CheckIfUserCanLeave", () => client.OrganizationSettings.CheckIfUserCanLeaveAsync(p, o, ct)); + await Probe("OrganizationSettings.ListOrganizationJoinRequests", () => client.OrganizationSettings.ListOrganizationJoinRequestsAsync(p, o, null, 5, null, ct)); + await Probe("OrganizationSettings.CheckSubmodules", () => client.OrganizationSettings.CheckSubmodulesAsync(p, o, ct)); + await Probe("OrganizationSettings.GetAiRiskCheckList", () => client.OrganizationSettings.GetAiRiskCheckListAsync(p, o, ct)); + await Probe("Billing.ListPaymentPlans", () => client.Billing.ListPaymentPlansAsync(ct)); + await Probe("Platform.ListConfiguredLoginIntegrations", () => client.Platform.ListConfiguredLoginIntegrationsAsync(null, 5, ct)); + await Probe("Platform.ListProviderIntegrations", () => client.Platform.ListProviderIntegrationsAsync(null, 5, ct)); + await Probe("Platform.GetConfigurationStatus", () => client.Platform.GetConfigurationStatusAsync(ct)); + await Probe("Platform.Health", () => client.Platform.HealthAsync(ct)); + await Probe("Enterprises.ListEnterprises", () => client.Enterprises.ListEnterprisesAsync(p, null, 5, ct)); + await Probe("Metrics.ReadyMetricsForOrganization", () => client.Metrics.ReadyMetricsForOrganizationAsync(p, o, ct)); + + // Tools and patterns + await Probe("Tools.ListLanguagesWithTools", () => client.Tools.ListLanguagesWithToolsAsync(ct)); + await Probe("Tools.ListDuplicationTools", () => client.Tools.ListDuplicationToolsAsync(ct)); + await Probe("Tools.ListMetricsTools", () => client.Tools.ListMetricsToolsAsync(ct)); + var tools = await Probe("Tools.ListTools", () => client.Tools.ListToolsAsync(null, 5, ct)); + if (tools?.Data.FirstOrDefault() is { } tool) + { + var patterns = await Probe("Tools.ListPatterns", () => client.Tools.ListPatternsAsync(tool.Uuid, null, 5, null, ct)); + if (patterns?.Data.FirstOrDefault() is { } pattern) + { + await Probe("Tools.GetPattern", () => client.Tools.GetPatternAsync(tool.Uuid, pattern.Id, ct)); + } + + await Probe("RepositoryToolPatterns.ListRepositoryToolPatterns", () => client.RepositoryToolPatterns.ListRepositoryToolPatternsAsync( + p, o, r, tool.Uuid, null, null, null, null, null, null, null, null, null, null, null, 5, ct)); + await Probe("RepositoryToolPatterns.ListRepositoryToolPatternConflicts", () => client.RepositoryToolPatterns.ListRepositoryToolPatternConflictsAsync(p, o, r, tool.Uuid, ct)); + } + + foreach (var line in _outcomes) + { + Output.WriteLine(line); + } + + // A passing test shows no output, so the outcomes can also be written to a file. + if (Environment.GetEnvironmentVariable("CODACY_SMOKE_REPORT") is { Length: > 0 } reportPath) + { + File.WriteAllLines(reportPath, _outcomes); + } + + var ok = _outcomes.Count(line => line.StartsWith("OK ", StringComparison.Ordinal)); + Output.WriteLine($"{ok} OK, {_outcomes.Count - ok} refused or server-side, {_failures.Count} failed"); + + _failures.Should().BeEmpty("a failure here is a request Codacy rejected as malformed or a model that cannot read the real response"); + ok.Should().BeGreaterThan(5, "if almost everything is refused the credentials or organization are probably wrong, and nothing was actually checked"); + } + + private async Task Probe(string name, Func> call) + { + try + { + var result = await call(); + _outcomes.Add($"OK {name}"); + return result; + } + catch (ApiException ex) when ((int)ex.StatusCode is 401 or 402 or 403 or 404 or 409) + { + _outcomes.Add($"REFUSED {name}: HTTP {(int)ex.StatusCode}"); + } + catch (ApiException ex) when ((int)ex.StatusCode >= 500) + { + _outcomes.Add($"SERVER {name}: HTTP {(int)ex.StatusCode}"); + } + catch (Exception ex) + { + // Message only: an ApiException's Content can hold response data we do not want in a log. + var detail = ex is ApiException api ? $"HTTP {(int)api.StatusCode} {api.Message}" : $"{ex.GetType().Name}: {ex.Message}"; + _failures.Add($"{name}: {detail}"); + _outcomes.Add($"FAILED {name}: {detail}"); + } + + return default; + } +} From f759b29bd4ebb4d37394db9bed47cd79b04cabd1 Mon Sep 17 00:00:00 2001 From: David Bond Date: Mon, 12 Oct 2026 13:18:32 +0100 Subject: [PATCH 2/2] Split the smoke test by area so no method exceeds Codacy's 50-line limit Six focused tests share the same helpers. Each asserts no failures and a per-area minimum of OK calls; CODACY_SMOKE_REPORT now appends, since several tests write to it. Co-Authored-By: Claude Sonnet 5.5 --- .../Integration/NewModulesSmokeTests.cs | 86 +++++++++++++++---- 1 file changed, 70 insertions(+), 16 deletions(-) diff --git a/Codacy.Api.Test/Integration/NewModulesSmokeTests.cs b/Codacy.Api.Test/Integration/NewModulesSmokeTests.cs index f95ad90..ffa18a9 100644 --- a/Codacy.Api.Test/Integration/NewModulesSmokeTests.cs +++ b/Codacy.Api.Test/Integration/NewModulesSmokeTests.cs @@ -24,18 +24,11 @@ public class NewModulesSmokeTests(ITestOutputHelper output) : TestBase(output) private string TestBranch => Configuration["CodacyApi:TestBranch"] is { Length: > 0 } branch ? branch : "main"; [Fact] - public async Task ReadOnlyEndpoints_DeserializeRealResponses() + public async Task RepositoryEndpoints_DeserializeRealResponses() { - var p = TestProvider; - var o = TestOrganization; - var r = TestRepository; - var ct = CancellationToken; + using var client = CreateQuietClient(); + var (p, o, r, ct) = (TestProvider, TestOrganization, TestRepository, CancellationToken); - // Logging off: the shared client logs response bodies, and some of these hold secrets or capability URLs. - var baseOptions = GetClientOptions(); - using var client = new CodacyClient(new CodacyClientOptions { ApiToken = baseOptions.ApiToken, BaseUrl = baseOptions.BaseUrl }); - - // Repository settings, files, coverage and analysis await Probe("RepositorySettings.GetQualitySettingsForRepository", () => client.RepositorySettings.GetQualitySettingsForRepositoryAsync(p, o, r, ct)); await Probe("RepositorySettings.GetBuildServerAnalysisSetting", () => client.RepositorySettings.GetBuildServerAnalysisSettingAsync(p, o, r, ct)); await Probe("RepositorySettings.GetRepositoryLanguages", () => client.RepositorySettings.GetRepositoryLanguagesAsync(p, o, r, ct)); @@ -44,12 +37,30 @@ public async Task ReadOnlyEndpoints_DeserializeRealResponses() await Probe("RepositoryFiles.ListDirectories", () => client.RepositoryFiles.ListDirectoriesAsync(p, o, r, null, null, null, null, null, 5, ct)); await Probe("RepositoryFiles.ListIgnoredFiles", () => client.RepositoryFiles.ListIgnoredFilesAsync(p, o, r, null, null, null, 5, ct)); await Probe("RepositoryCoverageReports.ListCoverageReports", () => client.RepositoryCoverageReports.ListCoverageReportsAsync(p, o, r, 5, ct)); + + Finish(minimumOk: 3); + } + + [Fact] + public async Task AnalysisAndSbomEndpoints_DeserializeRealResponses() + { + using var client = CreateQuietClient(); + var (p, o, r, ct) = (TestProvider, TestOrganization, TestRepository, CancellationToken); + await Probe("AnalysisActions.GetAutoconfigStatus", () => client.AnalysisActions.GetAutoconfigStatusAsync(p, o, r, ct)); await Probe("AnalysisActions.GetLatestAutoconfigRunSummary", () => client.AnalysisActions.GetLatestAutoconfigRunSummaryAsync(p, o, r, ct)); await Probe("AnalysisActions.HasQuickfixSuggestions", () => client.AnalysisActions.HasQuickfixSuggestionsAsync(p, o, r, TestBranch, ct)); await Probe("Sbom.GetRepositorySbomPresignedUrl", () => client.Sbom.GetRepositorySbomPresignedUrlAsync(p, o, r, ct)); - // Gate policies, segments, integrations + Finish(minimumOk: 0); + } + + [Fact] + public async Task PolicyAndSegmentEndpoints_DeserializeRealResponses() + { + using var client = CreateQuietClient(); + var (p, o, ct) = (TestProvider, TestOrganization, CancellationToken); + var policies = await Probe("GatePolicies.ListGatePolicies", () => client.GatePolicies.ListGatePoliciesAsync(p, o, null, 5, ct)); if (policies?.Data.FirstOrDefault() is { } policy) { @@ -60,12 +71,31 @@ public async Task ReadOnlyEndpoints_DeserializeRealResponses() await Probe("Segments.GetSegmentsSyncStatus", () => client.Segments.GetSegmentsSyncStatusAsync(p, o, ct)); await Probe("Segments.GetSegmentsKeys", () => client.Segments.GetSegmentsKeysAsync(p, o, null, 5, null, ct)); await Probe("Segments.GetSegmentsKeysWithIds", () => client.Segments.GetSegmentsKeysWithIdsAsync(p, o, null, 5, null, ct)); + + Finish(minimumOk: 1); + } + + [Fact] + public async Task IntegrationEndpoints_DeserializeRealResponses() + { + using var client = CreateQuietClient(); + var (p, o, ct) = (TestProvider, TestOrganization, CancellationToken); + await Probe("Jira.GetJiraIntegration", () => client.Jira.GetJiraIntegrationAsync(p, o, ct)); await Probe("Slack.GetSlackIntegration", () => client.Slack.GetSlackIntegrationAsync(p, o, ct)); await Probe("Dast.GetDastTargets", () => client.Dast.GetDastTargetsAsync(p, o, null, 5, ct)); await Probe("Images.ListOrganizationImages", () => client.Images.ListOrganizationImagesAsync(p, o, null, 5, ct)); - // Organization settings, billing and platform + // An organization may legitimately have none of these features. + Finish(minimumOk: 0); + } + + [Fact] + public async Task OrganizationAndPlatformEndpoints_DeserializeRealResponses() + { + using var client = CreateQuietClient(); + var (p, o, ct) = (TestProvider, TestOrganization, CancellationToken); + await Probe("OrganizationSettings.GetProviderSettings", () => client.OrganizationSettings.GetProviderSettingsAsync(p, o, ct)); await Probe("OrganizationSettings.RetrieveOrganizationOnboardingProgress", () => client.OrganizationSettings.RetrieveOrganizationOnboardingProgressAsync(p, o, ct)); await Probe("OrganizationSettings.GetGitProviderAppPermissions", () => client.OrganizationSettings.GetGitProviderAppPermissionsAsync(p, o, ct)); @@ -81,10 +111,19 @@ public async Task ReadOnlyEndpoints_DeserializeRealResponses() await Probe("Enterprises.ListEnterprises", () => client.Enterprises.ListEnterprisesAsync(p, null, 5, ct)); await Probe("Metrics.ReadyMetricsForOrganization", () => client.Metrics.ReadyMetricsForOrganizationAsync(p, o, ct)); - // Tools and patterns + Finish(minimumOk: 3); + } + + [Fact] + public async Task ToolAndPatternEndpoints_DeserializeRealResponses() + { + using var client = CreateQuietClient(); + var (p, o, r, ct) = (TestProvider, TestOrganization, TestRepository, CancellationToken); + await Probe("Tools.ListLanguagesWithTools", () => client.Tools.ListLanguagesWithToolsAsync(ct)); await Probe("Tools.ListDuplicationTools", () => client.Tools.ListDuplicationToolsAsync(ct)); await Probe("Tools.ListMetricsTools", () => client.Tools.ListMetricsToolsAsync(ct)); + var tools = await Probe("Tools.ListTools", () => client.Tools.ListToolsAsync(null, 5, ct)); if (tools?.Data.FirstOrDefault() is { } tool) { @@ -99,22 +138,37 @@ await Probe("RepositoryToolPatterns.ListRepositoryToolPatterns", () => client.Re await Probe("RepositoryToolPatterns.ListRepositoryToolPatternConflicts", () => client.RepositoryToolPatterns.ListRepositoryToolPatternConflictsAsync(p, o, r, tool.Uuid, ct)); } + Finish(minimumOk: 3); + } + + /// + /// A client with logging off: the shared one logs response bodies, and some of these hold secrets + /// or capability URLs. + /// + private CodacyClient CreateQuietClient() + { + var baseOptions = GetClientOptions(); + return new CodacyClient(new CodacyClientOptions { ApiToken = baseOptions.ApiToken, BaseUrl = baseOptions.BaseUrl }); + } + + private void Finish(int minimumOk) + { foreach (var line in _outcomes) { Output.WriteLine(line); } - // A passing test shows no output, so the outcomes can also be written to a file. + // A passing test shows no output, so the outcomes can also be appended to a file. if (Environment.GetEnvironmentVariable("CODACY_SMOKE_REPORT") is { Length: > 0 } reportPath) { - File.WriteAllLines(reportPath, _outcomes); + File.AppendAllLines(reportPath, _outcomes); } var ok = _outcomes.Count(line => line.StartsWith("OK ", StringComparison.Ordinal)); Output.WriteLine($"{ok} OK, {_outcomes.Count - ok} refused or server-side, {_failures.Count} failed"); _failures.Should().BeEmpty("a failure here is a request Codacy rejected as malformed or a model that cannot read the real response"); - ok.Should().BeGreaterThan(5, "if almost everything is refused the credentials or organization are probably wrong, and nothing was actually checked"); + ok.Should().BeGreaterThanOrEqualTo(minimumOk, "if almost everything is refused the credentials or organization are probably wrong, and nothing was actually checked"); } private async Task Probe(string name, Func> call)