Skip to content

Correct Observability Pipelines schema drift #4599

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dd-octo-sts merged 1 commit into from
Nov 5, 2026
Merged

Correct Observability Pipelines schema drift #4599

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions .generator/schemas/v2/openapi.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87185,7 +87185,7 @@ components:
type: string
type: array
processors:
description: Processors applied sequentially within this group. Events flow through each processor in order.
description: Processors applied sequentially within this group. Events flow through each processor in order. Omit or use an empty array for a group without processors.
example: [{"enabled": true, "fields": [{"name": "env", "value": "prod"}], "id": "add-fields-processor", "include": "*", "type": "add_fields"}, {"enabled": true, "id": "filter-processor", "include": "status:error", "type": "filter"}]
items:
$ref: "#/components/schemas/ObservabilityPipelineConfigProcessorItem"
Expand All @@ -87194,7 +87194,6 @@ components:
- id
- include
- inputs
- processors
- enabled
type: object
ObservabilityPipelineConfigProcessorItem:
Expand Down Expand Up @@ -90140,7 +90139,7 @@ components:
example: "service:my-service"
type: string
rules:
description: The list of Grok parsing rules selected by either source field or include query.
description: The list of custom Grok parsing rules selected by either source field or include query. Omit to use only the Datadog library rules.
items:
$ref: "#/components/schemas/ObservabilityPipelineParseGrokProcessorRuleItem"
type: array
Expand All @@ -90150,7 +90149,6 @@ components:
- id
- type
- include
- rules
- enabled
type: object
x-pipeline-types: [logs]
Expand Down Expand Up @@ -209147,7 +209145,11 @@ paths:
$ref: "#/components/schemas/ObservabilityPipeline"
description: OK
"400":
$ref: "#/components/responses/BadRequestResponse"
content:
application/json:
schema:
$ref: "#/components/schemas/JSONAPIErrorResponse"
description: Bad Request
"403":
$ref: "#/components/responses/NotAuthorizedResponse"
"409":
Expand Down Expand Up @@ -209219,7 +209221,11 @@ paths:
$ref: "#/components/schemas/ValidationResponse"
description: OK
"400":
$ref: "#/components/responses/BadRequestResponse"
content:
application/json:
schema:
$ref: "#/components/schemas/JSONAPIErrorResponse"
description: Bad Request
"403":
$ref: "#/components/responses/NotAuthorizedResponse"
"429":
Expand Down
76 changes: 76 additions & 0 deletions examples/v2/observability-pipelines/ValidatePipeline_2324504662.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
// Validate an observability pipeline with an empty processor group returns "OK" response

import com.datadog.api.client.ApiClient;
import com.datadog.api.client.ApiException;
import com.datadog.api.client.v2.api.ObservabilityPipelinesApi;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfig;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigDestinationItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigProcessorGroup;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigSourceItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineDataAttributes;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSource;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSourceType;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestination;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestinationType;
import com.datadog.api.client.v2.model.ObservabilityPipelineSpec;
import com.datadog.api.client.v2.model.ObservabilityPipelineSpecData;
import com.datadog.api.client.v2.model.ValidationResponse;
import java.util.Collections;

public class Example {
public static void main(String[] args) {
ApiClient defaultClient = ApiClient.getDefaultApiClient();
defaultClient.setAccessToken(System.getenv("DD_BEARER_TOKEN"));
ObservabilityPipelinesApi apiInstance = new ObservabilityPipelinesApi(defaultClient);

ObservabilityPipelineSpec body =
new ObservabilityPipelineSpec()
.data(
new ObservabilityPipelineSpecData()
.attributes(
new ObservabilityPipelineDataAttributes()
.config(
new ObservabilityPipelineConfig()
.destinations(
Collections.singletonList(
new ObservabilityPipelineConfigDestinationItem(
new ObservabilityPipelineDatadogLogsDestination()
.id("datadog-logs-destination")
.inputs(
Collections.singletonList(
"my-processor-group"))
.type(
ObservabilityPipelineDatadogLogsDestinationType
.DATADOG_LOGS))))
.processorGroups(
Collections.singletonList(
new ObservabilityPipelineConfigProcessorGroup()
.enabled(true)
.id("my-processor-group")
.include("*")
.inputs(
Collections.singletonList(
"datadog-agent-source"))))
.sources(
Collections.singletonList(
new ObservabilityPipelineConfigSourceItem(
new ObservabilityPipelineDatadogAgentSource()
.id("datadog-agent-source")
.type(
ObservabilityPipelineDatadogAgentSourceType
.DATADOG_AGENT)))))
.name("Pipeline with Empty Processor Group"))
.type("pipelines"));

try {
ValidationResponse result = apiInstance.validatePipeline(body);
System.out.println(result);
} catch (ApiException e) {
System.err.println("Exception when calling ObservabilityPipelinesApi#validatePipeline");
System.err.println("Status code: " + e.getCode());
System.err.println("Reason: " + e.getResponseBody());
System.err.println("Response headers: " + e.getResponseHeaders());
e.printStackTrace();
}
}
}
89 changes: 89 additions & 0 deletions examples/v2/observability-pipelines/ValidatePipeline_3415678462.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// Validate an observability pipeline with grok library rules returns "OK" response

import com.datadog.api.client.ApiClient;
import com.datadog.api.client.ApiException;
import com.datadog.api.client.v2.api.ObservabilityPipelinesApi;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfig;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigDestinationItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigProcessorGroup;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigProcessorItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineConfigSourceItem;
import com.datadog.api.client.v2.model.ObservabilityPipelineDataAttributes;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSource;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogAgentSourceType;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestination;
import com.datadog.api.client.v2.model.ObservabilityPipelineDatadogLogsDestinationType;
import com.datadog.api.client.v2.model.ObservabilityPipelineParseGrokProcessor;
import com.datadog.api.client.v2.model.ObservabilityPipelineParseGrokProcessorType;
import com.datadog.api.client.v2.model.ObservabilityPipelineSpec;
import com.datadog.api.client.v2.model.ObservabilityPipelineSpecData;
import com.datadog.api.client.v2.model.ValidationResponse;
import java.util.Collections;

public class Example {
public static void main(String[] args) {
ApiClient defaultClient = ApiClient.getDefaultApiClient();
defaultClient.setAccessToken(System.getenv("DD_BEARER_TOKEN"));
ObservabilityPipelinesApi apiInstance = new ObservabilityPipelinesApi(defaultClient);

ObservabilityPipelineSpec body =
new ObservabilityPipelineSpec()
.data(
new ObservabilityPipelineSpecData()
.attributes(
new ObservabilityPipelineDataAttributes()
.config(
new ObservabilityPipelineConfig()
.destinations(
Collections.singletonList(
new ObservabilityPipelineConfigDestinationItem(
new ObservabilityPipelineDatadogLogsDestination()
.id("datadog-logs-destination")
.inputs(
Collections.singletonList(
"my-processor-group"))
.type(
ObservabilityPipelineDatadogLogsDestinationType
.DATADOG_LOGS))))
.processorGroups(
Collections.singletonList(
new ObservabilityPipelineConfigProcessorGroup()
.enabled(true)
.id("my-processor-group")
.include("*")
.inputs(
Collections.singletonList(
"datadog-agent-source"))
.processors(
Collections.singletonList(
new ObservabilityPipelineConfigProcessorItem(
new ObservabilityPipelineParseGrokProcessor()
.enabled(true)
.id("parse-grok-processor")
.include("*")
.type(
ObservabilityPipelineParseGrokProcessorType
.PARSE_GROK))))))
.sources(
Collections.singletonList(
new ObservabilityPipelineConfigSourceItem(
new ObservabilityPipelineDatadogAgentSource()
.id("datadog-agent-source")
.type(
ObservabilityPipelineDatadogAgentSourceType
.DATADOG_AGENT)))))
.name("Pipeline with Grok Library Rules"))
.type("pipelines"));

try {
ValidationResponse result = apiInstance.validatePipeline(body);
System.out.println(result);
} catch (ApiException e) {
System.err.println("Exception when calling ObservabilityPipelinesApi#validatePipeline");
System.err.println("Status code: " + e.getCode());
System.err.println("Reason: " + e.getResponseBody());
System.err.println("Response headers: " + e.getResponseHeaders());
e.printStackTrace();
}
}
}
23 changes: 12 additions & 11 deletions src/main/java/com/datadog/api/client/v2/model/ObservabilityPipelineConfigProcessorGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ public class ObservabilityPipelineConfigProcessorGroup {
private List<String> inputs = new ArrayList<>();

public static final String JSON_PROPERTY_PROCESSORS = "processors";
private List<ObservabilityPipelineConfigProcessorItem> processors = new ArrayList<>();
private List<ObservabilityPipelineConfigProcessorItem> processors = null;

public ObservabilityPipelineConfigProcessorGroup() {}

Expand All @@ -57,17 +57,11 @@ public ObservabilityPipelineConfigProcessorGroup(
@JsonProperty(required = true, value = JSON_PROPERTY_ENABLED) Boolean enabled,
@JsonProperty(required = true, value = JSON_PROPERTY_ID) String id,
@JsonProperty(required = true, value = JSON_PROPERTY_INCLUDE) String include,
@JsonProperty(required = true, value = JSON_PROPERTY_INPUTS) List<String> inputs,
@JsonProperty(required = true, value = JSON_PROPERTY_PROCESSORS)
List<ObservabilityPipelineConfigProcessorItem> processors) {
@JsonProperty(required = true, value = JSON_PROPERTY_INPUTS) List<String> inputs) {
this.enabled = enabled;
this.id = id;
this.include = include;
this.inputs = inputs;
this.processors = processors;
for (ObservabilityPipelineConfigProcessorItem item : processors) {
this.unparsed |= item.unparsed;
}
}

public ObservabilityPipelineConfigProcessorGroup displayName(String displayName) {
Expand Down Expand Up @@ -179,26 +173,33 @@ public void setInputs(List<String> inputs) {
public ObservabilityPipelineConfigProcessorGroup processors(
List<ObservabilityPipelineConfigProcessorItem> processors) {
this.processors = processors;
for (ObservabilityPipelineConfigProcessorItem item : processors) {
this.unparsed |= item.unparsed;
if (processors != null) {
for (ObservabilityPipelineConfigProcessorItem item : processors) {
this.unparsed |= item.unparsed;
}
}
return this;
}

public ObservabilityPipelineConfigProcessorGroup addProcessorsItem(
ObservabilityPipelineConfigProcessorItem processorsItem) {
if (this.processors == null) {
this.processors = new ArrayList<>();
}
this.processors.add(processorsItem);
this.unparsed |= processorsItem.unparsed;
return this;
}

/**
* Processors applied sequentially within this group. Events flow through each processor in order.
* Omit or use an empty array for a group without processors.
*
* @return processors
*/
@jakarta.annotation.Nullable
@JsonProperty(JSON_PROPERTY_PROCESSORS)
@JsonInclude(value = JsonInclude.Include.ALWAYS)
@JsonInclude(value = JsonInclude.Include.USE_DEFAULTS)
public List<ObservabilityPipelineConfigProcessorItem> getProcessors() {
return processors;
}
Expand Down
23 changes: 12 additions & 11 deletions src/main/java/com/datadog/api/client/v2/model/ObservabilityPipelineParseGrokProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public class ObservabilityPipelineParseGrokProcessor {
private String include;

public static final String JSON_PROPERTY_RULES = "rules";
private List<ObservabilityPipelineParseGrokProcessorRuleItem> rules = new ArrayList<>();
private List<ObservabilityPipelineParseGrokProcessorRuleItem> rules = null;

public static final String JSON_PROPERTY_TYPE = "type";
private ObservabilityPipelineParseGrokProcessorType type =
Expand All @@ -71,17 +71,11 @@ public ObservabilityPipelineParseGrokProcessor(
@JsonProperty(required = true, value = JSON_PROPERTY_ENABLED) Boolean enabled,
@JsonProperty(required = true, value = JSON_PROPERTY_ID) String id,
@JsonProperty(required = true, value = JSON_PROPERTY_INCLUDE) String include,
@JsonProperty(required = true, value = JSON_PROPERTY_RULES)
List<ObservabilityPipelineParseGrokProcessorRuleItem> rules,
@JsonProperty(required = true, value = JSON_PROPERTY_TYPE)
ObservabilityPipelineParseGrokProcessorType type) {
this.enabled = enabled;
this.id = id;
this.include = include;
this.rules = rules;
for (ObservabilityPipelineParseGrokProcessorRuleItem item : rules) {
this.unparsed |= item.unparsed;
}
this.type = type;
this.unparsed |= !type.isValid();
}
Expand Down Expand Up @@ -212,26 +206,33 @@ public void setInclude(String include) {
public ObservabilityPipelineParseGrokProcessor rules(
List<ObservabilityPipelineParseGrokProcessorRuleItem> rules) {
this.rules = rules;
for (ObservabilityPipelineParseGrokProcessorRuleItem item : rules) {
this.unparsed |= item.unparsed;
if (rules != null) {
for (ObservabilityPipelineParseGrokProcessorRuleItem item : rules) {
this.unparsed |= item.unparsed;
}
}
return this;
}

public ObservabilityPipelineParseGrokProcessor addRulesItem(
ObservabilityPipelineParseGrokProcessorRuleItem rulesItem) {
if (this.rules == null) {
this.rules = new ArrayList<>();
}
this.rules.add(rulesItem);
this.unparsed |= rulesItem.unparsed;
return this;
}

/**
* The list of Grok parsing rules selected by either source field or include query.
* The list of custom Grok parsing rules selected by either source field or include query. Omit to
* use only the Datadog library rules.
*
* @return rules
*/
@jakarta.annotation.Nullable
@JsonProperty(JSON_PROPERTY_RULES)
@JsonInclude(value = JsonInclude.Include.ALWAYS)
@JsonInclude(value = JsonInclude.Include.USE_DEFAULTS)
public List<ObservabilityPipelineParseGrokProcessorRuleItem> getRules() {
return rules;
}
Expand Down
Loading
Loading