From dc63b6aa7706eaee3cb45fbc24f3bd231a99eb05 Mon Sep 17 00:00:00 2001 From: Alejandro Morera Date: Tue, 18 Aug 2026 16:30:29 -0600 Subject: [PATCH 1/6] Add DAG parse times via REST API how-to --- docs/how-tos/my_airflow/dag-parse-times.md | 93 ++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 docs/how-tos/my_airflow/dag-parse-times.md diff --git a/docs/how-tos/my_airflow/dag-parse-times.md b/docs/how-tos/my_airflow/dag-parse-times.md new file mode 100644 index 0000000..5ffab7a --- /dev/null +++ b/docs/how-tos/my_airflow/dag-parse-times.md @@ -0,0 +1,93 @@ +--- +title: "Find Slow-Parsing DAGs with the REST API" +sidebar_label: "DAG parsing times" +description: "Use the Airflow REST API to list DAG parse durations and find slow-parsing DAGs in Team Airflow and My Airflow on Datacoves." +sidebar_position: 79 +--- +# Find slow-parsing DAGs with the REST API + +Every DAG file is re-parsed continuously by the dag-processor. A DAG that is slow to parse (top-level code doing imports, API calls, or file reads) delays scheduling for every DAG in the environment, so parse times are one of the first things to check when Airflow feels sluggish. + +On Airflow 3 environments, the dag-processor records each DAG's most recent parse duration and the REST API exposes it, so you can get the real parsing-time report with a simple API call: no access to the dag-processor pod is needed. + +:::info +This works on Airflow 3 environments, for both **Team Airflow** and **My Airflow** (it is the same REST API). On My Airflow you can also use the [`datacoves my parse-logs`](/docs/reference/airflow/datacoves-commands#datacoves-my-parse-logs) command, which additionally shows the parse log of individual DAG files. + +Airflow 2 does not expose parse durations through its API; see [the note below](#airflow-2-environments). +::: + +## What you need + +1. An API key and the API URL for your target instance: follow [How to use the My Airflow API](/docs/how-tos/my_airflow/use-my-airflow-api). The same key works against Team Airflow if you have access; for Team Airflow, the API base is your Airflow URL (the one in your browser) plus `/api/v2`. +2. Store both in a `.env` file (and add it to `.gitignore`): + +```env +AIRFLOW_API_URL = "https://your-airflow-url/api/v2" +AIRFLOW_API_KEY = "your-api-key-here" +``` + +## Get the parsing-time report + +The `GET /dags` endpoint returns `last_parse_duration` (seconds) and `last_parsed_time` for every DAG. The script below fetches all DAGs (paginated) and prints them slowest-first, the same shape as the `datacoves my parse-logs` report: + +```python +# dag_parse_times.py +import os + +import requests +from dotenv import load_dotenv + +load_dotenv() +API_URL = os.getenv("AIRFLOW_API_URL").rstrip("/") +API_KEY = os.getenv("AIRFLOW_API_KEY") +HEADERS = {"Authorization": f"Token {API_KEY}"} + + +def fetch_all_dags(): + dags, offset = [], 0 + while True: + response = requests.get( + f"{API_URL}/dags", + headers=HEADERS, + params={"limit": 100, "offset": offset}, + ) + response.raise_for_status() + payload = response.json() + dags.extend(payload["dags"]) + offset += 100 + if offset >= payload["total_entries"]: + return dags + + +def print_parse_report(dags): + dags.sort(key=lambda d: d.get("last_parse_duration") or 0, reverse=True) + print(f"{'DAG':<50} {'Parse time':>12} {'Last parsed'}") + for dag in dags: + duration = dag.get("last_parse_duration") + duration = f"{duration:.3f}s" if duration is not None else "-" + parsed_at = dag.get("last_parsed_time") or "-" + print(f"{dag['dag_id']:<50} {duration:>12} {parsed_at}") + + +if __name__ == "__main__": + print_parse_report(fetch_all_dags()) +``` + +Example output: + +``` +DAG Parse time Last parsed +daily_loan_run 2.153s 2026-08-18T14:02:11.480217Z +variables_python_script 0.310s 2026-08-18T14:02:10.912304Z +simple_bash_dag 0.052s 2026-08-18T14:02:10.598721Z +``` + +A parse time consistently above ~1 second is worth investigating: the usual cause is work done at the top level of the DAG file (imports of heavy libraries, `Variable.get` outside a task, reading files or calling APIs during parsing). See [Airflow's best practices on top-level code](https://airflow.apache.org/docs/apache-airflow/stable/best-practices.html#top-level-python-code) for how to restructure it. + +:::tip +`last_parse_duration` reflects the most recent parse by the environment's dag-processor -- the number that actually matters for scheduling latency. Running `airflow dags report` inside a DAG task measures a worker's copy of the files under different conditions and can be misleading. +::: + +## Airflow 2 environments + +Airflow 2 records when each DAG was last parsed (`last_parsed_time`) but not how long it took, and its REST API does not expose parse durations. This is a known limitation; parse times there only exist in the dag-processor manager logs. Since new Datacoves environments run Airflow 3, no Airflow 2-specific tooling is provided -- upgrading is the path to this report. From 45c5fa2c3eb0dd98fbb9c4f229c7a9fb5ef92abb Mon Sep 17 00:00:00 2001 From: Alejandro Morera Date: Tue, 18 Aug 2026 16:52:57 -0600 Subject: [PATCH 2/6] Use the Airflow 3 JWT auth flow in the parse times guide --- docs/how-tos/my_airflow/dag-parse-times.md | 39 ++++++++++++++-------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/docs/how-tos/my_airflow/dag-parse-times.md b/docs/how-tos/my_airflow/dag-parse-times.md index 5ffab7a..d8071cb 100644 --- a/docs/how-tos/my_airflow/dag-parse-times.md +++ b/docs/how-tos/my_airflow/dag-parse-times.md @@ -11,24 +11,27 @@ Every DAG file is re-parsed continuously by the dag-processor. A DAG that is slo On Airflow 3 environments, the dag-processor records each DAG's most recent parse duration and the REST API exposes it, so you can get the real parsing-time report with a simple API call: no access to the dag-processor pod is needed. :::info -This works on Airflow 3 environments, for both **Team Airflow** and **My Airflow** (it is the same REST API). On My Airflow you can also use the [`datacoves my parse-logs`](/docs/reference/airflow/datacoves-commands#datacoves-my-parse-logs) command, which additionally shows the parse log of individual DAG files. +This guide targets **Team Airflow** on Airflow 3 environments. For **My Airflow**, use the [`datacoves my parse-logs`](/docs/reference/airflow/datacoves-commands#datacoves-my-parse-logs) command instead, which shows the same report plus the parse log of individual DAG files. Airflow 2 does not expose parse durations through its API; see [the note below](#airflow-2-environments). ::: ## What you need -1. An API key and the API URL for your target instance: follow [How to use the My Airflow API](/docs/how-tos/my_airflow/use-my-airflow-api). The same key works against Team Airflow if you have access; for Team Airflow, the API base is your Airflow URL (the one in your browser) plus `/api/v2`. -2. Store both in a `.env` file (and add it to `.gitignore`): +1. A Datacoves API key: generate one following [How to use the My Airflow API](/docs/how-tos/my_airflow/use-my-airflow-api). The same key authenticates you against the Team Airflow of **any environment you have access to**, with your own permissions. +2. The Airflow URL of the target environment: the same one you open in your browser (`https://airflow-.`). +3. Store both in a `.env` file (and add it to `.gitignore`): ```env -AIRFLOW_API_URL = "https://your-airflow-url/api/v2" -AIRFLOW_API_KEY = "your-api-key-here" +AIRFLOW_URL = "https://airflow-your-env.your-domain.com" +DATACOVES_API_KEY = "your-api-key-here" ``` +The script works the same from your laptop, from CI, or from a Datacoves workbench terminal -- the Airflow API endpoints are reachable from all of them. + ## Get the parsing-time report -The `GET /dags` endpoint returns `last_parse_duration` (seconds) and `last_parsed_time` for every DAG. The script below fetches all DAGs (paginated) and prints them slowest-first, the same shape as the `datacoves my parse-logs` report: +Airflow 3's API uses short-lived JWTs: you first exchange your Datacoves API key for a JWT at `POST /auth/token` (using the special `__datacoves_token__` username), then call the API with it. The `GET /api/v2/dags` endpoint returns `last_parse_duration` (seconds) and `last_parsed_time` for every DAG. The script below fetches all DAGs (paginated) and prints them slowest-first, the same shape as the `datacoves my parse-logs` report: ```python # dag_parse_times.py @@ -38,17 +41,26 @@ import requests from dotenv import load_dotenv load_dotenv() -API_URL = os.getenv("AIRFLOW_API_URL").rstrip("/") -API_KEY = os.getenv("AIRFLOW_API_KEY") -HEADERS = {"Authorization": f"Token {API_KEY}"} +BASE_URL = os.getenv("AIRFLOW_URL").rstrip("/") +API_KEY = os.getenv("DATACOVES_API_KEY") + + +def get_jwt(): + """Exchange the Datacoves API key for a short-lived Airflow JWT.""" + response = requests.post( + f"{BASE_URL}/auth/token", + json={"username": "__datacoves_token__", "password": API_KEY}, + ) + response.raise_for_status() + return response.json()["access_token"] -def fetch_all_dags(): +def fetch_all_dags(headers): dags, offset = [], 0 while True: response = requests.get( - f"{API_URL}/dags", - headers=HEADERS, + f"{BASE_URL}/api/v2/dags", + headers=headers, params={"limit": 100, "offset": offset}, ) response.raise_for_status() @@ -70,7 +82,8 @@ def print_parse_report(dags): if __name__ == "__main__": - print_parse_report(fetch_all_dags()) + headers = {"Authorization": f"Bearer {get_jwt()}"} + print_parse_report(fetch_all_dags(headers)) ``` Example output: From 93ef125014284232987954d3e29e89eca78bc4b7 Mon Sep 17 00:00:00 2001 From: Alejandro Morera Date: Wed, 19 Aug 2026 08:34:36 -0600 Subject: [PATCH 3/6] Move the DAG parse times guide under Team Airflow how-tos --- docs/how-tos/{my_airflow => airflow}/dag-parse-times.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/how-tos/{my_airflow => airflow}/dag-parse-times.md (99%) diff --git a/docs/how-tos/my_airflow/dag-parse-times.md b/docs/how-tos/airflow/dag-parse-times.md similarity index 99% rename from docs/how-tos/my_airflow/dag-parse-times.md rename to docs/how-tos/airflow/dag-parse-times.md index d8071cb..3a6a20c 100644 --- a/docs/how-tos/my_airflow/dag-parse-times.md +++ b/docs/how-tos/airflow/dag-parse-times.md @@ -2,7 +2,7 @@ title: "Find Slow-Parsing DAGs with the REST API" sidebar_label: "DAG parsing times" description: "Use the Airflow REST API to list DAG parse durations and find slow-parsing DAGs in Team Airflow and My Airflow on Datacoves." -sidebar_position: 79 +sidebar_position: 40 --- # Find slow-parsing DAGs with the REST API From f9795e87c7c24dc6f7c0aadecfb3eeadb56a040f Mon Sep 17 00:00:00 2001 From: Alejandro Morera Date: Wed, 19 Aug 2026 08:46:21 -0600 Subject: [PATCH 4/6] Use the environment Airflow API token as the primary auth path --- docs/how-tos/airflow/dag-parse-times.md | 50 ++++++++++++++----------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/docs/how-tos/airflow/dag-parse-times.md b/docs/how-tos/airflow/dag-parse-times.md index 3a6a20c..76172b2 100644 --- a/docs/how-tos/airflow/dag-parse-times.md +++ b/docs/how-tos/airflow/dag-parse-times.md @@ -18,20 +18,20 @@ Airflow 2 does not expose parse durations through its API; see [the note below]( ## What you need -1. A Datacoves API key: generate one following [How to use the My Airflow API](/docs/how-tos/my_airflow/use-my-airflow-api). The same key authenticates you against the Team Airflow of **any environment you have access to**, with your own permissions. -2. The Airflow URL of the target environment: the same one you open in your browser (`https://airflow-.`). +1. An Airflow API token for the target environment: in the launchpad, go to **Environments**, open your environment's **Keys**, and click **Generate New Airflow Token**. Copy the token immediately -- it is only shown once. +2. The **Airflow API URL** shown on that same page (`https://airflow-./api/v2/`). 3. Store both in a `.env` file (and add it to `.gitignore`): ```env -AIRFLOW_URL = "https://airflow-your-env.your-domain.com" -DATACOVES_API_KEY = "your-api-key-here" +AIRFLOW_API_URL = "https://airflow-your-env.your-domain.com/api/v2/" +AIRFLOW_API_TOKEN = "your-token-here" ``` The script works the same from your laptop, from CI, or from a Datacoves workbench terminal -- the Airflow API endpoints are reachable from all of them. ## Get the parsing-time report -Airflow 3's API uses short-lived JWTs: you first exchange your Datacoves API key for a JWT at `POST /auth/token` (using the special `__datacoves_token__` username), then call the API with it. The `GET /api/v2/dags` endpoint returns `last_parse_duration` (seconds) and `last_parsed_time` for every DAG. The script below fetches all DAGs (paginated) and prints them slowest-first, the same shape as the `datacoves my parse-logs` report: +The `GET /dags` endpoint returns `last_parse_duration` (seconds) and `last_parsed_time` for every DAG. The script below fetches all DAGs (paginated) and prints them slowest-first, the same shape as the `datacoves my parse-logs` report: ```python # dag_parse_times.py @@ -41,26 +41,16 @@ import requests from dotenv import load_dotenv load_dotenv() -BASE_URL = os.getenv("AIRFLOW_URL").rstrip("/") -API_KEY = os.getenv("DATACOVES_API_KEY") +API_URL = os.getenv("AIRFLOW_API_URL").rstrip("/") +HEADERS = {"Authorization": f"Bearer {os.getenv('AIRFLOW_API_TOKEN')}"} -def get_jwt(): - """Exchange the Datacoves API key for a short-lived Airflow JWT.""" - response = requests.post( - f"{BASE_URL}/auth/token", - json={"username": "__datacoves_token__", "password": API_KEY}, - ) - response.raise_for_status() - return response.json()["access_token"] - - -def fetch_all_dags(headers): +def fetch_all_dags(): dags, offset = [], 0 while True: response = requests.get( - f"{BASE_URL}/api/v2/dags", - headers=headers, + f"{API_URL}/dags", + headers=HEADERS, params={"limit": 100, "offset": offset}, ) response.raise_for_status() @@ -82,8 +72,7 @@ def print_parse_report(dags): if __name__ == "__main__": - headers = {"Authorization": f"Bearer {get_jwt()}"} - print_parse_report(fetch_all_dags(headers)) + print_parse_report(fetch_all_dags()) ``` Example output: @@ -101,6 +90,23 @@ A parse time consistently above ~1 second is worth investigating: the usual caus `last_parse_duration` reflects the most recent parse by the environment's dag-processor -- the number that actually matters for scheduling latency. Running `airflow dags report` inside a DAG task measures a worker's copy of the files under different conditions and can be misleading. ::: +## Alternative: authenticate as your own user + +The Airflow API token above belongs to the environment's service account and is a JWT with an expiration date, so long-lived automation would need re-generating it. As an alternative, you can authenticate with your personal Datacoves API key (generated under [User Settings > My Airflow API](/docs/how-tos/my_airflow/use-my-airflow-api) -- despite the name, it is a general Datacoves key): exchange it for a short-lived Airflow JWT at `POST /auth/token` using the special `__datacoves_token__` username. This authenticates you as **your own user** with your permissions, the key does not expire, and the same key works against **any environment you have access to**: + +```python +def get_jwt(base_url, datacoves_api_key): + """Exchange a Datacoves API key for a short-lived Airflow JWT.""" + response = requests.post( + f"{base_url}/auth/token", + json={"username": "__datacoves_token__", "password": datacoves_api_key}, + ) + response.raise_for_status() + return response.json()["access_token"] +``` + +Use the returned token in the same `Authorization: Bearer` header as above (`base_url` is the Airflow URL without `/api/v2`). + ## Airflow 2 environments Airflow 2 records when each DAG was last parsed (`last_parsed_time`) but not how long it took, and its REST API does not expose parse durations. This is a known limitation; parse times there only exist in the dag-processor manager logs. Since new Datacoves environments run Airflow 3, no Airflow 2-specific tooling is provided -- upgrading is the path to this report. From 1a1f5a8752c1c74d14e7d25f6ae39e1b375cb5b0 Mon Sep 17 00:00:00 2001 From: fmercadop <90359057+fmercadop@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:03:11 -0700 Subject: [PATCH 5/6] Clarify script execution environments in documentation Updated the description of script execution environments for clarity. --- docs/how-tos/airflow/dag-parse-times.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/how-tos/airflow/dag-parse-times.md b/docs/how-tos/airflow/dag-parse-times.md index 76172b2..07394d3 100644 --- a/docs/how-tos/airflow/dag-parse-times.md +++ b/docs/how-tos/airflow/dag-parse-times.md @@ -27,7 +27,7 @@ AIRFLOW_API_URL = "https://airflow-your-env.your-domain.com/api/v2/" AIRFLOW_API_TOKEN = "your-token-here" ``` -The script works the same from your laptop, from CI, or from a Datacoves workbench terminal -- the Airflow API endpoints are reachable from all of them. +The script works the same from CI, from Datacoves VSCode, or from any environment you use to reach Datacoves -- the Airflow API endpoints are reachable from all of them. ## Get the parsing-time report From bb685cd429f43b723cf497d0016f26b59e4d22e1 Mon Sep 17 00:00:00 2001 From: Noel Gomez Date: Fri, 9 Oct 2026 13:35:49 -0700 Subject: [PATCH 6/6] reorder dags how-tos and move dag parse time how to --- docs/how-tos/airflow/dags/create-dag-level-docs.md | 2 +- docs/how-tos/airflow/{ => dags}/dag-parse-times.md | 2 +- docs/how-tos/airflow/dags/dynamically-set-schedule.md | 2 +- docs/how-tos/airflow/dags/external-python-dag.md | 2 +- docs/how-tos/airflow/dags/generate-dags-from-yml.md | 2 +- docs/how-tos/airflow/dags/get-current-branch-name.md | 2 +- docs/how-tos/airflow/dags/retry-dbt-tasks.md | 2 +- docs/how-tos/airflow/dags/run-adf-pipeline.md | 2 +- docs/how-tos/airflow/dags/run-airbyte-sync-jobs.md | 2 +- docs/how-tos/airflow/dags/run-databricks-notebook.md | 2 +- docs/how-tos/airflow/dags/run-dbt.md | 2 +- docs/how-tos/airflow/dags/run-fivetran-sync-jobs.md | 2 +- docs/how-tos/airflow/dags/s3-to-snowflake.md | 2 +- docs/how-tos/airflow/dags/test-dags.md | 2 +- docs/how-tos/airflow/dags/use-variables-and-connections.md | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) rename docs/how-tos/airflow/{ => dags}/dag-parse-times.md (95%) diff --git a/docs/how-tos/airflow/dags/create-dag-level-docs.md b/docs/how-tos/airflow/dags/create-dag-level-docs.md index 28b2b49..b99cf4a 100644 --- a/docs/how-tos/airflow/dags/create-dag-level-docs.md +++ b/docs/how-tos/airflow/dags/create-dag-level-docs.md @@ -2,7 +2,7 @@ title: Add Documentation to Airflow DAGs in Datacoves sidebar_label: Add Documentation description: "How to add Markdown documentation directly to Airflow DAGs in Datacoves so operators can read context in the Airflow UI and Grid view." -sidebar_position: 20 +sidebar_position: 10 --- # How to Add Docs at the DAG Level in Airflow diff --git a/docs/how-tos/airflow/dag-parse-times.md b/docs/how-tos/airflow/dags/dag-parse-times.md similarity index 95% rename from docs/how-tos/airflow/dag-parse-times.md rename to docs/how-tos/airflow/dags/dag-parse-times.md index 07394d3..a0a8b28 100644 --- a/docs/how-tos/airflow/dag-parse-times.md +++ b/docs/how-tos/airflow/dags/dag-parse-times.md @@ -6,7 +6,7 @@ sidebar_position: 40 --- # Find slow-parsing DAGs with the REST API -Every DAG file is re-parsed continuously by the dag-processor. A DAG that is slow to parse (top-level code doing imports, API calls, or file reads) delays scheduling for every DAG in the environment, so parse times are one of the first things to check when Airflow feels sluggish. +Every DAG file is re-parsed regularly by the dag-processor. A DAG that is slow to parse (top-level code doing imports, API calls, or file reads) delays scheduling for every DAG in the environment, so parse times are one of the first things to check when DAGs randomly disappear from the UI. On Airflow 3 environments, the dag-processor records each DAG's most recent parse duration and the REST API exposes it, so you can get the real parsing-time report with a simple API call: no access to the dag-processor pod is needed. diff --git a/docs/how-tos/airflow/dags/dynamically-set-schedule.md b/docs/how-tos/airflow/dags/dynamically-set-schedule.md index cc6eae3..b4ec4cc 100644 --- a/docs/how-tos/airflow/dags/dynamically-set-schedule.md +++ b/docs/how-tos/airflow/dags/dynamically-set-schedule.md @@ -2,7 +2,7 @@ title: Dynamically Set Airflow Schedule Intervals sidebar_label: Dynamic Schedule description: "Use environment variables or Airflow Variables to dynamically configure DAG schedule intervals in Datacoves for flexible pipeline orchestration." -sidebar_position: 22 +sidebar_position: 30 --- # How to Dynamically set the schedule Interval diff --git a/docs/how-tos/airflow/dags/external-python-dag.md b/docs/how-tos/airflow/dags/external-python-dag.md index 47334ec..cae545a 100644 --- a/docs/how-tos/airflow/dags/external-python-dag.md +++ b/docs/how-tos/airflow/dags/external-python-dag.md @@ -2,7 +2,7 @@ title: Call External Python Scripts from Airflow DAGs sidebar_label: External Python Scripts description: "Use DatacovesBashOperator to run external Python scripts from Airflow DAGs in Datacoves, with custom packages pre-installed in the worker environment." -sidebar_position: 21 +sidebar_position: 20 --- # External Python DAG diff --git a/docs/how-tos/airflow/dags/generate-dags-from-yml.md b/docs/how-tos/airflow/dags/generate-dags-from-yml.md index 00659cb..47a4582 100644 --- a/docs/how-tos/airflow/dags/generate-dags-from-yml.md +++ b/docs/how-tos/airflow/dags/generate-dags-from-yml.md @@ -2,7 +2,7 @@ title: Generate Airflow DAGs from YAML with dbt-coves sidebar_label: Generate from YAML description: "Use dbt-coves to auto-generate Airflow Python DAG files from simple YAML definitions, reducing boilerplate and standardizing pipeline creation." -sidebar_position: 23 +sidebar_position: 50 --- # Generate DAGs from yml diff --git a/docs/how-tos/airflow/dags/get-current-branch-name.md b/docs/how-tos/airflow/dags/get-current-branch-name.md index 882595e..d5dcd0d 100644 --- a/docs/how-tos/airflow/dags/get-current-branch-name.md +++ b/docs/how-tos/airflow/dags/get-current-branch-name.md @@ -2,7 +2,7 @@ title: Get the Git Branch Name from an Airflow DAG Task sidebar_label: Get Git Branch Name description: "Retrieve the current git branch name from within an Airflow DAG task in Datacoves for branch-specific logic or conditional pipeline behavior." -sidebar_position: 24 +sidebar_position: 60 --- # Retrieving the Current Branch Name in a Git Repository diff --git a/docs/how-tos/airflow/dags/retry-dbt-tasks.md b/docs/how-tos/airflow/dags/retry-dbt-tasks.md index 590324b..7cfa992 100644 --- a/docs/how-tos/airflow/dags/retry-dbt-tasks.md +++ b/docs/how-tos/airflow/dags/retry-dbt-tasks.md @@ -2,7 +2,7 @@ title: Retry Failed dbt Tasks in Airflow DAGs sidebar_label: Retry dbt Tasks description: "Configure automatic retries for failed dbt tasks in Airflow DAGs in Datacoves, including retry count, delay settings, and email-on-retry behavior." -sidebar_position: 29 +sidebar_position: 80 --- # Retry a dbt task diff --git a/docs/how-tos/airflow/dags/run-adf-pipeline.md b/docs/how-tos/airflow/dags/run-adf-pipeline.md index fe1ada1..f5fa898 100644 --- a/docs/how-tos/airflow/dags/run-adf-pipeline.md +++ b/docs/how-tos/airflow/dags/run-adf-pipeline.md @@ -2,7 +2,7 @@ title: Trigger ADF Pipelines from Airflow in Datacoves sidebar_label: Run ADF Pipeline description: "Use Azure Data Factory operators in Airflow DAGs within Datacoves to trigger and monitor ADF pipelines as part of your data orchestration workflow." -sidebar_position: 26 +sidebar_position: 120 --- # Use Microsoft Azure Data Factory Operators diff --git a/docs/how-tos/airflow/dags/run-airbyte-sync-jobs.md b/docs/how-tos/airflow/dags/run-airbyte-sync-jobs.md index 9f2c824..21adf24 100644 --- a/docs/how-tos/airflow/dags/run-airbyte-sync-jobs.md +++ b/docs/how-tos/airflow/dags/run-airbyte-sync-jobs.md @@ -2,7 +2,7 @@ title: Trigger Airbyte Syncs from Airflow in Datacoves sidebar_label: Run Airbyte Syncs description: "Orchestrate Airbyte data syncs from Airflow DAGs in Datacoves using the Airbyte provider, with examples for triggering connections and monitoring status." -sidebar_position: 27 +sidebar_position: 130 --- # Run Airbyte sync jobs diff --git a/docs/how-tos/airflow/dags/run-databricks-notebook.md b/docs/how-tos/airflow/dags/run-databricks-notebook.md index 64c95ae..d681464 100644 --- a/docs/how-tos/airflow/dags/run-databricks-notebook.md +++ b/docs/how-tos/airflow/dags/run-databricks-notebook.md @@ -2,7 +2,7 @@ title: Run Databricks Notebooks from Airflow DAGs sidebar_label: Run Databricks Notebook description: "Trigger and monitor Databricks Notebook runs from Airflow DAGs in Datacoves using the Databricks provider operator with cluster configuration." -sidebar_position: 30 +sidebar_position: 90 --- # Run Databricks Notebooks diff --git a/docs/how-tos/airflow/dags/run-dbt.md b/docs/how-tos/airflow/dags/run-dbt.md index 771c092..b358d2d 100644 --- a/docs/how-tos/airflow/dags/run-dbt.md +++ b/docs/how-tos/airflow/dags/run-dbt.md @@ -2,7 +2,7 @@ title: "Run dbt from Airflow in Datacoves: DAG Examples" sidebar_label: Run dbt description: "Use the Datacoves dbt decorator or operator to run dbt commands from Airflow tasks, with full examples for dbt run, dbt test, and dbt build." -sidebar_position: 28 +sidebar_position: 100 --- # How to run dbt from an Airflow worker diff --git a/docs/how-tos/airflow/dags/run-fivetran-sync-jobs.md b/docs/how-tos/airflow/dags/run-fivetran-sync-jobs.md index 31f2a5b..351012e 100644 --- a/docs/how-tos/airflow/dags/run-fivetran-sync-jobs.md +++ b/docs/how-tos/airflow/dags/run-fivetran-sync-jobs.md @@ -2,7 +2,7 @@ title: Trigger Fivetran Syncs from Airflow in Datacoves sidebar_label: Run Fivetran Syncs description: "Orchestrate Fivetran connector syncs from Airflow DAGs in Datacoves using the Fivetran provider, with connection ID configuration and monitoring." -sidebar_position: 31 +sidebar_position: 140 --- # Run Fivetran sync jobs diff --git a/docs/how-tos/airflow/dags/s3-to-snowflake.md b/docs/how-tos/airflow/dags/s3-to-snowflake.md index 86925a2..b32f1d0 100644 --- a/docs/how-tos/airflow/dags/s3-to-snowflake.md +++ b/docs/how-tos/airflow/dags/s3-to-snowflake.md @@ -2,7 +2,7 @@ title: Load S3 Files into Snowflake with Airflow sidebar_label: S3 to Snowflake description: "Use Airflow in Datacoves to load files from Amazon S3 into Snowflake using the S3ToSnowflakeOperator, with staging area and copy options." -sidebar_position: 25 +sidebar_position: 70 --- # Loading S3 Files into Snowflake diff --git a/docs/how-tos/airflow/dags/test-dags.md b/docs/how-tos/airflow/dags/test-dags.md index f6ef734..13697d3 100644 --- a/docs/how-tos/airflow/dags/test-dags.md +++ b/docs/how-tos/airflow/dags/test-dags.md @@ -2,7 +2,7 @@ title: Test Airflow DAGs in Your CI/CD Pipeline sidebar_label: Test DAGs description: "How to run automated Airflow DAG tests in your CI/CD pipeline using pytest and the Datacoves CLI to catch import errors before deploying to production." -sidebar_position: 32 +sidebar_position: 110 --- # Run DAG tests in your CI/CD diff --git a/docs/how-tos/airflow/dags/use-variables-and-connections.md b/docs/how-tos/airflow/dags/use-variables-and-connections.md index ddd6d12..069b4c9 100644 --- a/docs/how-tos/airflow/dags/use-variables-and-connections.md +++ b/docs/how-tos/airflow/dags/use-variables-and-connections.md @@ -2,7 +2,7 @@ title: Use Airflow Variables and Connections in DAGs sidebar_label: Variables & Connections description: "How to access and use Airflow Variables and Connections in Datacoves DAGs, with best practices for avoiding performance issues during DAG parsing." -sidebar_position: 33 +sidebar_position: 150 --- # Variables and Connections