Merge dev changes

.github/workflows/main.yml

@@ -15,7 +15,7 @@ jobs:
       max-parallel: 1
       matrix:
         python-version: ["3.11", "3.14"]
-        sentieon-version: ["202503.02"]
+        sentieon-version: ["${{ vars.SENTIEON_VERSION }}"]
         os: [ubuntu-22.04] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -41,6 +41,10 @@ jobs:
           pip install multiqc
       - name: Install sentieon
         run: |
+          if [ -z "$SENTIEON_VERSION" ]; then
+            echo "::error::Repository variable SENTIEON_VERSION is not set." >&2
+            exit 1
+          fi
           url="https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz"
           bin_dir=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin
           machine_arch=$(uname -m)
@@ -98,7 +102,7 @@ jobs:
       max-parallel: 1
       matrix:
         python-version: ["3.11", "3.14"]
-        sentieon-version: ["202503.02"]
+        sentieon-version: ["${{ vars.SENTIEON_VERSION }}"]
         os: [ubuntu-22.04] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -136,6 +140,10 @@ jobs:
           sudo chmod ugo+x /usr/local/bin/mosdepth
       - name: Install sentieon
         run: |
+          if [ -z "$SENTIEON_VERSION" ]; then
+            echo "::error::Repository variable SENTIEON_VERSION is not set." >&2
+            exit 1
+          fi
           url="https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz"
           bin_dir=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin
           machine_arch=$(uname -m)
@@ -220,7 +228,7 @@ jobs:
       max-parallel: 1
       matrix:
         python-version: ["3.11", "3.14"]
-        sentieon-version: ["202503.02"]
+        sentieon-version: ["${{ vars.SENTIEON_VERSION }}"]
         os: [ubuntu-22.04] #, macos-latest, windows-latest]
     runs-on: ${{ matrix.os }}
     steps:
@@ -261,6 +269,10 @@ jobs:
           pip install multiqc
       - name: Install sentieon
         run: |
+          if [ -z "$SENTIEON_VERSION" ]; then
+            echo "::error::Repository variable SENTIEON_VERSION is not set." >&2
+            exit 1
+          fi
           url="https://s3.amazonaws.com/sentieon-release/software/sentieon-genomics-$SENTIEON_VERSION.tar.gz"
           bin_dir=$(pwd)/sentieon-genomics-$SENTIEON_VERSION/bin
           machine_arch=$(uname -m)
@@ -296,6 +308,7 @@ jobs:
             --sr_aln "tests/smoke/illumina.cram" \
             --lr_aln "tests/smoke/sam ple.cram" \
             -m "DNAscope hybrid IlmnPB1.1.bundle" \
+            --skip_cnv \
             --sr_duplicate_marking none \
             "output_hybrid1.vcf.gz"
           if [ ! -f "output_hybrid1.vcf.gz" ]; then
@@ -314,6 +327,7 @@ jobs:
             --lr_align_input \
             --lr_input_ref "tests/smoke/r ef.fa" \
             -m "DNAscope hybrid IlmnPB1.1.bundle" \
+            --skip_cnv \
             "output_hybrid2.vcf.gz"
           if [ ! -f "output_hybrid2.vcf.gz" ]; then
             exit 1
@@ -331,6 +345,7 @@ jobs:
             --sr_aln "tests/smoke/illumina.cram" "tests/smoke/illumina_2.bam" \
             --lr_aln "tests/smoke/sam ple.cram" \
             -m "DNAscope hybrid IlmnPB1.1.bundle" \
+            --skip_cnv \
             --sr_duplicate_marking none \
             "output_hybrid3.vcf.gz"
           if [ ! -f "output_hybrid3.vcf.gz" ]; then

Dockerfile

@@ -293,7 +293,7 @@ RUN sentieon driver --help && \
     perl -c "$(command -v run-t1k)" && \
     SENTIEON_LICENSE=placeholder segdup-caller --version && \
     SENTIEON_LICENSE=placeholder python -c "import sys; from packaging.version import Version; \
-from sentieon_cli.sentieon_pangenome import SEGDUP_MIN_VERSION; \
+from sentieon_cli.stages.segdup import SEGDUP_MIN_VERSIONS; \
 from sentieon_cli.util import check_version; \
-sys.exit(0 if all(check_version(c, v) for c, v in SEGDUP_MIN_VERSION.items()) else 1)" && \
+sys.exit(0 if all(check_version(c, v) for c, v in SEGDUP_MIN_VERSIONS.items()) else 1)" && \
     sentieon-cli -h

README.md

@@ -55,8 +55,12 @@ sentieon-cli ...
 
 ## Global arguments
 The `sentieon-cli` supports the following global arguments:
-- `--verbose`: verbose logging.
-- `--debug`: debugging mode for more verbose logging.
+- `--verbose` (`-v`): verbose logging. This is the default.
+- `--quiet` (`-q`): only log warnings and errors.
+- `--debug` (`-d`): debugging mode for more verbose logging. Takes precedence over `--verbose` and `--quiet`.
+
+## Logging
+Each run writes its log files to a directory next to the output VCF, named after the output file with the `.vcf.gz` suffix replaced by `_logs`, so `sample.vcf.gz` produces `sample_logs/`. The directory records the invocation in `command.txt`, the pipeline's own messages in `run.log`, and the output of each tool under `task_logs/`. Run metrics are also written as tab-separated tables for analysis: `process_metrics.txt` (one row per process, with its CPU time, peak memory and exit code) and `job_metrics.txt` (one row per pipeline job). Every pipeline accepts a `--log_dir` argument to write these files elsewhere. Rerunning a pipeline with the same output overwrites the logs of the previous run.
 
 ## Supported pipelines
 - [**DNAscope**](https://support.sentieon.com/docs/sentieon_cli/#dnascope) - DNAscope pipeline implementation for germline SNV and indel calling from short read data.

pyproject.toml

@@ -1,7 +1,7 @@
 
 [project]
 name = "sentieon_cli"
-version = "1.7.0"
+version = "2.0.0"
 description = "Pipeline implementations for the Sentieon software"
 authors = [
     {name = "Don Freed", email = "don.freed@sentieon.com"},
@@ -27,6 +27,10 @@ dependencies = [
 [project.scripts]
 sentieon-cli = "sentieon_cli:main"
 
+[tool.poetry]
+# Ship the packaged pseudo-autosomal region (PAR) BED files
+include = [{ path = "sentieon_cli/data/*.bed", format = ["sdist", "wheel"] }]
+
 [tool.mypy]
 ignore_missing_imports = true
 
@@ -44,6 +48,7 @@ exclude = [
     "sentieon_cli/scripts/hybrid_anno.py",
     "sentieon_cli/scripts/hybrid_select.py",
     "sentieon_cli/scripts/indel2cnv.py",
+    "sentieon_cli/scripts/sad_lad_update.py",
     "sentieon_cli/scripts/trimalt.py",
     "sentieon_cli/archive.py",
     "playground",

sentieon_cli/__init__.py

@@ -1,31 +1,50 @@
+import argparse
+
 from . import argh_parser
 from .dnascope import DNAscopePipeline
 from .dnascope_hybrid import DNAscopeHybridPipeline
 from .dnascope_longread import DNAscopeLRPipeline
+from .hybrid_pangenome import HybridPangenome
+from .job import Job
 from .sentieon_pangenome import SentieonPangenome
 from .util import __version__
 
 
-def main():
-    """main entry point for this project"""
-    parser = argh_parser.CustomArgparseParser()
+def add_logging_args(parser: argparse.ArgumentParser) -> None:
+    """Add the console verbosity flags"""
     parser.add_argument(
         "-v",
         "--verbose",
-        help="Verbose logging",
-        action="store_const",
-        dest="loglevel",
-        const="INFO",
-        default="WARNING",
+        help="Verbose logging (the default)",
+        action="store_true",
+    )
+    parser.add_argument(
+        "-q",
+        "--quiet",
+        help="Only log warnings and errors",
+        action="store_true",
     )
     parser.add_argument(
         "-d",
         "--debug",
         help="Print debugging info",
-        action="store_const",
-        dest="loglevel",
-        const="DEBUG",
+        action="store_true",
     )
+
+
+def resolve_loglevel(args: argparse.Namespace) -> str:
+    """The console log level implied by the verbosity flags"""
+    if args.debug:
+        return "DEBUG"
+    if args.quiet and not args.verbose:
+        return "WARNING"
+    return "INFO"
+
+
+def main():
+    """main entry point for this project"""
+    parser = argh_parser.CustomArgparseParser()
+    add_logging_args(parser)
     parser.add_argument(
         "--version",
         action="version",
@@ -58,7 +77,17 @@ def main():
     pipeline.add_arguments(dnascope_pangenome_subparser)
     dnascope_pangenome_subparser.set_defaults(pipeline=pipeline.main)
 
+    # Hybrid pangenome
+    pipeline = HybridPangenome()
+    hybrid_pangenome_subparser = subparsers.add_parser("hybrid-pangenome")
+    pipeline.add_arguments(hybrid_pangenome_subparser)
+    hybrid_pangenome_subparser.set_defaults(pipeline=pipeline.main)
+
     args = parser.parse_args()
+    args.loglevel = resolve_loglevel(args)
+    # Job ids must be unique for the whole run, which may execute more than
+    # one DAG, so numbering restarts here rather than per DAG.
+    Job.reset_ids()
     args.pipeline(args)