From 1e32d367c5887d0d8a2cc09fd52a8a34b9bccf7a Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Wed, 30 Sep 2026 18:10:40 +0000 Subject: [PATCH 1/2] Parse heredoc bodies with their instructions Heredoc payload lines currently become separate instructions. A FROM inside a RUN body changes parent_images, and replacing the RUN leaves its body behind. Consume bodies and terminators with their RUN, COPY, or ADD header, including ONBUILD instructions. Preserve body content and physical line bounds, and include the bodies in value and JSON output. Recognize quoted and multiple delimiters using Docker's frontend rules [1]. Unterminated bodies fail when reading structure, before edits that depend on that parse write. Reuse WordSplitter with optional separator preservation so operator spacing remains distinguishable without changing existing callers. Align cached line splitting with file reads so literal carriage returns in bodies do not change parsing or edit boundaries when caching is on. Fixes #145. Codex-assisted implementation with delegated independent review. [1]: https://docs.docker.com/reference/dockerfile/#here-documents Signed-off-by: Tamir Duberstein --- README.md | 20 ++++ dockerfile_parse/parser.py | 98 ++++++++++++++++++- dockerfile_parse/util.py | 7 +- tests/test_heredoc.py | 191 +++++++++++++++++++++++++++++++++++++ 4 files changed, 310 insertions(+), 6 deletions(-) create mode 100644 tests/test_heredoc.py diff --git a/README.md b/README.md index 1d92f06..e99449b 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,26 @@ dfp.baseimage = 'centos:7' print(dfp.content) ``` +### Here-documents + +The parser groups [Dockerfile here-documents] with their `RUN`, `COPY`, or +`ADD` instruction, including instructions nested in `ONBUILD`. Quoted +delimiters, `<<-` tab stripping, and multiple bodies in declaration order +are supported. Body lines are not parsed as instructions or comments. + +Each structure entry's `content` includes its complete heredoc bodies and +terminators, preserving their whitespace and line endings. Its `startline` +and `endline` cover the whole block, so inserting after, replacing, or +deleting the instruction also handles its bodies. The `value` and JSON +representation contain the normalized header followed by body and terminator +lines separated by `\n`, without a final newline. An unterminated heredoc +raises `ValueError` when reading `structure`, including edits that first +parse the instructions, such as setting `baseimage`. Cached and uncached +parsing use the same physical line boundaries; literal carriage returns +within a body do not create additional lines. + +[Dockerfile here-documents]: https://docs.docker.com/reference/dockerfile/#here-documents + [coveralls status badge]: https://coveralls.io/repos/containerbuildsystem/dockerfile-parse/badge.svg?branch=master [coveralls status link]: https://coveralls.io/r/containerbuildsystem/dockerfile-parse?branch=master [lgtm python badge]: https://img.shields.io/lgtm/grade/python/g/containerbuildsystem/dockerfile-parse.svg?logo=lgtm&logoWidth=18 diff --git a/dockerfile_parse/parser.py b/dockerfile_parse/parser.py index 1fe07f0..2be1efa 100644 --- a/dockerfile_parse/parser.py +++ b/dockerfile_parse/parser.py @@ -12,6 +12,7 @@ import os import re from contextlib import contextmanager +from io import StringIO from shlex import quote from .constants import DOCKERFILE_FILENAME, COMMENT_INSTRUCTION @@ -22,6 +23,73 @@ logger = logging.getLogger(__name__) +def _dequote_heredoc(word): + """Remove delimiter quotes without expanding variables or shell commands. + + Docker's literal delimiter decoding uses these double-quote escape rules: + https://github.com/moby/buildkit/blob/v0.32.2/frontend/dockerfile/shell/lex.go#L315-L330 + """ + result = [] + quote_char = None + characters = iter(word) + for character in characters: + if character == '\\' and quote_char != "'": + escaped = next(characters, '') + # Docker's double-quote lexer only unescapes these three characters. + if quote_char == '"' and escaped not in ('"', '$', '\\'): + result.append('\\') + result.append(escaped) + elif character == quote_char: + quote_char = None + elif quote_char is None and character in ('"', "'"): + quote_char = character + else: + result.append(character) + if quote_char is not None: + raise ValueError('Unterminated quote in heredoc delimiter: {0}'.format(word)) + return ''.join(result) + + +def _heredoc_delimiters(instruction, value): + """Return (delimiter, strip_tabs) pairs in the header's declaration order. + + Match Docker's whole-token recognition and whitespace after the operator: + https://github.com/moby/buildkit/blob/v0.32.2/frontend/dockerfile/parser/parser.go#L121 + https://github.com/moby/buildkit/blob/v0.32.2/frontend/dockerfile/shell/lex.go#L516-L532 + """ + if instruction == 'ONBUILD': + nested = value.split(None, 1) + if len(nested) != 2: + return [] + instruction, value = nested + if instruction.upper() not in ('RUN', 'COPY', 'ADD') or '<<' not in value: + return [] + + delimiters = [] + # Preserve quotes so a literal "< Date: Wed, 30 Sep 2026 21:30:25 +0000 Subject: [PATCH 2/2] Preserve escaped continuation characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Doubled terminal escape characters currently absorb the next Dockerfile instruction and lose a literal character from the value. Use Docker’s continuation rule for both instruction boundaries and value normalization, including the escape directive. Heredoc bodies then start at the same boundary as the builder. Signed-off-by: Tamir Duberstein --- dockerfile_parse/parser.py | 28 +++++++++---------- tests/test_continuations.py | 54 +++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 tests/test_continuations.py diff --git a/dockerfile_parse/parser.py b/dockerfile_parse/parser.py index 2be1efa..6711edc 100644 --- a/dockerfile_parse/parser.py +++ b/dockerfile_parse/parser.py @@ -311,11 +311,11 @@ def structure(self): Their value retains the normalized header followed by the literal body and terminator lines, joined with newlines and without a final newline. """ - def _rstrip_eol(text, line_continuation_char='\\'): - text = text.rstrip() - if text.endswith(line_continuation_char): - return text[:-1] - return text + def _continuation_regex(escape): + # Docker checks the immediately preceding character, not odd/even + # runs of escapes: even three trailing escapes end the instruction. + # https://github.com/moby/buildkit/blob/v0.32.2/frontend/dockerfile/parser/parser.go#L158-L168 + return re.compile(r'(?