From b21eaafdebbd50110fe60a0293f25cdbcad302af Mon Sep 17 00:00:00 2001 From: Zac Colbert Date: Mon, 12 Oct 2026 14:31:28 +1000 Subject: [PATCH] Preserve absolute NcML references with cdms3 base locations --- .../main/java/ucar/nc2/util/URLnaming.java | 26 ++- .../ncml/s3/TestNcmlAbsoluteReferences.java | 152 ++++++++++++++++++ .../java/ucar/nc2/util/TestURLnamingS3.java | 55 +++++++ 3 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/TestNcmlAbsoluteReferences.java create mode 100644 cdm/s3/src/test/java/ucar/nc2/util/TestURLnamingS3.java diff --git a/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java b/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java index 4d624f58d2..4d768c62bd 100644 --- a/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java +++ b/cdm/core/src/main/java/ucar/nc2/util/URLnaming.java @@ -90,7 +90,7 @@ public static String resolve(String baseUri, String relativeUri) { } if (baseUri.startsWith("cdms3:")) { - if (relativeUri.startsWith("cdms3:")) { + if (relativeUri.startsWith("cdms3:") || isAbsoluteUri(relativeUri)) { return relativeUri; } else { MFile absoluteMFile; @@ -121,11 +121,21 @@ public static String resolve(String baseUri, String relativeUri) { } } + /** + * Resolve a dataset reference against a document or directory location. + * Absolute URIs are preserved when the base location uses cdms3. + * + * @param baseDir document or directory location, or null to return the reference unchanged + * @param filepath absolute URI or relative file or object name, or null + * @return resolved location, or null when the reference is null + */ public static String resolveFile(String baseDir, String filepath) { if (baseDir == null) return filepath; if (filepath == null) return null; + if (baseDir.startsWith("cdms3:") && isAbsoluteUri(filepath)) + return filepath; File file = new File(filepath); if (file.isAbsolute()) return filepath; @@ -147,4 +157,18 @@ public static String resolveFile(String baseDir, String filepath) { return absoluteFile == null ? filepath : absoluteFile.getPath(); } + /** + * Check whether a dataset reference has an absolute URI scheme. + * + * @param reference non-null dataset reference or object name + * @return true for an absolute URI, false for a relative reference or invalid URI syntax + */ + private static boolean isAbsoluteUri(String reference) { + try { + return URI.create(reference).isAbsolute(); + } catch (IllegalArgumentException e) { + return false; + } + } + } diff --git a/cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/TestNcmlAbsoluteReferences.java b/cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/TestNcmlAbsoluteReferences.java new file mode 100644 index 0000000000..2be90788e9 --- /dev/null +++ b/cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/TestNcmlAbsoluteReferences.java @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ +package ucar.nc2.internal.ncml.s3; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertNotNull; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import java.io.IOException; +import java.io.StringReader; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Variable; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.internal.ncml.NcmlReader; +import ucar.nc2.write.NetcdfFormatWriter; + +/** Reads HTTP datasets referenced by NcML documents with a cdms3 base location. */ +public class TestNcmlAbsoluteReferences { + private static final String BASE = "cdms3://profile@localhost/bucket?site/wrapper.ncml#delimiter=/"; + private static final String NAMESPACE = "http://www.unidata.ucar.edu/namespaces/netcdf/ncml-2.2"; + + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private HttpServer server; + private String datasetUrl; + + /** + * Serve a generated NetCDF file on the loopback interface without external services. + * + * @throws IOException if the file or HTTP server cannot be created + * @throws InvalidRangeException if the fixture values do not match the time dimension + */ + @Before + public void setUp() throws IOException, InvalidRangeException { + Path fixture = temporaryFolder.newFile("data.nc").toPath(); + NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(fixture.toString()); + builder.addDimension("time", 2); + builder.addVariable("time", DataType.DOUBLE, "time"); + try (NetcdfFormatWriter writer = builder.build()) { + writer.write("time", Array.factory(DataType.DOUBLE, new int[] {2}, new double[] {1.0, 2.0})); + } + byte[] contents = Files.readAllBytes(fixture); + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/data.nc", exchange -> serveFixture(exchange, contents)); + server.start(); + datasetUrl = "http://127.0.0.1:" + server.getAddress().getPort() + "/data.nc"; + } + + /** Stop the HTTP server, including when setup or a dataset read fails. */ + @After + public void tearDown() { + if (server != null) { + server.stop(0); + } + } + + /** + * Read values through a wrapper containing an absolute HTTP reference. + * + * @throws IOException if the wrapper or referenced dataset cannot be read + */ + @Test + public void testWrapper() throws IOException { + String ncml = ""; + assertNcmlData(ncml, new double[] {1.0, 2.0}); + } + + /** + * Read all values from explicit aggregation members with absolute HTTP references. + * + * @throws IOException if the aggregation or referenced datasets cannot be read + */ + @Test + public void testAggregation() throws IOException { + String ncml = "" + "" + + "" + "" + ""; + assertNcmlData(ncml, new double[] {1.0, 2.0, 1.0, 2.0}); + } + + /** + * Check the complete time array returned by an NcML document with a cdms3 base location. + * + * @param ncml document containing absolute HTTP references to the generated NetCDF file + * @param expected complete time values expected from the document + * @throws IOException if the document or referenced dataset cannot be read + */ + private void assertNcmlData(String ncml, double[] expected) throws IOException { + try (NetcdfDataset dataset = NcmlReader.readNcml(new StringReader(ncml), BASE, null).build()) { + Variable time = dataset.findVariable("time"); + assertNotNull(time); + Array values = time.read(); + assertThat(values.getSize()).isEqualTo(expected.length); + for (int i = 0; i < expected.length; i++) { + assertThat(values.getDouble(i)).isEqualTo(expected[i]); + } + } + } + + /** + * Serve HEAD and single byte-range requests for the generated NetCDF file. + * + * @param exchange request from the NetCDF HTTP reader + * @param contents complete NetCDF file bytes + * @throws IOException if the response cannot be written + */ + private static void serveFixture(HttpExchange exchange, byte[] contents) throws IOException { + try { + exchange.getResponseHeaders().set("Content-Type", "application/x-netcdf"); + exchange.getResponseHeaders().set("Accept-Ranges", "bytes"); + if (exchange.getRequestMethod().equals("HEAD")) { + exchange.getResponseHeaders().set("Content-Length", Integer.toString(contents.length)); + exchange.sendResponseHeaders(200, -1); + return; + } + String range = exchange.getRequestHeaders().getFirst("Range"); + int start = 0; + int end = contents.length - 1; + if (range != null) { + String[] bounds = range.substring("bytes=".length()).split("-", -1); + start = Integer.parseInt(bounds[0]); + if (!bounds[1].isEmpty()) { + end = Math.min(end, Integer.parseInt(bounds[1])); + } + if (start > end) { + exchange.getResponseHeaders().set("Content-Range", "bytes */" + contents.length); + exchange.sendResponseHeaders(416, -1); + return; + } + exchange.getResponseHeaders().set("Content-Range", "bytes " + start + "-" + end + "/" + contents.length); + } + exchange.sendResponseHeaders(range == null ? 200 : 206, end - start + 1); + exchange.getResponseBody().write(contents, start, end - start + 1); + } finally { + exchange.close(); + } + } +} diff --git a/cdm/s3/src/test/java/ucar/nc2/util/TestURLnamingS3.java b/cdm/s3/src/test/java/ucar/nc2/util/TestURLnamingS3.java new file mode 100644 index 0000000000..020376b0b7 --- /dev/null +++ b/cdm/s3/src/test/java/ucar/nc2/util/TestURLnamingS3.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2026 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ +package ucar.nc2.util; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; + +/** Checks dataset reference resolution with the S3 file provider available. */ +public class TestURLnamingS3 { + private static final String BASE = "cdms3://profile@localhost/bucket?site/wrapper.ncml#delimiter=/"; + + /** Preserve absolute dataset URIs for both NcML wrappers and aggregation members. */ + @Test + public void testAbsoluteReferences() { + for (String base : new String[] {BASE, "cdms3:bucket?wrapper.ncml"}) { + for (String reference : new String[] {"http://localhost/data.nc", + "https://localhost/data.nc?selection=time#fragment", "dods://localhost/thredds/dodsC/data.nc", + "dap4://localhost/thredds/dap4/data.nc", "cdms3://profile@localhost/other-bucket?data.nc#delimiter=/", + "file:/tmp/data.nc"}) { + assertThat(URLnaming.resolve(base, reference)).isEqualTo(reference); + assertThat(URLnaming.resolveFile(base, reference)).isEqualTo(reference); + } + } + } + + /** Resolve relative object names within the containing object prefix. */ + @Test + public void testRelativeReferences() { + for (String reference : new String[] {"data.nc", "nested/data.nc", "data%20file.nc"}) { + String expected = "cdms3://profile@localhost/bucket?site/" + reference + "#delimiter=/"; + assertThat(URLnaming.resolve(BASE, reference)).isEqualTo(expected); + assertThat(URLnaming.resolveFile(BASE, reference)).isEqualTo(expected); + } + } + + /** Return references with invalid URI syntax when an object location cannot be constructed. */ + @Test + public void testInvalidReference() { + String reference = "data file.nc"; + assertThat(URLnaming.resolve(BASE, reference)).isEqualTo(reference); + assertThat(URLnaming.resolveFile(BASE, reference)).isEqualTo(reference); + } + + /** Preserve the null-base and null-reference return values. */ + @Test + public void testNullReferences() { + assertThat(URLnaming.resolve(BASE, null)).isNull(); + assertThat(URLnaming.resolveFile(BASE, null)).isNull(); + assertThat(URLnaming.resolve(null, "data.nc")).isEqualTo("data.nc"); + assertThat(URLnaming.resolveFile(null, "data.nc")).isEqualTo("data.nc"); + } +}