diff --git a/.github/workflows/builds.yml b/.github/workflows/builds.yml index 896c561190..48b9e61a40 100644 --- a/.github/workflows/builds.yml +++ b/.github/workflows/builds.yml @@ -1,8 +1,11 @@ name: Build Check +permissions: + contents: read on: schedule: - cron: '0 12 * * *' + workflow_dispatch: jobs: Verify: @@ -32,7 +35,7 @@ jobs: cache-jdk: true - uses: ./.github/actions/setup-maven - name: Run Tests - run: ./mvnw -B -ntp test -Ddocker.tests=true -Dexternal.tests=true + run: ./mvnw -B -ntp test -Ddocker.tests=true RunOnMacOs: runs-on: macos-latest @@ -47,7 +50,7 @@ jobs: cache-jdk: true - uses: ./.github/actions/setup-maven - name: Run Tests - run: ./mvnw -B -ntp test -Dexternal.tests=true + run: ./mvnw -B -ntp test -Dno.docker.tests=true RunOnWindows: runs-on: windows-latest @@ -62,4 +65,4 @@ jobs: cache-jdk: true - uses: ./.github/actions/setup-maven - name: Run Tests - run: ./mvnw.cmd -B -ntp test "-Dexternal.tests=true" + run: ./mvnw.cmd -B -ntp test "-Dno.docker.tests=true" diff --git a/AGENTS.md b/AGENTS.md index 8c2f5392c6..f96bf3b9c7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,7 +97,7 @@ Common commands: * Do not retry flaky tests. A flaky test is a bug. Find and fix the race. * Do not leak Netty `ByteBuf` instances. The leak detector extension will fail the test. * Keep the default test suite hermetic. -* Tests requiring public hosts must be tagged `external`. +* Tests must not reach public hosts or DNS. Use the in-process servers under `org.asynchttpclient.testserver`; only Docker-gated integration tests may pull images. * Docker-based integration tests must follow the existing Testcontainers gating properties. ## Coding Conventions diff --git a/client/src/main/java/org/asynchttpclient/DefaultAsyncHttpClientConfig.java b/client/src/main/java/org/asynchttpclient/DefaultAsyncHttpClientConfig.java index 14cf408725..a316cc2e92 100644 --- a/client/src/main/java/org/asynchttpclient/DefaultAsyncHttpClientConfig.java +++ b/client/src/main/java/org/asynchttpclient/DefaultAsyncHttpClientConfig.java @@ -1763,6 +1763,10 @@ public Builder setEventLoopGroup(EventLoopGroup eventLoopGroup) { * {@linkplain AddressResolverGroup#close() close} it when the client is shut down. * Do not pass a shared resolver group that is used by other clients unless you manage * its lifecycle independently. + *

+ * A {@link io.netty.resolver.dns.DnsAddressResolverGroup} must use the datagram channel of the transport the + * client selects (see {@link #setUseNativeTransport(boolean)}): {@code EpollDatagramChannel} on epoll, not + * {@code NioDatagramChannel}. Otherwise every DNS query fails with "incompatible event loop type". * * @param addressResolverGroup the resolver group, or {@code null} to use per-request resolvers * @return the same builder instance diff --git a/client/src/test/java/org/asynchttpclient/AddressResolverGroupTest.java b/client/src/test/java/org/asynchttpclient/AddressResolverGroupTest.java index 0a6f040adb..4d2db31ffd 100644 --- a/client/src/test/java/org/asynchttpclient/AddressResolverGroupTest.java +++ b/client/src/test/java/org/asynchttpclient/AddressResolverGroupTest.java @@ -15,48 +15,76 @@ */ package org.asynchttpclient; +import io.netty.channel.IoEventLoopGroup; +import io.netty.channel.epoll.EpollDatagramChannel; +import io.netty.channel.epoll.EpollIoHandler; +import io.netty.channel.kqueue.KQueueDatagramChannel; +import io.netty.channel.kqueue.KQueueIoHandler; +import io.netty.channel.socket.DatagramChannel; import io.netty.channel.socket.nio.NioDatagramChannel; +import io.netty.channel.uring.IoUringDatagramChannel; +import io.netty.channel.uring.IoUringIoHandler; +import io.netty.handler.codec.dns.DnsResponseCode; import io.netty.resolver.dns.DnsAddressResolverGroup; -import io.netty.resolver.dns.DnsServerAddressStreamProviders; +import io.netty.resolver.dns.DnsErrorCauseException; +import io.netty.resolver.dns.SingletonDnsServerAddressStreamProvider; import org.asynchttpclient.test.EventCollectingHandler; import org.asynchttpclient.testserver.HttpServer; import org.asynchttpclient.testserver.HttpTest; +import org.asynchttpclient.testserver.StubDnsServer; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.concurrent.ExecutionException; import static java.util.concurrent.TimeUnit.SECONDS; -import static org.asynchttpclient.Dsl.asyncHttpClient; import static org.asynchttpclient.Dsl.config; import static org.asynchttpclient.Dsl.get; -import static org.asynchttpclient.test.TestUtils.isExternalNetworkAvailable; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import static org.junit.jupiter.api.Assumptions.assumeTrue; public class AddressResolverGroupTest extends HttpTest { - private static final String GOOGLE_URL = "https://www.google.com/"; - private static final String EXAMPLE_URL = "https://www.example.com/"; + private static Class datagramChannelClass; private HttpServer server; + private StubDnsServer dns; + + // The resolver channel has to match whichever transport the client picks. + @BeforeAll + public static void probeTransport() { + try (DefaultAsyncHttpClient probe = new DefaultAsyncHttpClient()) { + IoEventLoopGroup group = (IoEventLoopGroup) probe.getEventLoopGroup(); + if (group.isIoType(EpollIoHandler.class)) { + datagramChannelClass = EpollDatagramChannel.class; + } else if (group.isIoType(KQueueIoHandler.class)) { + datagramChannelClass = KQueueDatagramChannel.class; + } else if (group.isIoType(IoUringIoHandler.class)) { + datagramChannelClass = IoUringDatagramChannel.class; + } else { + datagramChannelClass = NioDatagramChannel.class; + } + } + } @BeforeEach public void start() throws Throwable { server = new HttpServer(); server.start(); + dns = new StubDnsServer(); } @AfterEach public void stop() throws Throwable { + if (dns != null) { + dns.close(); + } server.close(); } @@ -64,11 +92,18 @@ private String getTargetUrl() { return server.getHttpUrl() + "/foo/bar"; } + // A name the hosts file cannot answer, so the resolver has to query the stub DNS server. + private String getTargetUrl(String host) { + return "http://" + host + ":" + server.getHttpPort() + "/foo/bar"; + } + + private DnsAddressResolverGroup newDnsResolverGroup() { + return new DnsAddressResolverGroup(datagramChannelClass, new SingletonDnsServerAddressStreamProvider(dns.getAddress())); + } + @Test public void requestWithDnsAddressResolverGroupSucceeds() throws Throwable { - DnsAddressResolverGroup resolverGroup = new DnsAddressResolverGroup( - NioDatagramChannel.class, - DnsServerAddressStreamProviders.platformDefault()); + DnsAddressResolverGroup resolverGroup = newDnsResolverGroup(); withClient(config().setAddressResolverGroup(resolverGroup)).run(client -> withServer(server).run(server -> { @@ -80,9 +115,7 @@ public void requestWithDnsAddressResolverGroupSucceeds() throws Throwable { @Test public void dnsResolverGroupFiresHostnameResolutionEvents() throws Throwable { - DnsAddressResolverGroup resolverGroup = new DnsAddressResolverGroup( - NioDatagramChannel.class, - DnsServerAddressStreamProviders.platformDefault()); + DnsAddressResolverGroup resolverGroup = newDnsResolverGroup(); withClient(config().setAddressResolverGroup(resolverGroup)).run(client -> withServer(server).run(server -> { @@ -119,56 +152,46 @@ public void defaultConfigDoesNotSetAddressResolverGroup() { @Test public void unknownHostWithDnsResolverGroupFails() throws Throwable { - DnsAddressResolverGroup resolverGroup = new DnsAddressResolverGroup( - NioDatagramChannel.class, - DnsServerAddressStreamProviders.platformDefault()); + DnsAddressResolverGroup resolverGroup = newDnsResolverGroup(); withClient(config().setAddressResolverGroup(resolverGroup)).run(client -> { try { - client.prepareGet("http://nonexistent.invalid/foo").execute().get(10, SECONDS); + client.prepareGet(getTargetUrl(StubDnsServer.UNKNOWN_HOST)).execute().get(10, SECONDS); fail("Request to nonexistent host should have thrown an exception"); } catch (ExecutionException e) { - assertNotNull(e.getCause(), "Should have a cause for the DNS failure"); + // The client surfaces the root cause, not Netty's UnknownHostException wrapper. + DnsErrorCauseException cause = assertInstanceOf(DnsErrorCauseException.class, e.getCause(), + "Should fail with a DNS failure"); + assertEquals(DnsResponseCode.NXDOMAIN, cause.getCode()); } }); } - @Tag("external") @Test - public void resolveRealDomainWithDnsResolverGroup() throws Throwable { - assumeTrue(isExternalNetworkAvailable(), "External network not available - skipping test"); - - DnsAddressResolverGroup resolverGroup = new DnsAddressResolverGroup( - NioDatagramChannel.class, - DnsServerAddressStreamProviders.platformDefault()); - - try (AsyncHttpClient client = asyncHttpClient(config().setAddressResolverGroup(resolverGroup))) { - Response response = client.prepareGet(GOOGLE_URL).execute().get(20, SECONDS); - assertNotNull(response); - assertTrue(response.getStatusCode() >= 200 && response.getStatusCode() < 400, - "Expected successful HTTP status but got " + response.getStatusCode()); - } + public void resolveHostWithDnsQuery() throws Throwable { + DnsAddressResolverGroup resolverGroup = newDnsResolverGroup(); + + withClient(config().setAddressResolverGroup(resolverGroup)).run(client -> + withServer(server).run(server -> { + server.enqueueOk(); + Response response = client.prepareGet(getTargetUrl("first.invalid")).execute().get(10, SECONDS); + assertEquals(200, response.getStatusCode()); + })); } - @Tag("external") @Test - public void resolveMultipleRealDomainsWithDnsResolverGroup() throws Throwable { - assumeTrue(isExternalNetworkAvailable(), "External network not available - skipping test"); - - DnsAddressResolverGroup resolverGroup = new DnsAddressResolverGroup( - NioDatagramChannel.class, - DnsServerAddressStreamProviders.platformDefault()); - - try (AsyncHttpClient client = asyncHttpClient(config().setAddressResolverGroup(resolverGroup))) { - Response response1 = client.prepareGet(GOOGLE_URL).execute().get(20, SECONDS); - assertNotNull(response1); - assertTrue(response1.getStatusCode() >= 200 && response1.getStatusCode() < 400, - "Expected successful HTTP status for google.com but got " + response1.getStatusCode()); - - Response response2 = client.prepareGet(EXAMPLE_URL).execute().get(20, SECONDS); - assertNotNull(response2); - assertTrue(response2.getStatusCode() >= 200 && response2.getStatusCode() < 400, - "Expected successful HTTP status for example.com but got " + response2.getStatusCode()); - } + public void resolveMultipleHostsWithDnsQueries() throws Throwable { + DnsAddressResolverGroup resolverGroup = newDnsResolverGroup(); + + withClient(config().setAddressResolverGroup(resolverGroup)).run(client -> + withServer(server).run(server -> { + server.enqueueOk(); + Response response1 = client.prepareGet(getTargetUrl("first.invalid")).execute().get(10, SECONDS); + assertEquals(200, response1.getStatusCode()); + + server.enqueueOk(); + Response response2 = client.prepareGet(getTargetUrl("second.invalid")).execute().get(10, SECONDS); + assertEquals(200, response2.getStatusCode()); + })); } } diff --git a/client/src/test/java/org/asynchttpclient/proxy/HttpsProxyTestcontainersIntegrationTest.java b/client/src/test/java/org/asynchttpclient/proxy/HttpsProxyTestcontainersIntegrationTest.java index 1346aa294d..08bd52c880 100644 --- a/client/src/test/java/org/asynchttpclient/proxy/HttpsProxyTestcontainersIntegrationTest.java +++ b/client/src/test/java/org/asynchttpclient/proxy/HttpsProxyTestcontainersIntegrationTest.java @@ -20,7 +20,6 @@ import org.asynchttpclient.Response; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +43,6 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; -@Tag("external") @Testcontainers public class HttpsProxyTestcontainersIntegrationTest { diff --git a/client/src/test/java/org/asynchttpclient/proxy/SocksProxyTestcontainersIntegrationTest.java b/client/src/test/java/org/asynchttpclient/proxy/SocksProxyTestcontainersIntegrationTest.java index d13b96691b..87d1399f89 100644 --- a/client/src/test/java/org/asynchttpclient/proxy/SocksProxyTestcontainersIntegrationTest.java +++ b/client/src/test/java/org/asynchttpclient/proxy/SocksProxyTestcontainersIntegrationTest.java @@ -20,7 +20,6 @@ import org.asynchttpclient.Response; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,7 +47,6 @@ * Integration tests for SOCKS proxy support using Dante SOCKS server in TestContainers. * This validates the fix for GitHub issue #1913. */ -@Tag("external") @Testcontainers public class SocksProxyTestcontainersIntegrationTest { diff --git a/client/src/test/java/org/asynchttpclient/test/TestUtils.java b/client/src/test/java/org/asynchttpclient/test/TestUtils.java index 7da1260241..313d713490 100644 --- a/client/src/test/java/org/asynchttpclient/test/TestUtils.java +++ b/client/src/test/java/org/asynchttpclient/test/TestUtils.java @@ -54,9 +54,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.InetSocketAddress; import java.net.ServerSocket; -import java.net.Socket; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -120,20 +118,6 @@ public static synchronized int findFreePort() throws IOException { } } - /** - * Probes whether outbound internet (and DNS) is reachable. Tests that genuinely require a real - * public host should be annotated {@code @Tag("external")} (excluded from the default build) and - * may additionally guard with this so they skip cleanly when run on an isolated machine. - */ - public static boolean isExternalNetworkAvailable() { - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress("www.google.com", 443), 3000); - return true; - } catch (Exception e) { - return false; - } - } - public static File resourceAsFile(String path) throws URISyntaxException, IOException { ClassLoader cl = TestUtils.class.getClassLoader(); URI uri = cl.getResource(path).toURI(); diff --git a/client/src/test/java/org/asynchttpclient/testserver/StubDnsServer.java b/client/src/test/java/org/asynchttpclient/testserver/StubDnsServer.java new file mode 100644 index 0000000000..0a0d93c67b --- /dev/null +++ b/client/src/test/java/org/asynchttpclient/testserver/StubDnsServer.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2026 AsyncHttpClient Project. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.asynchttpclient.testserver; + +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.Unpooled; +import io.netty.channel.Channel; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.ChannelInitializer; +import io.netty.channel.EventLoopGroup; +import io.netty.channel.MultiThreadIoEventLoopGroup; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.channel.nio.NioIoHandler; +import io.netty.channel.socket.nio.NioDatagramChannel; +import io.netty.handler.codec.dns.DatagramDnsQuery; +import io.netty.handler.codec.dns.DatagramDnsQueryDecoder; +import io.netty.handler.codec.dns.DatagramDnsResponse; +import io.netty.handler.codec.dns.DatagramDnsResponseEncoder; +import io.netty.handler.codec.dns.DefaultDnsRawRecord; +import io.netty.handler.codec.dns.DnsQuestion; +import io.netty.handler.codec.dns.DnsRecordType; +import io.netty.handler.codec.dns.DnsResponseCode; +import io.netty.handler.codec.dns.DnsSection; + +import java.io.Closeable; +import java.net.InetAddress; +import java.net.InetSocketAddress; + +import static java.util.concurrent.TimeUnit.SECONDS; + +/** + * In-process DNS server. Answers every A query with 127.0.0.1; {@link #UNKNOWN_HOST} gets NXDOMAIN. + */ +public final class StubDnsServer implements Closeable { + + public static final String UNKNOWN_HOST = "nonexistent.invalid"; + + private final EventLoopGroup group = new MultiThreadIoEventLoopGroup(1, NioIoHandler.newFactory()); + private final Channel channel; + + public StubDnsServer() throws InterruptedException { + channel = new Bootstrap() + .group(group) + .channel(NioDatagramChannel.class) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(NioDatagramChannel ch) { + ch.pipeline().addLast(new DatagramDnsQueryDecoder(), new DatagramDnsResponseEncoder(), new QueryHandler()); + } + }) + .bind(InetAddress.getLoopbackAddress(), 0) + .sync() + .channel(); + } + + public InetSocketAddress getAddress() { + return (InetSocketAddress) channel.localAddress(); + } + + @Override + public void close() { + channel.close().syncUninterruptibly(); + group.shutdownGracefully(0, 1, SECONDS).syncUninterruptibly(); + } + + private static final class QueryHandler extends SimpleChannelInboundHandler { + + @Override + protected void channelRead0(ChannelHandlerContext ctx, DatagramDnsQuery query) { + DnsQuestion question = query.recordAt(DnsSection.QUESTION); + DatagramDnsResponse response = new DatagramDnsResponse(query.recipient(), query.sender(), query.id()); + response.addRecord(DnsSection.QUESTION, question); + // Decoded names end with a dot and may carry a search domain. + if (question.name().startsWith(UNKNOWN_HOST)) { + response.setCode(DnsResponseCode.NXDOMAIN); + } else if (question.type() == DnsRecordType.A) { + response.addRecord(DnsSection.ANSWER, new DefaultDnsRawRecord(question.name(), DnsRecordType.A, 60, + Unpooled.wrappedBuffer(new byte[]{127, 0, 0, 1}))); + } + ctx.writeAndFlush(response); + } + } +} diff --git a/pom.xml b/pom.xml index 17b988d38a..19964bd368 100644 --- a/pom.xml +++ b/pom.xml @@ -56,10 +56,6 @@ 5.14.4 2.31.0 0.12.6 - - - external @@ -376,7 +372,6 @@ @{argLine} --add-exports java.base/jdk.internal.misc=ALL-UNNAMED - ${ahc.test.excludedGroups} @@ -552,21 +547,6 @@ - - - run-external-tests - - - external.tests - true - - - - - - -