Tests and Codgen fixes to build most AWS API models

.github/workflows/codegen-compatibility.yml

@@ -0,0 +1,58 @@
+name: Codegen Compatibility
+
+on:
+  # Run daily at 4 AM UTC
+  schedule:
+    - cron: '0 4 * * *'
+
+  # Allow manual trigger
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  codegen-compatibility:
+    runs-on: ubuntu-latest
+    name: AWS Model Codegen Compatibility
+
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Checkout api-models-aws
+        uses: actions/checkout@v6
+        with:
+          repository: aws/api-models-aws
+          path: api-models-aws
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v5
+        with:
+          java-version: 21
+          distribution: 'corretto'
+
+      - name: Cache compiled buildscripts
+        uses: actions/cache@v5
+        with:
+          key: ${{ runner.os }}-gradle-${{ hashFiles('buildSrc/**/*.kts') }}
+          path: |
+            ./buildSrc/build
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v5
+        with:
+          cache-encryption-key: ${{ secrets.GRADLE_ENCRYPTION_KEY }}
+          gradle-home-cache-includes: |
+            caches
+
+      - name: Run codegen compatibility tests
+        env:
+          API_MODELS_AWS_DIR: ${{ github.workspace }}/api-models-aws
+        run: |
+          ./gradlew :codegen:codegen-plugin:integ --tests "*.AwsModelCodegenCompilationTest" --stacktrace
+
+      - uses: actions/upload-artifact@v7
+        if: failure()
+        with:
+          name: codegen-compatibility-report
+          path: '**/build/reports/tests'

codegen/codegen-plugin/build.gradle.kts

@@ -36,6 +36,14 @@ dependencies {
     itImplementation(project(":codecs:json-codec", configuration = "shadow"))
     itImplementation(libs.smithy.aws.traits)
     itImplementation(libs.smithy.rules)
+
+    // Additional deps for AWS model codegen compilation test
+    itImplementation(project(":aws:aws-sigv4"))
+    itImplementation(project(":aws:aws-auth-api"))
+    itImplementation(project(":aws:client:aws-client-core"))
+    itImplementation(project(":aws:client:aws-client-http"))
+    itImplementation(project(":aws:client:aws-client-rulesengine"))
+    itImplementation(libs.smithy.aws.endpoints)
 }
 
 // Core codegen test runner
@@ -64,3 +72,14 @@ listOf("generateSourcesClient", "generateSourcesServer", "generateSourcesTypes")
 tasks.test {
     failOnNoDiscoveredTests = false
 }
+
+// If API_MODELS_AWS_DIR is set, add it as a task input so Gradle doesn't cache results.
+System.getenv("API_MODELS_AWS_DIR")?.let { modelsDir ->
+    val dir = file(modelsDir)
+    if (dir.isDirectory) {
+        tasks.named<Test>("integ") {
+            inputs.dir(dir).withPathSensitivity(PathSensitivity.RELATIVE)
+            maxHeapSize = "4g"
+        }
+    }
+}

codegen/codegen-plugin/src/it/java/software/amazon/smithy/java/codegen/AwsModelCodegenCompilationTest.java

@@ -0,0 +1,230 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package software.amazon.smithy.java.codegen;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.fail;
+import static org.junit.jupiter.api.Assumptions.abort;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import javax.tools.Diagnostic;
+import javax.tools.DiagnosticCollector;
+import javax.tools.FileObject;
+import javax.tools.ForwardingJavaFileManager;
+import javax.tools.JavaCompiler;
+import javax.tools.JavaFileManager;
+import javax.tools.JavaFileObject;
+import javax.tools.SimpleJavaFileObject;
+import javax.tools.StandardJavaFileManager;
+import javax.tools.ToolProvider;
+import org.junit.jupiter.api.Named;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.smithy.build.MockManifest;
+import software.amazon.smithy.build.PluginContext;
+import software.amazon.smithy.model.Model;
+import software.amazon.smithy.model.loader.ModelAssembler;
+import software.amazon.smithy.model.node.ArrayNode;
+import software.amazon.smithy.model.node.ObjectNode;
+import software.amazon.smithy.model.shapes.ServiceShape;
+
+/**
+ * Tests that Java code generation produces compilable code for all AWS service models.
+ *
+ * <p>Set the {@code API_MODELS_AWS_DIR} environment variable to the root of a local
+ * checkout of {@code aws/api-models-aws} to enable this test.
+ */
+@EnabledIfEnvironmentVariable(named = "API_MODELS_AWS_DIR", matches = ".*")
+class AwsModelCodegenCompilationTest {
+
+    private static final Set<String> IGNORED_SDK_IDS = Set.of(
+            "timestream-write",
+            "timestream-query",
+            "clouddirectory"
+
+    );
+
+    private static Path getModelsDir() {
+        var dir = System.getenv("API_MODELS_AWS_DIR");
+        return Paths.get(dir, "models");
+    }
+
+    static Stream<Named<Path>> awsModels() throws IOException {
+        var modelsDir = getModelsDir();
+        return Files.find(modelsDir,
+                4,
+                (p, a) -> p.toString().endsWith(".json")
+                        && p.getParent().getParent().getFileName().toString().equals("service"))
+                .map(p -> Named.of(sdkId(modelsDir, p), p));
+    }
+
+    @ParameterizedTest(name = "client: {0}")
+    @MethodSource("awsModels")
+    @Execution(ExecutionMode.CONCURRENT)
+    void compileGeneratedCode(Path modelPath) {
+        generateAndCompile(modelPath, "client", "server");
+    }
+
+    private void generateAndCompile(Path modelPath, String... modes) {
+        // 1. Load model
+        Model model = Model.assembler()
+                .addImport(modelPath)
+                .putProperty(ModelAssembler.ALLOW_UNKNOWN_TRAITS, true)
+                .disableValidation()
+                .assemble()
+                .unwrap();
+
+        ServiceShape service = model.getServiceShapes()
+                .stream()
+                .findFirst()
+                .orElseThrow(() -> new IllegalStateException("Could not find service shape"));
+
+        // 2. Run codegen with MockManifest (in-memory)
+        MockManifest manifest = new MockManifest();
+        PluginContext context = PluginContext.builder()
+                .fileManifest(manifest)
+                .settings(ObjectNode.builder()
+                        .withMember("service", service.getId().toString())
+                        .withMember("namespace", "test." + sanitize(service.getId().getName()))
+                        .withMember("modes", ArrayNode.fromStrings(modes))
+                        .build())
+                .model(model)
+                .build();
+        try {
+            new JavaCodegenPlugin().execute(context);
+
+            // 3. Validate all generated files
+            assertFalse(manifest.getFiles().isEmpty(), "No files generated for " + service.getId());
+
+            // 4. Collect generated files — Java sources for compilation, others for validation
+            List<JavaFileObject> sources = new ArrayList<>();
+            for (Path p : manifest.getFiles()) {
+                String content = manifest.expectFileString(p);
+                assertFalse(content.isBlank(), "Empty generated file: " + p);
+                if (p.toString().endsWith(".java")) {
+                    sources.add(new InMemoryJavaSource(p.toString(), content));
+                }
+            }
+
+            // 5. In-memory compile all generated Java sources
+            if (!sources.isEmpty()) {
+                JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+                DiagnosticCollector<JavaFileObject> diagnostics = new DiagnosticCollector<>();
+                try (var stdFm = compiler.getStandardFileManager(diagnostics, null, null)) {
+                    var fm = new InMemoryFileManager(stdFm);
+                    boolean ok = compiler.getTask(null,
+                            fm,
+                            diagnostics,
+                            List.of("-classpath", System.getProperty("java.class.path")),
+                            null,
+                            sources).call();
+                    if (!ok) {
+                        String errors = diagnostics.getDiagnostics()
+                                .stream()
+                                .filter(d -> d.getKind() == Diagnostic.Kind.ERROR)
+                                .map(Object::toString)
+                                .collect(Collectors.joining("\n"));
+                        Path dumpFile = dumpGeneratedSources(service.getId().toString(), sources, errors);
+                        fail(Arrays.toString(modes) + " compilation failed for " + service.getId()
+                                + ". Generated sources dumped to: " + dumpFile + "\n" + errors);
+                    }
+                }
+            }
+        } catch (Throwable t) {
+            if (IGNORED_SDK_IDS.contains(sdkId(getModelsDir(), modelPath))) {
+                abort("Known failure for " + service.getId() + ": " + t.getMessage());
+            }
+            sneakyThrow(t);
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static <T extends Throwable> void sneakyThrow(Throwable throwable) throws T {
+        throw (T) throwable;
+    }
+
+    private static Path dumpGeneratedSources(String serviceId, List<JavaFileObject> sources, String errors) {
+        try {
+            var dir = Files.createTempDirectory("codegen-fail-" + sanitize(serviceId) + "-");
+            for (var source : sources) {
+                // Strip the MockManifest "/test/" base dir prefix to get a clean relative path
+                var fileName = Path.of(source.getName()).getFileName();
+                var name = fileName != null ? fileName.toString() : source.getName();
+                Files.writeString(dir.resolve(name), source.getCharContent(true));
+            }
+            Files.writeString(dir.resolve("ERRORS.txt"), errors);
+            return dir;
+        } catch (IOException e) {
+            return Path.of("<dump failed: " + e.getMessage() + ">");
+        }
+    }
+
+    private static String sdkId(Path modelsDir, Path modelPath) {
+        return modelsDir.relativize(modelPath).getName(0).toString();
+    }
+
+    private static String sanitize(String name) {
+        return name.toLowerCase().replaceAll("[^a-z0-9]", "");
+    }
+
+    /**
+     * In-memory source file — wraps generated code string as JavaFileObject.
+     */
+    private static class InMemoryJavaSource extends SimpleJavaFileObject {
+        private final String code;
+
+        InMemoryJavaSource(String path, String code) {
+            super(URI.create("string:///" + path.replace('\\', '/')), Kind.SOURCE);
+            this.code = code;
+        }
+
+        @Override
+        public CharSequence getCharContent(boolean ignoreEncodingErrors) {
+            return code;
+        }
+    }
+
+    /**
+     * In-memory output manager — discards .class bytes.
+     */
+    private static class InMemoryFileManager extends ForwardingJavaFileManager<StandardJavaFileManager> {
+        InMemoryFileManager(StandardJavaFileManager delegate) {
+            super(delegate);
+        }
+
+        @Override
+        public JavaFileObject getJavaFileForOutput(
+                JavaFileManager.Location location,
+                String className,
+                JavaFileObject.Kind kind,
+                FileObject sibling
+        ) {
+            return new SimpleJavaFileObject(
+                    URI.create("mem:///" + className.replace('.', '/') + kind.extension),
+                    kind) {
+                @Override
+                public OutputStream openOutputStream() {
+                    return new ByteArrayOutputStream();
+                }
+            };
+        }
+    }
+}

codegen/codegen-plugin/src/it/resources/junit-platform.properties

@@ -0,0 +1 @@
+junit.jupiter.execution.parallel.enabled = true

codegen/codegen-plugin/src/main/java/software/amazon/smithy/java/codegen/CodeGenerationContext.java

@@ -54,7 +54,6 @@
 import software.amazon.smithy.model.traits.XmlFlattenedTrait;
 import software.amazon.smithy.model.traits.XmlNameTrait;
 import software.amazon.smithy.model.traits.XmlNamespaceTrait;
-import software.amazon.smithy.utils.SmithyInternalApi;
 import software.amazon.smithy.utils.SmithyUnstableApi;
 
 /**
@@ -114,16 +113,6 @@ public class CodeGenerationContext
     public CodeGenerationContext(
             CreateContextDirective<JavaCodegenSettings, JavaCodegenIntegration> directive,
             String plugin
-    ) {
-        this(directive, plugin, 64000);
-    }
-
-    //Visible for testing
-    @SmithyInternalApi
-    CodeGenerationContext(
-            CreateContextDirective<JavaCodegenSettings, JavaCodegenIntegration> directive,
-            String plugin,
-            int schemaPartitionThreshold
     ) {
         this.model = directive.model();
         this.settings = directive.settings();
@@ -137,7 +126,7 @@ public CodeGenerationContext(
         this.runtimeTraits = collectRuntimeTraits();
         this.traitInitializers = collectTraitInitializers();
         this.plugin = plugin;
-        this.schemaFieldOrder = new SchemaFieldOrder(directive, schemaPartitionThreshold, symbolProvider);
+        this.schemaFieldOrder = new SchemaFieldOrder(directive, this);
     }
 
     @Override

codegen/codegen-plugin/src/main/java/software/amazon/smithy/java/codegen/DirectedJavaCodegen.java

@@ -41,6 +41,7 @@
 import software.amazon.smithy.java.codegen.server.generators.ServiceGenerator;
 import software.amazon.smithy.model.shapes.Shape;
 import software.amazon.smithy.model.shapes.ShapeId;
+import software.amazon.smithy.utils.SmithyInternalApi;
 import software.amazon.smithy.utils.SmithyUnstableApi;
 
 /**
@@ -51,6 +52,9 @@ final class DirectedJavaCodegen
         implements DirectedCodegen<CodeGenerationContext, JavaCodegenSettings, JavaCodegenIntegration> {
 
     private final Set<CodegenMode> modes;
+    //Visible For Testing
+    @SmithyInternalApi
+    CodeGenerationContext context;
 
     DirectedJavaCodegen(Set<CodegenMode> modes) {
         this.modes = modes;
@@ -73,7 +77,8 @@ public CodeGenerationContext createContext(
             CreateContextDirective<JavaCodegenSettings, JavaCodegenIntegration> directive
     ) {
         String pluginName = getPluginName();
-        return new CodeGenerationContext(directive, pluginName);
+        this.context = new CodeGenerationContext(directive, pluginName);
+        return context;
     }
 
     private String getPluginName() {