diff --git a/codegen/build.gradle.kts b/codegen/build.gradle.kts index a0a733616..df6e5391c 100644 --- a/codegen/build.gradle.kts +++ b/codegen/build.gradle.kts @@ -15,5 +15,5 @@ allprojects { group = "software.amazon.smithy.python" - version = "0.3.0" + version = "0.3.1" } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java index b039dd582..a6def8968 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java @@ -31,6 +31,7 @@ import software.amazon.smithy.model.node.Node; import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.traits.DefaultTrait; import software.amazon.smithy.model.traits.ErrorTrait; import software.amazon.smithy.model.traits.TimestampFormatTrait; import software.amazon.smithy.model.traits.TimestampFormatTrait.Format; @@ -128,6 +129,18 @@ public static boolean isErrorMessage(Model model, MemberShape shape) { && model.expectShape(shape.getContainer()).hasTrait(ErrorTrait.class); } + /** + * Determines whether a member is required in the generated Python constructor — that is, + * neither nullable nor carrying a default value. + * + * @param index A nullable index for the model. + * @param member The member to check. + * @return Returns whether the member is required in generated code. + */ + public static boolean isRequiredMember(NullableIndex index, MemberShape member) { + return !index.isMemberNullable(member) && !member.hasTrait(DefaultTrait.class); + } + /** * Executes a given shell command in a given directory. * diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java index c432847bb..4d3c9333d 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java @@ -763,12 +763,15 @@ public Void nullNode(NullNode node) { @Override public Void numberNode(NumberNode node) { - // TODO: Add support for timestamp, int-enum, and others if (inputShape.isTimestampShape()) { var parsed = CodegenUtils.parseTimestampNode(model, inputShape, node); writer.writeInline(CodegenUtils.getDatetimeConstructor(writer, parsed)); } else if (inputShape.isFloatShape() || inputShape.isDoubleShape()) { writer.writeInline("float($L)", node.getValue()); + } else if (inputShape.isIntEnumShape()) { + var enumSymbol = + context.symbolProvider().toSymbol(inputShape); + writer.writeInline("$T($L)", enumSymbol, node.getValue()); } else { writer.writeInline("$L", node.getValue()); } @@ -800,6 +803,10 @@ public Void stringNode(StringNode node) { }; writer.writeInline("float($S)", value); + } else if (inputShape.isEnumShape()) { + var enumSymbol = + context.symbolProvider().toSymbol(inputShape); + writer.writeInline("$T($S)", enumSymbol, node.getValue()); } else { writer.writeInline("$S", node.getValue()); } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java index a44f7cb91..944d404da 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/PythonSymbolProvider.java @@ -329,16 +329,8 @@ public Symbol intEnumShape(IntEnumShape shape) { } private Symbol genericEnum(Shape shape) { - var enumSymbol = createGeneratedSymbolBuilder(shape, getDefaultShapeName(shape), SHAPES_FILE).build(); - - // We add this enum symbol as a property on a generic string/int symbol - // rather than returning the enum symbol directly because we only - // generate the enum constants for convenience. We actually want - // to pass around plain types rather than what is effectively - // a namespace class. - return createSymbolBuilder(shape, shape.isEnumShape() ? "str" : "int") - .putProperty(SymbolProperties.ENUM_SYMBOL, escaper.escapeSymbol(shape, enumSymbol)) - .build(); + Symbol symbol = createGeneratedSymbolBuilder(shape, getDefaultShapeName(shape), SHAPES_FILE).build(); + return escaper.escapeSymbol(shape, symbol); } @Override diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/RequiredMemberTargetIndex.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/RequiredMemberTargetIndex.java new file mode 100644 index 000000000..619056be4 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/RequiredMemberTargetIndex.java @@ -0,0 +1,48 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen; + +import java.util.HashSet; +import java.util.Set; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.KnowledgeIndex; +import software.amazon.smithy.model.knowledge.NullableIndex; +import software.amazon.smithy.model.shapes.ShapeId; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Knowledge index of all shapes that are the target of a python-required structure member, + * i.e. the shapes that may need a synthesized default during client error correction. + * + *

Computed once per model and cached via {@link Model#getKnowledge}. + */ +@SmithyInternalApi +public final class RequiredMemberTargetIndex implements KnowledgeIndex { + + private final Set targets = new HashSet<>(); + + private RequiredMemberTargetIndex(Model model) { + var index = NullableIndex.of(model); + for (var struct : model.getStructureShapes()) { + for (var member : struct.members()) { + if (CodegenUtils.isRequiredMember(index, member)) { + targets.add(member.getTarget()); + } + } + } + } + + public static RequiredMemberTargetIndex of(Model model) { + return model.getKnowledge(RequiredMemberTargetIndex.class, RequiredMemberTargetIndex::new); + } + + /** + * @param shape The shape to check. + * @return Returns whether the shape is the target of any python-required structure member. + */ + public boolean isRequiredMemberTarget(ShapeId shape) { + return targets.contains(shape); + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SymbolProperties.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SymbolProperties.java index aa35a1da0..992b8af21 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SymbolProperties.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/SymbolProperties.java @@ -35,13 +35,6 @@ public final class SymbolProperties { */ public static final Property STDLIB = Property.named("stdlib"); - /** - * Contains a symbol representing the class containing known enum values for the symbol. - * - *

In type signatures, the base int or str is used instead for forwards compatibility. - */ - public static final Property ENUM_SYMBOL = Property.named("enumSymbol"); - /** * Contains a symbol representing the unknown variant of a union. */ diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java index 354054503..b3cc99b4e 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/EnumGenerator.java @@ -9,11 +9,19 @@ import software.amazon.smithy.model.traits.DocumentationTrait; import software.amazon.smithy.model.traits.EnumValueTrait; import software.amazon.smithy.python.codegen.GenerationContext; -import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; import software.amazon.smithy.utils.SmithyInternalApi; /** - * Renders enums. + * Renders enums as a {@code StrEnum} subclass. + * + *

The {@code UnknownEnumMixin} base from smithy-core handles values that + * weren't known at generation time: deserializing an unrecognized value (or + * filling a missing required member during client error correction, see + * {@link MemberErrorCorrectionGenerator}) produces a pseudo-member with + * {@code is_unknown} set rather than raising. + * + * @see Smithy spec: enum */ @SmithyInternalApi public final class EnumGenerator implements Runnable { @@ -27,10 +35,12 @@ public EnumGenerator(GenerationContext context, EnumShape enumShape) { @Override public void run() { - var enumSymbol = context.symbolProvider().toSymbol(shape).expectProperty(SymbolProperties.ENUM_SYMBOL); + var enumSymbol = context.symbolProvider().toSymbol(shape); context.writerDelegator().useShapeWriter(shape, writer -> { writer.addStdlibImport("enum", "StrEnum"); - writer.openBlock("class $L(StrEnum):", "", enumSymbol.getName(), () -> { + writer.addDependency(SmithyPythonDependency.SMITHY_CORE); + writer.addImport("smithy_core.types", "UnknownEnumMixin"); + writer.openBlock("class $L(UnknownEnumMixin, StrEnum):", "", enumSymbol.getName(), () -> { shape.getTrait(DocumentationTrait.class).ifPresent(trait -> { writer.writeDocs(trait.getValue(), context); }); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java index 8816f9d38..33f8e4ee5 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/IntEnumGenerator.java @@ -10,9 +10,20 @@ import software.amazon.smithy.model.traits.EnumValueTrait; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; -import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.SmithyPythonDependency; import software.amazon.smithy.utils.SmithyInternalApi; +/** + * Renders intEnums as an {@code IntEnum} subclass. + * + *

The {@code UnknownEnumMixin} base from smithy-core handles values that + * weren't known at generation time: deserializing an unrecognized value (or + * filling a missing required member during client error correction, see + * {@link MemberErrorCorrectionGenerator}) produces a pseudo-member with + * {@code is_unknown} set rather than raising. + * + * @see Smithy spec: intEnum + */ @SmithyInternalApi public final class IntEnumGenerator implements Runnable { @@ -24,10 +35,12 @@ public IntEnumGenerator(GenerateIntEnumDirective { writer.addStdlibImport("enum", "IntEnum"); - writer.openBlock("class $L(IntEnum):", "", enumSymbol.getName(), () -> { + writer.addDependency(SmithyPythonDependency.SMITHY_CORE); + writer.addImport("smithy_core.types", "UnknownEnumMixin"); + writer.openBlock("class $L(UnknownEnumMixin, IntEnum):", "", enumSymbol.getName(), () -> { directive.shape().getTrait(DocumentationTrait.class).ifPresent(trait -> { writer.writeDocs(trait.getValue(), directive.context()); }); @@ -35,7 +48,7 @@ public void run() { for (MemberShape member : directive.shape().members()) { var name = directive.symbolProvider().toMemberName(member); var value = member.expectTrait(EnumValueTrait.class).expectIntValue(); - writer.write("$L = $L\n", name, value); + writer.write("$L = $L", name, value); member.getTrait(DocumentationTrait.class).ifPresent(trait -> { writer.writeDocs(trait.getValue(), directive.context()); }); diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberDeserializerGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberDeserializerGenerator.java index 53873b818..c14a93159 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberDeserializerGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberDeserializerGenerator.java @@ -135,7 +135,11 @@ public Void integerShape(IntegerShape shape) { @Override public Void intEnumShape(IntEnumShape shape) { - writeDeserializer("integer"); + pushMemberState(); + var enumSymbol = context.symbolProvider().toSymbol(shape); + writer.write("$T(${deserializer:L}.read_integer(${C|}))", + enumSymbol, + writer.consumer(w -> writeSchema())); return null; } @@ -183,7 +187,11 @@ public Void stringShape(StringShape shape) { @Override public Void enumShape(EnumShape shape) { - writeDeserializer("string"); + pushMemberState(); + var enumSymbol = context.symbolProvider().toSymbol(shape); + writer.write("$T(${deserializer:L}.read_string(${C|}))", + enumSymbol, + writer.consumer(w -> writeSchema())); return null; } diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberErrorCorrectionGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberErrorCorrectionGenerator.java new file mode 100644 index 000000000..372af8327 --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/MemberErrorCorrectionGenerator.java @@ -0,0 +1,215 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.generators; + +import java.util.function.Consumer; +import software.amazon.smithy.model.knowledge.NullableIndex; +import software.amazon.smithy.model.shapes.BigDecimalShape; +import software.amazon.smithy.model.shapes.BigIntegerShape; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.BooleanShape; +import software.amazon.smithy.model.shapes.ByteShape; +import software.amazon.smithy.model.shapes.DocumentShape; +import software.amazon.smithy.model.shapes.DoubleShape; +import software.amazon.smithy.model.shapes.EnumShape; +import software.amazon.smithy.model.shapes.FloatShape; +import software.amazon.smithy.model.shapes.IntEnumShape; +import software.amazon.smithy.model.shapes.IntegerShape; +import software.amazon.smithy.model.shapes.ListShape; +import software.amazon.smithy.model.shapes.LongShape; +import software.amazon.smithy.model.shapes.MapShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.ShapeVisitor; +import software.amazon.smithy.model.shapes.ShortShape; +import software.amazon.smithy.model.shapes.StringShape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.shapes.TimestampShape; +import software.amazon.smithy.model.shapes.UnionShape; +import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.python.codegen.CodegenUtils; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.python.codegen.SymbolProperties; +import software.amazon.smithy.python.codegen.writer.PythonWriter; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Produces the Python expression used to fill a missing required member during client error + * correction. + * + *

Visiting a shape returns a consumer that writes the default expression, or {@code null} + * if no default can be synthesized for the shape (a streaming shape, or a structure with a + * required member that itself has no synthesizable default). Callers decide what to do with + * unsynthesizable members; the visitor is the single source of truth for what is synthesizable. + * + * @see Smithy + * spec: Client error correction + */ +@SmithyInternalApi +public final class MemberErrorCorrectionGenerator extends ShapeVisitor.DataShapeVisitor> { + + private final GenerationContext context; + + public MemberErrorCorrectionGenerator(GenerationContext context) { + this.context = context; + } + + @Override + public Consumer booleanShape(BooleanShape shape) { + return writer -> writer.writeInline("False"); + } + + @Override + public Consumer byteShape(ByteShape shape) { + return writer -> writer.writeInline("0"); + } + + @Override + public Consumer shortShape(ShortShape shape) { + return writer -> writer.writeInline("0"); + } + + @Override + public Consumer integerShape(IntegerShape shape) { + return writer -> writer.writeInline("0"); + } + + @Override + public Consumer longShape(LongShape shape) { + return writer -> writer.writeInline("0"); + } + + @Override + public Consumer bigIntegerShape(BigIntegerShape shape) { + return writer -> writer.writeInline("0"); + } + + @Override + public Consumer floatShape(FloatShape shape) { + return writer -> writer.writeInline("0.0"); + } + + @Override + public Consumer doubleShape(DoubleShape shape) { + return writer -> writer.writeInline("0.0"); + } + + @Override + public Consumer bigDecimalShape(BigDecimalShape shape) { + return writer -> { + writer.addStdlibImport("decimal", "Decimal"); + writer.writeInline("Decimal(0)"); + }; + } + + @Override + public Consumer stringShape(StringShape shape) { + return writer -> writer.writeInline("\"\""); + } + + @Override + public Consumer blobShape(BlobShape shape) { + // Per Smithy spec § 13.3.1, a missing streaming blob is already handled by the + // deserializer (an empty HTTP body becomes a zero-length AsyncBytesReader), so + // client error correction is unnecessary. + if (shape.hasTrait(StreamingTrait.class)) { + return null; + } + return writer -> writer.writeInline("b\"\""); + } + + @Override + public Consumer timestampShape(TimestampShape shape) { + return writer -> { + writer.addStdlibImport("datetime", "datetime"); + writer.addStdlibImport("datetime", "timezone"); + writer.writeInline("datetime.fromtimestamp(0, tz=timezone.utc)"); + }; + } + + @Override + public Consumer documentShape(DocumentShape shape) { + return writer -> { + writer.addImport("smithy_core.documents", "Document"); + writer.writeInline("Document(None)"); + }; + } + + @Override + public Consumer listShape(ListShape shape) { + return writer -> writer.writeInline("[]"); + } + + @Override + public Consumer mapShape(MapShape shape) { + return writer -> writer.writeInline("{}"); + } + + @Override + public Consumer enumShape(EnumShape shape) { + var enumSymbol = context.symbolProvider().toSymbol(shape); + return writer -> { + writer.addImport(enumSymbol, enumSymbol.getName()); + writer.writeInline("$L._corrected(\"\")", enumSymbol.getName()); + }; + } + + @Override + public Consumer intEnumShape(IntEnumShape shape) { + var enumSymbol = context.symbolProvider().toSymbol(shape); + return writer -> { + writer.addImport(enumSymbol, enumSymbol.getName()); + // -1 is used (rather than 0) as the placeholder because it is least likely to + // collide with a real member value. + writer.writeInline("$L._corrected(-1)", enumSymbol.getName()); + }; + } + + @Override + public Consumer unionShape(UnionShape shape) { + // Streaming unions (event streams) have no synthesizable default; they also never + // appear as dataclass properties, so missing ones don't need correction. + if (shape.hasTrait(StreamingTrait.class)) { + return null; + } + var unknownSymbol = context.symbolProvider() + .toSymbol(shape) + .expectProperty(SymbolProperties.UNION_UNKNOWN); + return writer -> { + writer.addImport(unknownSymbol, unknownSymbol.getName()); + writer.writeInline("$L(tag=\"\")", unknownSymbol.getName()); + }; + } + + /** + * We can build a default for a struct only when we can build a default for each of its + * required members, so we have to recurse into nested structs. The recursion is safe + * because Smithy doesn't allow cycles where every member along the path is @required; + * we'll always reach a base case (a primitive, list, map, etc.) before looping back. + * + * See https://smithy.io/2.0/spec/aggregate-types.html#recursive-shape-definitions + */ + @Override + public Consumer structureShape(StructureShape shape) { + var index = NullableIndex.of(context.model()); + for (MemberShape member : shape.members()) { + if (!CodegenUtils.isRequiredMember(index, member)) { + continue; + } + if (context.model().expectShape(member.getTarget()).accept(this) == null) { + return null; + } + } + var symbol = context.symbolProvider().toSymbol(shape); + return writer -> { + writer.addImport(symbol, symbol.getName()); + writer.writeInline("$L._smithy_default()", symbol.getName()); + }; + } + + @Override + public Consumer memberShape(MemberShape shape) { + return context.model().expectShape(shape.getTarget()).accept(this); + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java index 55d4daf67..4f54d5c14 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java @@ -9,8 +9,11 @@ import java.time.ZonedDateTime; import java.util.ArrayList; import java.util.Collection; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.function.Consumer; import java.util.logging.Logger; import java.util.stream.Collectors; import software.amazon.smithy.codegen.core.SymbolProvider; @@ -29,6 +32,7 @@ import software.amazon.smithy.python.codegen.CodegenUtils; import software.amazon.smithy.python.codegen.GenerationContext; import software.amazon.smithy.python.codegen.PythonSettings; +import software.amazon.smithy.python.codegen.RequiredMemberTargetIndex; import software.amazon.smithy.python.codegen.SymbolProperties; import software.amazon.smithy.python.codegen.writer.PythonWriter; import software.amazon.smithy.utils.SmithyInternalApi; @@ -67,10 +71,10 @@ public StructureGenerator( var optional = new ArrayList(); var index = NullableIndex.of(context.model()); for (MemberShape member : shape.members()) { - if (index.isMemberNullable(member) || member.hasTrait(DefaultTrait.class)) { - optional.add(member); - } else { + if (CodegenUtils.isRequiredMember(index, member)) { required.add(member); + } else { + optional.add(member); } } this.requiredMembers = filterPropertyMembers(required); @@ -104,12 +108,15 @@ class $L: ${C|} + ${C|} + """, symbol.getName(), writer.consumer(w -> writeClassDocs()), writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), - writer.consumer(w -> generateDeserializeMethod())); + writer.consumer(w -> generateDeserializeMethod()), + writer.consumer(w -> generateSmithyDefaultMethod())); } private void renderError() { @@ -147,6 +154,8 @@ class $1L($2T): ${7C|} + ${8C|} + """, symbol.getName(), baseError, @@ -154,7 +163,8 @@ class $1L($2T): writer.consumer(w -> writeClassDocs()), writer.consumer(w -> writeProperties()), writer.consumer(w -> generateSerializeMethod()), - writer.consumer(w -> generateDeserializeMethod())); + writer.consumer(w -> generateDeserializeMethod()), + writer.consumer(w -> generateSmithyDefaultMethod())); } private void writeClassDocs() { @@ -272,6 +282,15 @@ private String getDefaultValue(PythonWriter writer, MemberShape member) { return CodegenUtils.getDatetimeConstructor(writer, value); } else if (target.isBlobShape()) { return String.format("b'%s'", defaultNode.expectStringNode().getValue()); + } else if (target.isEnumShape()) { + // Wrap rather than emit a bare string so the value matches the field type. + var enumSymbol = symbolProvider.toSymbol(target); + writer.addImport(enumSymbol, enumSymbol.getName()); + return String.format("%s(\"%s\")", enumSymbol.getName(), defaultNode.expectStringNode().getValue()); + } else if (target.isIntEnumShape()) { + var enumSymbol = symbolProvider.toSymbol(target); + writer.addImport(enumSymbol, enumSymbol.getName()); + return String.format("%s(%s)", enumSymbol.getName(), defaultNode.expectNumberNode().getValue()); } if (target.isDocumentShape()) { @@ -358,6 +377,9 @@ private void generateDeserializeMethod() { var schemaSymbol = symbolProvider.toSymbol(shape).expectProperty(SymbolProperties.SCHEMA); writer.putContext("schema", schemaSymbol); + var corrections = errorCorrections(); + writer.putContext("errorCorrection", !corrections.isEmpty()); + // TODO: either formalize deserialize_kwargs or remove it when http serde is converted writer.write(""" @classmethod @@ -375,14 +397,88 @@ def _consumer(schema: Schema, de: ShapeDeserializer) -> None: logger.debug("Unexpected member schema: %s", schema) deserializer.read_struct($T, consumer=_consumer) + ${?errorCorrection} + ${C|} + ${/errorCorrection} return kwargs """, writer.consumer(w -> deserializeMembers(shape.members())), - schemaSymbol); + schemaSymbol, + writer.consumer(w -> writeErrorCorrection(corrections))); writer.popState(); } + /** + * Collects client error correction defaults for required members the server failed + * to serialize, keyed by member name. Members whose targets have no synthesizable + * default (e.g. a streaming blob) are omitted; the dataclass will raise for those. + * + * @see Smithy + * spec: Client error correction + */ + private Map> errorCorrections() { + var visitor = new MemberErrorCorrectionGenerator(context); + var corrections = new LinkedHashMap>(); + for (MemberShape member : requiredMembers) { + var defaultExpression = model.expectShape(member.getTarget()).accept(visitor); + if (defaultExpression == null) { + continue; + } + corrections.put(symbolProvider.toMemberName(member), defaultExpression); + } + return corrections; + } + + private void writeErrorCorrection(Map> corrections) { + corrections.forEach((memberName, defaultExpression) -> { + writer.pushState(); + writer.putContext("memberName", memberName); + writer.write(""" + if ${memberName:S} not in kwargs: + kwargs[${memberName:S}] = ${C|}""", + writer.consumer(defaultExpression)); + writer.popState(); + }); + } + + /** + * Emits a {@code _smithy_default()} classmethod that constructs an instance with all + * required members filled in via client error correction. Used to fill nested structure + * members per the Smithy spec. Only emitted when this structure is actually referenced + * as the target of a required structure member elsewhere in the model. If the structure + * has any required member whose target has no synthesizable default (a streaming blob, + * or another structure whose own required members transitively have no default), + * {@code _smithy_default()} is also omitted. + */ + private void generateSmithyDefaultMethod() { + if (!RequiredMemberTargetIndex.of(model).isRequiredMemberTarget(shape.getId())) { + return; + } + var visitor = new MemberErrorCorrectionGenerator(context); + if (shape.accept(visitor) == null) { + return; + } + writer.write(""" + @classmethod + def _smithy_default(cls) -> Self: + return cls(${C|}) + """, + writer.consumer(w -> writeSmithyDefaultArguments(visitor))); + } + + private void writeSmithyDefaultArguments(MemberErrorCorrectionGenerator visitor) { + var first = true; + for (MemberShape member : requiredMembers) { + if (!first) { + writer.writeInline(", "); + } + first = false; + writer.writeInline("$L=", symbolProvider.toMemberName(member)); + model.expectShape(member.getTarget()).accept(visitor).accept(writer); + } + } + private void deserializeMembers(Collection members) { int index = -1; for (MemberShape member : members) { diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonDelegator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonDelegator.java index 0e6f5c7ca..cc10ce5c1 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonDelegator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/writer/PythonDelegator.java @@ -5,13 +5,9 @@ package software.amazon.smithy.python.codegen.writer; import software.amazon.smithy.build.FileManifest; -import software.amazon.smithy.codegen.core.Symbol; import software.amazon.smithy.codegen.core.SymbolProvider; import software.amazon.smithy.codegen.core.WriterDelegator; -import software.amazon.smithy.model.shapes.MemberShape; -import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.python.codegen.PythonSettings; -import software.amazon.smithy.python.codegen.SymbolProperties; import software.amazon.smithy.utils.SmithyInternalApi; /** @@ -27,30 +23,7 @@ public PythonDelegator( ) { super( fileManifest, - new EnumSymbolProviderWrapper(symbolProvider), + symbolProvider, new PythonWriter.PythonWriterFactory(settings)); } - - private static final class EnumSymbolProviderWrapper implements SymbolProvider { - - private final SymbolProvider wrapped; - - EnumSymbolProviderWrapper(SymbolProvider wrapped) { - this.wrapped = wrapped; - } - - @Override - public Symbol toSymbol(Shape shape) { - Symbol symbol = wrapped.toSymbol(shape); - if (shape.isEnumShape() || shape.isIntEnumShape()) { - symbol = symbol.expectProperty(SymbolProperties.ENUM_SYMBOL); - } - return symbol; - } - - @Override - public String toMemberName(MemberShape shape) { - return wrapped.toMemberName(shape); - } - } } diff --git a/packages/smithy-core/.changes/next-release/smithy-core-feature-a6b53cf89aa64daa8f9c25cc52c20875.json b/packages/smithy-core/.changes/next-release/smithy-core-feature-a6b53cf89aa64daa8f9c25cc52c20875.json new file mode 100644 index 000000000..4652c7839 --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-feature-a6b53cf89aa64daa8f9c25cc52c20875.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added `UnknownEnumMixin` for representing unknown and error-corrected enum/intEnum variants." +} \ No newline at end of file diff --git a/packages/smithy-core/src/smithy_core/types.py b/packages/smithy-core/src/smithy_core/types.py index b902e4e6a..e927b41fb 100644 --- a/packages/smithy-core/src/smithy_core/types.py +++ b/packages/smithy-core/src/smithy_core/types.py @@ -9,7 +9,7 @@ from datetime import datetime from email.utils import format_datetime, parsedate_to_datetime from enum import Enum -from typing import TYPE_CHECKING, Any, overload +from typing import TYPE_CHECKING, Any, ClassVar, Self, overload from .exceptions import ExpectationNotMetError from .interfaces import PropertyKey as _PropertyKey @@ -66,6 +66,64 @@ def from_json(j: Any) -> "JsonBlob": return json_string +class UnknownEnumMixin: + """Adds unknown-value support to a generated enum. + + Mixed into an :class:`enum.Enum` with a data type, e.g. + ``class Color(UnknownEnumMixin, StrEnum)``. Unrecognized values become + pseudo-members (flagged by :attr:`is_unknown`) instead of raising. A value + from the wire keeps native equality; an error-correction placeholder + (:meth:`_corrected`) equals only itself. + """ + + if TYPE_CHECKING: + # Set by EnumType when this is mixed into an enum with a data type. + _member_type_: ClassVar[type[Any]] + _name_: str + _value_: Any + + _smithy_unknown: bool = False + _smithy_corrected: bool = False + + @classmethod + def _unknown(cls, value: Any) -> Self: + member_type: Any = cls._member_type_ + pseudo: Self = member_type.__new__(cls, value) + pseudo._name_ = f"" + pseudo._value_ = value + pseudo._smithy_unknown = True + return pseudo + + @classmethod + def _corrected(cls, value: Any) -> Self: + pseudo = cls._unknown(value) + pseudo._smithy_corrected = True + return pseudo + + @classmethod + def _missing_(cls, value: object) -> Self | None: + if isinstance(value, cls._member_type_): + return cls._unknown(value) + return None + + def __eq__(self, other: object) -> bool: + if self._smithy_corrected or getattr(other, "_smithy_corrected", False): + return self is other + return super().__eq__(other) + + def __ne__(self, other: object) -> bool: + result = self.__eq__(other) + return result if result is NotImplemented else not result + + def __hash__(self) -> int: + return super().__hash__() + + @property + def is_unknown(self) -> bool: + """True if this value was not known at SDK generation time.""" + return self._smithy_unknown + + class TimestampFormat(Enum): """Smithy-defined timestamp formats with serialization and deserialization helpers. diff --git a/packages/smithy-core/tests/unit/test_types.py b/packages/smithy-core/tests/unit/test_types.py index 3dac29464..8debfbf4a 100644 --- a/packages/smithy-core/tests/unit/test_types.py +++ b/packages/smithy-core/tests/unit/test_types.py @@ -3,6 +3,7 @@ # pyright: reportPrivateUsage=false from datetime import UTC, datetime +from enum import IntEnum, StrEnum from typing import Any, assert_type import pytest @@ -14,6 +15,7 @@ PropertyKey, TimestampFormat, TypedProperties, + UnknownEnumMixin, ) @@ -351,3 +353,95 @@ def test_parametric_property() -> None: properties[parametric] = {"foo": "bar"} assert assert_type(properties[parametric], dict[str, str]) == {"foo": "bar"} + + +class _Color(UnknownEnumMixin, StrEnum): + RED = "RED" + GREEN = "GREEN" + + +class _Code(UnknownEnumMixin, IntEnum): + ZERO = 0 + OK = 200 + NOT_FOUND = 404 + + +def test_unknown_enum_known_values_resolve_to_members() -> None: + assert _Color("RED") is _Color.RED + assert _Code(200) is _Code.OK + assert not _Color.RED.is_unknown + assert not _Code.OK.is_unknown + + +def test_unknown_enum_unrecognized_value_does_not_raise() -> None: + color = _Color("CHARTREUSE") + assert color.is_unknown + assert color.value == "CHARTREUSE" + + code = _Code(418) + assert code.is_unknown + assert code.value == 418 + + +def test_unknown_enum_keeps_native_equality() -> None: + assert _Color("CHARTREUSE") == "CHARTREUSE" + assert _Code(418) == 418 + assert _Color("CHARTREUSE") == _Color("CHARTREUSE") + assert _Color("CHARTREUSE") != _Color.RED + + # Pseudo-members hash like their raw value, so dict lookups by raw value work. + codes: dict[Any, str] = {_Code(418): "teapot"} + assert codes[418] == "teapot" + + +def test_wire_unknown_keeps_native_equality() -> None: + # A value the SDK doesn't recognize keeps native equality (forwards-compatible). + wire = _Code(418) + assert wire.is_unknown + assert not wire._smithy_corrected + assert wire == 418 + + +def test_error_corrected_placeholder_is_equal_to_nothing() -> None: + # An invented placeholder for a missing required member equals only itself — + # not its raw value, not a known member sharing it, not another placeholder. + color = _Color._corrected("") + assert color.is_unknown + assert color._smithy_corrected + assert color != "" + assert color != _Color.RED + assert color != _Color._corrected("") + assert color == color + + code = _Code._corrected(0) + assert code.is_unknown + assert code._smithy_corrected + assert code != 0 + assert code != _Code.ZERO + assert code != _Code._corrected(0) + assert code == code + + assert {_Code.ZERO: "zero"}.get(code) is None + + +def test_wire_unknown_and_corrected_differ_on_equality() -> None: + # Both report is_unknown, but the wire-unknown value keeps native equality + # while the invented placeholder matches nothing. + wire = _Code(418) + corrected = _Code._corrected(418) + assert wire.is_unknown and corrected.is_unknown + assert wire == 418 + assert corrected != 418 + assert wire != corrected + + +def test_unknown_enum_rejects_mismatched_type() -> None: + with pytest.raises(ValueError): + _Color(0) # type: ignore[arg-type] + with pytest.raises(ValueError): + _Code("RED") # type: ignore[arg-type] + + +def test_unknown_enum_str_and_serialization_behavior() -> None: + assert str(_Color("CHARTREUSE")) == "CHARTREUSE" + assert int(_Code(418)) == 418 diff --git a/packages/smithy-json/.changes/next-release/smithy-json-bugfix-2eaf1f49875f4690b7c58f7735f1de7d.json b/packages/smithy-json/.changes/next-release/smithy-json-bugfix-2eaf1f49875f4690b7c58f7735f1de7d.json new file mode 100644 index 000000000..ebbb1b2e3 --- /dev/null +++ b/packages/smithy-json/.changes/next-release/smithy-json-bugfix-2eaf1f49875f4690b7c58f7735f1de7d.json @@ -0,0 +1,4 @@ +{ + "type": "bugfix", + "description": "Serialize `IntEnum` members as their underlying integer instead of their `repr`." +} \ No newline at end of file diff --git a/packages/smithy-json/src/smithy_json/_private/serializers.py b/packages/smithy-json/src/smithy_json/_private/serializers.py index 7146923e4..42f4ea476 100644 --- a/packages/smithy-json/src/smithy_json/_private/serializers.py +++ b/packages/smithy-json/src/smithy_json/_private/serializers.py @@ -300,7 +300,8 @@ def write_string(self, value: str) -> None: self._sink.write(b'"') def write_int(self, value: int) -> None: - self._sink.write(repr(value).encode("utf-8")) + # int() unwraps IntEnum members; otherwise repr would emit "". + self._sink.write(str(int(value)).encode("utf-8")) def write_float(self, value: float | Decimal) -> None: if not self._write_non_numeric_float(value=value):