diff --git a/aas_core_codegen/golang/tests/_generate_xmlization_test_common_test.py b/aas_core_codegen/golang/tests/_generate_xmlization_test_common_test.py index 106f80be6..b3ea0881b 100644 --- a/aas_core_codegen/golang/tests/_generate_xmlization_test_common_test.py +++ b/aas_core_codegen/golang/tests/_generate_xmlization_test_common_test.py @@ -23,6 +23,7 @@ INDENT3 as III, INDENT4 as IIII, INDENT5 as IIIII, + INDENT6 as IIIIII, ) @@ -173,6 +174,163 @@ def _generate_whitespace_round_trip_test( return None +def _generate_numeric_comparison_test( + symbol_table: intermediate.SymbolTable, +) -> Optional[Stripped]: + """Exercise numeric equivalence only at model-defined numeric positions.""" + paths_by_kind: MutableMapping[str, Tuple[str, str]] = {} + namespace = html.escape(symbol_table.meta_model.xml_namespace, quote=True) + for cls in symbol_table.our_types: + if not isinstance(cls, intermediate.ConcreteClass): + continue + for prop in cls.properties: + type_anno = intermediate.beneath_optional(prop.type_annotation) + if ( + intermediate.try_primitive_type(type_anno) + is intermediate.PrimitiveType.FLOAT + ): + kind = "double" + elif ( + intermediate.try_primitive_type(type_anno) + is intermediate.PrimitiveType.STR + ): + kind = "text" + elif isinstance(type_anno, intermediate.JsonValueTypeAnnotation): + kind = "json:double" + elif isinstance(type_anno, intermediate.JsonArrayTypeAnnotation): + kind = "json:array" + elif isinstance(type_anno, intermediate.JsonObjectTypeAnnotation): + kind = "json:object" + else: + continue + if kind in paths_by_kind: + continue + local = naming.xml_class_name(cls.name) + prefix = f'<{local} xmlns="{namespace}"><{prop.xml_name}>' + suffix = f"" + if kind == "json:double": + prefix += '' + suffix = "" + suffix + elif kind == "json:array": + prefix += '' + suffix = "" + suffix + elif kind == "json:object": + prefix += 'x' + suffix = "" + suffix + paths_by_kind[kind] = (prefix, suffix) + if not any(kind != "text" for kind in paths_by_kind): + return None + paths = ",\n".join( + "{" + + ", ".join(golang_common.string_literal(value) for value in (kind, *path)) + + "}" + for kind, path in sorted(paths_by_kind.items()) + ) + return Stripped( + f"""\ +func TestXmlNumericComparison(t *testing.T) {{ +{I}// These fragments exercise comparison at typed paths, without model verification. +{I}for _, path := range []struct {{ +{II}kind, prefix, suffix string +{I}}}{{ +{II}{indent_but_first_line(paths, II)}, +{I}}} {{ +{II}t.Run(path.kind, func(t *testing.T) {{ +{III}for _, c := range []struct {{ +{IIII}name, left, right string +{IIII}equal bool +{III}}}{{ +{IIII}{{"decimal", "42.0", "42", true}}, +{IIII}{{"exponent", "21668651.6", "2.16686516e+07", true}}, +{IIII}{{"leading_zeroes", "+0042.00", "42", true}}, +{IIII}{{"exponent_case", "4.2E1", "4.2e+01", true}}, +{IIII}{{"negative", "-42.00", "-4.2e1", true}}, +{IIII}{{"small", "0.0000000001", "1e-10", true}}, +{IIII}{{"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}}, +{IIII}{{"subnormal_vs_zero", "5e-324", "0", false}}, +{IIII}{{"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}}, +{IIII}{{"different", "42", "43", false}}, +{IIII}{{"adjacent_doubles", "1", "1.0000000000000002", false}}, +{IIII}{{"integer_precision", "9007199254740991", "9007199254740992", false}}, +{IIII}{{"signed_zero", "0", "-0", false}}, +{IIII}{{"negative_zero", "-0.0", "-0e0", true}}, +{IIII}{{"same_instruction", " y?>42.0", " y?>42", true}}, +{IIII}{{"different_instruction", " y?>42.0", " y?>42", false}}, +{IIII}{{"moved_instruction", "42", "42", false}}, +{IIII}{{"moved_comment", "42", "42", false}}, +{III}}} {{ +{IIII}t.Run(c.name, func(t *testing.T) {{ +{IIIII}left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) +{IIIII}if err != nil {{ +{IIIIII}t.Fatal(err) +{IIIII}}} +{IIIII}right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) +{IIIII}if err != nil {{ +{IIIIII}t.Fatal(err) +{IIIII}}} +{IIIII}want := c.equal && path.kind != "text" +{IIIII}if (left == right) != want {{ +{IIIIII}t.Fatalf("Unexpected comparison of %q and %q", left, right) +{IIIII}}} +{IIII}}}) +{III}}} +{III}if path.kind == "text" {{ +{IIII}return +{III}}} +{III}for _, c := range []struct{{ input, want string }}{{ +{IIII}{{"4", "42"}}, +{IIII}{{"42", "42"}}, +{IIII}{{" 42 ", "42"}}, +{IIII}{{"42", "42"}}, +{IIII}{{"42.0", "42"}}, +{IIII}{{" 42", "42"}}, +{IIII}{{"42", "42"}}, +{IIII}{{" 2 ", "42"}}, +{IIII}{{"42.0", "42"}}, +{III}}} {{ +{IIII}input := path.prefix + c.input + path.suffix +{IIII}instanceNumber, err := canonicalizeXmlForComparison(input) +{IIII}if err != nil {{ +{IIIII}t.Fatal(err) +{IIII}}} +{IIII}expected := path.prefix + c.want + path.suffix +{IIII}if instanceNumber != expected {{ +{IIIII}t.Fatalf("Expected %q, got %q", expected, instanceNumber) +{IIII}}} +{III}}} +{III}for _, spelling := range []string{{ +{IIII}"", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", +{IIII}"42", "4 2", "42" + string(rune(160)), +{III}}} {{ +{IIII}if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil {{ +{IIIII}t.Fatalf("Accepted invalid numeric content %q", spelling) +{IIII}}} +{III}}} +{III}for _, c := range []struct {{ +{IIII}spelling, want string +{III}}}{{ +{IIII}{{"INF", "INF"}}, {{"-INF", "-INF"}}, {{"NaN", "NaN"}}, {{"1e9999", "INF"}}, +{III}}} {{ +{IIII}canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) +{IIII}if strings.HasPrefix(path.kind, "json:") {{ +{IIIII}if err == nil {{ +{IIIIII}t.Fatalf("Accepted non-finite JSON number %q", c.spelling) +{IIIII}}} +{IIII}}} else {{ +{IIIII}if err != nil {{ +{IIIIII}t.Fatal(err) +{IIIII}}} +{IIIII}if canonical != path.prefix+c.want+path.suffix {{ +{IIIIII}t.Fatalf("Unexpected non-finite model number: %q", canonical) +{IIIII}}} +{IIII}}} +{III}}} +{II}}}) +{I}}} +}}""" + ) + + def _generate_text_context(symbol_table: intermediate.SymbolTable) -> Stripped: """Identify string content by its position in the model's XML structure.""" namespace = symbol_table.meta_model.xml_namespace @@ -219,6 +377,11 @@ def content(type_anno: intermediate.TypeAnnotationExceptOptional) -> str: """Distinguish whitespace-preserving strings from their enclosing containers.""" if intermediate.try_primitive_type(type_anno) is intermediate.PrimitiveType.STR: return "text" + if ( + intermediate.try_primitive_type(type_anno) + is intermediate.PrimitiveType.FLOAT + ): + return "double" if isinstance(type_anno, intermediate.PrimitiveTypeAnnotation): return "" if isinstance(type_anno, intermediate.OurTypeAnnotation): @@ -257,6 +420,7 @@ def content(type_anno: intermediate.TypeAnnotationExceptOptional) -> str: ("", "array"): "json:array", ("", "struct"): "json:object", ("", "string"): "text", + ("", "double"): "json:double", } children_by_type["json:array"] = {("", "data"): "json:data"} children_by_type["json:data"] = {("", "value"): "json:value"} @@ -306,7 +470,7 @@ def content(type_anno: intermediate.TypeAnnotationExceptOptional) -> str: joined = "\n".join(cases) return Stripped( f"""\ -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string {{ {I}switch parent {{ @@ -338,12 +502,15 @@ def generate(symbol_table: intermediate.SymbolTable, repo_url: Stripped) -> str: {I}"encoding/xml" {I}"fmt" {I}"io" +{I}"math" {I}"os" {I}"path/filepath" {I}"regexp" +{I}"strconv" {I}"strings" {I}"testing" +{I}xmlcommon "{repo_url}/internal/xmlcommon" {I}ourtesting "{repo_url}/ourtesting" {I}ourxmlization "{repo_url}/xmlization" )""" @@ -480,7 +647,64 @@ def generate(symbol_table: intermediate.SymbolTable, repo_url: Stripped) -> str: _generate_text_context(symbol_table), Stripped( f"""\ -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) {{ +{I}var content, layout bytes.Buffer +{I}var numberStart, numberEnd int +{I}for {{ +{II}start := decoder.InputOffset() +{II}token, err := decoder.Token() +{II}if err != nil {{ +{III}return "", err +{II}}} +{II}switch token := token.(type) {{ +{II}case xml.CharData: +{III}content.Write(token) +{III}if part := bytes.Trim(token, " \\t\\r\\n"); len(part) > 0 {{ +{IIII}if numberEnd == 0 {{ +{IIIII}numberStart = layout.Len() +{IIII}}} +{IIII}layout.Write(part) +{IIII}numberEnd = layout.Len() +{III}}} +{II}case xml.Comment, xml.ProcInst: +{III}layout.WriteString(source[start:decoder.InputOffset()]) +{II}case xml.EndElement: +{III}// Reuse the SDK reader's lexical validation and double-precision conversion. +{III}// The sentinel element terminates the supplied character data for ReadText. +{III}number, _, err := xmlcommon.ReadTextAs_double( +{IIII}xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) +{III}if err != nil {{ +{IIII}return "", err +{III}}} +{III}if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) {{ +{IIII}return "", fmt.Errorf("expected a finite JSON number") +{III}}} +{III}original := layout.String() +{III}endTag := source[start:decoder.InputOffset()] +{III}// In a valid number, '<' can only come from intervening markup. +{III}// Keep its position relative to the digits by retaining their spelling. +{III}if strings.Contains(original[numberStart:numberEnd], "<") {{ +{IIII}return original + endTag, nil +{III}}} +{III}// FormatFloat preserves every representable value, including signed zero. +{III}canonical := strconv.FormatFloat(number, 'g', -1, 64) +{III}if math.IsInf(number, 1) {{ +{IIII}canonical = "INF" +{III}}} else if math.IsInf(number, -1) {{ +{IIII}canonical = "-INF" +{III}}} +{III}return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil +{II}default: +{III}return "", fmt.Errorf("expected numeric text, got %T", token) +{II}}} +{I}}} +}}""" + ), + Stripped( + f"""\ +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) {{ {I}decoder := xml.NewDecoder(strings.NewReader(text)) @@ -495,8 +719,18 @@ def generate(symbol_table: intermediate.SymbolTable, repo_url: Stripped) -> str: {II}if err != nil {{ {III}return "", err {II}}} -{II}if start, ok := token.(xml.StartElement); ok {{ -{III}contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) +{II}if element, ok := token.(xml.StartElement); ok {{ +{III}context := xmlTextContext(contexts[len(contexts)-1], element.Name) +{III}if context == "double" || context == "json:double" {{ +{IIII}result.WriteString(text[start:decoder.InputOffset()]) +{IIII}normalized, err := normalizeXmlNumericContent(decoder, text, context) +{IIII}if err != nil {{ +{IIIII}return "", err +{IIII}}} +{IIII}result.WriteString(normalized) +{IIII}continue +{III}}} +{III}contexts = append(contexts, context) {II}}} {II}if data, ok := token.(xml.CharData); ok {{ {III}// Normalize strings consistently, even if they consist only of whitespace. @@ -522,6 +756,23 @@ def generate(symbol_table: intermediate.SymbolTable, repo_url: Stripped) -> str: {III}contexts = contexts[:len(contexts)-1] {II}}} {I}}} +}}""" + ), + Stripped( + f"""\ +func TestWritePositiveInfinity(t *testing.T) {{ +{I}var output strings.Builder +{I}encoder := xml.NewEncoder(&output) +{I}if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil {{ +{II}t.Fatal(err) +{I}}} +{I}if err := encoder.Flush(); err != nil {{ +{II}t.Fatal(err) +{I}}} +{I}// XSD 1.0 requires INF even though the reader also accepts +INF. +{I}if got := output.String(); got != "INF" {{ +{II}t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) +{I}}} }}""" ), Stripped( @@ -568,10 +819,44 @@ def generate(symbol_table: intermediate.SymbolTable, repo_url: Stripped) -> str: {I}if err != nil {{ {II}return "", err {I}}} +{I}// Protect processing instructions while the legacy regexes clean up markup. +{I}// The marker target is absent even after carriage-return removal. +{I}markerPrefix := "aasCoreComparisonPI" +{I}withoutCR := strings.ReplaceAll(text, "\\r", "") +{I}for strings.Contains(withoutCR, markerPrefix) {{ +{II}markerPrefix += "X" +{I}}} +{I}var instructions []string +{I}var protected bytes.Buffer +{I}decoder := xml.NewDecoder(strings.NewReader(text)) +{I}for {{ +{II}start := decoder.InputOffset() +{II}token, err := decoder.Token() +{II}if err == io.EOF {{ +{III}break +{II}}} +{II}if err != nil {{ +{III}return "", err +{II}}} +{II}original := text[start:decoder.InputOffset()] +{II}if _, ok := token.(xml.ProcInst); ok {{ +{III}fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) +{III}// XML 1.0 section 2.11 normalizes literal line endings before parsing. +{III}original = strings.ReplaceAll(strings.ReplaceAll(original, "\\r\\n", "\\n"), "\\r", "\\n") +{III}instructions = append(instructions, original) +{II}}} else {{ +{III}protected.WriteString(original) +{II}}} +{I}}} +{I}text = protected.String() {I}text = strings.ReplaceAll(text, "\\r", "") {I}text = removeWhitespaceBetweenTags(text) {I}text = forceSelfClosingTags(text) -{I}return removeWhitespaceWithinTags(text), nil +{I}text = removeWhitespaceWithinTags(text) +{I}for i, original := range instructions {{ +{II}text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) +{I}}} +{I}return text, nil }}""" ), Stripped( @@ -591,6 +876,13 @@ def generate(symbol_table: intermediate.SymbolTable, repo_url: Stripped) -> str: {II}{{"tab_vs_space", ` `, ` `, false}}, {II}{{"lost_tab", ` `, ``, false}}, {II}{{"space_vs_literal_tab", ` `, "\\t", false}}, +{II}{{"instruction_data", ` y?>`, ` y?>`, false}}, +{II}{{"instruction_markup", `?>`, `?>`, false}}, +{II}{{"instruction_cr", "", ``, false}}, +{II}{{"instruction_cr_as_lf", "", "", true}}, +{II}{{"instruction_crlf_as_lf", "", "", true}}, +{II}{{"instruction_indentation", "\\n y?>\\n", ` y?>`, true}}, +{II}{{"instruction_marker", ``, ``, false}}, {I}}} {{ {II}t.Run(c.name, func(t *testing.T) {{ {III}expected, err := canonicalizeXmlForComparison(c.expected) @@ -778,6 +1070,10 @@ def generate(symbol_table: intermediate.SymbolTable, repo_url: Stripped) -> str: if whitespace_test is not None: blocks.insert(-1, whitespace_test) + numeric_test = _generate_numeric_comparison_test(symbol_table) + if numeric_test is not None: + blocks.insert(-1, numeric_test) + writer = io.StringIO() for i, block in enumerate(blocks): if i > 0: diff --git a/dev/test_data/main/golang/expected/aas_core_meta.v3/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/aas_core_meta.v3/expected_output/xmlization/test/common_test.go index d8a32be97..f6652fa87 100644 --- a/dev/test_data/main/golang/expected/aas_core_meta.v3/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/aas_core_meta.v3/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/aas-core-works/aas-core3.0-golang/internal/xmlcommon" ourtesting "github.com/aas-core-works/aas-core3.0-golang/ourtesting" ourxmlization "github.com/aas-core-works/aas-core3.0-golang/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -1027,7 +1030,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -1042,8 +1100,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -1071,6 +1139,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -1111,10 +1194,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -1132,6 +1249,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/annotated_assignments/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/annotated_assignments/expected_output/xmlization/test/common_test.go index c7ccae158..bdf7f3b6f 100644 --- a/dev/test_data/main/golang/expected/annotated_assignments/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/annotated_assignments/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -218,7 +221,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -233,8 +291,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -262,6 +330,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -302,10 +385,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -323,6 +440,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/arithmetic/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/arithmetic/expected_output/xmlization/test/common_test.go index a1299d9a5..3d73d0244 100644 --- a/dev/test_data/main/golang/expected/arithmetic/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/arithmetic/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,12 +132,16 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { case "class:Something": switch name { + case (xml.Name{Space: "https://dummy.com", Local: "optionalRatio"}): + return "double" + case (xml.Name{Space: "https://dummy.com", Local: "ratio"}): + return "double" case (xml.Name{Space: "https://dummy.com", Local: "text"}): return "text" } @@ -147,7 +154,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -162,8 +224,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -191,6 +263,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -231,10 +318,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -252,6 +373,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -493,5 +621,107 @@ func TestWhitespaceStringRoundTrip(t *testing.T) { } } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"double", "", ""}, + {"text", "", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/assert_statements/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/assert_statements/expected_output/xmlization/test/common_test.go index 6da7af22f..20efce94d 100644 --- a/dev/test_data/main/golang/expected/assert_statements/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/assert_statements/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -169,7 +172,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -184,8 +242,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -213,6 +281,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -253,10 +336,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -274,6 +391,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/assignment_targets/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/assignment_targets/expected_output/xmlization/test/common_test.go index 7da70704d..6f0fea573 100644 --- a/dev/test_data/main/golang/expected/assignment_targets/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/assignment_targets/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -174,7 +177,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -189,8 +247,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -218,6 +286,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -258,10 +341,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -279,6 +396,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/collections_in_returns/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/collections_in_returns/expected_output/xmlization/test/common_test.go index 283515a72..494c473c0 100644 --- a/dev/test_data/main/golang/expected/collections_in_returns/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/collections_in_returns/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -178,7 +181,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -193,8 +251,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -222,6 +290,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -262,10 +345,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -283,6 +400,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/comparisons/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/comparisons/expected_output/xmlization/test/common_test.go index b88bac5b4..0cd9e87ea 100644 --- a/dev/test_data/main/golang/expected/comparisons/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/comparisons/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,12 +132,14 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { case "class:Something": switch name { + case (xml.Name{Space: "https://dummy.com", Local: "ratio"}): + return "double" case (xml.Name{Space: "https://dummy.com", Local: "text"}): return "text" } @@ -147,7 +152,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -162,8 +222,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -191,6 +261,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -231,10 +316,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -252,6 +371,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -493,5 +619,107 @@ func TestWhitespaceStringRoundTrip(t *testing.T) { } } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"double", "", ""}, + {"text", "", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/constrained_primitives/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/constrained_primitives/expected_output/xmlization/test/common_test.go index 888c93640..2872befbf 100644 --- a/dev/test_data/main/golang/expected/constrained_primitives/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/constrained_primitives/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,12 +132,14 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { case "class:Something": switch name { + case (xml.Name{Space: "https://dummy.com", Local: "someFloat"}): + return "double" case (xml.Name{Space: "https://dummy.com", Local: "someString"}): return "text" } @@ -147,7 +152,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -162,8 +222,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -191,6 +261,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -231,10 +316,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -252,6 +371,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -428,5 +554,107 @@ func assertIsDeserializationErrorAndEqualsExpectedOrRecord( return } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"double", "", ""}, + {"text", "", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/custom_serialization_names/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/custom_serialization_names/expected_output/xmlization/test/common_test.go index 56713bb75..a6142a225 100644 --- a/dev/test_data/main/golang/expected/custom_serialization_names/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/custom_serialization_names/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -149,7 +152,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -164,8 +222,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -193,6 +261,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -233,10 +316,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -254,6 +371,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/deep_class_hierarchy/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/deep_class_hierarchy/expected_output/xmlization/test/common_test.go index 90b5daa4f..5dae7f6df 100644 --- a/dev/test_data/main/golang/expected/deep_class_hierarchy/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/deep_class_hierarchy/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -207,7 +210,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -222,8 +280,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -251,6 +319,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -291,10 +374,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -312,6 +429,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/empty_class/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/empty_class/expected_output/xmlization/test/common_test.go index d04974fbf..a65a162d1 100644 --- a/dev/test_data/main/golang/expected/empty_class/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/empty_class/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -142,7 +145,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -157,8 +215,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -186,6 +254,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -226,10 +309,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -247,6 +364,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/enum/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/enum/expected_output/xmlization/test/common_test.go index 95fc94d86..7cb2d2417 100644 --- a/dev/test_data/main/golang/expected/enum/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/enum/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -147,7 +150,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -162,8 +220,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -191,6 +259,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -231,10 +314,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -252,6 +369,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/float_only/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/float_only/expected_output/xmlization/test/common_test.go index e4f52c3cb..f19eb8459 100644 --- a/dev/test_data/main/golang/expected/float_only/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/float_only/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,21 +132,32 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { case "class:Something": switch name { + case (xml.Name{Space: "https://dummy.com", Local: "someFloat"}): + return "double" case (xml.Name{Space: "https://dummy.com", Local: "someFloats"}): return "content:ListOf_double" + case (xml.Name{Space: "https://dummy.com", Local: "someOptionalFloat"}): + return "double" case (xml.Name{Space: "https://dummy.com", Local: "somePair"}): return "content:TupleOf2_string_double" } + case "content:ListOf_double": + switch name { + case (xml.Name{Space: "https://dummy.com", Local: "v"}): + return "double" + } case "content:TupleOf2_string_double": switch name { case (xml.Name{Space: "https://dummy.com", Local: "v1"}): return "text" + case (xml.Name{Space: "https://dummy.com", Local: "v2"}): + return "double" } case "root": switch name { @@ -154,7 +168,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -169,8 +238,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -198,6 +277,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -238,10 +332,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -259,6 +387,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -435,5 +570,106 @@ func assertIsDeserializationErrorAndEqualsExpectedOrRecord( return } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"double", "", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/for_statements/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/for_statements/expected_output/xmlization/test/common_test.go index 0ba5b099a..c243904cd 100644 --- a/dev/test_data/main/golang/expected/for_statements/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/for_statements/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -170,7 +173,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -185,8 +243,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -214,6 +282,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -254,10 +337,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -275,6 +392,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/if_statements/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/if_statements/expected_output/xmlization/test/common_test.go index 04bf20d9b..026a702ee 100644 --- a/dev/test_data/main/golang/expected/if_statements/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/if_statements/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -205,7 +208,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -220,8 +278,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -249,6 +317,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -289,10 +372,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -310,6 +427,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/implementation_specific/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/implementation_specific/expected_output/xmlization/test/common_test.go index 71ecaeeb2..04300583c 100644 --- a/dev/test_data/main/golang/expected/implementation_specific/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/implementation_specific/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -179,7 +182,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -194,8 +252,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -223,6 +291,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -263,10 +346,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -284,6 +401,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/int_length_interplay/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/int_length_interplay/expected_output/xmlization/test/common_test.go index e30d93b1e..a25523f00 100644 --- a/dev/test_data/main/golang/expected/int_length_interplay/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/int_length_interplay/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -154,7 +157,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -169,8 +227,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -198,6 +266,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -238,10 +321,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -259,6 +376,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/int_parsing/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/int_parsing/expected_output/xmlization/test/common_test.go index d9e6e573b..a1c35e657 100644 --- a/dev/test_data/main/golang/expected/int_parsing/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/int_parsing/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -155,7 +158,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -170,8 +228,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -199,6 +267,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -239,10 +322,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -260,6 +377,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/is_instance/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/is_instance/expected_output/xmlization/test/common_test.go index 30af50acc..db0e7d3ef 100644 --- a/dev/test_data/main/golang/expected/is_instance/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/is_instance/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -155,6 +158,11 @@ func xmlTextContext(parent string, name xml.Name) string { case (xml.Name{Space: "https://dummy.com", Local: "name"}): return "text" } + case "class:Number_literal": + switch name { + case (xml.Name{Space: "https://dummy.com", Local: "number"}): + return "double" + } case "class:Ordered_container": switch name { case (xml.Name{Space: "https://dummy.com", Local: "children"}): @@ -253,7 +261,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -268,8 +331,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -297,6 +370,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -337,10 +425,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -358,6 +480,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -599,5 +728,107 @@ func TestWhitespaceStringRoundTrip(t *testing.T) { } } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"double", "", ""}, + {"text", "", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/json_types/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/json_types/expected_output/xmlization/test/common_test.go index 04bdbb5eb..2b57cbe26 100644 --- a/dev/test_data/main/golang/expected/json_types/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/json_types/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -164,6 +167,8 @@ func xmlTextContext(parent string, name xml.Name) string { switch name { case (xml.Name{Space: "", Local: "array"}): return "json:array" + case (xml.Name{Space: "", Local: "double"}): + return "json:double" case (xml.Name{Space: "", Local: "string"}): return "text" case (xml.Name{Space: "", Local: "struct"}): @@ -195,6 +200,8 @@ func xmlTextContext(parent string, name xml.Name) string { switch name { case (xml.Name{Space: "", Local: "array"}): return "json:array" + case (xml.Name{Space: "", Local: "double"}): + return "json:double" case (xml.Name{Space: "", Local: "string"}): return "text" case (xml.Name{Space: "", Local: "struct"}): @@ -209,7 +216,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -224,8 +286,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -253,6 +325,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -293,10 +380,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -314,6 +435,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -490,5 +618,108 @@ func assertIsDeserializationErrorAndEqualsExpectedOrRecord( return } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"json:array", "", ""}, + {"json:double", "", ""}, + {"json:object", "x", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/json_types_in_collections/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/json_types_in_collections/expected_output/xmlization/test/common_test.go index 6142b256c..20776c560 100644 --- a/dev/test_data/main/golang/expected/json_types_in_collections/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/json_types_in_collections/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -179,6 +182,8 @@ func xmlTextContext(parent string, name xml.Name) string { switch name { case (xml.Name{Space: "", Local: "array"}): return "json:array" + case (xml.Name{Space: "", Local: "double"}): + return "json:double" case (xml.Name{Space: "", Local: "string"}): return "text" case (xml.Name{Space: "", Local: "struct"}): @@ -210,6 +215,8 @@ func xmlTextContext(parent string, name xml.Name) string { switch name { case (xml.Name{Space: "", Local: "array"}): return "json:array" + case (xml.Name{Space: "", Local: "double"}): + return "json:double" case (xml.Name{Space: "", Local: "string"}): return "text" case (xml.Name{Space: "", Local: "struct"}): @@ -224,7 +231,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -239,8 +301,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -268,6 +340,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -308,10 +395,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -329,6 +450,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -505,5 +633,106 @@ func assertIsDeserializationErrorAndEqualsExpectedOrRecord( return } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"json:object", "x", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/json_types_in_invariants/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/json_types_in_invariants/expected_output/xmlization/test/common_test.go index 833ee3135..c3a581cf3 100644 --- a/dev/test_data/main/golang/expected/json_types_in_invariants/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/json_types_in_invariants/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -158,6 +161,8 @@ func xmlTextContext(parent string, name xml.Name) string { switch name { case (xml.Name{Space: "", Local: "array"}): return "json:array" + case (xml.Name{Space: "", Local: "double"}): + return "json:double" case (xml.Name{Space: "", Local: "string"}): return "text" case (xml.Name{Space: "", Local: "struct"}): @@ -189,6 +194,8 @@ func xmlTextContext(parent string, name xml.Name) string { switch name { case (xml.Name{Space: "", Local: "array"}): return "json:array" + case (xml.Name{Space: "", Local: "double"}): + return "json:double" case (xml.Name{Space: "", Local: "string"}): return "text" case (xml.Name{Space: "", Local: "struct"}): @@ -203,7 +210,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -218,8 +280,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -247,6 +319,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -287,10 +374,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -308,6 +429,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -484,5 +612,108 @@ func assertIsDeserializationErrorAndEqualsExpectedOrRecord( return } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"json:array", "", ""}, + {"json:double", "", ""}, + {"json:object", "x", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/list_of_classes/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/list_of_classes/expected_output/xmlization/test/common_test.go index cba7d234f..3a4d92f8f 100644 --- a/dev/test_data/main/golang/expected/list_of_classes/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/list_of_classes/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -179,7 +182,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -194,8 +252,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -223,6 +291,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -263,10 +346,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -284,6 +401,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/list_of_constrained_primitives/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/list_of_constrained_primitives/expected_output/xmlization/test/common_test.go index 8722998ac..b919ae1e5 100644 --- a/dev/test_data/main/golang/expected/list_of_constrained_primitives/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/list_of_constrained_primitives/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -152,7 +155,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -167,8 +225,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -196,6 +264,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -236,10 +319,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -257,6 +374,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/list_of_enums/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/list_of_enums/expected_output/xmlization/test/common_test.go index 025385ed1..57f084842 100644 --- a/dev/test_data/main/golang/expected/list_of_enums/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/list_of_enums/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -152,7 +155,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -167,8 +225,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -196,6 +264,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -236,10 +319,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -257,6 +374,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/list_of_primitives/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/list_of_primitives/expected_output/xmlization/test/common_test.go index c478e614c..2513ec6b0 100644 --- a/dev/test_data/main/golang/expected/list_of_primitives/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/list_of_primitives/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -146,6 +149,11 @@ func xmlTextContext(parent string, name xml.Name) string { case (xml.Name{Space: "https://dummy.com", Local: "someStrings"}): return "content:ListOf_string" } + case "content:ListOf_double": + switch name { + case (xml.Name{Space: "https://dummy.com", Local: "v"}): + return "double" + } case "content:ListOf_string": switch name { case (xml.Name{Space: "https://dummy.com", Local: "v"}): @@ -160,7 +168,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -175,8 +238,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -204,6 +277,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -244,10 +332,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -265,6 +387,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/method_references/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/method_references/expected_output/xmlization/test/common_test.go index aa9ec68c0..c30ceb164 100644 --- a/dev/test_data/main/golang/expected/method_references/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/method_references/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -147,7 +150,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -162,8 +220,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -191,6 +259,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -231,10 +314,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -252,6 +369,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/nested_any_in_all/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/nested_any_in_all/expected_output/xmlization/test/common_test.go index 1254c482b..7300af664 100644 --- a/dev/test_data/main/golang/expected/nested_any_in_all/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/nested_any_in_all/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -205,7 +208,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -220,8 +278,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -249,6 +317,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -289,10 +372,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -310,6 +427,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/nested_collections/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/nested_collections/expected_output/xmlization/test/common_test.go index d813a92c7..0904ce7c3 100644 --- a/dev/test_data/main/golang/expected/nested_collections/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/nested_collections/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -239,7 +242,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -254,8 +312,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -283,6 +351,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -323,10 +406,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -344,6 +461,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/optional_arguments/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/optional_arguments/expected_output/xmlization/test/common_test.go index c9e522c13..b8cd47139 100644 --- a/dev/test_data/main/golang/expected/optional_arguments/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/optional_arguments/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -190,7 +193,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -205,8 +263,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -234,6 +302,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -274,10 +357,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -295,6 +412,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/primitive_types/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/primitive_types/expected_output/xmlization/test/common_test.go index 7690353d4..5e930d73a 100644 --- a/dev/test_data/main/golang/expected/primitive_types/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/primitive_types/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,12 +132,14 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { case "class:Something": switch name { + case (xml.Name{Space: "https://dummy.com", Local: "someFloat"}): + return "double" case (xml.Name{Space: "https://dummy.com", Local: "someString"}): return "text" } @@ -147,7 +152,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -162,8 +222,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -191,6 +261,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -231,10 +316,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -252,6 +371,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) @@ -493,5 +619,107 @@ func TestWhitespaceStringRoundTrip(t *testing.T) { } } +func TestXmlNumericComparison(t *testing.T) { + // These fragments exercise comparison at typed paths, without model verification. + for _, path := range []struct { + kind, prefix, suffix string + }{ + {"double", "", ""}, + {"text", "", ""}, + } { + t.Run(path.kind, func(t *testing.T) { + for _, c := range []struct { + name, left, right string + equal bool + }{ + {"decimal", "42.0", "42", true}, + {"exponent", "21668651.6", "2.16686516e+07", true}, + {"leading_zeroes", "+0042.00", "42", true}, + {"exponent_case", "4.2E1", "4.2e+01", true}, + {"negative", "-42.00", "-4.2e1", true}, + {"small", "0.0000000001", "1e-10", true}, + {"smallest_subnormal", "5e-324", "4.9406564584124654e-324", true}, + {"subnormal_vs_zero", "5e-324", "0", false}, + {"largest_finite", "1.7976931348623157e308", "1.7976931348623157E+308", true}, + {"different", "42", "43", false}, + {"adjacent_doubles", "1", "1.0000000000000002", false}, + {"integer_precision", "9007199254740991", "9007199254740992", false}, + {"signed_zero", "0", "-0", false}, + {"negative_zero", "-0.0", "-0e0", true}, + {"same_instruction", " y?>42.0", " y?>42", true}, + {"different_instruction", " y?>42.0", " y?>42", false}, + {"moved_instruction", "42", "42", false}, + {"moved_comment", "42", "42", false}, + } { + t.Run(c.name, func(t *testing.T) { + left, err := canonicalizeXmlForComparison(path.prefix + c.left + path.suffix) + if err != nil { + t.Fatal(err) + } + right, err := canonicalizeXmlForComparison(path.prefix + c.right + path.suffix) + if err != nil { + t.Fatal(err) + } + want := c.equal && path.kind != "text" + if (left == right) != want { + t.Fatalf("Unexpected comparison of %q and %q", left, right) + } + }) + } + if path.kind == "text" { + return + } + for _, c := range []struct{ input, want string }{ + {"4", "42"}, + {"42", "42"}, + {" 42 ", "42"}, + {"42", "42"}, + {"42.0", "42"}, + {" 42", "42"}, + {"42", "42"}, + {" 2 ", "42"}, + {"42.0", "42"}, + } { + input := path.prefix + c.input + path.suffix + instanceNumber, err := canonicalizeXmlForComparison(input) + if err != nil { + t.Fatal(err) + } + expected := path.prefix + c.want + path.suffix + if instanceNumber != expected { + t.Fatalf("Expected %q, got %q", expected, instanceNumber) + } + } + for _, spelling := range []string{ + "", " ", "nan", "Infinity", "0x1p0", "1_0", "1 2", + "42", "4 2", "42" + string(rune(160)), + } { + if _, err := canonicalizeXmlForComparison(path.prefix + spelling + path.suffix); err == nil { + t.Fatalf("Accepted invalid numeric content %q", spelling) + } + } + for _, c := range []struct { + spelling, want string + }{ + {"INF", "INF"}, {"-INF", "-INF"}, {"NaN", "NaN"}, {"1e9999", "INF"}, + } { + canonical, err := canonicalizeXmlForComparison(path.prefix + c.spelling + path.suffix) + if strings.HasPrefix(path.kind, "json:") { + if err == nil { + t.Fatalf("Accepted non-finite JSON number %q", c.spelling) + } + } else { + if err != nil { + t.Fatal(err) + } + if canonical != path.prefix+c.want+path.suffix { + t.Fatalf("Unexpected non-finite model number: %q", canonical) + } + } + } + }) + } +} + // This code has been automatically generated by aas-core-codegen. // Do NOT edit or append. diff --git a/dev/test_data/main/golang/expected/problematic_keywords/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/problematic_keywords/expected_output/xmlization/test/common_test.go index 38c756d7a..03ddaddbd 100644 --- a/dev/test_data/main/golang/expected/problematic_keywords/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/problematic_keywords/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -153,7 +156,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -168,8 +226,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -197,6 +265,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -237,10 +320,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -258,6 +375,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/property_named_like_class/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/property_named_like_class/expected_output/xmlization/test/common_test.go index f0c57b925..f988905b1 100644 --- a/dev/test_data/main/golang/expected/property_named_like_class/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/property_named_like_class/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -147,7 +150,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -162,8 +220,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -191,6 +259,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -231,10 +314,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -252,6 +369,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/sets/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/sets/expected_output/xmlization/test/common_test.go index 6998dfe41..9ad4f36e9 100644 --- a/dev/test_data/main/golang/expected/sets/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/sets/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -202,7 +205,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -217,8 +275,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -246,6 +314,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -286,10 +369,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -307,6 +424,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/string_operations/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/string_operations/expected_output/xmlization/test/common_test.go index bd1881039..53a3c3463 100644 --- a/dev/test_data/main/golang/expected/string_operations/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/string_operations/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -155,7 +158,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -170,8 +228,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -199,6 +267,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -239,10 +322,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -260,6 +377,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/switch_statements/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/switch_statements/expected_output/xmlization/test/common_test.go index 3e9cbc881..90dbd7dbb 100644 --- a/dev/test_data/main/golang/expected/switch_statements/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/switch_statements/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -151,7 +154,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -166,8 +224,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -195,6 +263,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -235,10 +318,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -256,6 +373,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/transpiled_methods/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/transpiled_methods/expected_output/xmlization/test/common_test.go index e62aab500..caaa57cb8 100644 --- a/dev/test_data/main/golang/expected/transpiled_methods/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/transpiled_methods/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -165,7 +168,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -180,8 +238,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -209,6 +277,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -249,10 +332,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -270,6 +387,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/tuples/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/tuples/expected_output/xmlization/test/common_test.go index d6cd5d32c..eaf7b4d48 100644 --- a/dev/test_data/main/golang/expected/tuples/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/tuples/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -192,7 +195,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -207,8 +265,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -236,6 +304,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -276,10 +359,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -297,6 +414,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/unions/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/unions/expected_output/xmlization/test/common_test.go index 125012e88..24049b5ba 100644 --- a/dev/test_data/main/golang/expected/unions/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/unions/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -318,7 +321,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -333,8 +391,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -362,6 +430,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -402,10 +485,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -423,6 +540,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/variance_of_containers/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/variance_of_containers/expected_output/xmlization/test/common_test.go index 043026ee9..0d6bb4514 100644 --- a/dev/test_data/main/golang/expected/variance_of_containers/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/variance_of_containers/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -195,7 +198,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -210,8 +268,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -239,6 +307,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -279,10 +362,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -300,6 +417,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected) diff --git a/dev/test_data/main/golang/expected/visibility/expected_output/xmlization/test/common_test.go b/dev/test_data/main/golang/expected/visibility/expected_output/xmlization/test/common_test.go index 8cfcc61d0..b4ba4c69c 100644 --- a/dev/test_data/main/golang/expected/visibility/expected_output/xmlization/test/common_test.go +++ b/dev/test_data/main/golang/expected/visibility/expected_output/xmlization/test/common_test.go @@ -8,12 +8,15 @@ import ( "encoding/xml" "fmt" "io" + "math" "os" "path/filepath" "regexp" + "strconv" "strings" "testing" + xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon" ourtesting "github.com/dummy-works/dummy/ourtesting" ourxmlization "github.com/dummy-works/dummy/xmlization" ) @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string { return result } -// Follow the model's XML nesting to recognize string values, including collection +// Follow the model's XML nesting to recognize scalar values, including collection // items. Container whitespace remains available to the existing indentation cleanup. func xmlTextContext(parent string, name xml.Name) string { switch parent { @@ -149,7 +152,62 @@ func xmlTextContext(parent string, name xml.Name) string { return "" } -// Normalize escaping in character data while retaining the original markup. +// Consume a numeric element's content, retaining comments, processing instructions and the end tag. +// CDATA and character references may split a single number across text tokens. +func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) { + var content, layout bytes.Buffer + var numberStart, numberEnd int + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err != nil { + return "", err + } + switch token := token.(type) { + case xml.CharData: + content.Write(token) + if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 { + if numberEnd == 0 { + numberStart = layout.Len() + } + layout.Write(part) + numberEnd = layout.Len() + } + case xml.Comment, xml.ProcInst: + layout.WriteString(source[start:decoder.InputOffset()]) + case xml.EndElement: + // Reuse the SDK reader's lexical validation and double-precision conversion. + // The sentinel element terminates the supplied character data for ReadText. + number, _, err := xmlcommon.ReadTextAs_double( + xml.NewDecoder(strings.NewReader("")), xml.CharData(content.Bytes())) + if err != nil { + return "", err + } + if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) { + return "", fmt.Errorf("expected a finite JSON number") + } + original := layout.String() + endTag := source[start:decoder.InputOffset()] + // In a valid number, '<' can only come from intervening markup. + // Keep its position relative to the digits by retaining their spelling. + if strings.Contains(original[numberStart:numberEnd], "<") { + return original + endTag, nil + } + // FormatFloat preserves every representable value, including signed zero. + canonical := strconv.FormatFloat(number, 'g', -1, 64) + if math.IsInf(number, 1) { + canonical = "INF" + } else if math.IsInf(number, -1) { + canonical = "-INF" + } + return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil + default: + return "", fmt.Errorf("expected numeric text, got %T", token) + } + } +} + +// Normalize character escaping and numeric spelling while retaining the markup. // Copying tags verbatim avoids changing namespace declarations or prefixes. func normalizeXmlTextEscaping(text string) (string, error) { decoder := xml.NewDecoder(strings.NewReader(text)) @@ -164,8 +222,18 @@ func normalizeXmlTextEscaping(text string) (string, error) { if err != nil { return "", err } - if start, ok := token.(xml.StartElement); ok { - contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name)) + if element, ok := token.(xml.StartElement); ok { + context := xmlTextContext(contexts[len(contexts)-1], element.Name) + if context == "double" || context == "json:double" { + result.WriteString(text[start:decoder.InputOffset()]) + normalized, err := normalizeXmlNumericContent(decoder, text, context) + if err != nil { + return "", err + } + result.WriteString(normalized) + continue + } + contexts = append(contexts, context) } if data, ok := token.(xml.CharData); ok { // Normalize strings consistently, even if they consist only of whitespace. @@ -193,6 +261,21 @@ func normalizeXmlTextEscaping(text string) (string, error) { } } +func TestWritePositiveInfinity(t *testing.T) { + var output strings.Builder + encoder := xml.NewEncoder(&output) + if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil { + t.Fatal(err) + } + if err := encoder.Flush(); err != nil { + t.Fatal(err) + } + // XSD 1.0 requires INF even though the reader also accepts +INF. + if got := output.String(); got != "INF" { + t.Fatalf("Expected positive infinity as %q, got %q", "INF", got) + } +} + func TestNormalizeXmlTextEscaping(t *testing.T) { cases := []struct { name string @@ -233,10 +316,44 @@ func canonicalizeXmlForComparison(text string) (string, error) { if err != nil { return "", err } + // Protect processing instructions while the legacy regexes clean up markup. + // The marker target is absent even after carriage-return removal. + markerPrefix := "aasCoreComparisonPI" + withoutCR := strings.ReplaceAll(text, "\r", "") + for strings.Contains(withoutCR, markerPrefix) { + markerPrefix += "X" + } + var instructions []string + var protected bytes.Buffer + decoder := xml.NewDecoder(strings.NewReader(text)) + for { + start := decoder.InputOffset() + token, err := decoder.Token() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + original := text[start:decoder.InputOffset()] + if _, ok := token.(xml.ProcInst); ok { + fmt.Fprintf(&protected, "", markerPrefix, len(instructions)) + // XML 1.0 section 2.11 normalizes literal line endings before parsing. + original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n") + instructions = append(instructions, original) + } else { + protected.WriteString(original) + } + } + text = protected.String() text = strings.ReplaceAll(text, "\r", "") text = removeWhitespaceBetweenTags(text) text = forceSelfClosingTags(text) - return removeWhitespaceWithinTags(text), nil + text = removeWhitespaceWithinTags(text) + for i, original := range instructions { + text = strings.ReplaceAll(text, fmt.Sprintf("", markerPrefix, i), original) + } + return text, nil } func TestXmlComparisonPreservesText(t *testing.T) { @@ -254,6 +371,13 @@ func TestXmlComparisonPreservesText(t *testing.T) { {"tab_vs_space", ` `, ` `, false}, {"lost_tab", ` `, ``, false}, {"space_vs_literal_tab", ` `, "\t", false}, + {"instruction_data", ` y?>`, ` y?>`, false}, + {"instruction_markup", `?>`, `?>`, false}, + {"instruction_cr", "", ``, false}, + {"instruction_cr_as_lf", "", "", true}, + {"instruction_crlf_as_lf", "", "", true}, + {"instruction_indentation", "\n y?>\n", ` y?>`, true}, + {"instruction_marker", ``, ``, false}, } { t.Run(c.name, func(t *testing.T) { expected, err := canonicalizeXmlForComparison(c.expected)