Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
306 changes: 301 additions & 5 deletions aas_core_codegen/golang/tests/_generate_xmlization_test_common_test.py

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@ import (
"encoding/xml"
"fmt"
"io"
"math"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"

xmlcommon "github.com/aas-core-works/aas-core3.0-golang/internal/xmlcommon"
ourtesting "github.com/aas-core-works/aas-core3.0-golang/ourtesting"
ourxmlization "github.com/aas-core-works/aas-core3.0-golang/xmlization"
)
Expand Down Expand Up @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string {
return result
}

// Follow the model's XML nesting to recognize string values, including collection
// Follow the model's XML nesting to recognize scalar values, including collection

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The values are not scalar -- I see "class:Reference" below?

// items. Container whitespace remains available to the existing indentation cleanup.
func xmlTextContext(parent string, name xml.Name) string {
switch parent {
Expand Down Expand Up @@ -1027,7 +1030,62 @@ func xmlTextContext(parent string, name xml.Name) string {
return ""
}

// Normalize escaping in character data while retaining the original markup.
// Consume a numeric element's content, retaining comments, processing instructions and the end tag.
// CDATA and character references may split a single number across text tokens.
func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) {
var content, layout bytes.Buffer
var numberStart, numberEnd int
for {
start := decoder.InputOffset()
token, err := decoder.Token()
if err != nil {
return "", err
}
switch token := token.(type) {
case xml.CharData:
content.Write(token)
if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 {
if numberEnd == 0 {
numberStart = layout.Len()
}
layout.Write(part)
numberEnd = layout.Len()
}
case xml.Comment, xml.ProcInst:
layout.WriteString(source[start:decoder.InputOffset()])
case xml.EndElement:
// Reuse the SDK reader's lexical validation and double-precision conversion.
// The sentinel element terminates the supplied character data for ReadText.
number, _, err := xmlcommon.ReadTextAs_double(
xml.NewDecoder(strings.NewReader("<end/>")), xml.CharData(content.Bytes()))
if err != nil {
return "", err
}
if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) {
return "", fmt.Errorf("expected a finite JSON number")
}
original := layout.String()
endTag := source[start:decoder.InputOffset()]
// In a valid number, '<' can only come from intervening markup.
// Keep its position relative to the digits by retaining their spelling.
if strings.Contains(original[numberStart:numberEnd], "<") {
return original + endTag, nil
}
// FormatFloat preserves every representable value, including signed zero.
canonical := strconv.FormatFloat(number, 'g', -1, 64)
if math.IsInf(number, 1) {
canonical = "INF"
} else if math.IsInf(number, -1) {
canonical = "-INF"
}
return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil
default:
return "", fmt.Errorf("expected numeric text, got %T", token)
}
}
}

// Normalize character escaping and numeric spelling while retaining the markup.
// Copying tags verbatim avoids changing namespace declarations or prefixes.
func normalizeXmlTextEscaping(text string) (string, error) {
decoder := xml.NewDecoder(strings.NewReader(text))
Expand All @@ -1042,8 +1100,18 @@ func normalizeXmlTextEscaping(text string) (string, error) {
if err != nil {
return "", err
}
if start, ok := token.(xml.StartElement); ok {
contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name))
if element, ok := token.(xml.StartElement); ok {
context := xmlTextContext(contexts[len(contexts)-1], element.Name)
if context == "double" || context == "json:double" {
result.WriteString(text[start:decoder.InputOffset()])
normalized, err := normalizeXmlNumericContent(decoder, text, context)
if err != nil {
return "", err
}
result.WriteString(normalized)
continue
}
contexts = append(contexts, context)
}
if data, ok := token.(xml.CharData); ok {
// Normalize strings consistently, even if they consist only of whitespace.
Expand Down Expand Up @@ -1071,6 +1139,21 @@ func normalizeXmlTextEscaping(text string) (string, error) {
}
}

func TestWritePositiveInfinity(t *testing.T) {
var output strings.Builder
encoder := xml.NewEncoder(&output)
if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil {
t.Fatal(err)
}
if err := encoder.Flush(); err != nil {
t.Fatal(err)
}
// XSD 1.0 requires INF even though the reader also accepts +INF.
if got := output.String(); got != "INF" {
t.Fatalf("Expected positive infinity as %q, got %q", "INF", got)
}
}

func TestNormalizeXmlTextEscaping(t *testing.T) {
cases := []struct {
name string
Expand Down Expand Up @@ -1111,10 +1194,44 @@ func canonicalizeXmlForComparison(text string) (string, error) {
if err != nil {
return "", err
}
// Protect processing instructions while the legacy regexes clean up markup.
// The marker target is absent even after carriage-return removal.
markerPrefix := "aasCoreComparisonPI"
withoutCR := strings.ReplaceAll(text, "\r", "")
for strings.Contains(withoutCR, markerPrefix) {
markerPrefix += "X"
}
var instructions []string
var protected bytes.Buffer
decoder := xml.NewDecoder(strings.NewReader(text))
for {
start := decoder.InputOffset()
token, err := decoder.Token()
if err == io.EOF {
break
}
if err != nil {
return "", err
}
original := text[start:decoder.InputOffset()]
if _, ok := token.(xml.ProcInst); ok {
fmt.Fprintf(&protected, "<?%s%d?>", markerPrefix, len(instructions))
// XML 1.0 section 2.11 normalizes literal line endings before parsing.
original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n")
instructions = append(instructions, original)
} else {
protected.WriteString(original)
}
}
text = protected.String()
text = strings.ReplaceAll(text, "\r", "")
text = removeWhitespaceBetweenTags(text)
text = forceSelfClosingTags(text)
return removeWhitespaceWithinTags(text), nil
text = removeWhitespaceWithinTags(text)
for i, original := range instructions {
text = strings.ReplaceAll(text, fmt.Sprintf("<?%s%d?>", markerPrefix, i), original)
}
return text, nil
}

func TestXmlComparisonPreservesText(t *testing.T) {
Expand All @@ -1132,6 +1249,13 @@ func TestXmlComparisonPreservesText(t *testing.T) {
{"tab_vs_space", `<v>&#x9;</v>`, `<v>&#32;</v>`, false},
{"lost_tab", `<v>&#x9;</v>`, `<v></v>`, false},
{"space_vs_literal_tab", `<v>&#32;</v>`, "<v>\t</v>", false},
{"instruction_data", `<v><?p x > y?></v>`, `<v><?p x> y?></v>`, false},
{"instruction_markup", `<v><?p <a></a>?></v>`, `<v><?p <a/>?></v>`, false},
{"instruction_cr", "<v><?p x\ry?></v>", `<v><?p xy?></v>`, false},
{"instruction_cr_as_lf", "<v><?p x\ry?></v>", "<v><?p x\ny?></v>", true},
{"instruction_crlf_as_lf", "<v><?p x\r\ny?></v>", "<v><?p x\ny?></v>", true},
{"instruction_indentation", "<v>\n<?p x > y?>\n</v>", `<v><?p x > y?></v>`, true},
{"instruction_marker", `<v><?p aasCoreComparisonPI0?></v>`, `<v><?p aasCoreComparisonPI1?></v>`, false},
} {
t.Run(c.name, func(t *testing.T) {
expected, err := canonicalizeXmlForComparison(c.expected)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,15 @@ import (
"encoding/xml"
"fmt"
"io"
"math"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"testing"

xmlcommon "github.com/dummy-works/dummy/internal/xmlcommon"
ourtesting "github.com/dummy-works/dummy/ourtesting"
ourxmlization "github.com/dummy-works/dummy/xmlization"
)
Expand Down Expand Up @@ -129,7 +132,7 @@ func removeWhitespaceWithinTags(text string) string {
return result
}

// Follow the model's XML nesting to recognize string values, including collection
// Follow the model's XML nesting to recognize scalar values, including collection
// items. Container whitespace remains available to the existing indentation cleanup.
func xmlTextContext(parent string, name xml.Name) string {
switch parent {
Expand Down Expand Up @@ -218,7 +221,62 @@ func xmlTextContext(parent string, name xml.Name) string {
return ""
}

// Normalize escaping in character data while retaining the original markup.
// Consume a numeric element's content, retaining comments, processing instructions and the end tag.
// CDATA and character references may split a single number across text tokens.
func normalizeXmlNumericContent(decoder *xml.Decoder, source string, context string) (string, error) {
var content, layout bytes.Buffer
var numberStart, numberEnd int
for {
start := decoder.InputOffset()
token, err := decoder.Token()
if err != nil {
return "", err
}
switch token := token.(type) {
case xml.CharData:
content.Write(token)
if part := bytes.Trim(token, " \t\r\n"); len(part) > 0 {
if numberEnd == 0 {
numberStart = layout.Len()
}
layout.Write(part)
numberEnd = layout.Len()
}
case xml.Comment, xml.ProcInst:
layout.WriteString(source[start:decoder.InputOffset()])
case xml.EndElement:
// Reuse the SDK reader's lexical validation and double-precision conversion.
// The sentinel element terminates the supplied character data for ReadText.
number, _, err := xmlcommon.ReadTextAs_double(
xml.NewDecoder(strings.NewReader("<end/>")), xml.CharData(content.Bytes()))
if err != nil {
return "", err
}
if context == "json:double" && (math.IsInf(number, 0) || math.IsNaN(number)) {
return "", fmt.Errorf("expected a finite JSON number")
}
original := layout.String()
endTag := source[start:decoder.InputOffset()]
// In a valid number, '<' can only come from intervening markup.
// Keep its position relative to the digits by retaining their spelling.
if strings.Contains(original[numberStart:numberEnd], "<") {
return original + endTag, nil
}
// FormatFloat preserves every representable value, including signed zero.
canonical := strconv.FormatFloat(number, 'g', -1, 64)
if math.IsInf(number, 1) {
canonical = "INF"
} else if math.IsInf(number, -1) {
canonical = "-INF"
}
return original[:numberStart] + canonical + original[numberEnd:] + endTag, nil
default:
return "", fmt.Errorf("expected numeric text, got %T", token)
}
}
}

// Normalize character escaping and numeric spelling while retaining the markup.
// Copying tags verbatim avoids changing namespace declarations or prefixes.
func normalizeXmlTextEscaping(text string) (string, error) {
decoder := xml.NewDecoder(strings.NewReader(text))
Expand All @@ -233,8 +291,18 @@ func normalizeXmlTextEscaping(text string) (string, error) {
if err != nil {
return "", err
}
if start, ok := token.(xml.StartElement); ok {
contexts = append(contexts, xmlTextContext(contexts[len(contexts)-1], start.Name))
if element, ok := token.(xml.StartElement); ok {
context := xmlTextContext(contexts[len(contexts)-1], element.Name)
if context == "double" || context == "json:double" {
result.WriteString(text[start:decoder.InputOffset()])
normalized, err := normalizeXmlNumericContent(decoder, text, context)
if err != nil {
return "", err
}
result.WriteString(normalized)
continue
}
contexts = append(contexts, context)
}
if data, ok := token.(xml.CharData); ok {
// Normalize strings consistently, even if they consist only of whitespace.
Expand Down Expand Up @@ -262,6 +330,21 @@ func normalizeXmlTextEscaping(text string) (string, error) {
}
}

func TestWritePositiveInfinity(t *testing.T) {
var output strings.Builder
encoder := xml.NewEncoder(&output)
if err := xmlcommon.WriteAsText_double(encoder, math.Inf(1)); err != nil {
t.Fatal(err)
}
if err := encoder.Flush(); err != nil {
t.Fatal(err)
}
// XSD 1.0 requires INF even though the reader also accepts +INF.
if got := output.String(); got != "INF" {
t.Fatalf("Expected positive infinity as %q, got %q", "INF", got)
}
}

func TestNormalizeXmlTextEscaping(t *testing.T) {
cases := []struct {
name string
Expand Down Expand Up @@ -302,10 +385,44 @@ func canonicalizeXmlForComparison(text string) (string, error) {
if err != nil {
return "", err
}
// Protect processing instructions while the legacy regexes clean up markup.
// The marker target is absent even after carriage-return removal.
markerPrefix := "aasCoreComparisonPI"
withoutCR := strings.ReplaceAll(text, "\r", "")
for strings.Contains(withoutCR, markerPrefix) {
markerPrefix += "X"
}
var instructions []string
var protected bytes.Buffer
decoder := xml.NewDecoder(strings.NewReader(text))
for {
start := decoder.InputOffset()
token, err := decoder.Token()
if err == io.EOF {
break
}
if err != nil {
return "", err
}
original := text[start:decoder.InputOffset()]
if _, ok := token.(xml.ProcInst); ok {
fmt.Fprintf(&protected, "<?%s%d?>", markerPrefix, len(instructions))
// XML 1.0 section 2.11 normalizes literal line endings before parsing.
original = strings.ReplaceAll(strings.ReplaceAll(original, "\r\n", "\n"), "\r", "\n")
instructions = append(instructions, original)
} else {
protected.WriteString(original)
}
}
text = protected.String()
text = strings.ReplaceAll(text, "\r", "")
text = removeWhitespaceBetweenTags(text)
text = forceSelfClosingTags(text)
return removeWhitespaceWithinTags(text), nil
text = removeWhitespaceWithinTags(text)
for i, original := range instructions {
text = strings.ReplaceAll(text, fmt.Sprintf("<?%s%d?>", markerPrefix, i), original)
}
return text, nil
}

func TestXmlComparisonPreservesText(t *testing.T) {
Expand All @@ -323,6 +440,13 @@ func TestXmlComparisonPreservesText(t *testing.T) {
{"tab_vs_space", `<v>&#x9;</v>`, `<v>&#32;</v>`, false},
{"lost_tab", `<v>&#x9;</v>`, `<v></v>`, false},
{"space_vs_literal_tab", `<v>&#32;</v>`, "<v>\t</v>", false},
{"instruction_data", `<v><?p x > y?></v>`, `<v><?p x> y?></v>`, false},
{"instruction_markup", `<v><?p <a></a>?></v>`, `<v><?p <a/>?></v>`, false},
{"instruction_cr", "<v><?p x\ry?></v>", `<v><?p xy?></v>`, false},
{"instruction_cr_as_lf", "<v><?p x\ry?></v>", "<v><?p x\ny?></v>", true},
{"instruction_crlf_as_lf", "<v><?p x\r\ny?></v>", "<v><?p x\ny?></v>", true},
{"instruction_indentation", "<v>\n<?p x > y?>\n</v>", `<v><?p x > y?></v>`, true},
{"instruction_marker", `<v><?p aasCoreComparisonPI0?></v>`, `<v><?p aasCoreComparisonPI1?></v>`, false},
} {
t.Run(c.name, func(t *testing.T) {
expected, err := canonicalizeXmlForComparison(c.expected)
Expand Down
Loading
Loading