From 6b7c2b08d29230713c3290c3398ee3f2ccbca89f Mon Sep 17 00:00:00 2001 From: MatloaItumeleng Date: Tue, 5 May 2026 10:13:39 +0200 Subject: [PATCH 1/3] workaround , subtracting single-quote characters --- .../time/DateTimePattern.scala | 35 ++++++++++++++----- .../time/DateTimePatternSuite.scala | 11 ++++++ .../types/parsers/DateTimeParserSuite.scala | 13 +++++++ 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala b/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala index 068f071..a0d4132 100644 --- a/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala +++ b/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala @@ -129,24 +129,41 @@ object DateTimePattern { override val defaultTimeZone: Option[String] = assignedDefaultTimeZone.filterNot(_ => timeZoneInPattern) override val isTimeZoned: Boolean = timeZoneInPattern || defaultTimeZone.nonEmpty - val (millisecondsPosition, microsecondsPosition, nanosecondsPosition) = analyzeSecondFractionsPositions(pattern) + private val (patternMilliPos, patternMicroPos, patternNanoPos) = scanSecondFractionsInPattern(pattern) + override val patternWithoutSecondFractions: String = { + val patternSections = Section.mergeTouchingSectionsAndSort(Seq(patternMilliPos, patternMicroPos, patternNanoPos).flatten) + Section.removeMultipleFrom(pattern, patternSections) + } + + val (millisecondsPosition, microsecondsPosition, nanosecondsPosition) = adjustPositionsForValue(pattern, patternMilliPos, patternMicroPos, patternNanoPos) override val secondFractionsSections: Seq[Section] = Section.mergeTouchingSectionsAndSort(Seq(millisecondsPosition, microsecondsPosition, nanosecondsPosition).flatten) - override val patternWithoutSecondFractions: String = Section.removeMultipleFrom(pattern, secondFractionsSections) private def scanForPlaceholder(withinString: String, placeHolder: Char): Option[Section] = { val start = withinString.findFirstUnquoted(Set(placeHolder), Set('\'')) start.map(index => Section.ofSameChars(withinString, index)) } - private def analyzeSecondFractionsPositions(withinString: String): (Option[Section], Option[Section], Option[Section]) = { - val clearedPattern = withinString - - // TODO as part of #7 fix (originally Enceladus#677) - val milliSP = scanForPlaceholder(clearedPattern, patternMilliSecondChar) - val microSP = scanForPlaceholder(clearedPattern, patternMicroSecondChar) - val nanoSP = scanForPlaceholder(clearedPattern, patternNanoSecondChat) + private def scanSecondFractionsInPattern(withinString: String): (Option[Section], Option[Section], Option[Section]) = { + val milliSP = scanForPlaceholder(withinString, patternMilliSecondChar) + val microSP = scanForPlaceholder(withinString, patternMicroSecondChar) + val nanoSP = scanForPlaceholder(withinString, patternNanoSecondChat) (milliSP, microSP, nanoSP) } + + private def adjustPositionsForValue( + pat: String, + milliPos: Option[Section], + microPos: Option[Section], + nanoPos: Option[Section] + ): (Option[Section], Option[Section], Option[Section]) = { + def adjust(sectionOpt: Option[Section]): Option[Section] = { + sectionOpt.map { s => + val quotesBefore = pat.substring(0, s.start).count(_ == '\'') + s.copy(start = s.start - quotesBefore) + } + } + (adjust(milliPos), adjust(microPos), adjust(nanoPos)) + } } private final case class StandardDTPattern(override val pattern: String, diff --git a/src/test/scala/za/co/absa/standardization/time/DateTimePatternSuite.scala b/src/test/scala/za/co/absa/standardization/time/DateTimePatternSuite.scala index 6945d8b..f589b9e 100644 --- a/src/test/scala/za/co/absa/standardization/time/DateTimePatternSuite.scala +++ b/src/test/scala/za/co/absa/standardization/time/DateTimePatternSuite.scala @@ -268,4 +268,15 @@ class DateTimePatternSuite extends AnyFunSuite { assert(dtp.patternWithoutSecondFractions == "yyyy-MM-dd HH:mm:ss") assert(!dtp.containsSecondFractions) } + + test("Second fractions detection in regular pattern with quoted literal - milliseconds") { + val pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX" + val dtp = DateTimePattern(pattern) + assert(dtp.millisecondsPosition.contains(Section(20, 3))) + assert(dtp.microsecondsPosition.isEmpty) + assert(dtp.nanosecondsPosition.isEmpty) + assert(dtp.secondFractionsSections == Seq(Section(20, 3))) + assert(dtp.patternWithoutSecondFractions == "yyyy-MM-dd'T'HH:mm:ss.XXX") + assert(dtp.containsSecondFractions) + } } diff --git a/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala b/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala index 9511eae..8afa34d 100644 --- a/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala +++ b/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala @@ -186,6 +186,19 @@ class DateTimeParserSuite extends AnyFunSuite{ assert(parser7.format(t) == "(789) 1970-01-02 (123) 01:00:00 (456)") } + test("DateParser class actual pattern with quoted literal and timezone and milliseconds") { + val parser = DateTimeParser("yyyy-MM-dd'T'HH:mm:ss.SSSXXX") + val str = "2025-07-18T16:07:51.569+02:00" + + val resultDate: Date = parser.parseDate(str) + val expectedDate: Date = Date.valueOf("2025-07-18") + assert(resultDate == expectedDate) + + val resultTimestamp: Timestamp = parser.parseTimestamp(str) + val expectedTimestamp: Timestamp = Timestamp.valueOf("2025-07-18 14:07:51.569") + assert(resultTimestamp == expectedTimestamp) + } + test("Lenient interpretation is not accepted") { //first lenient interpretation val pattern = "dd-MM-yyyy" From 660c8414d17a78be31b79a7e74de719247a76b10 Mon Sep 17 00:00:00 2001 From: MatloaItumeleng Date: Fri, 8 May 2026 14:32:53 +0200 Subject: [PATCH 2/3] Review recommendations, added TimeZoneNormalizer and refactored adjustPositionsForValue --- .../za/co/absa/standardization/time/DateTimePattern.scala | 3 ++- .../standardization/types/parsers/DateTimeParserSuite.scala | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala b/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala index a0d4132..1f6328e 100644 --- a/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala +++ b/src/main/scala/za/co/absa/standardization/time/DateTimePattern.scala @@ -158,7 +158,8 @@ object DateTimePattern { ): (Option[Section], Option[Section], Option[Section]) = { def adjust(sectionOpt: Option[Section]): Option[Section] = { sectionOpt.map { s => - val quotesBefore = pat.substring(0, s.start).count(_ == '\'') + val prefix = pat.substring(0, s.start) + val quotesBefore = prefix.countUnquoted(Set('\''), Set.empty).getOrElse('\'', 0) s.copy(start = s.start - quotesBefore) } } diff --git a/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala b/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala index 8afa34d..7d50f53 100644 --- a/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala +++ b/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala @@ -20,11 +20,14 @@ import java.sql.{Date, Timestamp} import java.text.{ParseException, SimpleDateFormat} import org.scalatest.funsuite.AnyFunSuite +import za.co.absa.standardization.testing.TimeZoneNormalizer case class TestInputRow(id: Int, stringField: String) class DateTimeParserSuite extends AnyFunSuite{ + TimeZoneNormalizer.normalizeJVMTimeZone() + test("DateParser class epoch") { val parser = DateTimeParser("epoch") From 0263e3d18eeac3347ad1a68c86f11ccdc9115b12 Mon Sep 17 00:00:00 2001 From: MatloaItumeleng Date: Mon, 11 May 2026 12:37:02 +0200 Subject: [PATCH 3/3] review recomm, set default timezone as UTC --- .../standardization/types/parsers/DateTimeParserSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala b/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala index 7d50f53..086bb1a 100644 --- a/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala +++ b/src/test/scala/za/co/absa/standardization/types/parsers/DateTimeParserSuite.scala @@ -18,6 +18,7 @@ package za.co.absa.standardization.types.parsers import java.sql.{Date, Timestamp} import java.text.{ParseException, SimpleDateFormat} +import java.util.TimeZone import org.scalatest.funsuite.AnyFunSuite import za.co.absa.standardization.testing.TimeZoneNormalizer @@ -26,6 +27,7 @@ case class TestInputRow(id: Int, stringField: String) class DateTimeParserSuite extends AnyFunSuite{ + TimeZone.setDefault(TimeZone.getTimeZone("UTC")) TimeZoneNormalizer.normalizeJVMTimeZone() test("DateParser class epoch") {