From 97cb3a706d6f7f9895f7f81bef5d03c436a0bb74 Mon Sep 17 00:00:00 2001 From: 283375 Date: Thu, 12 Nov 2026 02:46:41 +0800 Subject: [PATCH 01/12] extract general OcrScenario, fix cropping and layout detection --- .../arcaeaoffline/helpers/DeviceOcrHelper.kt | 44 +------------- core/AGENTS.md | 4 +- .../arcaeaoffline/core/ocr/OcrScenario.kt | 9 +++ .../core/ocr/device/DeviceOcrScenario.kt | 60 +++++++++++++++++++ .../ocr/device/OcrPerformanceBenchmark.kt | 20 ++----- .../ocr/device/rois/DeviceRoisAutoSelector.kt | 27 +++++++++ 6 files changed, 106 insertions(+), 58 deletions(-) create mode 100644 core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt create mode 100644 core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt diff --git a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt index 279c90ba..db7db484 100644 --- a/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt +++ b/app/src/main/java/xyz/sevive/arcaeaoffline/helpers/DeviceOcrHelper.kt @@ -23,16 +23,8 @@ import org.opencv.core.MatOfByte import org.opencv.imgcodecs.Imgcodecs import xyz.sevive.arcaeaoffline.core.database.entities.PlayResult import xyz.sevive.arcaeaoffline.core.ocr.ImageHashesDatabase -import xyz.sevive.arcaeaoffline.core.ocr.device.CropBlackEdges -import xyz.sevive.arcaeaoffline.core.ocr.device.DeviceOcr import xyz.sevive.arcaeaoffline.core.ocr.device.DeviceOcrResult -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelector -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelectorResult -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT1 -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT2 -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.DeviceRoisExtractor -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT1 -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT2 +import xyz.sevive.arcaeaoffline.core.ocr.device.DeviceOcrScenario import xyz.sevive.arcaeaoffline.core.ocr.device.toPlayResult import xyz.sevive.arcaeaoffline.core.ocr.opencv.use import xyz.sevive.arcaeaoffline.helpers.context.getFilename @@ -63,38 +55,8 @@ object DeviceOcrHelper { return MatOfByte(*byteArray).use { matOfBytes -> Imgcodecs.imdecode(matOfBytes, Imgcodecs.IMREAD_COLOR).use { img -> - val roisAutoType = DeviceRoisAutoSelector.select(img) - - CropBlackEdges.crop(img).use { imgCropped -> - val rois = - when (roisAutoType) { - DeviceRoisAutoSelectorResult.T1 -> { - DeviceRoisAutoT1( - imgCropped.width(), - imgCropped.height(), - ) - } - - else -> { - DeviceRoisAutoT2( - imgCropped.width(), - imgCropped.height(), - ) - } - } - val extractor = DeviceRoisExtractor(rois, imgCropped) - val masker = - when (roisAutoType) { - DeviceRoisAutoSelectorResult.T1 -> DeviceRoisMaskerAutoT1() - else -> DeviceRoisMaskerAutoT2() - } - - DeviceOcr( - extractor = extractor, - masker = masker, - ortSession = ortSession, - hashesDb = imageHashesDatabase, - ).ocr() + DeviceOcrScenario.forImage(img).use { scenario -> + scenario.ocr(ortSession, imageHashesDatabase) } } } diff --git a/core/AGENTS.md b/core/AGENTS.md index 5bcdb8ff..9f86ed9b 100644 --- a/core/AGENTS.md +++ b/core/AGENTS.md @@ -10,8 +10,8 @@ Kotlin sources live under `src/main/kotlin/`. ``` xyz.sevive.arcaeaoffline.core/ ├── ocr/ -│ ├── ImageHashers, ImageHashesDatabase(+Builder), ImageHashItem -│ ├── device/ DeviceOcr, ONNX helper, ScreenshotDetect, performance benchmark +│ ├── ImageHashers, ImageHashesDatabase(+Builder), ImageHashItem, OcrScenario +│ ├── device/ DeviceOcr, DeviceOcrScenario, ONNX helper, ScreenshotDetect, performance benchmark │ │ └── rois/ ROI definition / extractor / masker │ └── opencv/ OpenCV helpers └── di/CoreModule.kt diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt new file mode 100644 index 00000000..69c25ee0 --- /dev/null +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt @@ -0,0 +1,9 @@ +package xyz.sevive.arcaeaoffline.core.ocr + +/** + * Marker interface for OCR scenarios. + * + * Scenario branches define their own result contracts, so no callable behavior lives here. + * This interface provides only a common supertype for scenario implementations. + */ +interface OcrScenario diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt new file mode 100644 index 00000000..b14c9ac2 --- /dev/null +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt @@ -0,0 +1,60 @@ +package xyz.sevive.arcaeaoffline.core.ocr.device + +import ai.onnxruntime.OrtSession +import org.opencv.core.Mat +import xyz.sevive.arcaeaoffline.core.ocr.ImageHashesDatabase +import xyz.sevive.arcaeaoffline.core.ocr.OcrScenario +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelector +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelectorResult +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.DeviceRoisExtractor +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMasker + +/** + * A device screenshot OCR scenario: owns the black-edge-cropped image and the extractor + * and masker built from it. The [extractor]'s ROI properties are submat views of the + * cropped image and are invalid after [close]. + */ +class DeviceOcrScenario private constructor( + private val imgCropped: Mat, + val extractor: DeviceRoisExtractor, + private val masker: DeviceRoisMasker, +) : OcrScenario, + AutoCloseable { + companion object { + /** + * Builds a scenario from a raw BGR screenshot. The rois type (T1/T2) is auto-detected + * on the black-edge-cropped image, and the matching rois/masker pair is constructed. + * + * @param fallback Selected when every detector reports zero confidence. + * @throws IllegalArgumentException when the rois type cannot be determined and + * [fallback] is null. + */ + fun forImage( + imgBgr: Mat, + fallback: DeviceRoisAutoSelectorResult? = null, + ): DeviceOcrScenario { + val imgCropped = CropBlackEdges.cropOrOriginal(imgBgr) + val roisType = DeviceRoisAutoSelector.select(imgCropped, fallback) + require(roisType != DeviceRoisAutoSelectorResult.UNKNOWN) { + "Cannot determine device rois type; pass an explicit fallback" + } + + val (rois, masker) = + DeviceRoisAutoSelector.createRoisAndMasker( + roisType, + imgCropped.width(), + imgCropped.height(), + ) + return DeviceOcrScenario(imgCropped, DeviceRoisExtractor(rois, imgCropped), masker) + } + } + + fun ocr( + ortSession: OrtSession, + hashesDb: ImageHashesDatabase, + ): DeviceOcrResult = DeviceOcr(extractor, masker, ortSession, hashesDb).ocr() + + override fun close() { + imgCropped.release() + } +} diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/OcrPerformanceBenchmark.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/OcrPerformanceBenchmark.kt index 075bbf41..8334cbda 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/OcrPerformanceBenchmark.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/OcrPerformanceBenchmark.kt @@ -8,11 +8,7 @@ import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.launch import org.opencv.core.Mat -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelector -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.DeviceRoisAutoSelectorResult -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT1 -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT2 -import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.DeviceRoisExtractor +import xyz.sevive.arcaeaoffline.core.ocr.device.DeviceOcrScenario import xyz.sevive.arcaeaoffline.core.ocr.opencv.use import java.util.Locale import kotlin.system.measureTimeMillis @@ -45,16 +41,10 @@ object OcrPerformanceBenchmark { * ownership transfers to the caller. */ fun extractRois(img: Mat): List = - CropBlackEdges.crop(img).use { imgCropped -> - val rois = - when (DeviceRoisAutoSelector.select(img)) { - DeviceRoisAutoSelectorResult.T1 -> DeviceRoisAutoT1(imgCropped.width(), imgCropped.height()) - else -> DeviceRoisAutoT2(imgCropped.width(), imgCropped.height()) - } - val extractor = DeviceRoisExtractor(rois, imgCropped) - - // extractor properties return submat views of imgCropped; clone them - // so the returned Mats stay valid after imgCropped is released. + DeviceOcrScenario.forImage(img).use { scenario -> + // extractor properties return submat views of the scenario's cropped image + // clone them so the returned Mats stay valid after the scenario closes + val extractor = scenario.extractor listOf(extractor.pure, extractor.far, extractor.lost, extractor.score, extractor.maxRecall).map { view -> view.use { it.clone() } } diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt index db38a005..c4ced1c6 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt @@ -6,10 +6,13 @@ import org.opencv.core.Mat import org.opencv.core.Rect import org.opencv.core.Scalar import org.opencv.imgproc.Imgproc +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRois import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT1 import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT2 import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.doubleArrayRoundToRect +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMasker import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT1 +import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT2 enum class DeviceRoisAutoSelectorResult { T1, T2, UNKNOWN } @@ -202,4 +205,28 @@ object DeviceRoisAutoSelector { if (results.all { it.value == 0.0 }) return fallback ?: DeviceRoisAutoSelectorResult.UNKNOWN return results.maxBy { it.value }.key.result } + + /** + * Creates the rois/masker pair matching the selector [result]. + * + * @throws IllegalArgumentException when [result] is [DeviceRoisAutoSelectorResult.UNKNOWN]. + */ + fun createRoisAndMasker( + result: DeviceRoisAutoSelectorResult, + w: Int, + h: Int, + ): Pair = + when (result) { + DeviceRoisAutoSelectorResult.T1 -> { + DeviceRoisAutoT1(w, h) to DeviceRoisMaskerAutoT1() + } + + DeviceRoisAutoSelectorResult.T2 -> { + DeviceRoisAutoT2(w, h) to DeviceRoisMaskerAutoT2() + } + + DeviceRoisAutoSelectorResult.UNKNOWN -> { + throw IllegalArgumentException("No rois/masker pair for $result") + } + } } From 4c97e879ebd7f5e71f58bc15deee883658f4ed5e Mon Sep 17 00:00:00 2001 From: 283375 Date: Thu, 12 Nov 2026 03:31:43 +0800 Subject: [PATCH 02/12] fix boundary problems --- .../core/ocr/device/CropBlackEdges.kt | 29 +++++++++++-------- .../core/ocr/device/DeviceOcr.kt | 17 +++++++---- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt index cf04df84..2ea2af1b 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt @@ -6,7 +6,6 @@ import org.opencv.core.Mat import org.opencv.core.Rect import org.opencv.core.Scalar import org.opencv.imgproc.Imgproc -import xyz.sevive.arcaeaoffline.core.ocr.device.CropBlackEdges.Companion.cropOrOriginal import xyz.sevive.arcaeaoffline.core.ocr.opencv.use class CropBlackEdges { @@ -42,42 +41,48 @@ class CropBlackEdges { var top = 0 var bottom = height - // submat views are passed directly instead of pixel-copying clones; + // submat views are passed directly instead of pixel-copying clones. // isBlackEdge only reads the slice - for (i in 0..width) { + for (i in 0 until width) { val isBlack = imgGray.submat(Rect(i, 0, 1, height)).use { isBlackEdge(it, blackPixelThreshold) } if (!isBlack) break left += 1 } - for (i in width downTo 0) { - val isBlack = imgGray.submat(Rect(i - 1, 0, 1, height)).use { isBlackEdge(it, blackPixelThreshold) } + // never scans past left + 1, so the crop spans at least two columns once a + // non-black column exists + for (i in width - 1 downTo 0) { + if (i <= left + 1) break + val isBlack = imgGray.submat(Rect(i, 0, 1, height)).use { isBlackEdge(it, blackPixelThreshold) } if (!isBlack) break right -= 1 } - for (i in 0..height) { + for (i in 0 until height) { val isBlack = imgGray.submat(Rect(0, i, width, 1)).use { isBlackEdge(it, blackPixelThreshold) } if (!isBlack) break top += 1 } - for (i in height downTo 0) { - val isBlack = imgGray.submat(Rect(0, i - 1, width, 1)).use { isBlackEdge(it, blackPixelThreshold) } + for (i in height - 1 downTo 0) { + if (i <= top + 1) break + val isBlack = imgGray.submat(Rect(0, i, width, 1)).use { isBlackEdge(it, blackPixelThreshold) } if (!isBlack) break bottom -= 1 } - assert(right > left) { "CropBlackEdges: cropped width <= 0" } - assert(bottom > top) { "CropBlackEdges: cropped height <= 0" } + require(right > left) { "CropBlackEdges: cropped width <= 0" } + require(bottom > top) { "CropBlackEdges: cropped height <= 0" } return Rect(left, top, right - left, bottom - top) } /** - * This function would fail if the cropped image has a width or height below 0. + * Crops the black edges of [img]. + * + * @throws IllegalArgumentException when the image is fully black. * - * Use [cropOrOriginal] if you don't want to handle errors. + * Use [cropOrOriginal] to fall back to the original image instead of throwing. */ fun crop( img: Mat, diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt index caf2cd38..1b70bb4f 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt @@ -158,14 +158,21 @@ class DeviceOcr( preprocessPartnerIcon(roiGray).use { hashesDb.lookupPartnerIcon(it) } } + /** Parses the ROI's OCR text as an Int. Returns `null` when the OCR result is empty. */ + private fun ocrIntOrNull(roiBgr: Mat): Int? = + roiBgr + .use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession) } + .takeIf { it.isNotEmpty() } + ?.toInt() + fun ocr(): DeviceOcrResult = DeviceOcrResult( ratingClass = ratingClass(), - pure = extractor.pure.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, - far = extractor.far.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, - lost = extractor.lost.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, - score = extractor.score.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, - maxRecall = extractor.maxRecall.use { DeviceOcrOnnxHelper.ocrBgrMat(it, ortSession).toInt() }, + pure = ocrIntOrNull(extractor.pure) ?: 0, + far = ocrIntOrNull(extractor.far) ?: 0, + lost = ocrIntOrNull(extractor.lost) ?: 0, + score = ocrIntOrNull(extractor.score) ?: 0, + maxRecall = ocrIntOrNull(extractor.maxRecall), songIdResults = lookupSongId(), clearStatus = clearStatus(), partnerIdResults = lookupPartnerId(), From 5d0999788d81ab15de3519a02759b08ecb1563f4 Mon Sep 17 00:00:00 2001 From: 283375 Date: Thu, 12 Nov 2026 05:03:58 +0800 Subject: [PATCH 03/12] optimize hash weights --- .../arcaeaoffline/core/ocr/ImageHashItem.kt | 45 ++++++++++++------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashItem.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashItem.kt index a514d3d5..dacf1caf 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashItem.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashItem.kt @@ -25,28 +25,41 @@ data class ImageHashItem( val confidence: Double, ) +/** + * Picks the winning label across per-hash-type lookup results and returns its best entry. + * + * A label's score is the weighted average of its entries' confidences, weighting DCT 4, + * DIFFERENCE 2 and AVERAGE 1. Score ties resolve to the earliest-appearing label. + * + * @return The winning label's highest-confidence entry, with [ImageHashItem.confidence] + * replaced by the label's score rather than the entry's raw confidence. `null` on an + * empty list. + */ fun List.getMostConfidentItem(): ImageHashItem? { if (isEmpty()) return null - val labelConfidenceMap = - this - .map { it.label } - .distinct() - .associateWith { 0.0 } - .toMutableMap() + data class Accumulator( + val weightedConfidenceSum: Double, + val weightSum: Double, + ) - this.forEach { + val labelAccumulators = linkedMapOf() + for ((hashType, _, label, confidence) in this) { val weight = - when (it.hashType) { - ImageHashItemHashType.AVERAGE -> 0.3 - ImageHashItemHashType.DCT -> 0.2 - ImageHashItemHashType.DIFFERENCE -> 0.2 + when (hashType) { + ImageHashItemHashType.AVERAGE -> 1.0 + ImageHashItemHashType.DIFFERENCE -> 2.0 + ImageHashItemHashType.DCT -> 4.0 } - val value = labelConfidenceMap[it.label] ?: 0.0 - labelConfidenceMap[it.label] = value + it.confidence * weight + val acc = labelAccumulators[label] ?: Accumulator(0.0, 0.0) + labelAccumulators[label] = + Accumulator(acc.weightedConfidenceSum + confidence * weight, acc.weightSum + weight) } - return this - .sortedByDescending { it.confidence } - .first { it.label == labelConfidenceMap.maxBy { mapItem -> mapItem.value }.key } + val winner = + labelAccumulators.entries.maxByOrNull { it.value.weightedConfidenceSum / it.value.weightSum } + ?: return null + val fused = winner.value.weightedConfidenceSum / winner.value.weightSum + + return this.filter { it.label == winner.key }.maxByOrNull { it.confidence }?.copy(confidence = fused) } From 54ef3852983893e8f20f8338f23234222df71504 Mon Sep 17 00:00:00 2001 From: 283375 Date: Thu, 12 Nov 2026 05:04:19 +0800 Subject: [PATCH 04/12] fix comments --- .../kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt | 1 - .../xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt | 3 +-- .../sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt index 69c25ee0..ceec53ad 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/OcrScenario.kt @@ -4,6 +4,5 @@ package xyz.sevive.arcaeaoffline.core.ocr * Marker interface for OCR scenarios. * * Scenario branches define their own result contracts, so no callable behavior lives here. - * This interface provides only a common supertype for scenario implementations. */ interface OcrScenario diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt index 2ea2af1b..f2e0c079 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/CropBlackEdges.kt @@ -49,8 +49,7 @@ class CropBlackEdges { left += 1 } - // never scans past left + 1, so the crop spans at least two columns once a - // non-black column exists + // keeps at least a two-column crop whenever a non-black column exists for (i in width - 1 downTo 0) { if (i <= left + 1) break val isBlack = imgGray.submat(Rect(i, 0, 1, height)).use { isBlackEdge(it, blackPixelThreshold) } diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt index b14c9ac2..57b3ac05 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt @@ -23,7 +23,7 @@ class DeviceOcrScenario private constructor( companion object { /** * Builds a scenario from a raw BGR screenshot. The rois type (T1/T2) is auto-detected - * on the black-edge-cropped image, and the matching rois/masker pair is constructed. + * on the black-edge-cropped image. * * @param fallback Selected when every detector reports zero confidence. * @throws IllegalArgumentException when the rois type cannot be determined and From 5edaeccf342b468489710fefb0fff1d1756a91c0 Mon Sep 17 00:00:00 2001 From: 283375 Date: Thu, 12 Nov 2026 05:17:37 +0800 Subject: [PATCH 05/12] fix submat misuse --- .../core/ocr/device/ScreenshotDetect.kt | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/ScreenshotDetect.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/ScreenshotDetect.kt index 91551fcd..4cdc86d0 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/ScreenshotDetect.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/ScreenshotDetect.kt @@ -5,6 +5,8 @@ import org.opencv.core.Mat import org.opencv.core.Rect import org.opencv.core.Scalar import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRoisAutoT2 +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use +import kotlin.math.roundToInt object ScreenshotDetect { private val PURPLE_HSV_LOWER = Scalar(110.0, 45.0, 75.0) @@ -20,17 +22,15 @@ object ScreenshotDetect { ): Boolean { val rois = DeviceRoisAutoT2(imgHsv.width(), imgHsv.height()) - val roiRect = - Rect( - 0, - (rois.layoutAreaHMid + ROI_HEIGHT_OFFSET * rois.factor).toInt(), - (rois.wMid + ROI_WIDTH_OFFSET * rois.factor).toInt(), - (ROI_HEIGHT * rois.factor).toInt(), - ) + val y = (rois.layoutAreaHMid + ROI_HEIGHT_OFFSET * rois.factor).roundToInt() + val w = (rois.wMid + ROI_WIDTH_OFFSET * rois.factor).roundToInt() + val h = (ROI_HEIGHT * rois.factor).roundToInt() - val roi = imgHsv.submat(roiRect) - Core.inRange(roi, PURPLE_HSV_LOWER, PURPLE_HSV_UPPER, roi) - - return Core.countNonZero(roi) >= roi.width() * roi.height() * ratio + Mat().use { mask -> + imgHsv.submat(Rect(0, y, w, h)).use { roi -> + Core.inRange(roi, PURPLE_HSV_LOWER, PURPLE_HSV_UPPER, mask) + } + return Core.countNonZero(mask) >= w * h * ratio + } } } From df071733975b66cd5d62fcdf6554f9e4d930bcdc Mon Sep 17 00:00:00 2001 From: 283375 Date: Thu, 12 Nov 2026 23:56:05 +0800 Subject: [PATCH 06/12] port json layout definitions --- core/AGENTS.md | 1 + .../ocr/device/rois/DeviceRoisAutoSelector.kt | 119 +++----- .../ocr/device/rois/definition/DeviceRois.kt | 8 + .../device/rois/definition/DeviceRoisAuto.kt | 284 +++++------------- .../core/ocr/layouts/DeviceLayout.kt | 64 ++++ .../main/resources/ocr/layouts/device/t1.json | 21 ++ .../main/resources/ocr/layouts/device/t2.json | 21 ++ .../core/ocr/layouts/DeviceLayoutTest.kt | 54 ++++ 8 files changed, 285 insertions(+), 287 deletions(-) create mode 100644 core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayout.kt create mode 100644 core/src/main/resources/ocr/layouts/device/t1.json create mode 100644 core/src/main/resources/ocr/layouts/device/t2.json create mode 100644 core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayoutTest.kt diff --git a/core/AGENTS.md b/core/AGENTS.md index 9f86ed9b..e01d0cfb 100644 --- a/core/AGENTS.md +++ b/core/AGENTS.md @@ -13,6 +13,7 @@ xyz.sevive.arcaeaoffline.core/ │ ├── ImageHashers, ImageHashesDatabase(+Builder), ImageHashItem, OcrScenario │ ├── device/ DeviceOcr, DeviceOcrScenario, ONNX helper, ScreenshotDetect, performance benchmark │ │ └── rois/ ROI definition / extractor / masker +│ ├── layouts/ DeviceLayout: bundled T1/T2 layout JSON and its screen mapping │ └── opencv/ OpenCV helpers └── di/CoreModule.kt ``` diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt index c4ced1c6..5c02acd4 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/DeviceRoisAutoSelector.kt @@ -3,7 +3,6 @@ package xyz.sevive.arcaeaoffline.core.ocr.device.rois import co.touchlab.kermit.Logger import org.opencv.core.Core import org.opencv.core.Mat -import org.opencv.core.Rect import org.opencv.core.Scalar import org.opencv.imgproc.Imgproc import xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition.DeviceRois @@ -13,6 +12,7 @@ import xyz.sevive.arcaeaoffline.core.ocr.device.rois.extractor.doubleArrayRoundT import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMasker import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT1 import xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker.DeviceRoisMaskerAutoT2 +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use enum class DeviceRoisAutoSelectorResult { T1, T2, UNKNOWN } @@ -38,7 +38,6 @@ private object T1Detector : DeviceRoisAutoSelectorDetector { private const val INTEREST_POINTS = 3.0 - private const val PFL_LABEL_WIDTH = 85 private val pureLabelHsvLower = Scalar(80.0, 60.0, 125.0) private val pureLabelHsvUpper = Scalar(110.0, 200.0, 225.0) @@ -46,49 +45,20 @@ private object T1Detector : DeviceRoisAutoSelectorDetector { private const val FAR_LABEL_NON_ZERO_RATIO_THRESHOLD = 0.04 private const val LOST_LABEL_NON_ZERO_RATIO_THRESHOLD = 0.055 - private val masker = DeviceRoisMaskerAutoT1() - - private fun getPflLabelRectFromPflRoiRect( - array: DoubleArray, - labelWidth: Double, - ): Rect = - doubleArrayRoundToRect( - doubleArrayOf(array[0] - labelWidth, array[1], labelWidth, array[3]), - ) - - private fun getPflLabelBgr( - imgBgr: Mat, - roiRect: DoubleArray, - ): Mat { - val rois = DeviceRoisAutoT1(imgBgr.width(), imgBgr.height()) - val pflLabelWidth = PFL_LABEL_WIDTH * rois.factor - - val labelRect = getPflLabelRectFromPflRoiRect(roiRect, pflLabelWidth) - return imgBgr.submat(labelRect).clone() - } + private val grayMasker = DeviceRoisMaskerAutoT1() private fun maskPureLabel(roiBgr: Mat): Mat { - val roiHsv = Mat() - Imgproc.cvtColor(roiBgr, roiHsv, Imgproc.COLOR_BGR2HSV) val roiMasked = Mat() - Core.inRange(roiHsv, pureLabelHsvLower, pureLabelHsvUpper, roiMasked) + Mat().use { roiHsv -> + Imgproc.cvtColor(roiBgr, roiHsv, Imgproc.COLOR_BGR2HSV) + Core.inRange(roiHsv, pureLabelHsvLower, pureLabelHsvUpper, roiMasked) + } return roiMasked } - private fun maskFarLostLabel(roiBgr: Mat): Mat = masker.gray(roiBgr) - - private fun pflLabelMatches( - labelMasked: Mat, - nonZeroRatioThreshold: Double, - ): Boolean { - val labelSize = labelMasked.size().height * labelMasked.size().width - val nonZeroRatio = Core.countNonZero(labelMasked) / labelSize - return nonZeroRatio >= nonZeroRatioThreshold - } - @Suppress("ArrayInDataClass") - private data class PflLabelInterest( - val roiRect: DoubleArray, + private data class Interest( + val rect: DoubleArray, val masker: (Mat) -> Mat, val nonZeroRatioThreshold: Double, ) @@ -99,13 +69,16 @@ private object T1Detector : DeviceRoisAutoSelectorDetector { val rois = DeviceRoisAutoT1(imgBgr.width(), imgBgr.height()) listOf( - PflLabelInterest(rois.pure, ::maskPureLabel, PURE_LABEL_NON_ZERO_RATIO_THRESHOLD), - PflLabelInterest(rois.far, ::maskFarLostLabel, FAR_LABEL_NON_ZERO_RATIO_THRESHOLD), - PflLabelInterest(rois.lost, ::maskFarLostLabel, LOST_LABEL_NON_ZERO_RATIO_THRESHOLD), - ).forEach { - val labelBgr = getPflLabelBgr(imgBgr, it.roiRect) - val labelMasked = it.masker(labelBgr) - if (pflLabelMatches(labelMasked, it.nonZeroRatioThreshold)) matches += 1 + Interest(rois.pureLabel, ::maskPureLabel, PURE_LABEL_NON_ZERO_RATIO_THRESHOLD), + Interest(rois.farLabel, grayMasker::gray, FAR_LABEL_NON_ZERO_RATIO_THRESHOLD), + Interest(rois.lostLabel, grayMasker::gray, LOST_LABEL_NON_ZERO_RATIO_THRESHOLD), + ).forEach { (rect, masker, threshold) -> + imgBgr.submat(doubleArrayRoundToRect(rect)).use { labelBgr -> + masker(labelBgr).use { labelMasked -> + val labelSize = labelMasked.size().height * labelMasked.size().width + if (Core.countNonZero(labelMasked) / labelSize >= threshold) matches += 1 + } + } } return matches / INTEREST_POINTS @@ -117,7 +90,6 @@ private object T2Detector : DeviceRoisAutoSelectorDetector { private const val INTEREST_POINTS = 3.0 - private const val PFL_LABEL_WIDTH = 180 private val pureLabelHsvLower = Scalar(110.0, 25.0, 90.0) private val pureLabelHsvUpper = Scalar(160.0, 150.0, 230.0) private val farLabelHsvLower = Scalar(5.0, 25.0, 120.0) @@ -127,52 +99,35 @@ private object T2Detector : DeviceRoisAutoSelectorDetector { private const val PFL_LABEL_NON_ZERO_RATIO_THRESHOLD = 0.3 - private fun getPflLabelRectFromPflRoiRect( - array: DoubleArray, - labelWidth: Double, - ): Rect = - doubleArrayRoundToRect( - doubleArrayOf(array[0] - labelWidth, array[1], labelWidth, array[3]), - ) - - private fun pflLabelMatches( - imgBgr: Mat, - roiRect: DoubleArray, - hsvLower: Scalar, - hsvUpper: Scalar, - ): Boolean { - val rois = DeviceRoisAutoT2(imgBgr.width(), imgBgr.height()) - val pflLabelWidth = PFL_LABEL_WIDTH * rois.factor - val labelRect = getPflLabelRectFromPflRoiRect(roiRect, pflLabelWidth) - - val labelBgr = imgBgr.submat(labelRect).clone() - val labelHsv = Mat() - Imgproc.cvtColor(labelBgr, labelHsv, Imgproc.COLOR_BGR2HSV) - val labelMasked = Mat() - Core.inRange(labelHsv, hsvLower, hsvUpper, labelMasked) - - val labelSize = labelMasked.size().height * labelMasked.size().width - val nonZeroRatio = Core.countNonZero(labelMasked) / labelSize - return nonZeroRatio >= PFL_LABEL_NON_ZERO_RATIO_THRESHOLD - } - @Suppress("ArrayInDataClass") - private data class PflLabelInterest( - val roiRect: DoubleArray, + private data class Interest( + val rect: DoubleArray, val hsvLower: Scalar, val hsvUpper: Scalar, ) override fun confidence(imgBgr: Mat): Double { var matches = 0 + val rois = DeviceRoisAutoT2(imgBgr.width(), imgBgr.height()) listOf( - PflLabelInterest(rois.pure, pureLabelHsvLower, pureLabelHsvUpper), - PflLabelInterest(rois.far, farLabelHsvLower, farLabelHsvUpper), - PflLabelInterest(rois.lost, lostLabelHsvLower, lostLabelHsvUpper), - ).forEach { - if (pflLabelMatches(imgBgr, it.roiRect, it.hsvLower, it.hsvUpper)) matches += 1 + Interest(rois.pureLabel, pureLabelHsvLower, pureLabelHsvUpper), + Interest(rois.farLabel, farLabelHsvLower, farLabelHsvUpper), + Interest(rois.lostLabel, lostLabelHsvLower, lostLabelHsvUpper), + ).forEach { (rect, hsvLower, hsvUpper) -> + imgBgr.submat(doubleArrayRoundToRect(rect)).use { labelBgr -> + Mat().use { labelHsv -> + Imgproc.cvtColor(labelBgr, labelHsv, Imgproc.COLOR_BGR2HSV) + Mat().use { labelMasked -> + Core.inRange(labelHsv, hsvLower, hsvUpper, labelMasked) + val labelSize = labelMasked.size().height * labelMasked.size().width + if (Core.countNonZero(labelMasked) / labelSize >= PFL_LABEL_NON_ZERO_RATIO_THRESHOLD) { + matches += 1 + } + } + } + } } return matches / INTEREST_POINTS diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRois.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRois.kt index ea171c82..36a4ab51 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRois.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRois.kt @@ -1,5 +1,6 @@ package xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition +/** ROI windows on a screenshot. Every value is `[x, y, w, h]`. */ interface DeviceRois { val pure: DoubleArray val far: DoubleArray @@ -10,4 +11,11 @@ interface DeviceRois { val jacket: DoubleArray val clearStatus: DoubleArray val partnerIcon: DoubleArray + val highScore: DoubleArray + val highScoreDelta: DoubleArray + val pureShiny: DoubleArray + val earlyLate: DoubleArray + val pureLabel: DoubleArray + val farLabel: DoubleArray + val lostLabel: DoubleArray } diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRoisAuto.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRoisAuto.kt index d808a159..b4ea9b29 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRoisAuto.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/definition/DeviceRoisAuto.kt @@ -1,6 +1,6 @@ package xyz.sevive.arcaeaoffline.core.ocr.device.rois.definition -import kotlin.math.max +import xyz.sevive.arcaeaoffline.core.ocr.layouts.DeviceLayout abstract class DeviceRoisAuto( open val w: Int, @@ -9,240 +9,114 @@ abstract class DeviceRoisAuto( abstract val factor: Double } -class DeviceRoisAutoT1( - override val w: Int, - override val h: Int, +/** + * Rois whose field windows come from a [DeviceLayout] mapped onto the screenshot. + * + * The one exception is [partnerIcon]: it pins to the physical screen top and has no + * window on the layout canvas. + */ +abstract class DeviceRoisFromLayout( + w: Int, + h: Int, ) : DeviceRoisAuto(w, h) { - override val factor: Double - get() { - return if (this.w.toDouble() / this.h.toDouble() < 16.0 / 9.0) ((this.w / 16.0) * 9.0) / 720.0 else this.h / 720.0 - } + protected abstract val layout: DeviceLayout - private val wMid: Double - get() = this.w / 2.0 - -// val hMid: Double -// get() = this.h / 2.0 - - private val topBar: List - get() = listOf(0.0, 0.0, this.w.toDouble(), 50.0 * this.factor) - - private val layoutAreaHMid: Double - get() = this.h / 2.0 + this.topBar[3] - - private val pflLeftFromWMid: Double - get() = 5 * this.factor - - private val pflX: Double - get() = this.wMid + this.pflLeftFromWMid - - private val pflW: Double - get() = 76 * this.factor - - private val pflH: Double - get() = 26 * this.factor - - override val pure - get() = - doubleArrayOf( - this.pflX, - this.layoutAreaHMid + 110.0 * this.factor, - this.pflW, - this.pflH, - ) - - override val far - get() = - doubleArrayOf( - this.pflX, - this.pure[1] + this.pure[3] + 12.0 * this.factor, - this.pflW, - this.pflH, - ) - - override val lost - get() = - doubleArrayOf( - this.pflX, - this.far[1] + this.far[3] + 10.0 * this.factor, - this.pflW, - this.pflH, - ) + override val pure: DoubleArray + get() = layout.screenRect("pure", w, h) + + override val far: DoubleArray + get() = layout.screenRect("far", w, h) + + override val lost: DoubleArray + get() = layout.screenRect("lost", w, h) override val score: DoubleArray - get() { - val w = 280 * this.factor - val h = 45 * this.factor - return doubleArrayOf( - this.wMid - w / 2, - this.layoutAreaHMid - 75 * this.factor - h, - w, - h, - ) - } + get() = layout.screenRect("score", w, h) override val ratingClass: DoubleArray - get() = - doubleArrayOf( - this.wMid - 610 * this.factor, - this.layoutAreaHMid - 180 * this.factor, - 265 * this.factor, - 35 * this.factor, - ) + get() = layout.screenRect("rating_class", w, h) override val maxRecall: DoubleArray - get() = - doubleArrayOf( - this.wMid - 465 * this.factor, - this.layoutAreaHMid - 215 * this.factor, - 150 * this.factor, - 35 * this.factor, - ) + get() = layout.screenRect("max_recall", w, h) override val jacket: DoubleArray - get() = - doubleArrayOf( - this.wMid - 610 * this.factor, - this.layoutAreaHMid - 143 * this.factor, - 375 * this.factor, - 375 * this.factor, - ) + get() = layout.screenRect("jacket", w, h) override val clearStatus: DoubleArray - get() { - val w = 550 * this.factor - val h = 60 * this.factor - return doubleArrayOf( - this.wMid - w / 2, - this.layoutAreaHMid - 155 * this.factor - h, - w, - h, - ) - } - - override val partnerIcon: DoubleArray - get() { - val w = 90 * this.factor - val h = 75 * this.factor - return doubleArrayOf(this.wMid - w / 2, 0.0, w, h) - } -} - -class DeviceRoisAutoT2( - override val w: Int, - override val h: Int, -) : DeviceRoisAuto(w, h) { - override val factor: Double - get() { - return if (this.w.toDouble() / this.h.toDouble() < 16.0 / 9.0) ((this.w / 16.0) * 9.0) / 1080.0 else this.h / 1080.0 - } - - val wMid: Double - get() = this.w / 2.0 - -// val hMid: Double -// get() = this.h / 2.0 + get() = layout.screenRect("clear_status", w, h) - private val topBar: List - get() = listOf(0.0, 0.0, this.w.toDouble(), 75.0 * this.factor) + override val highScore: DoubleArray + get() = layout.screenRect("high_score", w, h) - val layoutAreaHMid: Double - get() = this.h / 2.0 + this.topBar[3] + override val highScoreDelta: DoubleArray + get() = layout.screenRect("high_score_delta", w, h) -// val pflMidFromWMid: Double -// get() = 60 * this.factor + override val pureShiny: DoubleArray + get() = layout.screenRect("pure_shiny", w, h) - private val pflX: Double - get() = this.wMid + 10 * this.factor + override val earlyLate: DoubleArray + get() = layout.screenRect("early_late", w, h) - private val pflW: Double - get() = 100 * this.factor + override val pureLabel: DoubleArray + get() = layout.screenRect("pure_label", w, h) - private val pflH: Double - get() = 24 * this.factor + override val farLabel: DoubleArray + get() = layout.screenRect("far_label", w, h) - override val pure: DoubleArray - get() = - doubleArrayOf( - this.pflX, - this.layoutAreaHMid + 175.0 * this.factor, - this.pflW, - this.pflH, - ) + override val lostLabel: DoubleArray + get() = layout.screenRect("lost_label", w, h) +} - override val far: DoubleArray - get() = - doubleArrayOf( - this.pflX, - this.pure[1] + this.pure[3] + 30.0 * this.factor, - this.pflW, - this.pflH, - ) +class DeviceRoisAutoT1( + override val w: Int, + override val h: Int, +) : DeviceRoisFromLayout(w, h) { + companion object { + private val LAYOUT = DeviceLayout.loadBuiltin("t1") + } - override val lost: DoubleArray - get() = - doubleArrayOf( - this.pflX, - this.far[1] + this.far[3] + 35.0 * this.factor, - this.pflW, - this.pflH, - ) + override val layout: DeviceLayout + get() = LAYOUT - override val score: DoubleArray + override val partnerIcon: DoubleArray get() { - val w = 420.0 * this.factor - val h = 70.0 * this.factor - return doubleArrayOf( - this.wMid - w / 2.0, - this.layoutAreaHMid - 110 * this.factor - h, - w, - h, - ) + val w = 90.0 * factor + val h = 75.0 * factor + return doubleArrayOf(wMid - w / 2, 0.0, w, h) } - override val ratingClass: DoubleArray - get() = - doubleArrayOf( - max(0.0, this.wMid - 965 * this.factor), - this.layoutAreaHMid - 330 * this.factor, - 350 * this.factor, - 110 * this.factor, - ) + override val factor: Double + get() = if (w.toDouble() / h < 16.0 / 9.0) (w / 16.0) * 9.0 / 720.0 else h / 720.0 - override val maxRecall: DoubleArray - get() = - doubleArrayOf( - this.wMid - 625 * this.factor, - this.layoutAreaHMid - 275 * this.factor, - 150 * this.factor, - 50 * this.factor, - ) + private val wMid: Double + get() = w / 2.0 +} - override val jacket: DoubleArray - get() = - doubleArrayOf( - this.wMid - 915 * this.factor, - this.layoutAreaHMid - 215 * this.factor, - 565 * this.factor, - 565 * this.factor, - ) +class DeviceRoisAutoT2( + override val w: Int, + override val h: Int, +) : DeviceRoisFromLayout(w, h) { + companion object { + private val LAYOUT = DeviceLayout.loadBuiltin("t2") + } - override val clearStatus: DoubleArray - get() { - val w = 825 * this.factor - val h = 90 * this.factor - return doubleArrayOf( - this.wMid - w / 2, - this.layoutAreaHMid - 235 * this.factor - h, - w, - h, - ) - } + override val layout: DeviceLayout + get() = LAYOUT override val partnerIcon: DoubleArray get() { - val w = 135 * this.factor - val h = 110 * this.factor - return doubleArrayOf(this.wMid - w / 2, 0.0, w, h) + val w = 135.0 * factor + val h = 110.0 * factor + return doubleArrayOf(wMid - w / 2, 0.0, w, h) } + + // screen-anchored helpers: ScreenshotDetect positions its probe against them + override val factor: Double + get() = if (w.toDouble() / h < 16.0 / 9.0) (w / 16.0) * 9.0 / 1080.0 else h / 1080.0 + + val wMid: Double + get() = w / 2.0 + + val layoutAreaHMid: Double + get() = h / 2.0 + 75.0 * factor } diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayout.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayout.kt new file mode 100644 index 00000000..d803ea35 --- /dev/null +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayout.kt @@ -0,0 +1,64 @@ +package xyz.sevive.arcaeaoffline.core.ocr.layouts + +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import kotlin.math.max +import kotlin.math.min + +/** + * A device layout definition: ROI field windows authored on a design canvas. + * + * Values in [fields] are `[x, y, w, h]` from the top-left corner of the canvas sized + * [design], e.g. `[1280, 720]` for a 16:9 design. + */ +@Serializable +data class DeviceLayout( + val design: List, + val fields: Map>, +) { + /** + * Maps a field window onto a `w * h` screenshot. + * + * Screens with the canvas aspect ratio or wider scale the canvas by height and + * center it horizontally. Narrower ones scale by width and center it vertically. + * The mapped x is never negative. + * + * @throws NoSuchElementException when [field] is not in [fields]. + */ + fun screenRect( + field: String, + w: Int, + h: Int, + ): DoubleArray { + val (xDesign, yDesign, wDesign, hDesign) = fields.getValue(field) + val (designW, designH) = design + val factor = min(w.toDouble() / designW, h.toDouble() / designH) + val x = max(0.0, w / 2.0 + (xDesign - designW / 2.0) * factor) + val y = + if (w.toDouble() / h >= designW.toDouble() / designH) { + yDesign * factor + } else { + h / 2.0 + (yDesign - designH / 2.0) * factor + } + return doubleArrayOf(x, y, wDesign * factor, hDesign * factor) + } + + companion object { + private val json = Json { ignoreUnknownKeys = true } + + /** + * Parses the bundled layout JSON of [name] from `ocr/layouts/device/`. + * + * @throws IllegalStateException when no layout of [name] is bundled. + */ + fun loadBuiltin(name: String): DeviceLayout { + val text = + DeviceLayout::class.java + .getResourceAsStream("/ocr/layouts/device/$name.json") + ?.readBytes() + ?.decodeToString() + ?: throw IllegalStateException("No bundled layout named $name") + return json.decodeFromString(text) + } + } +} diff --git a/core/src/main/resources/ocr/layouts/device/t1.json b/core/src/main/resources/ocr/layouts/device/t1.json new file mode 100644 index 00000000..f9fa3949 --- /dev/null +++ b/core/src/main/resources/ocr/layouts/device/t1.json @@ -0,0 +1,21 @@ +{ + "id": "t1", + "design": [1280, 720], + "fields": { + "pure": [645, 520, 76, 26], + "far": [645, 558, 76, 26], + "lost": [645, 594, 76, 26], + "score": [500, 290, 280, 45], + "rating_class": [30, 230, 265, 35], + "max_recall": [175, 195, 150, 35], + "jacket": [30, 267, 375, 375], + "clear_status": [365, 195, 220, 60], + "high_score": [576, 353, 186, 40], + "high_score_delta": [770, 353, 118, 40], + "pure_shiny": [716, 521, 62, 30], + "early_late": [714, 556, 275, 32], + "pure_label": [572, 516.57, 61, 37], + "far_label": [589, 552.8, 44, 37], + "lost_label": [574, 589.57, 59, 37] + } +} diff --git a/core/src/main/resources/ocr/layouts/device/t2.json b/core/src/main/resources/ocr/layouts/device/t2.json new file mode 100644 index 00000000..466f082b --- /dev/null +++ b/core/src/main/resources/ocr/layouts/device/t2.json @@ -0,0 +1,21 @@ +{ + "id": "t2", + "design": [1280, 720], + "fields": { + "pure": [646.6666666666666, 526.6666666666666, 66.66666666666667, 16], + "far": [646.6666666666666, 562.6666666666666, 66.66666666666667, 16], + "lost": [646.6666666666666, 602, 66.66666666666667, 16], + "score": [500, 290, 280, 46.666666666666664], + "rating_class": [-3.3333333333333712, 190, 233.33333333333334, 73.33333333333333], + "max_recall": [223.33333333333334, 226.66666666666666, 100, 33.333333333333336], + "jacket": [30, 266.6666666666667, 376.66666666666663, 376.66666666666663], + "clear_status": [365, 193.33333333333334, 220, 60], + "high_score": [626.6666666666666, 353.3333333333333, 126.66666666666667, 36.666666666666664], + "high_score_delta": [766.6666666666666, 353.3333333333333, 106.66666666666667, 36.666666666666664], + "pure_shiny": [715.3333333333334, 515.3333333333334, 72, 37.333333333333336], + "early_late": [714.6666666666666, 552, 183.33333333333334, 41.333333333333336], + "pure_label": [574, 519, 55, 32], + "far_label": [582, 555.47, 40, 32], + "lost_label": [575, 592.5, 54, 32] + } +} diff --git a/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayoutTest.kt b/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayoutTest.kt new file mode 100644 index 00000000..4f41ee91 --- /dev/null +++ b/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/layouts/DeviceLayoutTest.kt @@ -0,0 +1,54 @@ +package xyz.sevive.arcaeaoffline.core.ocr.layouts + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Test + +class DeviceLayoutTest { + companion object { + private const val DELTA = 1e-6 + } + + @Test + fun builtinLayoutsParseWithAllFields() { + for (name in listOf("t1", "t2")) { + val layout = DeviceLayout.loadBuiltin(name) + assertEquals(listOf(1280, 720), layout.design) + assertEquals(15, layout.fields.size) + } + } + + @Test + fun screenRectOnCanvasAspectRatio() { + // 1920x1080 is the design canvas scaled by 1.5 + val layout = DeviceLayout.loadBuiltin("t1") + assertArrayEquals( + doubleArrayOf(967.5, 780.0, 114.0, 39.0), + layout.screenRect("pure", 1920, 1080), + DELTA, + ) + } + + @Test + fun screenRectOnNarrowerScreenCentersVertically() { + // rating_class has a negative design x, so the mapped x clamps to 0 + val layout = DeviceLayout.loadBuiltin("t2") + assertArrayEquals( + doubleArrayOf(0.0, 248.0, 186.666667, 58.666667), + layout.screenRect("rating_class", 1024, 768), + DELTA, + ) + } + + @Test + fun screenRectOnWiderScreenScalesByHeight() { + // wider than the canvas aspect, so the canvas scales by height (1.5) and centers + // horizontally + val layout = DeviceLayout.loadBuiltin("t1") + assertArrayEquals( + doubleArrayOf(1287.5, 780.0, 114.0, 39.0), + layout.screenRect("pure", 2560, 1080), + DELTA, + ) + } +} From 62b3c299dd3d26ef92e88ce0370564871ff9b196 Mon Sep 17 00:00:00 2001 From: 283375 Date: Fri, 13 Nov 2026 01:39:57 +0800 Subject: [PATCH 07/12] DeviceRoisMaskerAuto rework * Rewrite gray masking * Extract hsv masker * Align hsv ranges with upstream --- .../rois/masker/DeviceRoisMaskerAuto.kt | 197 ++++++++---------- 1 file changed, 87 insertions(+), 110 deletions(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt index ac7690bf..49f50358 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt @@ -1,11 +1,25 @@ package xyz.sevive.arcaeaoffline.core.ocr.device.rois.masker import org.opencv.core.Core -import org.opencv.core.CvType import org.opencv.core.Mat import org.opencv.core.Scalar import org.opencv.core.Size import org.opencv.imgproc.Imgproc +import xyz.sevive.arcaeaoffline.core.ocr.opencv.use + +/** Returns a newly allocated mask of [roiBgr] pixels inside the HSV range. Ownership passes to the caller. */ +private fun maskBgrInHsv( + roiBgr: Mat, + hsvLower: Scalar, + hsvUpper: Scalar, +): Mat { + val masked = Mat() + Mat().use { roiHsv -> + Imgproc.cvtColor(roiBgr, roiHsv, Imgproc.COLOR_BGR2HSV) + Core.inRange(roiHsv, hsvLower, hsvUpper, masked) + } + return masked +} interface DeviceRoisMaskerAuto : DeviceRoisMasker @@ -43,100 +57,85 @@ class DeviceRoisMaskerAutoT1 : DeviceRoisMaskerAuto { private val pureMemoryHsvLower = Scalar(90.0, 70.0, 80.0) private val pureMemoryHsvUpper = Scalar(110.0, 200.0, 175.0) - fun gray(roiBgr: Mat): Mat { - val bgrValueEqualMask = Mat.zeros(roiBgr.rows(), roiBgr.cols(), CvType.CV_8UC1) - for (h in 0 until roiBgr.height()) { - for (w in 0 until roiBgr.width()) { - val pixel = roiBgr[h, w] - if (pixel.max() - pixel.min() <= 5.0) { - bgrValueEqualMask.put(h, w, 255.0) - } - } + /** Returns a newly allocated 8UC1 mask with 255 where the BGR channel spread is at most [threshold]. */ + private fun channelSpreadMask( + roiBgr: Mat, + threshold: Double = 5.0, + ): Mat { + val b = Mat() + val g = Mat() + val r = Mat() + val bgDiff = Mat() + val brDiff = Mat() + val grDiff = Mat() + val maxDiff = Mat() + val mask = Mat() + try { + Core.extractChannel(roiBgr, b, 0) + Core.extractChannel(roiBgr, g, 1) + Core.extractChannel(roiBgr, r, 2) + // the largest pairwise diff is the spread (max − min), so this caps it at [threshold] + Core.absdiff(b, g, bgDiff) + Core.absdiff(b, r, brDiff) + Core.absdiff(g, r, grDiff) + Core.max(bgDiff, brDiff, maxDiff) + Core.max(maxDiff, grDiff, maxDiff) + Core.compare(maxDiff, Scalar(threshold), mask, Core.CMP_LE) + } finally { + listOf(b, g, r, bgDiff, brDiff, grDiff, maxDiff).forEach { it.release() } } - - val roiBgrMasked = Mat() - Core.bitwise_and(roiBgr, roiBgr, roiBgrMasked, bgrValueEqualMask) - Core.inRange(roiBgrMasked, grayBgrLower, grayBgrUpper, roiBgrMasked) - return roiBgrMasked + return mask } - private fun pfl(roiBgr: Mat): Mat { - val grayMasked = gray(roiBgr) - Imgproc.dilate( - grayMasked, - grayMasked, - Imgproc.getStructuringElement(Imgproc.MORPH_RECT, Size(2.0, 2.0)), - ) - Imgproc.erode( - grayMasked, - grayMasked, - Imgproc.getStructuringElement(Imgproc.MORPH_RECT, Size(1.0, 1.0)), - ) - return grayMasked + /** + * Masks gray pixels: BGR values within [grayBgrLower, grayBgrUpper] whose channel spread + * (max − min) is at most 5. Pixels outside the spread are zeroed and the image eroded 2×2 + * before the range check. + */ + fun gray(roiBgr: Mat): Mat { + val masked = Mat() + channelSpreadMask(roiBgr).use { mask -> + Core.bitwise_and(roiBgr, roiBgr, masked, mask) + } + Imgproc.getStructuringElement(Imgproc.MORPH_RECT, Size(2.0, 2.0)).use { kernel -> + Imgproc.erode(masked, masked, kernel) + } + Core.inRange(masked, grayBgrLower, grayBgrUpper, masked) + return masked } - override fun pure(roiBgr: Mat): Mat = pfl(roiBgr) + override fun pure(roiBgr: Mat): Mat = gray(roiBgr) - override fun far(roiBgr: Mat): Mat = pfl(roiBgr) + override fun far(roiBgr: Mat): Mat = gray(roiBgr) - override fun lost(roiBgr: Mat): Mat = pfl(roiBgr) + override fun lost(roiBgr: Mat): Mat = gray(roiBgr) override fun maxRecall(roiBgr: Mat): Mat = gray(roiBgr) - private fun maskHsv( - roiBgr: Mat, - hsvLower: Scalar, - hsvUpper: Scalar, - ): Mat { - val roiHsv = Mat() - val dst = Mat() - Imgproc.cvtColor(roiBgr, roiHsv, Imgproc.COLOR_BGR2HSV) - Core.inRange(roiHsv, hsvLower, hsvUpper, dst) - return dst - } - - override fun score(roiBgr: Mat): Mat = maskHsv(roiBgr, whiteHsvLower, whiteHsvUpper) + override fun score(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, whiteHsvLower, whiteHsvUpper) - override fun ratingClassPst(roiBgr: Mat): Mat = maskHsv(roiBgr, pstHsvLower, pstHsvUpper) + override fun ratingClassPst(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pstHsvLower, pstHsvUpper) - override fun ratingClassPrs(roiBgr: Mat): Mat = maskHsv(roiBgr, prsHsvLower, prsHsvUpper) + override fun ratingClassPrs(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, prsHsvLower, prsHsvUpper) - override fun ratingClassFtr(roiBgr: Mat): Mat = maskHsv(roiBgr, ftrHsvLower, ftrHsvUpper) + override fun ratingClassFtr(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, ftrHsvLower, ftrHsvUpper) - override fun ratingClassByd(roiBgr: Mat): Mat = maskHsv(roiBgr, bydHsvLower, bydHsvUpper) + override fun ratingClassByd(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, bydHsvLower, bydHsvUpper) - override fun ratingClassEtr(roiBgr: Mat): Mat = maskHsv(roiBgr, etrHsvMin, etrHsvMax) + override fun ratingClassEtr(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, etrHsvMin, etrHsvMax) - override fun clearStatusTrackLost(roiBgr: Mat): Mat = maskHsv(roiBgr, trackLostHsvLower, trackLostHsvUpper) + override fun clearStatusTrackLost(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, trackLostHsvLower, trackLostHsvUpper) - override fun clearStatusTrackComplete(roiBgr: Mat): Mat = maskHsv(roiBgr, trackCompleteHsvLower, trackCompleteHsvUpper) + override fun clearStatusTrackComplete(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, trackCompleteHsvLower, trackCompleteHsvUpper) - override fun clearStatusFullRecall(roiBgr: Mat): Mat = maskHsv(roiBgr, fullRecallHsvLower, fullRecallHsvUpper) + override fun clearStatusFullRecall(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, fullRecallHsvLower, fullRecallHsvUpper) - override fun clearStatusPureMemory(roiBgr: Mat): Mat = maskHsv(roiBgr, pureMemoryHsvLower, pureMemoryHsvUpper) + override fun clearStatusPureMemory(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pureMemoryHsvLower, pureMemoryHsvUpper) } class DeviceRoisMaskerAutoT2 : DeviceRoisMaskerAuto { - private val pflHsvLower = Scalar(0.0, 0.0, 245.0) - private val pflHsvUpper = Scalar(179.0, 30.0, 255.0) - -// private val pureSupplementHsvLower = Scalar(120.0, 0.0, 180.0) -// private val pureSupplementHsvUpper = Scalar(135.0, 30.0, 200.0) -// -// private val farSupplementHsvLower = Scalar(0.0, 0.0, 160.0) -// private val farSupplementHsvUpper = Scalar(30.0, 90.0, 225.0) -// -// private val lostSupplementHsvLower = Scalar(145.0, 0.0, 130.0) -// private val lostSupplementHsvUpper = Scalar(175.0, 50.0, 155.0) - -// private val pureSupplementHsvLower = Scalar(0.0, 0.0, 0.0) -// private val pureSupplementHsvUpper = Scalar(0.0, 0.0, 0.0) -// -// private val farSupplementHsvLower = Scalar(0.0, 0.0, 0.0) -// private val farSupplementHsvUpper = Scalar(0.0, 0.0, 0.0) -// -// private val lostSupplementHsvLower = Scalar(0.0, 0.0, 0.0) -// private val lostSupplementHsvUpper = Scalar(0.0, 0.0, 0.0) + private val pflHsvLower = Scalar(0.0, 0.0, 248.0) + private val pflHsvUpper = Scalar(179.0, 40.0, 255.0) private val scoreHsvLower = Scalar(0.0, 0.0, 180.0) private val scoreHsvUpper = Scalar(179.0, 255.0, 255.0) @@ -171,55 +170,33 @@ class DeviceRoisMaskerAutoT2 : DeviceRoisMaskerAuto { private val pureMemoryHsvLower = Scalar(90.0, 70.0, 80.0) private val pureMemoryHsvUpper = Scalar(110.0, 200.0, 175.0) - private fun maskHsv( - roiBgr: Mat, - hsvLower: Scalar, - hsvUpper: Scalar, - ): Mat { - val roiHsv = Mat() - val dst = Mat() - Imgproc.cvtColor(roiBgr, roiHsv, Imgproc.COLOR_BGR2HSV) - Core.inRange(roiHsv, hsvLower, hsvUpper, dst) - return dst - } - -// private fun pfl(roiBgr: Mat, supplementLower: Scalar, supplementUpper: Scalar): Mat { -// val result = Mat() -// -// val common = maskHsv(roiBgr, this.pflHsvLower, this.pflHsvUpper) -// val supplement = maskHsv(roiBgr, supplementLower, supplementUpper) -// Core.bitwise_or(common, supplement, result) -// -// return result -// } - - private fun pfl(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.pflHsvLower, this.pflHsvUpper) + private fun pfl(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pflHsvLower, pflHsvUpper) - override fun pure(roiBgr: Mat): Mat = this.pfl(roiBgr) + override fun pure(roiBgr: Mat): Mat = pfl(roiBgr) - override fun far(roiBgr: Mat): Mat = this.pfl(roiBgr) + override fun far(roiBgr: Mat): Mat = pfl(roiBgr) - override fun lost(roiBgr: Mat): Mat = this.pfl(roiBgr) + override fun lost(roiBgr: Mat): Mat = pfl(roiBgr) - override fun score(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.scoreHsvLower, this.scoreHsvUpper) + override fun score(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, scoreHsvLower, scoreHsvUpper) - override fun ratingClassPst(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.pstHsvLower, this.pstHsvUpper) + override fun ratingClassPst(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pstHsvLower, pstHsvUpper) - override fun ratingClassPrs(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.prsHsvLower, this.prsHsvUpper) + override fun ratingClassPrs(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, prsHsvLower, prsHsvUpper) - override fun ratingClassFtr(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.ftrHsvLower, this.ftrHsvUpper) + override fun ratingClassFtr(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, ftrHsvLower, ftrHsvUpper) - override fun ratingClassByd(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.bydHsvLower, this.bydHsvUpper) + override fun ratingClassByd(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, bydHsvLower, bydHsvUpper) - override fun ratingClassEtr(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.etrHsvMin, this.etrHsvMax) + override fun ratingClassEtr(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, etrHsvMin, etrHsvMax) - override fun maxRecall(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.maxRecallHsvLower, this.maxRecallHsvUpper) + override fun maxRecall(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, maxRecallHsvLower, maxRecallHsvUpper) - override fun clearStatusTrackLost(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.trackLostHsvLower, this.trackLostHsvUpper) + override fun clearStatusTrackLost(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, trackLostHsvLower, trackLostHsvUpper) - override fun clearStatusTrackComplete(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.trackCompleteHsvLower, this.trackCompleteHsvUpper) + override fun clearStatusTrackComplete(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, trackCompleteHsvLower, trackCompleteHsvUpper) - override fun clearStatusFullRecall(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.fullRecallHsvLower, this.fullRecallHsvUpper) + override fun clearStatusFullRecall(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, fullRecallHsvLower, fullRecallHsvUpper) - override fun clearStatusPureMemory(roiBgr: Mat): Mat = this.maskHsv(roiBgr, this.pureMemoryHsvLower, this.pureMemoryHsvUpper) + override fun clearStatusPureMemory(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pureMemoryHsvLower, pureMemoryHsvUpper) } From a08c56c96d3ee4a6af392a90cc5fd65ebaf7ae1c Mon Sep 17 00:00:00 2001 From: 283375 Date: Fri, 13 Nov 2026 02:05:15 +0800 Subject: [PATCH 08/12] add hsv votes --- .../core/ocr/device/DeviceOcr.kt | 29 ++------- .../device/rois/masker/DeviceRoisMasker.kt | 20 ++++++ .../rois/masker/DeviceRoisMaskerAuto.kt | 63 +++++++++++++++++++ 3 files changed, 87 insertions(+), 25 deletions(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt index 1b70bb4f..a8b17e01 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt @@ -115,35 +115,14 @@ class DeviceOcr( fun ratingClass(): ArcaeaRatingClass = extractor.ratingClass.use { roi -> - val results = - listOf( - masker.ratingClassPst(roi), - masker.ratingClassPrs(roi), - masker.ratingClassFtr(roi), - masker.ratingClassByd(roi), - masker.ratingClassEtr(roi), - ) - try { - ArcaeaRatingClass.fromInt(results.indices.maxBy { Core.countNonZero(results[it]) }) - } finally { - results.forEach { it.release() } - } + val votes = masker.ratingClassVotes(roi) + ArcaeaRatingClass.fromInt(votes.indices.maxBy { votes[it] }) } private fun clearStatus(): Int = extractor.clearStatus.use { roi -> - val results = - listOf( - masker.clearStatusTrackLost(roi), - masker.clearStatusTrackComplete(roi), - masker.clearStatusFullRecall(roi), - masker.clearStatusPureMemory(roi), - ) - try { - results.indices.maxBy { Core.countNonZero(results[it]) } - } finally { - results.forEach { it.release() } - } + val votes = masker.clearStatusVotes(roi) + votes.indices.maxBy { votes[it] } } private fun lookupSongId(): List = diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt index 1d09c728..e8bb8560 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMasker.kt @@ -31,4 +31,24 @@ interface DeviceRoisMasker { fun clearStatusFullRecall(roiBgr: Mat): Mat fun clearStatusPureMemory(roiBgr: Mat): Mat + + /** + * Votes for a rating class by the non-zero pixel count of [roiBgr] masked under each + * class's HSV range, with a single BGR→HSV conversion. + * + * Index i is the vote count for `ArcaeaRatingClass.fromInt(i)`. + */ + fun ratingClassVotes(roiBgr: Mat): IntArray + + /** + * Votes for a clear status by the non-zero pixel count of [roiBgr] masked under each + * status's HSV range, with a single BGR→HSV conversion. + * + * Index order is part of the persisted result contract: + * * 0 = [xyz.sevive.arcaeaoffline.core.constants.ArcaeaPlayResultClearStatus.TRACK_LOST] + * * 1 = [xyz.sevive.arcaeaoffline.core.constants.ArcaeaPlayResultClearStatus.CLEARED] + * * 2 = [xyz.sevive.arcaeaoffline.core.constants.ArcaeaPlayResultClearStatus.FULL_RECALL] + * * 3 = [xyz.sevive.arcaeaoffline.core.constants.ArcaeaPlayResultClearStatus.PURE_MEMORY] + */ + fun clearStatusVotes(roiBgr: Mat): IntArray } diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt index 49f50358..bec3bba1 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/rois/masker/DeviceRoisMaskerAuto.kt @@ -21,6 +21,27 @@ private fun maskBgrInHsv( return masked } +/** + * Counts non-zero pixels of [roiBgr] masked under each HSV range in order, with a single + * BGR -> HSV conversion. + */ +private fun hsvVote( + roiBgr: Mat, + ranges: List>, +): IntArray { + val votes = IntArray(ranges.size) + Mat().use { roiHsv -> + Imgproc.cvtColor(roiBgr, roiHsv, Imgproc.COLOR_BGR2HSV) + Mat().use { mask -> + ranges.forEachIndexed { index, (lower, upper) -> + Core.inRange(roiHsv, lower, upper, mask) + votes[index] = Core.countNonZero(mask) + } + } + } + return votes +} + interface DeviceRoisMaskerAuto : DeviceRoisMasker class DeviceRoisMaskerAutoT1 : DeviceRoisMaskerAuto { @@ -57,6 +78,23 @@ class DeviceRoisMaskerAutoT1 : DeviceRoisMaskerAuto { private val pureMemoryHsvLower = Scalar(90.0, 70.0, 80.0) private val pureMemoryHsvUpper = Scalar(110.0, 200.0, 175.0) + private val ratingClassRanges = + listOf( + pstHsvLower to pstHsvUpper, + prsHsvLower to prsHsvUpper, + ftrHsvLower to ftrHsvUpper, + bydHsvLower to bydHsvUpper, + etrHsvMin to etrHsvMax, + ) + + private val clearStatusRanges = + listOf( + trackLostHsvLower to trackLostHsvUpper, + trackCompleteHsvLower to trackCompleteHsvUpper, + fullRecallHsvLower to fullRecallHsvUpper, + pureMemoryHsvLower to pureMemoryHsvUpper, + ) + /** Returns a newly allocated 8UC1 mask with 255 where the BGR channel spread is at most [threshold]. */ private fun channelSpreadMask( roiBgr: Mat, @@ -131,6 +169,10 @@ class DeviceRoisMaskerAutoT1 : DeviceRoisMaskerAuto { override fun clearStatusFullRecall(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, fullRecallHsvLower, fullRecallHsvUpper) override fun clearStatusPureMemory(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pureMemoryHsvLower, pureMemoryHsvUpper) + + override fun ratingClassVotes(roiBgr: Mat): IntArray = hsvVote(roiBgr, ratingClassRanges) + + override fun clearStatusVotes(roiBgr: Mat): IntArray = hsvVote(roiBgr, clearStatusRanges) } class DeviceRoisMaskerAutoT2 : DeviceRoisMaskerAuto { @@ -170,6 +212,23 @@ class DeviceRoisMaskerAutoT2 : DeviceRoisMaskerAuto { private val pureMemoryHsvLower = Scalar(90.0, 70.0, 80.0) private val pureMemoryHsvUpper = Scalar(110.0, 200.0, 175.0) + private val ratingClassRanges = + listOf( + pstHsvLower to pstHsvUpper, + prsHsvLower to prsHsvUpper, + ftrHsvLower to ftrHsvUpper, + bydHsvLower to bydHsvUpper, + etrHsvMin to etrHsvMax, + ) + + private val clearStatusRanges = + listOf( + trackLostHsvLower to trackLostHsvUpper, + trackCompleteHsvLower to trackCompleteHsvUpper, + fullRecallHsvLower to fullRecallHsvUpper, + pureMemoryHsvLower to pureMemoryHsvUpper, + ) + private fun pfl(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pflHsvLower, pflHsvUpper) override fun pure(roiBgr: Mat): Mat = pfl(roiBgr) @@ -199,4 +258,8 @@ class DeviceRoisMaskerAutoT2 : DeviceRoisMaskerAuto { override fun clearStatusFullRecall(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, fullRecallHsvLower, fullRecallHsvUpper) override fun clearStatusPureMemory(roiBgr: Mat): Mat = maskBgrInHsv(roiBgr, pureMemoryHsvLower, pureMemoryHsvUpper) + + override fun ratingClassVotes(roiBgr: Mat): IntArray = hsvVote(roiBgr, ratingClassRanges) + + override fun clearStatusVotes(roiBgr: Mat): IntArray = hsvVote(roiBgr, clearStatusRanges) } From ff496dc8e8eaef76feaad19999e7a0f96e443c83 Mon Sep 17 00:00:00 2001 From: 283375 Date: Fri, 13 Nov 2026 02:23:30 +0800 Subject: [PATCH 09/12] optimize lookupHash performance --- .../core/ocr/ImageHashesDatabase.kt | 40 ++++++++++++++++--- .../arcaeaoffline/core/ocr/StableTopKTest.kt | 38 ++++++++++++++++++ 2 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/StableTopKTest.kt diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt index 710eee4b..1a7ff4fe 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/ImageHashesDatabase.kt @@ -20,6 +20,33 @@ private fun hammingDistance( return distance } +/** + * Returns the [limit] items with the smallest [scoreOf] value as `(item, score)` pairs, + * ordered by score ascending. + * + * Equal scores keep their original relative order, like a stable sort truncated to [limit]. + * [scoreOf] is called once per item. + */ +internal fun stableTopKBy( + source: Sequence, + limit: Int, + scoreOf: (T) -> Int, +): List> { + require(limit > 0) { "limit must be positive" } + + val top = ArrayList>(limit) + for (item in source) { + val score = scoreOf(item) + if (top.size == limit && score >= top.last().second) continue + + // insert after equal scores so earlier items keep priority + val pos = top.indexOfFirst { it.second > score } + top.add(if (pos == -1) top.size else pos, item to score) + if (top.size > limit) top.removeAt(top.size - 1) + } + return top +} + class ImageHashesDatabase( private val conn: SQLiteConnection, ) { @@ -27,6 +54,8 @@ class ImageHashesDatabase( const val PROP_HASH_SIZE_KEY = "hash_size" const val PROP_HIGH_FREQ_FACTOR_KEY = "high_freq_factor" const val PROP_BUILT_TIMESTAMP_KEY = "built_timestamp" + + private const val LOOKUP_LIMIT = 10 } private var hashSize: Int by Delegates.notNull() @@ -145,13 +174,12 @@ class ImageHashesDatabase( hash: ByteArray, ): List { val hashLength = hashSize.toDouble().pow(2) - return allHashes - .filter { it.type == type && it.hashType == hashType } - .sortedBy { it.distance(hash) } - .take(10) - .map { entry -> - val distance = entry.distance(hash) + return stableTopKBy( + allHashes.asSequence().filter { it.type == type && it.hashType == hashType }, + LOOKUP_LIMIT, + ) { it.distance(hash) } + .map { (entry, distance) -> ImageHashItem( hashType = hashType, type = type, diff --git a/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/StableTopKTest.kt b/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/StableTopKTest.kt new file mode 100644 index 00000000..9c3c74ff --- /dev/null +++ b/core/src/test/kotlin/xyz/sevive/arcaeaoffline/core/ocr/StableTopKTest.kt @@ -0,0 +1,38 @@ +package xyz.sevive.arcaeaoffline.core.ocr + +import org.junit.Assert.assertEquals +import org.junit.Test +import kotlin.random.Random + +class StableTopKTest { + @Test + fun matchesStableSortTruncatedToLimit() { + val random = Random(20261112) + repeat(100) { + val items = + List(random.nextInt(0, 100)) { index -> + index to random.nextInt(0, 5) + } + val expected = + items + .sortedBy { it.second } + .take(10) + val actual = stableTopKBy(items.asSequence(), 10) { it.second }.map { it.first } + assertEquals(expected, actual) + } + } + + @Test + fun equalScoresKeepOriginalOrder() { + val items = List(50) { it to 7 } + val actual = stableTopKBy(items.asSequence(), 10) { it.second }.map { it.first } + assertEquals(items.take(10), actual) + } + + @Test + fun fewerItemsThanLimitReturnsAllSorted() { + val items = listOf(3 to 30, 1 to 10, 2 to 20) + val actual = stableTopKBy(items.asSequence(), 10) { it.second }.map { it.first } + assertEquals(listOf(1 to 10, 2 to 20, 3 to 30), actual) + } +} From 830a9f562d69428458180fb406593113c4854ce1 Mon Sep 17 00:00:00 2001 From: 283375 Date: Fri, 13 Nov 2026 02:39:38 +0800 Subject: [PATCH 10/12] fix max recall fallback --- .../xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt index a8b17e01..0e0f6fb2 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt @@ -151,7 +151,7 @@ class DeviceOcr( far = ocrIntOrNull(extractor.far) ?: 0, lost = ocrIntOrNull(extractor.lost) ?: 0, score = ocrIntOrNull(extractor.score) ?: 0, - maxRecall = ocrIntOrNull(extractor.maxRecall), + maxRecall = ocrIntOrNull(extractor.maxRecall) ?: 0, songIdResults = lookupSongId(), clearStatus = clearStatus(), partnerIdResults = lookupPartnerId(), From 44f78e54268c750f2e5c5f7cd85d267c8ca8ad0f Mon Sep 17 00:00:00 2001 From: 283375 Date: Fri, 13 Nov 2026 03:05:17 +0800 Subject: [PATCH 11/12] fix cropped image mat lifecycle --- .../core/ocr/device/DeviceOcrScenario.kt | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt index 57b3ac05..352d6ea3 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcrScenario.kt @@ -25,6 +25,9 @@ class DeviceOcrScenario private constructor( * Builds a scenario from a raw BGR screenshot. The rois type (T1/T2) is auto-detected * on the black-edge-cropped image. * + * The scenario owns its cropped image and never aliases [imgBgr]. The caller stays + * responsible for releasing it. + * * @param fallback Selected when every detector reports zero confidence. * @throws IllegalArgumentException when the rois type cannot be determined and * [fallback] is null. @@ -33,19 +36,27 @@ class DeviceOcrScenario private constructor( imgBgr: Mat, fallback: DeviceRoisAutoSelectorResult? = null, ): DeviceOcrScenario { - val imgCropped = CropBlackEdges.cropOrOriginal(imgBgr) - val roisType = DeviceRoisAutoSelector.select(imgCropped, fallback) - require(roisType != DeviceRoisAutoSelectorResult.UNKNOWN) { - "Cannot determine device rois type; pass an explicit fallback" - } + val cropped = CropBlackEdges.cropOrOriginal(imgBgr) + // cropOrOriginal returns imgBgr itself when cropping fails, so clone to keep + // close() off the caller's Mat + val imgCropped = if (cropped === imgBgr) cropped.clone() else cropped + try { + val roisType = DeviceRoisAutoSelector.select(imgCropped, fallback) + require(roisType != DeviceRoisAutoSelectorResult.UNKNOWN) { + "Cannot determine device rois type; pass an explicit fallback" + } - val (rois, masker) = - DeviceRoisAutoSelector.createRoisAndMasker( - roisType, - imgCropped.width(), - imgCropped.height(), - ) - return DeviceOcrScenario(imgCropped, DeviceRoisExtractor(rois, imgCropped), masker) + val (rois, masker) = + DeviceRoisAutoSelector.createRoisAndMasker( + roisType, + imgCropped.width(), + imgCropped.height(), + ) + return DeviceOcrScenario(imgCropped, DeviceRoisExtractor(rois, imgCropped), masker) + } catch (e: Exception) { + imgCropped.release() + throw e + } } } From dda13f03645aa1f7c6ab3ea0861582924c8990d4 Mon Sep 17 00:00:00 2001 From: 283375 Date: Fri, 13 Nov 2026 09:38:25 +0800 Subject: [PATCH 12/12] Revert "fix max recall fallback" This reverts commit 830a9f562d69428458180fb406593113c4854ce1. --- .../xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt index 0e0f6fb2..a8b17e01 100644 --- a/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt +++ b/core/src/main/kotlin/xyz/sevive/arcaeaoffline/core/ocr/device/DeviceOcr.kt @@ -151,7 +151,7 @@ class DeviceOcr( far = ocrIntOrNull(extractor.far) ?: 0, lost = ocrIntOrNull(extractor.lost) ?: 0, score = ocrIntOrNull(extractor.score) ?: 0, - maxRecall = ocrIntOrNull(extractor.maxRecall) ?: 0, + maxRecall = ocrIntOrNull(extractor.maxRecall), songIdResults = lookupSongId(), clearStatus = clearStatus(), partnerIdResults = lookupPartnerId(),