From f91bc9d394527cee3c4de82811951620f7607b07 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:27:43 +0000 Subject: [PATCH 01/33] feat: hero catalog sync, slot extraction and recognition v0 Rone Arena is the primary catalog (GET /api/heroes, paginated, HTTPS only); MLBBDex is used for the completeness cross-check and roles/lanes by slug. Provider-independent hero identity is a name-derived slug. A file-based snapshot plus a private image cache serve offline browsing; the catalog and its images are downloaded only on the user's explicit Sync. Slot extraction crops the source rectangle out of the frame upright and is covered by rotation, edge, minimum-size and payload-mismatch tests. Matching is a dHash perceptual hash plus a normalized 3x3 mean-RGB descriptor, combined into an explicit non-probability similarity; below threshold the result is no-match, and the user confirms or corrects manually. --- app/src/main/AndroidManifest.xml | 2 + .../main/kotlin/com/draftlens/DraftLensApp.kt | 21 ++ .../main/kotlin/com/draftlens/MainActivity.kt | 2 + .../datastore/HeroRecognitionStore.kt | 80 +++++ .../feature/visionlab/HeroRecognitionCard.kt | 280 ++++++++++++++++++ .../feature/visionlab/VisionLabScreen.kt | 6 + .../feature/visionlab/VisionLabUiState.kt | 29 +- .../feature/visionlab/VisionLabViewModel.kt | 185 +++++++++++- .../heroes/AndroidBitmapSlotImageExtractor.kt | 77 +++++ .../heroes/AndroidHeroCatalogTransport.kt | 84 ++++++ .../heroes/FileHeroCatalogRepository.kt | 157 ++++++++++ .../vision/heroes/HeroCatalogCodec.kt | 49 +++ .../vision/heroes/HeroCatalogRepository.kt | 30 ++ .../vision/heroes/HeroCatalogSync.kt | 246 +++++++++++++++ .../vision/heroes/HeroCatalogTransport.kt | 13 + .../com/draftlens/vision/heroes/HeroIds.kt | 16 + .../draftlens/vision/heroes/HeroMatcher.kt | 188 ++++++++++++ .../com/draftlens/vision/heroes/HeroModels.kt | 114 +++++++ .../vision/heroes/HeroRecognition.kt | 48 +++ .../vision/heroes/HeroReferenceImages.kt | 30 ++ .../vision/heroes/SlotImageExtractor.kt | 128 ++++++++ .../datastore/HeroRecognitionStoreTest.kt | 93 ++++++ .../visionlab/VisionLabViewModelTest.kt | 193 ++++++++++-- .../heroes/FileHeroCatalogRepositoryTest.kt | 163 ++++++++++ .../draftlens/heroes/HeroCatalogCodecTest.kt | 93 ++++++ .../draftlens/heroes/HeroCatalogSyncTest.kt | 198 +++++++++++++ .../com/draftlens/heroes/HeroMatcherTest.kt | 123 ++++++++ .../draftlens/heroes/HeroRecognitionTest.kt | 76 +++++ .../heroes/SlotImageExtractorTest.kt | 146 +++++++++ 29 files changed, 2845 insertions(+), 25 deletions(-) create mode 100644 app/src/main/kotlin/com/draftlens/datastore/HeroRecognitionStore.kt create mode 100644 app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogCodec.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogRepository.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogSync.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogTransport.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroIds.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroMatcher.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroModels.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt create mode 100644 app/src/test/kotlin/com/draftlens/datastore/HeroRecognitionStoreTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/HeroCatalogCodecTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/HeroCatalogSyncTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 3a38ef0..66e937a 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -5,6 +5,8 @@ + + VisionLabScreen( calibrationRepository = app.calibrationProfiles, source = app.visionSource, + heroCatalog = app.heroCatalog, + heroRecognitionStore = app.heroRecognitionStore, ) } } diff --git a/app/src/main/kotlin/com/draftlens/datastore/HeroRecognitionStore.kt b/app/src/main/kotlin/com/draftlens/datastore/HeroRecognitionStore.kt new file mode 100644 index 0000000..0b39c85 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/datastore/HeroRecognitionStore.kt @@ -0,0 +1,80 @@ +package com.draftlens.datastore + +import androidx.datastore.core.DataStore +import androidx.datastore.preferences.core.Preferences +import androidx.datastore.preferences.core.edit +import androidx.datastore.preferences.core.stringPreferencesKey +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.flow.map +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json + +/** One manually confirmed hero, keyed by the slot's stable id. Metadata only, never pixels. */ +@Serializable +data class HeroConfirmation( + val heroId: String, + val heroName: String, + val confirmedAtMs: Long, +) + +@Serializable +private data class HeroConfirmationDocument( + val schemaVersion: Int = CURRENT_SCHEMA, + val entries: Map = emptyMap(), +) { + companion object { + const val CURRENT_SCHEMA = 1 + } +} + +/** + * Persists the user's per-slot hero confirmations in the preferences DataStore under + * `hero_confirmations_json`. Separate from the annotation schema on purpose: the slot codec + * stays v1-compatible, and a corrupt document here never blocks annotations. + */ +class HeroRecognitionStore(private val dataStore: DataStore) { + + private val key = stringPreferencesKey("hero_confirmations_json") + + /** Emits the confirmations map on every change; empty on a missing or invalid document. */ + val confirmations: Flow> = dataStore.data.map { prefs -> + decode(prefs[key]) + } + + suspend fun load(): Map = + dataStore.data.first()[key].let(::decode) + + suspend fun confirm(slotId: String, confirmation: HeroConfirmation) { + dataStore.edit { prefs -> + val current = decode(prefs[key]).toMutableMap() + current[slotId] = confirmation + prefs[key] = encode(current) + } + } + + suspend fun clear(slotId: String) { + dataStore.edit { prefs -> + val current = decode(prefs[key]).toMutableMap() + current.remove(slotId) + prefs[key] = encode(current) + } + } + + private fun decode(raw: String?): Map { + if (raw == null) return emptyMap() + return try { + val document = Json { ignoreUnknownKeys = false }.decodeFromString(HeroConfirmationDocument.serializer(), raw) + if (document.schemaVersion != HeroConfirmationDocument.CURRENT_SCHEMA) return emptyMap() + document.entries + } catch (e: SerializationException) { + emptyMap() + } catch (e: IllegalArgumentException) { + emptyMap() + } + } + + private fun encode(entries: Map): String = + Json { encodeDefaults = true }.encodeToString(HeroConfirmationDocument.serializer(), HeroConfirmationDocument(entries = entries)) +} diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt new file mode 100644 index 0000000..a1dc451 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt @@ -0,0 +1,280 @@ +package com.draftlens.feature.visionlab + +import androidx.compose.foundation.Image +import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.Column +import androidx.compose.foundation.layout.Row +import androidx.compose.foundation.layout.fillMaxWidth +import androidx.compose.foundation.layout.height +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.layout.size +import androidx.compose.foundation.clickable +import androidx.compose.material3.Card +import androidx.compose.material3.CardDefaults +import androidx.compose.material3.CircularProgressIndicator +import androidx.compose.material3.HorizontalDivider +import androidx.compose.material3.MaterialTheme +import androidx.compose.material3.OutlinedButton +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.Text +import androidx.compose.material3.TextButton +import androidx.compose.runtime.Composable +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.produceState +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.ImageBitmap +import androidx.compose.ui.graphics.asImageBitmap +import androidx.compose.ui.layout.ContentScale +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.unit.dp +import com.draftlens.vision.heroes.HeroCatalogState +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext + +/** + * Hero recognition for the selected slot: explicit sync, explicit Identify, manual confirm, + * manual correction. Nothing runs continuously and nothing selects a hero automatically. + */ +@Composable +fun HeroRecognitionCard(state: VisionLabUiState, vm: VisionLabViewModel) { + val selectedSlot = state.slots.firstOrNull { it.id == state.selectedSlotId } + val recognition = selectedSlot?.let { state.heroRecognition[it.id] } + + Card(modifier = Modifier.fillMaxWidth()) { + Column( + modifier = Modifier.padding(12.dp), + verticalArrangement = Arrangement.spacedBy(8.dp), + ) { + Text("Hero recognition", style = MaterialTheme.typography.titleMedium) + Text( + "Unofficial community data. Mobile Legends: Bang Bang is a trademark of " + + "Shanghai Moonton Technology Co., Ltd.; DraftLens is not affiliated with or endorsed by Moonton.", + style = MaterialTheme.typography.bodySmall, + ) + + CatalogSyncRow(state.heroCatalog, onSync = vm::syncHeroCatalog) + + if (selectedSlot == null) { + Text("Select a slot to identify the hero in it.", style = MaterialTheme.typography.bodySmall) + } else { + Text( + "Selected slot: ${selectedSlot.id.take(8)} · ${selectedSlot.type} · ${selectedSlot.team} · #${selectedSlot.index}", + style = MaterialTheme.typography.bodySmall, + fontFamily = FontFamily.Monospace, + ) + HeroPreview(state) + HeroResultArea(slotId = selectedSlot.id, state = recognition, vm = vm) + } + + HorizontalDivider() + Text( + "Hero data: Rone Arena API (arena.rone.dev, unofficial community project, BSD-3-Clause). " + + "Roles and lanes: MLBBDex (mlbbdex.com, community wiki data, CC BY-SA). " + + "Images: courtesy of their respective rights holders. Nothing is uploaded.", + style = MaterialTheme.typography.bodySmall, + ) + } + } +} + +@Composable +private fun CatalogSyncRow(catalog: HeroCatalogState, onSync: () -> Unit) { + Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + when (catalog) { + HeroCatalogState.NotSynced -> Text("Catalog not synced yet.", modifier = Modifier.weight(1f)) + is HeroCatalogState.Syncing -> { + CircularProgressIndicator(modifier = Modifier.size(18.dp)) + Text(catalog.stage, modifier = Modifier.weight(1f), style = MaterialTheme.typography.bodySmall) + } + + is HeroCatalogState.Ready -> Text( + "Catalog ready: ${catalog.entryCount} heroes.", + modifier = Modifier.weight(1f), + ) + + is HeroCatalogState.Error -> Text( + "Catalog error: ${catalog.message}", + modifier = Modifier.weight(1f), + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + } + val syncing = catalog is HeroCatalogState.Syncing + OutlinedButton(onClick = onSync, enabled = !syncing) { Text("Sync catalog") } + } +} + +@Composable +private fun HeroPreview(state: VisionLabUiState) { + val bitmap = state.heroPreview ?: return + Image( + bitmap = bitmap.asImageBitmap(), + contentDescription = "Extracted slot image", + modifier = Modifier + .fillMaxWidth() + .height(140.dp), + contentScale = ContentScale.Fit, + ) +} + +@Composable +private fun HeroResultArea( + slotId: String, + state: HeroSlotUiState?, + vm: VisionLabViewModel, +) { + when (state?.phase) { + null, HeroSlotPhase.IDLE -> { + OutlinedButton( + onClick = vm::identifySelectedSlot, + modifier = Modifier.fillMaxWidth(), + ) { Text("Identify hero") } + } + + HeroSlotPhase.PROCESSING -> Row( + modifier = Modifier.fillMaxWidth(), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + CircularProgressIndicator(modifier = Modifier.size(18.dp)) + Text("Extracting and matching…", style = MaterialTheme.typography.bodySmall) + } + + HeroSlotPhase.CANDIDATES -> { + val candidates = state?.candidates.orEmpty() + Text("Candidates (similarity, not a probability):", style = MaterialTheme.typography.labelMedium) + candidates.forEach { candidate -> + CandidateRow( + candidate = candidate, + onConfirm = { vm.confirmHero(slotId, candidate.hero.id.value) }, + vm = vm, + ) + } + TextButton(onClick = vm::identifySelectedSlot) { Text("Try again") } + } + + HeroSlotPhase.NO_MATCH -> { + val bestSimilarity = state?.bestSimilarity + Text( + "No confident match (best similarity ${bestSimilarity?.let { "%.2f".format(it) } ?: "—"}). " + + "Confirm the hero manually below.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + ManualHeroSearch(slotId = slotId, vm = vm) + TextButton(onClick = vm::identifySelectedSlot) { Text("Try again") } + } + + HeroSlotPhase.CONFIRMED -> { + Text( + "Confirmed hero: ${state?.message}", + style = MaterialTheme.typography.bodyLarge, + ) + Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { + OutlinedButton(onClick = { vm.clearHeroConfirmation(slotId) }) { Text("Clear") } + TextButton(onClick = vm::identifySelectedSlot) { Text("Change…") } + } + } + + HeroSlotPhase.FAILED -> { + Text( + state?.message ?: "Recognition failed.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + OutlinedButton(onClick = vm::identifySelectedSlot) { Text("Try again") } + ManualHeroSearch(slotId = slotId, vm = vm) + } + } +} + +@Composable +private fun CandidateRow(candidate: com.draftlens.vision.heroes.HeroCandidate, onConfirm: () -> Unit, vm: VisionLabViewModel) { + Card( + modifier = Modifier.fillMaxWidth(), + colors = CardDefaults.cardColors(containerColor = MaterialTheme.colorScheme.surfaceVariant), + ) { + Row( + modifier = Modifier.fillMaxWidth().padding(8.dp), + horizontalArrangement = Arrangement.spacedBy(8.dp), + verticalAlignment = Alignment.CenterVertically, + ) { + HeroFileAvatar(resolveFile = { vm.heroAssetFile(candidate.hero) }, name = candidate.hero.canonicalName, size = 56.dp) + Column(modifier = Modifier.weight(1f)) { + Text(candidate.hero.canonicalName, style = MaterialTheme.typography.titleSmall) + val ambiguousNote = if (candidate.ambiguous) " · ambiguous" else "" + Text( + "Similarity ${"%.2f".format(candidate.similarity)}$ambiguousNote", + style = MaterialTheme.typography.bodySmall, + ) + } + TextButton(onClick = onConfirm) { Text("Confirm") } + } + } +} + +/** Loads the reference image from the local catalog cache; never from the network. */ +@Composable +private fun HeroFileAvatar( + resolveFile: () -> java.io.File?, + name: String, + size: androidx.compose.ui.unit.Dp, +) { + val bitmap = produceState(initialValue = null) { + value = withContext(Dispatchers.IO) { + resolveFile()?.let { file -> + runCatching { android.graphics.BitmapFactory.decodeFile(file.absolutePath)?.asImageBitmap() } + .getOrNull() + } + } + } + val loaded = bitmap.value + if (loaded != null) { + Image( + bitmap = loaded, + contentDescription = name, + modifier = Modifier.size(size), + contentScale = ContentScale.Crop, + ) + } else { + Box( + modifier = Modifier + .size(size) + .padding(2.dp), + contentAlignment = Alignment.Center, + ) { Text(name.take(2).uppercase(), style = MaterialTheme.typography.titleSmall) } + } +} + +/** Manual correction: find a hero by name and confirm it, no matching involved. */ +@Composable +private fun ManualHeroSearch(slotId: String, vm: VisionLabViewModel) { + var query by remember { mutableStateOf("") } + val results = remember(query) { vm.searchCatalog(query) } + OutlinedTextField( + value = query, + onValueChange = { query = it }, + label = { Text("Search hero by name") }, + singleLine = true, + modifier = Modifier.fillMaxWidth(), + ) + results.forEach { hero -> + Row( + modifier = Modifier + .fillMaxWidth() + .clickable { vm.confirmHero(slotId, hero.id.value) } + .padding(vertical = 6.dp), + ) { + Text(hero.canonicalName, modifier = Modifier.padding(start = 4.dp)) + } + } +} diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt index 6ff0e12..d82789e 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt @@ -47,12 +47,16 @@ import com.draftlens.vision.preprocessing.AndroidFramePreprocessor fun VisionLabScreen( calibrationRepository: CalibrationRepository, source: VisionFrameSource, + heroCatalog: com.draftlens.vision.heroes.HeroCatalogRepository, + heroRecognitionStore: com.draftlens.datastore.HeroRecognitionStore, ) { val vm: VisionLabViewModel = viewModel { VisionLabViewModel( source = source, preprocessor = AndroidFramePreprocessor, calibrationRepository = calibrationRepository, + heroCatalog = heroCatalog, + heroRecognitionStore = heroRecognitionStore, ) } val state by vm.uiState.collectAsState() @@ -100,6 +104,8 @@ fun VisionLabScreen( SlotEditor(state = state, vm = vm) + HeroRecognitionCard(state = state, vm = vm) + ProcessingInspectorCard(state = state, onMaxDimensionChange = vm::setMaxProcessedDimension) ProfileControls(state = state, vm = vm) diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt index 815d820..b91002b 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt @@ -3,6 +3,8 @@ package com.draftlens.feature.visionlab import com.draftlens.vision.annotation.DraftSlot import com.draftlens.vision.annotation.SlotIssue import com.draftlens.vision.calibration.CalibrationLoadResult +import com.draftlens.vision.heroes.HeroCandidate +import com.draftlens.vision.heroes.HeroCatalogState import com.draftlens.vision.model.OriginalFrame import com.draftlens.vision.model.ProcessedFrame import com.draftlens.vision.model.VisionSourceState @@ -11,7 +13,8 @@ import com.draftlens.vision.model.VisionSourceState enum class ProcessStatus { IDLE, RUNNING, SUCCESS, FAILED } /** - * Everything the Vision Lab renders. No detection data: this sprint has no recognizer. + * Everything the Vision Lab renders. Recognition runs only on the user's explicit + * Identify action; nothing here is a continuous detection. */ data class VisionLabUiState( val liveFrame: OriginalFrame? = null, @@ -35,6 +38,12 @@ data class VisionLabUiState( val canRedo: Boolean = false, /** True while the fullscreen annotation editor is open. */ val fullscreenEditor: Boolean = false, + /** The hero catalog state as the UI shows it. */ + val heroCatalog: HeroCatalogState = HeroCatalogState.NotSynced, + /** Recognition state per slot id. Only slots the user ran Identify on appear here. */ + val heroRecognition: Map = emptyMap(), + /** The extracted crop for preview; Android only, null for synthetic payloads. */ + val heroPreview: android.graphics.Bitmap? = null, ) { /** The frame the canvas draws: the pinned one when frozen, otherwise the live one. */ val displayFrame: OriginalFrame? get() = frozenFrame ?: liveFrame @@ -48,3 +57,21 @@ data class VisionLabUiState( const val DEFAULT_MAX_PROCESSED_DIMENSION = 1280 } } + +/** Phase of a slot's hero recognition flow. */ +enum class HeroSlotPhase { IDLE, PROCESSING, CANDIDATES, NO_MATCH, CONFIRMED, FAILED } + +/** Recognition state of one slot. */ +data class HeroSlotUiState( + val phase: HeroSlotPhase, + /** Ranked candidates; meaningful only in [HeroSlotPhase.CANDIDATES]. */ + val candidates: List = emptyList(), + /** Best similarity seen in [HeroSlotPhase.NO_MATCH]; null elsewhere. */ + val bestSimilarity: Float? = null, + /** User-facing message for [HeroSlotPhase.FAILED]; also used for the confirmed hero's name in [HeroSlotPhase.CONFIRMED]. */ + val message: String? = null, + /** The confirmed hero id in [HeroSlotPhase.CONFIRMED]. */ + val confirmedHeroId: String? = null, + /** Wall-clock confirmation time in [HeroSlotPhase.CONFIRMED]. */ + val confirmedAtMs: Long? = null, +) diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt index 1db8e74..20c2b61 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt @@ -2,6 +2,17 @@ package com.draftlens.feature.visionlab import androidx.lifecycle.ViewModel import androidx.lifecycle.viewModelScope +import com.draftlens.vision.heroes.HeroCatalogEntry +import com.draftlens.vision.heroes.HeroCatalogRepository +import com.draftlens.vision.heroes.HeroReference +import com.draftlens.vision.heroes.HeroRecognition +import com.draftlens.vision.heroes.HeroReferenceImages +import com.draftlens.vision.heroes.HeroSlotPhase +import com.draftlens.vision.heroes.SlotImageExtractor +import com.draftlens.vision.heroes.SlotImageExtractors +import com.draftlens.vision.heroes.pixelsOf +import com.draftlens.datastore.HeroConfirmation +import com.draftlens.datastore.HeroRecognitionStore import com.draftlens.vision.annotation.DraftSlot import com.draftlens.vision.annotation.SlotAnnotationStore import com.draftlens.vision.annotation.SlotTeam @@ -32,14 +43,18 @@ import java.util.concurrent.atomic.AtomicLong * State holder for the Vision Lab. * * Frames are consumed through [VisionFrameSource]; processing is explicit and - * happens only when the user asks for it. There is no recognizer in this sprint: - * the processed frame is pixels plus geometry, never a detection. + * happens only when the user asks for it. There is no recognizer wired to automatic + * processing: recognition runs only on the user's explicit Identify action. */ class VisionLabViewModel( private val source: VisionFrameSource, private val preprocessor: FramePreprocessor, private val calibrationRepository: CalibrationRepository, private val annotationStore: SlotAnnotationStore = SlotAnnotationStore(idGenerator = { UUID.randomUUID().toString() }), + private val heroCatalog: HeroCatalogRepository, + private val heroRecognitionStore: HeroRecognitionStore, + private val extractorFor: (OriginalFrame) -> SlotImageExtractor = SlotImageExtractors::forFrame, + private val referenceLoader: (java.io.File, HeroCatalogEntry) -> HeroReference? = HeroReferenceImages::loadReference, ) : ViewModel() { private val _uiState = MutableStateFlow(VisionLabUiState()) @@ -63,9 +78,33 @@ class VisionLabViewModel( viewModelScope.launch { calibrationRepository.profiles.collect { result -> _uiState.update { it.copy(calibration = result) } } } + viewModelScope.launch { + heroCatalog.state.collect { state -> _uiState.update { it.copy(heroCatalog = state) } } + } + viewModelScope.launch { + loadConfirmedHeroes() + } refreshSlots() } + /** Restores per-slot hero confirmations from the store so they survive app restarts. */ + private suspend fun loadConfirmedHeroes() { + val confirmations = heroRecognitionStore.load() + if (confirmations.isEmpty()) return + _uiState.update { + it.copy( + heroRecognition = it.heroRecognition + confirmations.mapValues { (_, confirmation) -> + com.draftlens.feature.visionlab.HeroSlotUiState( + phase = HeroSlotPhase.CONFIRMED, + message = confirmation.heroName, + confirmedHeroId = confirmation.heroId, + confirmedAtMs = confirmation.confirmedAtMs, + ) + }, + ) + } + } + // region frame control /** Pins the frame that is on screen; live frames keep arriving but are not shown. */ @@ -349,6 +388,148 @@ class VisionLabViewModel( // endregion + // region hero recognition + + /** Downloads the hero catalog and its reference images. Only on the user's action. */ + fun syncHeroCatalog() { + viewModelScope.launch(Dispatchers.IO) { + heroCatalog.refresh() + } + } + + /** + * Extracts the selected slot's image and matches it against the cached catalog. + * Explicit user action; never runs continuously. + */ + fun identifySelectedSlot() { + val state = _uiState.value + val slot = state.selectedSlotId?.let { id -> annotationStore.slots().firstOrNull { it.id == id } } + if (slot == null) { + report("Select a slot first") + return + } + val frame = state.frozenFrame ?: source.frames.value + if (frame == null) { + report("There is no frame to recognize") + return + } + _uiState.update { + it.copy( + heroRecognition = it.heroRecognition + (slot.id to com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.PROCESSING)), + heroPreview = null, + ) + } + viewModelScope.launch(Dispatchers.Default) { + when (val computation = recognize(frame, slot)) { + is IdentifyComputation.Failed -> updateHeroSlot(slot.id) { + com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.FAILED, message = computation.message) + } + + is IdentifyComputation.Done -> when (val outcome = computation.outcome) { + is com.draftlens.vision.heroes.HeroIdentifyOutcome.Candidates -> updateHeroSlot(slot.id) { + com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.CANDIDATES, candidates = outcome.heroCandidates) + } + + is com.draftlens.vision.heroes.HeroIdentifyOutcome.NoMatch -> updateHeroSlot(slot.id) { + com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.NO_MATCH, bestSimilarity = outcome.bestSimilarity) + } + + is com.draftlens.vision.heroes.HeroIdentifyOutcome.Failed -> updateHeroSlot(slot.id) { + com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.FAILED, message = outcome.message) + } + }.also { + _uiState.update { it.copy(heroPreview = computation.preview) } + } + } + } + } + + /** Records the user's manual confirmation of a hero for a slot. */ + fun confirmHero(slotId: String, heroId: String) { + val candidateName = _uiState.value.heroRecognition[slotId]?.candidates + ?.firstOrNull { it.hero.id.value == heroId }?.hero?.canonicalName + val snapshotName = heroCatalog.snapshot()?.entries?.firstOrNull { it.id.value == heroId }?.canonicalName + val name = candidateName ?: snapshotName ?: "Unknown hero" + val confirmation = HeroConfirmation(heroId = heroId, heroName = name, confirmedAtMs = System.currentTimeMillis()) + viewModelScope.launch { + val result = runCatching { heroRecognitionStore.confirm(slotId, confirmation) } + if (result.isFailure) { + report("The hero confirmation could not be saved: ${result.exceptionOrNull()?.message}") + return@launch + } + _uiState.update { + it.copy( + heroRecognition = it.heroRecognition + ( + slotId to com.draftlens.feature.visionlab.HeroSlotUiState( + HeroSlotPhase.CONFIRMED, + message = name, + confirmedHeroId = heroId, + confirmedAtMs = confirmation.confirmedAtMs, + ) + ), + heroPreview = null, + ) + } + } + } + + /** Removes a confirmed hero so the slot can be identified again. */ + fun clearHeroConfirmation(slotId: String) { + viewModelScope.launch { + runCatching { heroRecognitionStore.clear(slotId) } + _uiState.update { + it.copy(heroRecognition = it.heroRecognition - slotId, heroPreview = null) + } + } + } + + /** The cached HEAD reference file for a hero, if any; never triggers a download. */ + fun heroAssetFile(hero: HeroCatalogEntry): java.io.File? = + heroCatalog.assetFile(hero.id, com.draftlens.vision.heroes.HeroAssetVariant.HEAD) + + /** Filters the cached catalog by name for manual correction. */ + fun searchCatalog(query: String, limit: Int = 8): List { + if (query.isBlank()) return emptyList() + val needle = query.trim().lowercase() + return heroCatalog.snapshot()?.entries + ?.filter { it.canonicalName.lowercase().contains(needle) } + ?.take(limit) + ?: emptyList() + } + + private sealed interface IdentifyComputation { + data class Done(val outcome: com.draftlens.vision.heroes.HeroIdentifyOutcome, val preview: android.graphics.Bitmap?) : IdentifyComputation + data class Failed(val message: String) : IdentifyComputation + } + + private fun recognize(frame: OriginalFrame, slot: DraftSlot): IdentifyComputation { + val extractor = extractorFor(frame) + val image = extractor.extract(frame, slot).getOrElse { + return IdentifyComputation.Failed(it.message ?: "The slot image could not be extracted") + } + val pixels = pixelsOf(image) + ?: return IdentifyComputation.Failed("The extracted image could not be read for comparison") + val snapshot = heroCatalog.snapshot() + ?: return IdentifyComputation.Failed("Sync the hero catalog first") + val references = snapshot.entries.mapNotNull { entry -> + val asset = entry.assets.firstOrNull { it.variant == com.draftlens.vision.heroes.HeroAssetVariant.HEAD } + ?: return@mapNotNull null + val file = heroCatalog.assetFile(entry.id, asset.variant) ?: return@mapNotNull null + referenceLoader(file, entry) + } + val preview = (image.payload as? com.draftlens.vision.model.BitmapFramePayload)?.bitmap + return IdentifyComputation.Done( + HeroRecognition.identify(pixels.first, pixels.second, pixels.third, references), + preview, + ) + } + + private fun updateHeroSlot(slotId: String, transform: () -> com.draftlens.feature.visionlab.HeroSlotUiState) { + _uiState.update { it.copy(heroRecognition = it.heroRecognition + (slotId to transform())) } + } + + // endregion + private fun report(message: String?) { if (message == null) return _uiState.update { it.copy(message = message) } diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt new file mode 100644 index 0000000..c8d1d2d --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt @@ -0,0 +1,77 @@ +package com.draftlens.vision.heroes + +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Matrix +import android.os.SystemClock +import com.draftlens.vision.annotation.DraftSlot +import com.draftlens.vision.model.BitmapFramePayload +import com.draftlens.vision.model.FrameRotation +import com.draftlens.vision.model.FrameId +import com.draftlens.vision.model.OriginalFrame +import com.draftlens.vision.model.PixelArrayFramePayload +import com.draftlens.vision.model.SlotImage + +/** Android extractor for Bitmap payloads: axis-aligned crop, then upright rotation. */ +object AndroidBitmapSlotImageExtractor : SlotImageExtractor { + + override fun extract(frame: OriginalFrame, slot: DraftSlot): Result { + val payload = frame.payload as? BitmapFramePayload + ?: return Result.failure(IllegalArgumentException("AndroidBitmapSlotImageExtractor requires a BitmapFramePayload")) + val bitmap = payload.bitmap + if (bitmap.isRecycled) { + return Result.failure(IllegalStateException("The frame bitmap has been recycled")) + } + if (slot.rect.width * frame.sourceWidth < 1f || slot.rect.height * frame.sourceHeight < 1f) { + return Result.failure(IllegalArgumentException("The slot rectangle is smaller than one pixel")) + } + val crop = slotCropBounds(slot.rect, frame.sourceWidth, frame.sourceHeight) + val cropped = try { + Bitmap.createBitmap(bitmap, crop.left, crop.top, crop.width, crop.height) + } catch (e: Exception) { + return Result.failure(IllegalStateException("The slot could not be cropped: ${e.message}", e)) + } + val upright = rotateToUpright(cropped, frame.rotation) + if (upright !== cropped) cropped.recycle() + return Result.success( + SlotImage( + id = FrameId.random(), + sourceId = frame.id, + slotId = slot.id, + payload = BitmapFramePayload(upright), + sourceRect = crop, + createdAtElapsedRealtimeMs = SystemClock.elapsedRealtime(), + ), + ) + } + + private fun rotateToUpright(source: Bitmap, rotation: FrameRotation): Bitmap { + if (rotation == FrameRotation.ROTATION_0) return source + val swapsAxes = rotation == FrameRotation.ROTATION_90 || rotation == FrameRotation.ROTATION_270 + val outWidth = if (swapsAxes) source.height else source.width + val outHeight = if (swapsAxes) source.width else source.height + val out = Bitmap.createBitmap(outWidth, outHeight, Bitmap.Config.ARGB_8888) + val matrix = Matrix().apply { + postTranslate(-source.width / 2f, -source.height / 2f) + postRotate( + when (rotation) { + FrameRotation.ROTATION_90 -> 90f + FrameRotation.ROTATION_180 -> 180f + FrameRotation.ROTATION_270 -> 270f + else -> 0f + }, + ) + postTranslate(outWidth / 2f, outHeight / 2f) + } + Canvas(out).drawBitmap(source, matrix, null) + return out + } +} + +/** Picks the extractor that can read [frame]'s payload. */ +object SlotImageExtractors { + fun forFrame(frame: OriginalFrame): SlotImageExtractor = when (frame.payload) { + is PixelArrayFramePayload -> PixelSlotImageExtractor + else -> AndroidBitmapSlotImageExtractor + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt new file mode 100644 index 0000000..e5e661b --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt @@ -0,0 +1,84 @@ +package com.draftlens.vision.heroes + +import java.io.BufferedInputStream +import java.net.HttpURLConnection +import java.net.URL + +/** + * HTTPS-only transport over the platform HTTP client. No third-party dependency. + * One retry on I/O errors and 5xx; 429 fails fast with its code. The size cap is + * enforced while reading, so a hostile or broken endpoint cannot exhaust memory. + */ +class AndroidHeroCatalogTransport( + private val connectTimeoutMs: Int = 15_000, + private val readTimeoutMs: Int = 30_000, +) : HeroCatalogTransport { + + override fun get(url: String, maxBytes: Long, accept: String?): HeroHttpResult { + if (maxBytes <= 0L) return HeroHttpResult.Failure("The size cap must be positive") + if (!url.startsWith("https://")) { + return HeroHttpResult.Failure("Only HTTPS URLs are allowed: $url") + } + var lastFailure: HeroHttpResult.Failure? = null + repeat(2) { attempt -> + when (val outcome = attemptOnce(url, maxBytes, accept)) { + is HeroHttpResult.Success -> return outcome + is HeroHttpResult.Failure -> { + lastFailure = outcome + val retryable = outcome.httpCode == null || outcome.httpCode >= 500 + if (!retryable) return outcome // 4xx, including 429: fail fast, do not parallel-hammer + if (attempt == 1) return outcome + } + } + } + return lastFailure ?: HeroHttpResult.Failure("The request failed") + } + + private fun attemptOnce(url: String, maxBytes: Long, accept: String?): HeroHttpResult { + val connection = try { + (URL(url).openConnection() as? HttpURLConnection) + ?: return HeroHttpResult.Failure("Only HTTP(S) URLs are supported: $url") + } catch (e: Exception) { + return HeroHttpResult.Failure("The URL could not be opened: ${e.message}") + } + return try { + connection.requestMethod = "GET" + connection.connectTimeout = connectTimeoutMs + connection.readTimeout = readTimeoutMs + if (accept != null) connection.setRequestProperty("Accept", accept) + connection.setRequestProperty("User-Agent", "DraftLens-HeroCatalog/1.0") + val code = connection.responseCode + if (code in 200..299) { + val bytes = readCapped(connection.inputStream, maxBytes) + ?: return HeroHttpResult.Failure("The response exceeds $maxBytes bytes", code) + val contentType = connection.contentType?.substringBefore(';')?.trim()?.lowercase() + HeroHttpResult.Success(HeroHttpResponse(bytes, contentType)) + } else { + HeroHttpResult.Failure("HTTP $code from $url", code) + } + } catch (e: Exception) { + HeroHttpResult.Failure("The request failed: ${e.message}") + } finally { + try { + connection.disconnect() + } catch (_: Exception) { + } + } + } + + private fun readCapped(input: java.io.InputStream, maxBytes: Long): ByteArray? { + val buffer = ByteArray(8192) + val out = java.io.ByteArrayOutputStream() + BufferedInputStream(input).use { stream -> + var total = 0L + while (true) { + val read = stream.read(buffer) + if (read == -1) break + total += read + if (total > maxBytes) return null + out.write(buffer, 0, read) + } + } + return out.toByteArray() + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt new file mode 100644 index 0000000..51bf9b8 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt @@ -0,0 +1,157 @@ +package com.draftlens.vision.heroes + +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import java.io.File + +/** + * File-based catalog: one atomic snapshot JSON plus one file per reference image. + * Writes go to a `.tmp` sibling first and are renamed into place, so an interrupted + * sync never leaves a half-read snapshot behind. + */ +class FileHeroCatalogRepository( + private val dataDir: File, + private val cacheDir: File, + private val transport: HeroCatalogTransport, + private val clock: () -> Long = System::currentTimeMillis, +) : HeroCatalogRepository { + + private val snapshotFile = File(dataDir, "hero-catalog/catalog.json") + private val assetsDir = File(cacheDir, "hero-assets") + + @Volatile + private var cached: CatalogSnapshot? = null + + private val _state = MutableStateFlow(HeroCatalogState.NotSynced) + override val state: StateFlow = _state.asStateFlow() + + init { + cached = readDiskSnapshot() + _state.value = cached?.let { HeroCatalogState.Ready(it.entries.size, it.syncedAtMs) } + ?: HeroCatalogState.NotSynced + } + + override fun snapshot(): CatalogSnapshot? { + cached?.let { return it } + val disk = readDiskSnapshot() ?: return null + cached = disk + return disk + } + + override suspend fun refresh(): Result { + _state.value = HeroCatalogState.Syncing("Downloading catalog") + val sync = HeroCatalogSync(transport) + val report = sync.sync().getOrElse { failure -> + _state.value = HeroCatalogState.Error( + message = failure.message ?: "The hero catalog could not be downloaded", + cachedEntryCount = cached?.entries?.size, + ) + return Result.failure(failure) + } + _state.value = HeroCatalogState.Syncing("Downloading hero images") + val assetFailures = downloadAssets(report.entries) + val snapshot = CatalogSnapshot( + schemaVersion = CatalogSnapshot.CURRENT_SCHEMA, + syncedAtMs = clock(), + sourceAttribution = listOf(SourceAttribution.RONE_ARENA, SourceAttribution.MLBBDEX), + entries = report.entries, + assetFailures = assetFailures, + crossCheck = (report.crossCheck as? CrossCheckResult.Available)?.crossCheck, + ) + if (!writeAtomically(snapshotFile, HeroCatalogCodec.encode(snapshot).toByteArray())) { + val failure = IllegalStateException("The hero catalog snapshot could not be written") + _state.value = HeroCatalogState.Error(failure.message!!, cached?.entries?.size) + return Result.failure(failure) + } + cached = snapshot + _state.value = HeroCatalogState.Ready(snapshot.entries.size, snapshot.syncedAtMs) + return Result.success(snapshot) + } + + override fun assetFile(id: HeroId, variant: HeroAssetVariant): File? { + val file = assetFileFor(id, variant) + return file.takeIf { it.exists() && it.length() > 0L } + } + + // region internals + + private fun assetFileFor(id: HeroId, variant: HeroAssetVariant): File = + File(assetsDir, "${id.value}-${variant.name.lowercase()}.img") + + /** Returns the names of heroes whose images failed to download or validate. */ + private fun downloadAssets(entries: List): List { + val failures = mutableListOf() + for (entry in entries) { + val asset = entry.assets.firstOrNull { it.variant == HeroAssetVariant.HEAD } ?: continue + val target = assetFileFor(entry.id, HeroAssetVariant.HEAD) + if (target.exists() && target.length() > 0L) continue // keep the cache; refresh is about the catalog + val ok = downloadAsset(asset, target) + if (!ok) { + runCatching { target.delete() } + failures += entry.canonicalName + } + } + return failures + } + + /** HTTPS + image content check via magic bytes, with a size cap enforced by the transport. */ + private fun downloadAsset(asset: HeroAsset, target: File): Boolean { + val response = when (val result = transport.get(asset.url, MAX_IMAGE_BYTES, null)) { + is HeroHttpResult.Success -> result.response + is HeroHttpResult.Failure -> return false + } + if (asset.url.startsWith("https://").not()) return false + if (response.bytes.size > MAX_IMAGE_BYTES) return false + val contentType = response.contentType?.lowercase() + if (contentType != null && !contentType.startsWith("image/")) return false + if (!looksLikeImage(response.bytes)) return false + return writeAtomically(target, response.bytes) + } + + private fun looksLikeImage(bytes: ByteArray): Boolean { + if (bytes.size < 8) return false + val isPng = bytes[0] == 0x89.toByte() && bytes[1] == 'P'.code.toByte() && bytes[2] == 'N'.code.toByte() + val isJpeg = bytes[0] == 0xFF.toByte() && bytes[1] == 0xD8.toByte() && bytes[2] == 0xFF.toByte() + val isWebp = bytes.size >= 12 && + bytes[0] == 'R'.code.toByte() && bytes[1] == 'I'.code.toByte() && + bytes[2] == 'F'.code.toByte() && bytes[3] == 'F'.code.toByte() && + bytes[8] == 'W'.code.toByte() && bytes[9] == 'E'.code.toByte() && + bytes[10] == 'B'.code.toByte() && bytes[11] == 'P'.code.toByte() + val isGif = bytes[0] == 'G'.code.toByte() && bytes[1] == 'I'.code.toByte() && bytes[2] == 'F'.code.toByte() + return isPng || isJpeg || isWebp || isGif + } + + private fun readDiskSnapshot(): CatalogSnapshot? { + val file = snapshotFile + if (!file.exists()) return null + return try { + HeroCatalogCodec.decode(file.readText()) + } catch (e: Exception) { + _state.value = HeroCatalogState.Error( + "The cached hero catalog is invalid and must be re-downloaded: ${e.message}", + cachedEntryCount = null, + ) + null + } + } + + private fun writeAtomically(target: File, bytes: ByteArray): Boolean = runCatching { + val parent = target.parentFile ?: return false + parent.mkdirs() + val tmp = File(parent, "${target.name}.tmp") + tmp.writeBytes(bytes) + if (!tmp.renameTo(target)) { + // renameTo can fail across file systems; fall back to a copy+delete. + target.writeBytes(bytes) + tmp.delete() + } + true + }.getOrDefault(false) + + // endregion + + private companion object { + const val MAX_IMAGE_BYTES = 4L * 1024L * 1024L // 4 MiB per reference image + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogCodec.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogCodec.kt new file mode 100644 index 0000000..fff69bc --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogCodec.kt @@ -0,0 +1,49 @@ +package com.draftlens.vision.heroes + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.json.Json + +/** Strict codec for the cached catalog snapshot. Unknown fields are rejected. */ +object HeroCatalogCodec { + + private val json = Json { + ignoreUnknownKeys = false + encodeDefaults = true + } + + fun encode(snapshot: CatalogSnapshot): String { + validate(snapshot) + return json.encodeToString(CatalogSnapshot.serializer(), snapshot) + } + + fun decode(text: String): CatalogSnapshot { + val snapshot = try { + json.decodeFromString(CatalogSnapshot.serializer(), text) + } catch (e: SerializationException) { + throw IllegalArgumentException("The cached hero catalog is not valid or uses an unknown field. ${e.message}", e) + } catch (e: IllegalArgumentException) { + throw IllegalArgumentException("The cached hero catalog is invalid: ${e.message}", e) + } + validate(snapshot) + return snapshot + } + + private fun validate(snapshot: CatalogSnapshot) { + if (snapshot.schemaVersion != CatalogSnapshot.CURRENT_SCHEMA) { + throw IllegalArgumentException( + "Unsupported hero catalog schema version ${snapshot.schemaVersion}. Expected ${CatalogSnapshot.CURRENT_SCHEMA}.", + ) + } + val seen = mutableSetOf() + for (entry in snapshot.entries) { + if (entry.id.value.isBlank()) throw IllegalArgumentException("A hero entry has a blank id.") + if (entry.canonicalName.isBlank()) throw IllegalArgumentException("Hero '${entry.id.value}' has a blank name.") + if (!seen.add(entry.id)) throw IllegalArgumentException("Duplicate hero id '${entry.id.value}' in the cached catalog.") + for (asset in entry.assets) { + if (!asset.url.startsWith("https://")) { + throw IllegalArgumentException("Hero '${entry.id.value}' asset is not HTTPS: ${asset.url}") + } + } + } + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogRepository.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogRepository.kt new file mode 100644 index 0000000..08b5107 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogRepository.kt @@ -0,0 +1,30 @@ +package com.draftlens.vision.heroes + +import kotlinx.coroutines.flow.StateFlow +import java.io.File + +/** What the UI knows about the catalog. */ +sealed interface HeroCatalogState { + data object NotSynced : HeroCatalogState + data class Syncing(val stage: String) : HeroCatalogState + data class Ready(val entryCount: Int, val syncedAtMs: Long) : HeroCatalogState + data class Error(val message: String, val cachedEntryCount: Int?) : HeroCatalogState +} + +/** + * The hero catalog: a provider-independent model, a file-based atomic snapshot and a + * private image cache. No network happens until [refresh] is called; [snapshot] serves + * cached offline browsing. + */ +interface HeroCatalogRepository { + val state: StateFlow + + /** The in-memory or on-disk snapshot, or null when nothing usable is cached. */ + fun snapshot(): CatalogSnapshot? + + /** Downloads the catalog and its reference images; returns the stored snapshot. */ + suspend fun refresh(): Result + + /** The cached image file for [id]/[variant] when it exists and is non-empty. */ + fun assetFile(id: HeroId, variant: HeroAssetVariant): File? +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogSync.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogSync.kt new file mode 100644 index 0000000..98f4ef6 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogSync.kt @@ -0,0 +1,246 @@ +package com.draftlens.vision.heroes + +import kotlinx.serialization.SerializationException +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +/** + * Pulls the hero catalog from Rone Arena and cross-checks it against MLBBDex. + * + * Pure logic against [HeroCatalogTransport], so tests run with fake responses. + * HTTPS only, one page size, a page cap, and duplicate-ID protection are enforced here. + */ +class HeroCatalogSync( + private val transport: HeroCatalogTransport, + private val roneArenaBases: List = listOf( + "https://arena.fastapicloud.dev/api", + "https://arena.rone.dev/api", + ), + private val mlbbdexBase: String = "https://mlbbdex.com/api/v1", + private val pageSize: Int = 100, + private val maxPages: Int = 20, +) { + + data class SyncReport( + val entries: List, + val invalidRecords: Int, + val duplicateIds: Int, + val crossCheck: CrossCheckResult, + ) + + fun sync(): Result { + val mlbbdex = fetchMlbbdexNames() // null on failure: cross-check degrades, sync continues + val list = when (val outcome = fetchRoneArenaList(mlbbdex)) { + is ResultData.Success -> outcome.value + is ResultData.Failure -> return Result.failure(outcome.cause) + } + val (entries, invalid, duplicates) = list + return Result.success( + SyncReport( + entries = entries, + invalidRecords = invalid, + duplicateIds = duplicates, + crossCheck = crossCheck(mlbbdex, entries), + ), + ) + } + + // region fetching + + private sealed interface ResultData { + data class Success(val value: T) : ResultData + data class Failure(val cause: Exception) : ResultData + } + + /** Rone Arena list with a one-shot mirror fallback: same code and data on the other host. */ + private fun fetchRoneArenaList(mlbbdex: List?): ResultData, Int, Int>> { + for (base in roneArenaBases) { + when (val outcome = fetchRoneArenaListFrom(base, mlbbdex)) { + is ResultData.Success -> return outcome + is ResultData.Failure -> continue // try the mirror, if there is one + } + } + return ResultData.Failure(IllegalStateException("No Rone Arena host responded with a usable catalog")) + } + + private fun fetchRoneArenaListFrom( + base: String, + mlbbdex: List?, + ): ResultData, Int, Int>> { + val seen = LinkedHashMap() + var invalid = 0 + var duplicates = 0 + var total = Int.MAX_VALUE + var page = 1 + while (page <= maxPages && seen.size < total) { + val url = "$base/heroes?size=$pageSize&index=$page&lang=en" + val raw = when (val result = transport.get(url, MAX_JSON_BYTES, "application/json")) { + is HeroHttpResult.Success -> result.response.bytes + is HeroHttpResult.Failure -> + return ResultData.Failure(IllegalStateException("Rone Arena catalog request failed: ${result.message}")) + } + val parsed = try { + json.decodeFromString(RoneHeroesResponse.serializer(), raw.decodeToString()) + } catch (e: SerializationException) { + return ResultData.Failure(IllegalArgumentException("Rone Arena catalog page $page is not valid JSON: ${e.message}", e)) + } catch (e: IllegalArgumentException) { + return ResultData.Failure(IllegalArgumentException("Rone Arena catalog page $page is invalid: ${e.message}", e)) + } + if (parsed.code != 0) { + return ResultData.Failure(IllegalStateException("Rone Arena catalog page $page returned code ${parsed.code} (${parsed.message})")) + } + total = parsed.data.total.coerceAtLeast(0) + val records = parsed.data.records + if (records.isEmpty()) break + for (record in records) { + val heroId = record.data.heroId + val hero = record.data.hero?.data + if (heroId == null || hero == null || hero.name.isNullOrBlank() || + (hero.head.isNullOrBlank() && hero.smallmap.isNullOrBlank()) + ) { + invalid++ + continue + } + if (seen.putIfAbsent(heroId, record) != null) { + duplicates++ + } + } + if (records.size < pageSize) break + page++ + } + if (seen.isEmpty()) { + return ResultData.Failure(IllegalStateException("Rone Arena catalog returned no usable hero records")) + } + val bySlug = mlbbdex?.associateBy { it.slug } ?: emptyMap() + val entries = seen.values.mapNotNull { record -> + val hero = record.data.hero!!.data!! + val name = hero.name!!.trim() + val id = HeroId(HeroIds.slugOf(name)) + val reference = bySlug[id.value] + buildList { + hero.head?.takeIf { it.isNotBlank() }?.let { add(HeroAsset(HeroAssetVariant.HEAD, it)) } + hero.smallmap?.takeIf { it.isNotBlank() }?.let { add(HeroAsset(HeroAssetVariant.SMALLMAP, it)) } + }.takeIf { it.isNotEmpty() }?.let { assets -> + HeroCatalogEntry( + id = id, + canonicalName = name, + aliases = emptyList(), + roles = reference?.roles.orEmpty(), + lanes = reference?.lanes.orEmpty(), + specialties = reference?.specialties.orEmpty(), + assets = assets, + externalIds = mapOf( + "roneArenaHeroId" to record.data.heroId!!.toString(), + "mlbbdexSlug" to id.value, + ), + source = HeroCatalogSource.RONE_ARENA, + ) + } + } + return ResultData.Success(Triple(entries, invalid, duplicates)) + } + + private data class MlbbdexName( + val slug: String, + val name: String, + val roles: List, + val lanes: List, + val specialties: List, + ) + + /** Names from MLBBDex for the cross-check and for roles/lanes. Null when unreachable. */ + private fun fetchMlbbdexNames(): List? { + val raw = when (val result = transport.get("$mlbbdexBase/heroes", MAX_JSON_BYTES, "application/json")) { + is HeroHttpResult.Success -> result.response.bytes + is HeroHttpResult.Failure -> return null + } + return try { + json.decodeFromString(MlbbdexResponse.serializer(), raw.decodeToString()) + .data + .filter { it.slug.isNotBlank() && it.name.isNotBlank() } + .map { + MlbbdexName( + slug = it.slug, + name = it.name, + roles = it.roles, + lanes = it.lanes, + specialties = it.specialties, + ) + } + } catch (e: Exception) { + null + } + } + + private fun crossCheck(mlbbdex: List?, entries: List): CrossCheckResult { + val names = mlbbdex ?: return CrossCheckResult.Unavailable("MLBBDex /api/v1/heroes could not be read") + val roneSlugs = entries.map { it.id.value }.toSet() + val dexSlugs = names.map { it.slug }.toSet() + return CrossCheckResult.Available( + CatalogCrossCheck( + roneArenaCount = roneSlugs.size, + mlbbdexCount = dexSlugs.size, + matchedNames = roneSlugs.intersect(dexSlugs).size, + missingInRoneArena = names.filter { it.slug !in roneSlugs }.map { it.name }, + missingInMlbbdex = entries.filter { it.id.value !in dexSlugs }.map { it.canonicalName }, + ), + ) + } + + // endregion + + private companion object { + const val MAX_JSON_BYTES = 16L * 1024L * 1024L // 16 MiB cap on catalog pages + val json = Json { ignoreUnknownKeys = true; isLenient = true } + } + + // region wire formats (Rone Arena list and MLBBDex list) + + @Serializable + private data class RoneHeroesResponse( + val code: Int = 0, + val message: String = "", + val data: RoneListData = RoneListData(), + ) + + @Serializable + private data class RoneListData( + val records: List = emptyList(), + val total: Int = 0, + ) + + @Serializable + private data class RoneHeroRecord(val data: RoneRecordData = RoneRecordData()) + + @Serializable + private data class RoneRecordData( + val hero: RoneHero? = null, + val hero_id: Long? = null, + ) { + val heroId: Long? get() = hero_id + } + + @Serializable + private data class RoneHero(val data: RoneHeroDetails? = null) + + @Serializable + private data class RoneHeroDetails( + val head: String? = null, + val name: String? = null, + val smallmap: String? = null, + ) + + @Serializable + private data class MlbbdexResponse(val data: List = emptyList()) + + @Serializable + private data class MlbbdexHero( + val slug: String = "", + val name: String = "", + val roles: List = emptyList(), + val lanes: List = emptyList(), + val specialties: List = emptyList(), + ) + + // endregion +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogTransport.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogTransport.kt new file mode 100644 index 0000000..1288724 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroCatalogTransport.kt @@ -0,0 +1,13 @@ +package com.draftlens.vision.heroes + +/** One HTTPS GET. Implementations enforce the size cap while reading. */ +interface HeroCatalogTransport { + fun get(url: String, maxBytes: Long, accept: String?): HeroHttpResult +} + +data class HeroHttpResponse(val bytes: ByteArray, val contentType: String?) + +sealed interface HeroHttpResult { + data class Success(val response: HeroHttpResponse) : HeroHttpResult + data class Failure(val message: String, val httpCode: Int? = null) : HeroHttpResult +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroIds.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroIds.kt new file mode 100644 index 0000000..cb45af6 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroIds.kt @@ -0,0 +1,16 @@ +package com.draftlens.vision.heroes + +object HeroIds { + /** + * Normalized English name -> provider-independent slug. + * + * Lower-cased, every run of non-alphanumeric characters becomes one hyphen, leading and + * trailing hyphens removed. MLBBDex slugs follow the same rule for the names checked in + * this sprint ("Yi Sun-shin" -> "yi-sun-shin"). + */ + fun slugOf(name: String): String = + name.lowercase() + .replace(Regex("[^a-z0-9]+"), "-") + .trim('-') + .also { require(it.isNotEmpty()) { "hero name '$name' normalizes to an empty slug" } } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroMatcher.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroMatcher.kt new file mode 100644 index 0000000..1411e93 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroMatcher.kt @@ -0,0 +1,188 @@ +package com.draftlens.vision.heroes + +import kotlin.math.abs + +/** + * Hero matching v0: a dHash perceptual hash plus a normalized 3x3 mean-RGB appearance + * descriptor, combined into one deterministic similarity in [0, 1]. + * + * The score is a heuristic, **not** a calibrated confidence or a probability. Treat it as a + * ranking aid; the user always confirms. Comparison happens on fixed-size grids, so the + * reference image resolution does not need to match the crop. + */ + +/** Tunables for matching. The threshold is a starting point, calibrated on synthetic fixtures only. */ +data class HeroMatchConfig( + /** Below this similarity the result is "no match". */ + val noMatchThreshold: Float = 0.55f, + val topK: Int = 3, + /** Two candidates closer than this in similarity are reported as ambiguous. */ + val ambiguousGap: Float = 0.05f, + /** Weight of the perceptual hash against the color descriptor in the combined score. */ + val hashWeight: Float = 0.6f, +) { + init { + require(noMatchThreshold in 0f..1f) { "noMatchThreshold must be within [0, 1]" } + require(topK >= 1) { "topK must be at least 1" } + require(ambiguousGap >= 0f) { "ambiguousGap must not be negative" } + require(hashWeight in 0f..1f) { "hashWeight must be within [0, 1]" } + } +} + +data class HeroReference( + val entry: HeroCatalogEntry, + val pixels: IntArray, + val width: Int, + val height: Int, +) + +data class HeroMatch( + val entry: HeroCatalogEntry, + /** Similarity in [0, 1]. Not a probability. */ + val similarity: Float, + /** True when this candidate is within ambiguousGap of the next-best candidate. */ + val ambiguous: Boolean, +) + +sealed interface HeroMatchResult { + data class Candidates(val matches: List) : HeroMatchResult + /** Evidence is weak; [bestSimilarity] is reported so the UI can say how far off it was. */ + data class NoMatch(val bestSimilarity: Float) : HeroMatchResult +} + +object HeroMatcher { + + private const val HASH_WIDTH = 9 + private const val HASH_HEIGHT = 8 + private const val GRID = 3 + + fun match( + queryPixels: IntArray, + queryWidth: Int, + queryHeight: Int, + references: List, + config: HeroMatchConfig = HeroMatchConfig(), + ): HeroMatchResult { + if (queryPixels.size != queryWidth * queryHeight || queryWidth <= 0 || queryHeight <= 0) { + throw IllegalArgumentException("query pixels do not match the query dimensions") + } + val usable = references.filter { it.pixels.size == it.width * it.height && it.width > 0 && it.height > 0 } + if (usable.isEmpty()) throw IllegalStateException("no usable hero reference images are available") + + val queryHash = perceptualHash(queryPixels, queryWidth, queryHeight) + val queryColor = colorDescriptor(queryPixels, queryWidth, queryHeight) + val ranked = usable + .map { reference -> + val hash = perceptualHash(reference.pixels, reference.width, reference.height) + val color = colorDescriptor(reference.pixels, reference.width, reference.height) + val hashScore = 1f - hammingDistance(queryHash, hash).toFloat() / 64f + val colorScore = 1f - colorDistance(queryColor, color) + val similarity = (config.hashWeight * hashScore + (1f - config.hashWeight) * colorScore) + .coerceIn(0f, 1f) + reference.entry to similarity + } + .sortedByDescending { it.second } + + val best = ranked.first().second + if (best < config.noMatchThreshold) return HeroMatchResult.NoMatch(best) + val gapToNext = ranked.getOrNull(1)?.let { best - it.second } ?: Float.MAX_VALUE + val ambiguous = gapToNext < config.ambiguousGap + val matches = ranked.take(config.topK).mapIndexed { index, (entry, similarity) -> + // The gap to the *runner-up* marks the top candidate; the gap to the *next* marks later ones. + val entryAmbiguous = if (index == 0) { + ambiguous + } else { + val gap = similarity - (ranked.getOrNull(index + 1)?.second ?: 0f) + gap < config.ambiguousGap + } + HeroMatch(entry, similarity, entryAmbiguous) + } + return HeroMatchResult.Candidates(matches) + } + + /** 64-bit dHash: downsampled grayscale, bit c of row r = gray > right neighbor. */ + fun perceptualHash(pixels: IntArray, width: Int, height: Int): Long { + val gray = downsampleGray(pixels, width, height, HASH_WIDTH, HASH_HEIGHT) + var hash = 0L + for (row in 0 until HASH_HEIGHT) { + for (col in 0 until HASH_WIDTH - 1) { + hash = hash shl 1 + if (gray[row][col] > gray[row][col + 1]) hash = hash or 1L + } + } + return hash + } + + fun hammingDistance(a: Long, b: Long): Int = (a xor b).countOneBits() + + /** 27 floats: the mean RGB of a 3x3 grid, normalized to [0, 1]. */ + fun colorDescriptor(pixels: IntArray, width: Int, height: Int): FloatArray { + val grid = Array(GRID) { FloatArray(GRID * 3) } + val counts = IntArray(GRID * GRID) + for (y in 0 until height) { + val gy = (y * GRID) / height + for (x in 0 until width) { + val gx = (x * GRID) / width + val cell = gy * GRID + gx + val p = pixels[y * width + x] + grid[gy][gx * 3 + 0] += ((p shr 16) and 0xFF).toFloat() + grid[gy][gx * 3 + 1] += ((p shr 8) and 0xFF).toFloat() + grid[gy][gx * 3 + 2] += (p and 0xFF).toFloat() + counts[cell]++ + } + } + val out = FloatArray(GRID * GRID * 3) + for (gy in 0 until GRID) { + for (gx in 0 until GRID) { + val cell = gy * GRID + gx + val count = counts[cell].coerceAtLeast(1) + out[cell * 3 + 0] = grid[gy][gx * 3 + 0] / (255f * count) + out[cell * 3 + 1] = grid[gy][gx * 3 + 1] / (255f * count) + out[cell * 3 + 2] = grid[gy][gx * 3 + 2] / (255f * count) + } + } + return out + } + + /** Mean absolute difference of two descriptors, in [0, 1]. */ + fun colorDistance(a: FloatArray, b: FloatArray): Float { + require(a.size == b.size) { "descriptors must have the same size" } + if (a.isEmpty()) return 0f + var sum = 0f + for (i in a.indices) sum += abs(a[i] - b[i]) + return sum / a.size + } + + /** Area-average downsample to [columns] x [rows] grayscale values. */ + private fun downsampleGray( + pixels: IntArray, + width: Int, + height: Int, + columns: Int, + rows: Int, + ): Array { + val out = Array(rows) { FloatArray(columns) } + for (row in 0 until rows) { + val y0 = row * height / rows + val y1 = (row + 1) * height / rows + for (col in 0 until columns) { + val x0 = col * width / columns + val x1 = (col + 1) * width / columns + var sum = 0f + var count = 0 + for (y in y0 until y1.coerceAtMost(height)) { + for (x in x0 until x1.coerceAtMost(width)) { + val p = pixels[y * width + x] + val r = (p shr 16) and 0xFF + val g = (p shr 8) and 0xFF + val b = p and 0xFF + sum += 0.299f * r + 0.587f * g + 0.114f * b + count++ + } + } + out[row][col] = if (count > 0) sum / count else 0f + } + } + return out + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroModels.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroModels.kt new file mode 100644 index 0000000..fab8d1f --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroModels.kt @@ -0,0 +1,114 @@ +package com.draftlens.vision.heroes + +import kotlinx.serialization.Serializable + +/** + * The hero catalog. + * + * Provider-independent identity is the normalized English name turned into a slug + * ([HeroId]). Provider-specific ids (Rone Arena `hero_id`, MLBBDex `slug`) are stored in + * [HeroCatalogEntry.externalIds]; display names are never used as identity. + */ + +/** Stable, provider-independent identity: lower-case hyphenated slug. */ +@Serializable +@JvmInline +value class HeroId(val value: String) { + init { + require(value.isNotBlank()) { "hero id must not be blank" } + } +} + +/** A localized name for a hero. Only English is synced in this sprint. */ +@Serializable +data class HeroAlias(val language: String, val name: String) + +@Serializable +enum class HeroAssetVariant { HEAD, HEAD_BIG, SMALLMAP, SQUAREHEAD, SQUAREHEAD_BIG } + +/** + * A downloadable reference image. [contentType] and [sizeBytes] are filled in after the + * asset has been downloaded once; they stay null until then. + */ +@Serializable +data class HeroAsset( + val variant: HeroAssetVariant, + val url: String, + val contentType: String? = null, + val sizeBytes: Long? = null, +) + +@Serializable +enum class HeroCatalogSource { RONE_ARENA, MLBBDEX } + +@Serializable +data class SourceAttribution( + val name: String, + val url: String, + val license: String, + val notice: String, +) { + companion object { + val RONE_ARENA = SourceAttribution( + name = "Rone Arena API (unofficial community project)", + url = "https://arena.rone.dev", + license = "BSD-3-Clause", + notice = "Rone Arena is not affiliated with or endorsed by Moonton; the game is named only to describe the data. " + + "Game names and visual assets remain the property of their respective rights holders.", + ) + val MLBBDEX = SourceAttribution( + name = "MLBBDex (independent fan project)", + url = "https://mlbbdex.com", + license = "CC BY-SA (community wiki data); written content MIT", + notice = "Game visuals remain the property of Shanghai Moonton Technology Co., Ltd.", + ) + } +} + +/** One hero. Identity is [id]; names are for display only. */ +@Serializable +data class HeroCatalogEntry( + val id: HeroId, + val canonicalName: String, + val aliases: List, + val roles: List, + val lanes: List, + val specialties: List, + val assets: List, + val externalIds: Map, + val source: HeroCatalogSource, +) + +/** + * Completeness cross-check of the Rone Arena catalog against MLBBDex. + * Not evidence; just a signal that the name sets roughly agree. + */ +@Serializable +data class CatalogCrossCheck( + val roneArenaCount: Int, + val mlbbdexCount: Int, + val matchedNames: Int, + val missingInRoneArena: List, + val missingInMlbbdex: List, +) + +/** What a full sync produced. */ +@Serializable +data class CatalogSnapshot( + val schemaVersion: Int, + val syncedAtMs: Long, + val sourceAttribution: List, + val entries: List, + val assetFailures: List, + val crossCheck: CatalogCrossCheck?, +) { + companion object { + const val CURRENT_SCHEMA = 1 + } +} + +/** Result of the MLBBDex completeness check. */ +sealed interface CrossCheckResult { + data class Available(val crossCheck: CatalogCrossCheck) : CrossCheckResult + data class Unavailable(val reason: String) : CrossCheckResult +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt new file mode 100644 index 0000000..cc28583 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt @@ -0,0 +1,48 @@ +package com.draftlens.vision.heroes + +/** One ranked hero shown after an Identify action. Similarity is [0, 1], never a probability. */ +data class HeroCandidate( + val hero: HeroCatalogEntry, + val similarity: Float, + val ambiguous: Boolean, +) + +/** What the Identify button produced. */ +sealed interface HeroIdentifyOutcome { + data class Candidates(val candidates: List) : HeroIdentifyOutcome + data class NoMatch(val bestSimilarity: Float) : HeroIdentifyOutcome + data class Failed(val message: String) : HeroIdentifyOutcome +} + +/** + * Recognition v0 orchestration. Pure: pixels and preloaded references in, outcome out. + * Decoding assets and building references happen on the caller's side so tests can inject + * synthetic references. + */ +object HeroRecognition { + + fun identify( + pixels: IntArray, + width: Int, + height: Int, + references: List, + config: HeroMatchConfig = HeroMatchConfig(), + ): HeroIdentifyOutcome { + val usable = references.filter { it.pixels.size == it.width * it.height && it.width > 0 && it.height > 0 } + if (width <= 0 || height <= 0 || pixels.size != width * height) { + return HeroIdentifyOutcome.Failed("The extracted image is empty or invalid.") + } + if (usable.isEmpty()) { + return HeroIdentifyOutcome.Failed( + "No usable hero reference images are cached. Sync the hero catalog first.", + ) + } + return when (val result = HeroMatcher.match(pixels, width, height, usable, config)) { + is HeroMatchResult.Candidates -> HeroIdentifyOutcome.Candidates( + result.matches.map { HeroCandidate(it.entry, it.similarity, it.ambiguous) }, + ) + + is HeroMatchResult.NoMatch -> HeroIdentifyOutcome.NoMatch(result.bestSimilarity) + } + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt new file mode 100644 index 0000000..a32ab55 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt @@ -0,0 +1,30 @@ +package com.draftlens.vision.heroes + +import android.graphics.BitmapFactory +import com.draftlens.vision.model.BitmapFramePayload +import java.io.File + +/** + * Android-side decoding for the matcher. Everything stays in memory and nothing is + * written to disk by these functions. + */ +object HeroReferenceImages { + + /** Decode a cached reference asset, downsampled so matching stays cheap. Null when missing/corrupt. */ + fun loadReference(entry: HeroCatalogEntry, file: File): HeroReference? = runCatching { + val options = BitmapFactory.Options().apply { + inJustDecodeBounds = true + } + BitmapFactory.decodeFile(file.absolutePath, options) + if (options.outWidth <= 0 || options.outHeight <= 0) return null + val sample = (maxOf(options.outWidth, options.outHeight) / MAX_REFERENCE_DIMENSION).coerceAtLeast(1) + val decoded = BitmapFactory.Options().apply { inSampleSize = sample } + val bitmap = BitmapFactory.decodeFile(file.absolutePath, decoded) ?: return null + val pixels = IntArray(bitmap.width * bitmap.height) + bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) + bitmap.recycle() + HeroReference(entry, pixels, bitmap.width, bitmap.height) + }.getOrNull() + + private const val MAX_REFERENCE_DIMENSION = 256 +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt new file mode 100644 index 0000000..9920f6b --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt @@ -0,0 +1,128 @@ +package com.draftlens.vision.heroes + +import com.draftlens.vision.annotation.DraftSlot +import com.draftlens.vision.geometry.NormalizedRect +import com.draftlens.vision.geometry.PixelRect +import com.draftlens.vision.geometry.toPixelRect +import com.draftlens.vision.model.FrameId +import com.draftlens.vision.model.FrameRotation +import com.draftlens.vision.model.OriginalFrame +import com.draftlens.vision.model.PixelArrayFramePayload +import com.draftlens.vision.model.SlotImage + +/** + * Pulls the image contained in a slot out of a captured frame, upright as the user sees it. + * Failures are returned, never thrown; nothing is written to disk. + */ +fun interface SlotImageExtractor { + fun extract(frame: OriginalFrame, slot: DraftSlot): Result +} + +/** + * The crop bounds of a slot rectangle on the source image, in whole source pixels. + * Same rounding rule as the canvas ([NormalizedRect.toPixelRect]): floors the origin, + * ceils the far edge, clamps to the image, always at least one pixel per axis. + */ +internal fun slotCropBounds(rect: NormalizedRect, sourceWidth: Int, sourceHeight: Int): PixelRect = + rect.toPixelRect(sourceWidth, sourceHeight) + +/** + * Reads an extracted slot image back into pixels for the matcher. + * Pure logic for PixelArray payloads; Bitmap payloads are read on Android. + */ +fun pixelsOf(image: SlotImage): Triple? = when (val payload = image.payload) { + is com.draftlens.vision.model.PixelArrayFramePayload -> { + val pixels = payload.pixels() + Triple(pixels, payload.width, payload.height) + } + + is com.draftlens.vision.model.BitmapFramePayload -> { + val bitmap = payload.bitmap + if (bitmap.isRecycled) { + null + } else { + val pixels = IntArray(bitmap.width * bitmap.height) + bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) + Triple(pixels, bitmap.width, bitmap.height) + } + } + + else -> null +} + +/** Pure Kotlin extractor for synthetic/JVM payloads. */ +object PixelSlotImageExtractor : SlotImageExtractor { + + override fun extract(frame: OriginalFrame, slot: DraftSlot): Result { + val payload = frame.payload as? PixelArrayFramePayload + ?: return Result.failure( + IllegalArgumentException("PixelSlotImageExtractor requires a PixelArrayFramePayload"), + ) + if (payload.width != frame.sourceWidth || payload.height != frame.sourceHeight) { + return Result.failure( + IllegalArgumentException( + "payload ${payload.width}x${payload.height} does not match source ${frame.sourceWidth}x${frame.sourceHeight}", + ), + ) + } + if (slot.rect.width * frame.sourceWidth < 1f || slot.rect.height * frame.sourceHeight < 1f) { + return Result.failure(IllegalArgumentException("The slot rectangle is smaller than one pixel")) + } + val crop = slotCropBounds(slot.rect, frame.sourceWidth, frame.sourceHeight) + val source = payload.pixels() // defensive copy; the frame is never mutated or recycled + val width = crop.width + val height = crop.height + val cropped = IntArray(width * height) + for (y in 0 until height) { + val srcRow = (crop.top + y) * frame.sourceWidth + crop.left + System.arraycopy(source, srcRow, cropped, y * width, width) + } + val (pixels, outWidth, outHeight) = rotateToUpright(cropped, width, height, frame.rotation) + val output = SlotImage( + id = FrameId.random(), + sourceId = frame.id, + slotId = slot.id, + payload = PixelArrayFramePayload(outWidth, outHeight, pixels), + sourceRect = crop, + createdAtElapsedRealtimeMs = System.currentTimeMillis(), + ) + return Result.success(output) + } + + /** Rotates the axis-aligned crop so the result matches what the canvas shows. */ + private fun rotateToUpright( + pixels: IntArray, + width: Int, + height: Int, + rotation: FrameRotation, + ): Triple = when (rotation) { + FrameRotation.ROTATION_0 -> Triple(pixels, width, height) + FrameRotation.ROTATION_90 -> { + // oriented(xx, yy) = source(col = yy, row = height - 1 - xx) + val out = IntArray(width * height) + for (xx in 0 until height) { + for (yy in 0 until width) { + out[xx * width + yy] = pixels[(height - 1 - xx) * width + yy] + } + } + Triple(out, height, width) + } + + FrameRotation.ROTATION_180 -> { + val out = IntArray(width * height) + for (i in pixels.indices) out[pixels.size - 1 - i] = pixels[i] + Triple(out, width, height) + } + + FrameRotation.ROTATION_270 -> { + // oriented(xx, yy) = source(col = width - 1 - yy, row = xx) + val out = IntArray(width * height) + for (xx in 0 until height) { + for (yy in 0 until width) { + out[xx * width + yy] = pixels[xx * width + (width - 1 - yy)] + } + } + Triple(out, height, width) + } + } +} diff --git a/app/src/test/kotlin/com/draftlens/datastore/HeroRecognitionStoreTest.kt b/app/src/test/kotlin/com/draftlens/datastore/HeroRecognitionStoreTest.kt new file mode 100644 index 0000000..ce5dd18 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/datastore/HeroRecognitionStoreTest.kt @@ -0,0 +1,93 @@ +package com.draftlens.datastore + +import androidx.datastore.preferences.core.PreferenceDataStoreFactory +import androidx.datastore.preferences.core.edit +import androidx.datastore.preferences.core.preferencesOf +import androidx.datastore.preferences.core.stringPreferencesKey +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.cancel +import kotlinx.coroutines.runBlocking +import org.junit.After +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import java.io.File +import java.nio.file.Files + +class HeroRecognitionStoreTest { + + private lateinit var directory: File + private lateinit var scope: CoroutineScope + private lateinit var store: HeroRecognitionStore + + @Before fun setUp() { + directory = Files.createTempDirectory("hero-confirmations").toFile() + scope = CoroutineScope(Dispatchers.IO + SupervisorJob()) + val dataStore = PreferenceDataStoreFactory.create(scope = scope) { + File(directory, "confirmations.preferences_pb") + } + store = HeroRecognitionStore(dataStore) + } + + @After fun tearDown() { + scope.cancel() + directory.deleteRecursively() + } + + @Test fun `confirm then load round-trips the hero and the timestamp`() = runBlocking { + store.confirm("slot-1", HeroConfirmation("miya", "Miya", 1000L)) + store.confirm("slot-2", HeroConfirmation("layla", "Layla", 2000L)) + + assertEquals( + mapOf( + "slot-1" to HeroConfirmation("miya", "Miya", 1000L), + "slot-2" to HeroConfirmation("layla", "Layla", 2000L), + ), + store.load(), + ) + } + + @Test fun `confirming the same slot replaces the previous hero`() = runBlocking { + store.confirm("slot-1", HeroConfirmation("miya", "Miya", 1000L)) + store.confirm("slot-1", HeroConfirmation("layla", "Layla", 2000L)) + + assertEquals(HeroConfirmation("layla", "Layla", 2000L), store.load()["slot-1"]) + } + + @Test fun `clear removes only the slot it is given`() = runBlocking { + store.confirm("slot-1", HeroConfirmation("miya", "Miya", 1000L)) + store.confirm("slot-2", HeroConfirmation("layla", "Layla", 2000L)) + + store.clear("slot-1") + + assertEquals(mapOf("slot-2" to HeroConfirmation("layla", "Layla", 2000L)), store.load()) + } + + @Test fun `a corrupt confirmation document reads as empty, never as garbage`() = runBlocking { + val dataStore = androidx.datastore.preferences.core.PreferenceDataStoreFactory.create(scope = scope) { + File(directory, "corrupt.preferences_pb") + } + dataStore.edit { it[stringPreferencesKey("hero_confirmations_json")] = "{ this is not json" } + + val corrupt = HeroRecognitionStore(dataStore) + + assertTrue(corrupt.load().isEmpty()) + } + + @Test fun `a document with an unsupported schema version reads as empty`() = runBlocking { + val dataStore = androidx.datastore.preferences.core.PreferenceDataStoreFactory.create(scope = scope) { + File(directory, "future.preferences_pb") + } + dataStore.edit { + it[stringPreferencesKey("hero_confirmations_json")] = + """{"schemaVersion":99,"entries":{"slot-1":{"heroId":"miya","heroName":"Miya","confirmedAtMs":1000}}}""" + } + + val future = HeroRecognitionStore(dataStore) + + assertTrue(future.load().isEmpty()) + } +} diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 8c2b5e2..7a5356c 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -160,7 +160,7 @@ class VisionLabViewModelTest { @Test fun `clear cancels in flight processing and drops the late result`() = runBlocking { val source = FakeVisionFrameSource(originalFrame(BLOCKED_FRAME_ID)) val preprocessor = BlockingFakePreprocessor() - val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.processFrame() assertTrue("the request never reached the preprocessor", preprocessor.awaitEntered()) @@ -178,7 +178,7 @@ class VisionLabViewModelTest { @Test fun `save profile requires a frozen frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository() - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.saveProfile("Ranked") @@ -197,7 +197,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.updateProfile("p1", "Ranked") @@ -216,7 +216,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.applyProfile("p1") @@ -227,7 +227,7 @@ class VisionLabViewModelTest { @Test fun `a frozen frame satisfies the profile frame requirement`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository() - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.freeze() vm.saveProfile("Ranked") @@ -237,7 +237,7 @@ class VisionLabViewModelTest { @Test fun `live frame is exposed in the ui state`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) assertEquals("live", vm.uiState.value.liveFrame?.id?.value) assertEquals(VisionSourceState.CAPTURING, vm.uiState.value.sourceState) @@ -454,15 +454,37 @@ class VisionLabViewModelTest { assertEquals(first.id, vm.uiState.value.selectedSlotId) } + // region hero recognition fixtures + + private object EmptyTransport : com.draftlens.vision.heroes.HeroCatalogTransport { + override fun get(url: String, maxBytes: Long, accept: String?): com.draftlens.vision.heroes.HeroHttpResult = + com.draftlens.vision.heroes.HeroHttpResult.Failure("no catalog in this test") + } + + private fun heroCatalog(): com.draftlens.vision.heroes.FileHeroCatalogRepository = + com.draftlens.vision.heroes.FileHeroCatalogRepository( + java.nio.file.Files.createTempDirectory("hero-catalog").toFile(), + java.nio.file.Files.createTempDirectory("hero-assets").toFile(), + EmptyTransport, + ) + + private fun heroStore(): com.draftlens.datastore.HeroRecognitionStore = + com.draftlens.datastore.HeroRecognitionStore( + androidx.datastore.preferences.core.PreferenceDataStoreFactory.create( + scope = kotlinx.coroutines.CoroutineScope(kotlinx.coroutines.Dispatchers.IO + kotlinx.coroutines.SupervisorJob()), + ) { java.io.File(java.nio.file.Files.createTempDirectory("hero-confirmations").toFile(), "confirmations.preferences_pb") }, + ) + + // endregion + private fun slotVm(): VisionLabViewModel = VisionLabViewModel( FakeVisionFrameSource(originalFrame("live")), EchoPreprocessor(), - FakeCalibrationRepository(), - ) + FakeCalibrationRepository(),, heroCatalog(), heroStore()) @Test fun `freeze pins the current frame while live frames continue`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("first")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.freeze() source.emit(originalFrame("second")) @@ -474,7 +496,7 @@ class VisionLabViewModelTest { @Test fun `resume releases the pinned frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("first")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.freeze() vm.resume() @@ -485,7 +507,7 @@ class VisionLabViewModelTest { @Test fun `clear releases the pinned frame and the processed frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("first")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.freeze() vm.processFrame() @@ -499,7 +521,7 @@ class VisionLabViewModelTest { @Test fun `process frame publishes the processed frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.processFrame() val state = withTimeout(10_000) { vm.uiState.first { it.processed != null } } @@ -511,7 +533,7 @@ class VisionLabViewModelTest { @Test fun `stale processing result is discarded after a newer request`() = runBlocking { val source = FakeVisionFrameSource(originalFrame(BLOCKED_FRAME_ID)) val preprocessor = BlockingFakePreprocessor() - val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.processFrame() // The first request must be inside the preprocessor before the second one supersedes it; @@ -532,7 +554,7 @@ class VisionLabViewModelTest { @Test fun `processing failure is reported honestly`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, FailingPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, FailingPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.processFrame() val state = withTimeout(10_000) { vm.uiState.first { it.processStatus == ProcessStatus.FAILED } } @@ -543,7 +565,7 @@ class VisionLabViewModelTest { @Test fun `process frame without a frame reports an error`() = runBlocking { val source = FakeVisionFrameSource(null) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.processFrame() @@ -554,7 +576,7 @@ class VisionLabViewModelTest { @Test fun `save profile is refused when the document schema is unsupported`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.UnsupportedSchema("too new")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.saveProfile("Ranked") @@ -565,7 +587,7 @@ class VisionLabViewModelTest { @Test fun `delete all calibration data stays available when the schema is unsupported`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.UnsupportedSchema("too new")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.deleteAllProfiles() @@ -575,7 +597,7 @@ class VisionLabViewModelTest { @Test fun `save profile stores the current slots and the oriented aspect ratio`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository() - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.1f, 0.1f, 0.2f, 0.2f)) vm.freeze() @@ -599,7 +621,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live", width = 10, height = 10)) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.freeze() vm.applyProfile("p1") @@ -628,7 +650,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.6f, 0.6f, 0.7f, 0.7f)) vm.freeze() @@ -660,7 +682,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.6f, 0.6f, 0.7f, 0.7f)) vm.freeze() @@ -674,7 +696,7 @@ class VisionLabViewModelTest { @Test fun `slot index that duplicates another slot in the same scope is rejected`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.1f, 0.1f, 0.2f, 0.2f)) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.4f, 0.4f, 0.5f, 0.5f)) @@ -686,6 +708,133 @@ class VisionLabViewModelTest { assertEquals(2, vm.uiState.value.slots[1].index) } + // region hero recognition + + private class RoutesTransport( + private val routes: Map, + ) : com.draftlens.vision.heroes.HeroCatalogTransport { + override fun get(url: String, maxBytes: Long, accept: String?): com.draftlens.vision.heroes.HeroHttpResult = + routes[url] ?: com.draftlens.vision.heroes.HeroHttpResult.Failure("no route", 404) + } + + private val fakePng = byteArrayOf(0x89.toByte(), 'P'.code.toByte(), 'N'.code.toByte(), 'G'.code.toByte(), + 0x0D, 0x0A, 0x1A, 0x0A) + ByteArray(64) { 0x01 } + + private fun gradient(size: Int = 32): IntArray = + IntArray(size * size) { i -> + val x = i % size + val y = i / size + 0xFF000000.toInt() or (x * 255 / size) or ((y * 255 / size) shl 8) or (((x + y) * 255 / (size + size)) shl 16) + } + + private fun heroEntryJson(): String = """ + {"code":0,"message":"OK","data":{"records":[ + {"data":{"hero":{"data":{"head":"https://wsrv.nl/?url=miya.head","name":"Miya","smallmap":"https://wsrv.nl/?url=miya.map"}},"hero_id":1}}, + {"data":{"hero":{"data":{"head":"https://wsrv.nl/?url=layla.head","name":"Layla","smallmap":"https://wsrv.nl/?url=layla.map"}},"hero_id":2}} + ],"total":2}} + """.trimIndent() + + private fun happyRoutes(): Map = mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to + com.draftlens.vision.heroes.HeroHttpResult.Success(com.draftlens.vision.heroes.HeroHttpResponse(heroEntryJson().toByteArray(), "application/json")), + "https://wsrv.nl/?url=miya.head" to + com.draftlens.vision.heroes.HeroHttpResult.Success(com.draftlens.vision.heroes.HeroHttpResponse(fakePng, "image/png")), + "https://wsrv.nl/?url=layla.head" to + com.draftlens.vision.heroes.HeroHttpResult.Success(com.draftlens.vision.heroes.HeroHttpResponse(fakePng, "image/png")), + "https://mlbbdex.com/api/v1/heroes" to + com.draftlens.vision.heroes.HeroHttpResult.Failure("offline", 503), + ) + + private fun recognitionVm(referencePixels: IntArray = gradient()): VisionLabViewModel { + val repo = com.draftlens.vision.heroes.FileHeroCatalogRepository( + java.nio.file.Files.createTempDirectory("hero-catalog").toFile(), + java.nio.file.Files.createTempDirectory("hero-assets").toFile(), + RoutesTransport(happyRoutes()), + ) + return VisionLabViewModel( + FakeVisionFrameSource(originalFrame("live")), + EchoPreprocessor(), + FakeCalibrationRepository(), + heroCatalog = repo, + heroRecognitionStore = heroStore(), + referenceLoader = { _, entry -> com.draftlens.vision.heroes.HeroReference(entry, referencePixels, 32, 32) }, + ) + } + + @Test fun `identify without a selected slot reports a message`() = runBlocking { + val vm = recognitionVm() + + vm.identifySelectedSlot() + + assertTrue("message was: ${vm.uiState.value.message}", vm.uiState.value.message!!.contains("Select a slot")) + } + + @Test fun `identify without a synced catalog fails honestly instead of guessing`() = runBlocking { + val vm = recognitionVm() + vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.3f, 0.3f)) + + vm.identifySelectedSlot() + val state = withTimeout(5_000) { vm.uiState.first { it.heroRecognition[vm.uiState.value.slots.single().id] != null } } + + assertEquals(HeroSlotPhase.FAILED, state.heroRecognition[state.slots.single().id]!!.phase) + assertTrue( + state.heroRecognition[state.slots.single().id]!!.message!!.contains("Sync the hero catalog"), + ) + } + + @Test fun `a synced catalog produces ranked candidates for the selected slot`() = runBlocking { + val vm = recognitionVm() + vm.syncHeroCatalog() + withTimeout(10_000) { vm.uiState.first { it.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } } + vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.4f, 0.4f)) + val slotId = vm.uiState.value.slots.single().id + + vm.identifySelectedSlot() + val state = withTimeout(10_000) { + vm.uiState.first { it.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } + } + + val candidates = state.heroRecognition[slotId]!!.candidates + assertTrue(candidates.isNotEmpty()) + assertEquals(2, candidates.size.coerceAtMost(2)) + } + + @Test fun `confirming a candidate persists the hero and clearing removes it`() = runBlocking { + val vm = recognitionVm() + vm.syncHeroCatalog() + withTimeout(10_000) { vm.uiState.first { it.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } } + vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.4f, 0.4f)) + val slotId = vm.uiState.value.slots.single().id + vm.identifySelectedSlot() + withTimeout(10_000) { vm.uiState.first { it.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } } + + vm.confirmHero(slotId, "miya") + val confirmed = withTimeout(5_000) { + vm.uiState.first { it.heroRecognition[slotId]?.phase == HeroSlotPhase.CONFIRMED } + } + assertEquals("Miya", confirmed.heroRecognition[slotId]!!.message) + + vm.clearHeroConfirmation(slotId) + val cleared = withTimeout(5_000) { + vm.uiState.first { it.heroRecognition[slotId] == null } + } + assertNull(cleared.heroRecognition[slotId]) + } + + @Test fun `the search filters the cached catalog by name`() = runBlocking { + val vm = recognitionVm() + vm.syncHeroCatalog() + withTimeout(10_000) { vm.uiState.first { it.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } } + + val results = vm.searchCatalog("mi") + + assertEquals(listOf("Miya"), results.map { it.canonicalName }) + assertTrue(vm.searchCatalog("").isEmpty()) + assertTrue(vm.searchCatalog("zzz-nobody").isEmpty()) + } + + // endregion + private companion object { /** The frame id whose processing call the fake blocks. */ const val BLOCKED_FRAME_ID = "first" diff --git a/app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt b/app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt new file mode 100644 index 0000000..6ceed97 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt @@ -0,0 +1,163 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.FileHeroCatalogRepository +import com.draftlens.vision.heroes.HeroAssetVariant +import com.draftlens.vision.heroes.HeroCatalogState +import com.draftlens.vision.heroes.HeroHttpResponse +import com.draftlens.vision.heroes.HeroHttpResult +import com.draftlens.vision.heroes.HeroId +import kotlinx.coroutines.runBlocking +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.File +import java.nio.file.Files + +private const val PNG_MAGIC = 0x89.toByte() + +private fun fakePng(): ByteArray { + val header = byteArrayOf(PNG_MAGIC, 'P'.code.toByte(), 'N'.code.toByte(), 'G'.code.toByte(), 0x0D, 0x0A, 0x1A, 0x0A) + return header + ByteArray(64) { 0x01 } +} + +private fun listRoute(): String = "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" + +private fun listBody(): String = """ + {"code":0,"message":"OK","data":{"records":[ + {"data":{"hero":{"data":{"head":"https://wsrv.nl/?url=miya.head","name":"Miya","smallmap":"https://wsrv.nl/?url=miya.map"}},"hero_id":1}}, + {"data":{"hero":{"data":{"head":"https://wsrv.nl/?url=layla.head","name":"Layla","smallmap":"https://wsrv.nl/?url=layla.map"}},"hero_id":2}} + ],"total":2}} +""".trimIndent() + +private fun syncRoutes(pngBytes: ByteArray = fakePng(), contentType: String? = "image/png"): Map = mapOf( + listRoute() to HeroHttpResult.Success(HeroHttpResponse(listBody().toByteArray(), "application/json")), + "https://wsrv.nl/?url=miya.head" to HeroHttpResult.Success(HeroHttpResponse(pngBytes, contentType)), + "https://wsrv.nl/?url=layla.head" to HeroHttpResult.Success(HeroHttpResponse(pngBytes, contentType)), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("offline", 503), +) + +class FileHeroCatalogRepositoryTest { + + private fun tempDirs(): Pair = + Files.createTempDirectory("hero-catalog").toFile() to Files.createTempDirectory("hero-assets").toFile() + + @Test fun `refresh stores the snapshot and downloads the assets`() = runBlocking { + val (data, cache) = tempDirs() + val repo = FileHeroCatalogRepository(data, cache, FakeTransport(syncRoutes())) { 1234L } + + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertEquals(1234L, result.getOrThrow().syncedAtMs) + assertEquals(HeroCatalogState.Ready(2, 1234L), repo.state.value) + assertEquals(2, repo.snapshot()?.entries?.size) + assertNotNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + assertTrue(File(data, "hero-catalog/catalog.json").exists()) + } + + @Test fun `cached snapshot is served offline after a restart`() = runBlocking { + val (data, cache) = tempDirs() + val first = FileHeroCatalogRepository(data, cache, FakeTransport(syncRoutes())) { 1234L } + first.refresh() + + // A fresh repository instance with a dead network must serve the cache. + val second = FileHeroCatalogRepository(data, cache, FakeTransport()) { 9999L } + + assertEquals(HeroCatalogState.Ready(2, 1234L), second.state.value) + assertEquals(2, second.snapshot()?.entries?.size) + assertEquals("Layla", second.snapshot()?.entries?.last()?.canonicalName) + } + + @Test fun `a magic-byte-invalid asset is rejected but the catalog is stored`() = runBlocking { + val (data, cache) = tempDirs() + val repo = FileHeroCatalogRepository( + data, + cache, + FakeTransport(syncRoutes(pngBytes = "not an image".toByteArray())), + ) { 1234L } + + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertEquals(listOf("Miya", "Layla"), result.getOrThrow().assetFailures) + assertNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `an oversized asset is rejected`() = runBlocking { + val (data, cache) = tempDirs() + val big = fakePng() + ByteArray(5 * 1024 * 1024) + val repo = FileHeroCatalogRepository( + data, + cache, + FakeTransport(syncRoutes(pngBytes = big)), + ) { 1234L } + + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertEquals(listOf("Miya", "Layla"), result.getOrThrow().assetFailures) + } + + @Test fun `a non-image content type is rejected`() = runBlocking { + val (data, cache) = tempDirs() + val repo = FileHeroCatalogRepository( + data, + cache, + FakeTransport(syncRoutes(contentType = "text/html")), + ) { 1234L } + + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertEquals(listOf("Miya", "Layla"), result.getOrThrow().assetFailures) + } + + @Test fun `sync failure surfaces an error state and keeps the cache`() = runBlocking { + val (data, cache) = tempDirs() + val good = FileHeroCatalogRepository(data, cache, FakeTransport(syncRoutes())) { 1234L } + good.refresh() + + val broken = FileHeroCatalogRepository(data, cache, FakeTransport()) { 9999L } + val result = broken.refresh() + + assertTrue(result.isFailure) + val state = broken.state.value + assertTrue(state is HeroCatalogState.Error) + assertEquals(2, (state as HeroCatalogState.Error).cachedEntryCount) + // The cache is still usable for offline browsing. + assertEquals(2, broken.snapshot()?.entries?.size) + } + + @Test fun `a corrupt cached snapshot reports an error and is not used`() { + val (data, cache) = tempDirs() + val snapshotDir = File(data, "hero-catalog").also { it.mkdirs() } + File(snapshotDir, "catalog.json").writeText("{ not json") + + val repo = FileHeroCatalogRepository(data, cache, FakeTransport()) { 1234L } + + assertTrue(repo.state.value is HeroCatalogState.Error) + assertNull(repo.snapshot()) + } + + @Test fun `duplicate hero ids in a cached snapshot are refused`() { + val (data, cache) = tempDirs() + val snapshotDir = File(data, "hero-catalog").also { it.mkdirs() } + // encode() would refuse a duplicate, so the cache can only contain this raw document + // if something else wrote it; the codec must still refuse it. + File(snapshotDir, "catalog.json").writeText( + """ + {"schemaVersion":1,"syncedAtMs":1000,"sourceAttribution":[],"entries":[ + {"id":"miya","canonicalName":"Miya","aliases":[],"roles":[],"lanes":[],"specialties":[],"assets":[],"externalIds":{},"source":"RONE_ARENA"}, + {"id":"miya","canonicalName":"Miya","aliases":[],"roles":[],"lanes":[],"specialties":[],"assets":[],"externalIds":{},"source":"RONE_ARENA"} + ],"assetFailures":[],"crossCheck":null} + """.trimIndent(), + ) + + val repo = FileHeroCatalogRepository(data, cache, FakeTransport()) { 1234L } + + assertTrue(repo.state.value is HeroCatalogState.Error) + assertNull(repo.snapshot()) + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroCatalogCodecTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroCatalogCodecTest.kt new file mode 100644 index 0000000..8ac1192 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroCatalogCodecTest.kt @@ -0,0 +1,93 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.CatalogSnapshot +import com.draftlens.vision.heroes.HeroAlias +import com.draftlens.vision.heroes.HeroAsset +import com.draftlens.vision.heroes.HeroAssetVariant +import com.draftlens.vision.heroes.HeroCatalogCodec +import com.draftlens.vision.heroes.HeroCatalogEntry +import com.draftlens.vision.heroes.HeroCatalogSource +import com.draftlens.vision.heroes.HeroId +import com.draftlens.vision.heroes.SourceAttribution +import org.junit.Assert.assertEquals +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Test + +private fun entry(id: String, name: String) = HeroCatalogEntry( + id = HeroId(id), + canonicalName = name, + aliases = listOf(HeroAlias("en", name)), + roles = listOf("Marksman"), + lanes = listOf("Gold"), + specialties = listOf("Finisher"), + assets = listOf(HeroAsset(HeroAssetVariant.HEAD, "https://example.com/$id.png")), + externalIds = mapOf("roneArenaHeroId" to "1", "mlbbdexSlug" to id), + source = HeroCatalogSource.RONE_ARENA, +) + +private fun snapshot() = CatalogSnapshot( + schemaVersion = CatalogSnapshot.CURRENT_SCHEMA, + syncedAtMs = 1000L, + sourceAttribution = listOf(SourceAttribution.RONE_ARENA, SourceAttribution.MLBBDEX), + entries = listOf(entry("miya", "Miya"), entry("layla", "Layla")), + assetFailures = emptyList(), + crossCheck = null, +) + +class HeroCatalogCodecTest { + + @Test fun `a snapshot round-trips byte for byte`() { + val decoded = HeroCatalogCodec.decode(HeroCatalogCodec.encode(snapshot())) + + assertEquals(snapshot(), decoded) + } + + @Test fun `encoding the current schema succeeds`() { + assertTrue(HeroCatalogCodec.encode(snapshot()).contains("\"schemaVersion\":1")) + } + + @Test fun `a newer schema version is refused`() { + val raw = HeroCatalogCodec.encode(snapshot()).replace("\"schemaVersion\":1", "\"schemaVersion\":2") + + assertThrows(IllegalArgumentException::class.java) { HeroCatalogCodec.decode(raw) } + } + + @Test fun `a blank hero id is refused at the model boundary`() { + assertThrows(IllegalArgumentException::class.java) { HeroId("") } + assertThrows(IllegalArgumentException::class.java) { HeroId(" ") } + } + + @Test fun `a blank hero name is refused on encode`() { + assertThrows(IllegalArgumentException::class.java) { + HeroCatalogCodec.encode(snapshot().copy(entries = listOf(entry("miya", " ")))) + } + } + + @Test fun `a non-https asset url is refused on encode`() { + val insecure = entry("miya", "Miya").copy( + assets = listOf(HeroAsset(HeroAssetVariant.HEAD, "http://example.com/miya.png")), + ) + + assertThrows(IllegalArgumentException::class.java) { + HeroCatalogCodec.encode(snapshot().copy(entries = listOf(insecure))) + } + } + + @Test fun `duplicate hero ids are refused on decode`() { + val raw = """ + {"schemaVersion":1,"syncedAtMs":1000,"sourceAttribution":[],"entries":[ + {"id":"miya","canonicalName":"Miya","aliases":[],"roles":[],"lanes":[],"specialties":[],"assets":[{"variant":"HEAD","url":"https://e.com/1.png"}],"externalIds":{},"source":"RONE_ARENA"}, + {"id":"miya","canonicalName":"Miya","aliases":[],"roles":[],"lanes":[],"specialties":[],"assets":[{"variant":"HEAD","url":"https://e.com/2.png"}],"externalIds":{},"source":"RONE_ARENA"} + ],"assetFailures":[],"crossCheck":null} + """.trimIndent() + + assertThrows(IllegalArgumentException::class.java) { HeroCatalogCodec.decode(raw) } + } + + @Test fun `unknown fields are rejected`() { + val raw = HeroCatalogCodec.encode(snapshot()).replace("\"entries\":[", "\"surprise\":true,\"entries\":[") + + assertThrows(Exception::class.java) { HeroCatalogCodec.decode(raw) } + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroCatalogSyncTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroCatalogSyncTest.kt new file mode 100644 index 0000000..99c08ad --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroCatalogSyncTest.kt @@ -0,0 +1,198 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.HeroCatalogEntry +import com.draftlens.vision.heroes.HeroCatalogSource +import com.draftlens.vision.heroes.HeroCatalogSync +import com.draftlens.vision.heroes.HeroHttpResponse +import com.draftlens.vision.heroes.HeroHttpResult +import com.draftlens.vision.heroes.HeroId +import com.draftlens.vision.heroes.HeroIds +import com.draftlens.vision.heroes.CrossCheckResult +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class FakeTransport( + private val routes: Map = emptyMap(), +) : com.draftlens.vision.heroes.HeroCatalogTransport { + val requested = mutableListOf() + + override fun get(url: String, maxBytes: Long, accept: String?): HeroHttpResult { + requested += url + return routes[url] + ?: routes.entries.firstOrNull { (route, _) -> url.startsWith(route) }?.value + ?: HeroHttpResult.Failure("no route for $url", 404) + } +} + +private fun ronePage(records: List>, total: Int): HeroHttpResult.Success { + val body = records.joinToString(",") { (id, name) -> + """{"data":{"hero":{"data":{"head":"https://wsrv.nl/?url=$id.head","name":"$name","smallmap":"https://wsrv.nl/?url=$id.map"}},"hero_id":$id}}""" + } + return HeroHttpResult.Success( + HeroHttpResponse( + """{"code":0,"message":"OK","data":{"records":[$body],"total":$total}}""".trimIndent().toByteArray(), + "application/json", + ), + ) +} + +class HeroCatalogSyncTest { + + @Test fun `a hero id is the slugified English name`() { + assertEquals("yi-sun-shin", HeroIds.slugOf("Yi Sun-shin")) + assertEquals("chang-e", HeroIds.slugOf("Chang'e")) + assertEquals("x-borg", HeroIds.slugOf("X.Borg")) + assertEquals("miya", HeroIds.slugOf("Miya")) + } + + @Test fun `pagination fetches every page until the total is reached`() { + val page1 = (1..100).map { it.toLong() to "Hero-$it" } + val page2 = (101..134).map { it.toLong() to "Hero-$it" } + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to ronePage(page1, total = 134), + "https://arena.fastapicloud.dev/api/heroes?size=100&index=2&lang=en" to ronePage(page2, total = 134), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Success( + HeroHttpResponse("""{"data":[],"total":0,"source":{"name":"w","url":"u","license":"CC BY-SA"}}""".toByteArray(), "application/json"), + ), + ), + ) + val report = HeroCatalogSync(transport).sync().getOrThrow() + + assertEquals(134, report.entries.size) + assertEquals(2, transport.requested.count { it.contains("heroes?size=") }) + assertEquals("roneArenaHeroId", report.entries.first().externalIds.keys.first()) + } + + @Test fun `duplicate hero ids are collapsed and counted`() { + val records = listOf(1L to "Miya", 1L to "Miya", 2L to "Layla") + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to ronePage(records, total = 3), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("offline", 503), + ), + ) + val report = HeroCatalogSync(transport).sync().getOrThrow() + + assertEquals(2, report.entries.size) + assertEquals(1, report.duplicateIds) + assertTrue(report.crossCheck is CrossCheckResult.Unavailable) + } + + @Test fun `invalid records are skipped and counted`() { + val body = """ + {"data":{"hero":{"data":{"head":"","name":"","smallmap":""}},"hero_id":7}}, + {"data":{"hero":{"data":{"head":"https://wsrv.nl/?url=a","name":"Miya","smallmap":"https://wsrv.nl/?url=b"}},"hero_id":8}}, + {"data":{"hero":null,"hero_id":9}} + """.trimIndent() + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to HeroHttpResult.Success( + HeroHttpResponse( + """{"code":0,"message":"OK","data":{"records":[$body],"total":3}}""".toByteArray(), + "application/json", + ), + ), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("offline", 503), + ), + ) + val report = HeroCatalogSync(transport).sync().getOrThrow() + + assertEquals(1, report.entries.size) + assertEquals(2, report.invalidRecords) + assertEquals("miya", report.entries.single().id.value) + assertEquals(HeroCatalogSource.RONE_ARENA, report.entries.single().source) + assertEquals(listOf("HEAD", "SMALLMAP").toSet(), report.entries.single().assets.map { it.variant.name }.toSet()) + } + + @Test fun `roles and lanes come from MLBBDex by slug`() { + val rone = ronePage(listOf(1L to "Luo Yi"), total = 1) + val mlbbdex = HeroHttpResult.Success( + HeroHttpResponse( + """ + {"data":[{"slug":"luo-yi","name":"Luo Yi","roles":["Mage"],"lanes":["Mid"],"specialties":["Poke"]}, + {"slug":"miya","name":"Miya","roles":["Marksman"],"lanes":["Gold"],"specialties":["Damage"]}], + "total":2,"source":{"name":"w","url":"u","license":"CC BY-SA"}} + """.trimIndent().toByteArray(), + "application/json", + ), + ) + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to rone, + "https://mlbbdex.com/api/v1/heroes" to mlbbdex, + ), + ) + val report = HeroCatalogSync(transport).sync().getOrThrow() + + val entry = report.entries.single() + assertEquals(listOf("Mage"), entry.roles) + assertEquals(listOf("Mid"), entry.lanes) + assertEquals(listOf("Poke"), entry.specialties) + val cross = report.crossCheck as CrossCheckResult.Available + assertEquals(1, cross.crossCheck.matchedNames) + assertEquals(1, cross.crossCheck.missingInRoneArena.size) + assertEquals(0, cross.crossCheck.missingInMlbbdex.size) + } + + @Test fun `a full outage returns failure, not a bogus empty catalog`() { + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes" to HeroHttpResult.Failure("down", 503), + "https://arena.rone.dev/api/heroes" to HeroHttpResult.Failure("down", 503), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("down", 503), + ), + ) + assertTrue(HeroCatalogSync(transport).sync().isFailure) + } + + @Test fun `the mirror host is tried when the primary host fails`() { + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to HeroHttpResult.Failure("down", 503), + "https://arena.rone.dev/api/heroes?size=100&index=1&lang=en" to ronePage(listOf(1L to "Miya"), total = 1), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("down", 503), + ), + ) + val report = HeroCatalogSync(transport).sync().getOrThrow() + + assertEquals(1, report.entries.size) + assertTrue(transport.requested.any { it.startsWith("https://arena.rone.dev/api/heroes") }) + } + + @Test fun `a non-zero API code is a failure`() { + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to HeroHttpResult.Success( + HeroHttpResponse( + """{"code":1001,"message":"bad","data":{"records":[],"total":0}}""".toByteArray(), + "application/json", + ), + ), + "https://arena.rone.dev/api/heroes?size=100&index=1&lang=en" to HeroHttpResult.Success( + HeroHttpResponse( + """{"code":1001,"message":"bad","data":{"records":[],"total":0}}""".toByteArray(), + "application/json", + ), + ), + ), + ) + assertTrue(HeroCatalogSync(transport).sync().isFailure) + } + + @Test fun `aliases and external ids are provider-specific and never reused as identity`() { + val transport = FakeTransport( + mapOf( + "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" to ronePage(listOf(42L to "Yi Sun-shin"), total = 1), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("down", 503), + ), + ) + val report = HeroCatalogSync(transport).sync().getOrThrow() + val entry = report.entries.single() + + assertEquals(HeroId("yi-sun-shin"), entry.id) + assertEquals("42", entry.externalIds["roneArenaHeroId"]) + assertEquals("yi-sun-shin", entry.externalIds["mlbbdexSlug"]) + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt new file mode 100644 index 0000000..04c7e9f --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt @@ -0,0 +1,123 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.HeroCatalogEntry +import com.draftlens.vision.heroes.HeroAsset +import com.draftlens.vision.heroes.HeroAssetVariant +import com.draftlens.vision.heroes.HeroCatalogSource +import com.draftlens.vision.heroes.HeroId +import com.draftlens.vision.heroes.HeroMatchConfig +import com.draftlens.vision.heroes.HeroMatchResult +import com.draftlens.vision.heroes.HeroMatcher +import com.draftlens.vision.heroes.HeroReference +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +private fun solid(width: Int, height: Int, rgb: Int): IntArray = + IntArray(width * height) { (0xFF000000.toInt() or rgb) } + +private fun gradient(width: Int, height: Int): IntArray = + IntArray(width * height) { i -> + val x = i % width + val y = i / width + 0xFF000000.toInt() or (x * 255 / width) or ((y * 255 / height) shl 8) or (((x + y) * 255 / (width + height)) shl 16) + } + +private fun entry(id: String, name: String) = HeroCatalogEntry( + id = HeroId(id), + canonicalName = name, + aliases = emptyList(), + roles = emptyList(), + lanes = emptyList(), + specialties = emptyList(), + assets = listOf(HeroAsset(HeroAssetVariant.HEAD, "https://example.com/$id.png")), + externalIds = emptyMap(), + source = HeroCatalogSource.RONE_ARENA, +) + +private fun reference(id: String, pixels: IntArray, width: Int, height: Int) = + HeroReference(entry(id, id), pixels, width, height) + +class HeroMatcherTest { + + private val miya = reference("miya", gradient(32, 32), 32, 32) + private val layla = reference("layla", solid(32, 32, 0x0000FF), 32, 32) + private val laylaLike = reference("layla-like", solid(32, 32, 0x0000FE), 32, 32) + + @Test fun `identical images are a perfect match`() { + val result = HeroMatcher.match( + queryPixels = gradient(32, 32), + queryWidth = 32, + queryHeight = 32, + references = listOf(miya, layla), + ) as HeroMatchResult.Candidates + + assertEquals("miya", result.matches.first().entry.id.value) + assertEquals(1f, result.matches.first().similarity, 0.001f) + } + + @Test fun `ranking order follows similarity`() { + val query = solid(16, 16, 0x0000FF) + val result = HeroMatcher.match(query, 16, 16, listOf(miya, layla, laylaLike)) as HeroMatchResult.Candidates + + assertEquals("layla", result.matches[0].entry.id.value) + assertEquals("layla-like", result.matches[1].entry.id.value) + assertTrue(result.matches[0].similarity >= result.matches[1].similarity) + } + + @Test fun `evidence below the threshold yields no match with the best similarity reported`() { + val black = solid(16, 16, 0x000000) + val result = HeroMatcher.match(black, 16, 16, listOf(miya), HeroMatchConfig(noMatchThreshold = 0.99f)) + + assertTrue(result is HeroMatchResult.NoMatch) + assertTrue((result as HeroMatchResult.NoMatch).bestSimilarity < 0.99f) + } + + @Test fun `an ambiguous gap marks the top candidate`() { + val query = solid(16, 16, 0x0000FF) + val result = HeroMatcher.match( + query, + 16, + 16, + listOf(layla, laylaLike), + HeroMatchConfig(ambiguousGap = 0.10f), + ) as HeroMatchResult.Candidates + + assertTrue(result.matches[0].ambiguous) + } + + @Test fun `references with corrupt dimensions are skipped`() { + val corrupt = reference("corrupt", IntArray(10), 3, 3) + val result = HeroMatcher.match(solid(8, 8, 0x0000FF), 8, 8, listOf(corrupt, layla)) + + assertTrue(result is HeroMatchResult.Candidates) + assertEquals("layla", (result as HeroMatchResult.Candidates).matches.first().entry.id.value) + } + + @Test fun `hamming distance counts differing bits`() { + assertEquals(0, HeroMatcher.hammingDistance(0b1010, 0b1010)) + assertEquals(2, HeroMatcher.hammingDistance(0b1010, 0b0101)) + assertEquals(64, HeroMatcher.hammingDistance(Long.MAX_VALUE, Long.MIN_VALUE + 1)) + } + + @Test fun `color distance is zero for identical descriptors`() { + val a = HeroMatcher.colorDescriptor(gradient(16, 16), 16, 16) + + assertEquals(0f, HeroMatcher.colorDistance(a, a), 0f) + } + + @Test fun `a malformed query throws instead of lying`() { + org.junit.Assert.assertThrows(IllegalArgumentException::class.java) { + HeroMatcher.match(IntArray(5), 2, 2, listOf(miya)) + } + } + + @Test fun `ranking is deterministic for repeated calls`() { + val query = solid(16, 16, 0x00FF00) + val a = HeroMatcher.match(query, 16, 16, listOf(miya, layla, laylaLike)) + val b = HeroMatcher.match(query, 16, 16, listOf(miya, layla, laylaLike)) + + assertEquals(a, b) + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt new file mode 100644 index 0000000..5ffebae --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt @@ -0,0 +1,76 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.HeroCatalogEntry +import com.draftlens.vision.heroes.HeroAsset +import com.draftlens.vision.heroes.HeroAssetVariant +import com.draftlens.vision.heroes.HeroCatalogSource +import com.draftlens.vision.heroes.HeroId +import com.draftlens.vision.heroes.HeroIdentifyOutcome +import com.draftlens.vision.heroes.HeroRecognition +import com.draftlens.vision.heroes.HeroReference +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +private fun entry(id: String) = HeroCatalogEntry( + id = HeroId(id), + canonicalName = id, + aliases = emptyList(), + roles = emptyList(), + lanes = emptyList(), + specialties = emptyList(), + assets = listOf(HeroAsset(HeroAssetVariant.HEAD, "https://example.com/$id.png")), + externalIds = emptyMap(), + source = HeroCatalogSource.RONE_ARENA, +) + +class HeroRecognitionTest { + + private fun gradient(width: Int, height: Int): IntArray = + IntArray(width * height) { i -> + val x = i % width + val y = i / width + 0xFF000000.toInt() or (x * 255 / width) or ((y * 255 / height) shl 8) or (((x + y) * 255 / (width + height)) shl 16) + } + + private val miyaReference = HeroReference(entry("miya"), gradient(32, 32), 32, 32) + + @Test fun `an exact reference answers with the hero name and similarity`() { + val outcome = HeroRecognition.identify( + pixels = gradient(32, 32), + width = 32, + height = 32, + references = listOf(miyaReference), + ) + + val candidates = (outcome as HeroIdentifyOutcome.Candidates).heroCandidates + assertEquals(1, candidates.size) + assertEquals("miya", candidates.single().hero.canonicalName) + assertEquals(1f, candidates.single().similarity, 0.001f) + } + + @Test fun `no usable references produce an honest failure, not a guess`() { + val outcome = HeroRecognition.identify( + pixels = gradient(16, 16), + width = 16, + height = 16, + references = listOf(HeroReference(entry("broken"), IntArray(3), 1, 1)), + ) + + assertTrue(outcome is HeroIdentifyOutcome.Failed) + assertTrue((outcome as HeroIdentifyOutcome.Failed).message.contains("Sync the hero catalog")) + } + + @Test fun `evidence below the threshold reports no match instead of a wrong hero`() { + val black = IntArray(32 * 32) { 0xFF000000.toInt() } + val outcome = HeroRecognition.identify( + pixels = black, + width = 32, + height = 32, + references = listOf(miyaReference), + config = com.draftlens.vision.heroes.HeroMatchConfig(noMatchThreshold = 0.9f), + ) + + assertTrue(outcome is HeroIdentifyOutcome.NoMatch) + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt b/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt new file mode 100644 index 0000000..f565fc6 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt @@ -0,0 +1,146 @@ +package com.draftlens.heroes + +import com.draftlens.vision.annotation.DraftSlot +import com.draftlens.vision.annotation.SlotTeam +import com.draftlens.vision.annotation.SlotType +import com.draftlens.vision.geometry.NormalizedRect +import com.draftlens.vision.heroes.PixelSlotImageExtractor +import com.draftlens.vision.heroes.slotCropBounds +import com.draftlens.vision.model.FrameId +import com.draftlens.vision.model.FrameRotation +import com.draftlens.vision.model.OriginalFrame +import com.draftlens.vision.model.PixelArrayFramePayload +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +private fun frame(width: Int, height: Int, rotation: FrameRotation, pixels: IntArray = IntArray(width * height) { it }) = + OriginalFrame( + id = FrameId("frame-$width-$height-$rotation"), + payload = PixelArrayFramePayload(width, height, pixels), + sourceWidth = width, + sourceHeight = height, + displayRotation = FrameRotation.ROTATION_0, + rotation = rotation, + capturedAtElapsedRealtimeMs = 1, + wallClockTimestampMs = 1, + captureLatencyMs = 1, + ) + +private fun slot(rect: NormalizedRect, id: String = "slot-1") = DraftSlot( + id = id, + type = SlotType.BAN, + team = SlotTeam.ALLY, + index = 1, + rect = rect, +) + +class SlotImageExtractorTest { + + @Test fun `portrait and landscape frames extract without mutating the source`() { + val portrait = frame(4, 8, FrameRotation.ROTATION_0) + val landscape = frame(8, 4, FrameRotation.ROTATION_0) + val before = portrait.payload as PixelArrayFramePayload + + val result = PixelSlotImageExtractor.extract(portrait, slot(NormalizedRect(0.25f, 0.25f, 0.75f, 0.75f))) + + assertTrue(result.isSuccess) + assertEquals(2, result.getOrThrow().payload.width) + assertEquals(4, result.getOrThrow().payload.height) + assertEquals(before.pixels().toList(), (portrait.payload as PixelArrayFramePayload).pixels().toList()) + + val wide = PixelSlotImageExtractor.extract(landscape, slot(NormalizedRect(0.25f, 0.25f, 0.75f, 0.75f))) + assertTrue(wide.isSuccess) + assertEquals(4, wide.getOrThrow().payload.width) + assertEquals(2, wide.getOrThrow().payload.height) + } + + @Test fun `the extracted image matches the oriented appearance for every rotation`() { + // Asymmetric image so orientation mistakes are visible. + val pixels = intArrayOf( + 0xFF000000.toInt(), 0xFF000001.toInt(), 0xFF000002.toInt(), 0xFF000003.toInt(), + 0xFF000010.toInt(), 0xFF000011.toInt(), 0xFF000012.toInt(), 0xFF000013.toInt(), + 0xFF000100.toInt(), 0xFF000101.toInt(), 0xFF000102.toInt(), 0xFF000103.toInt(), + 0xFF001000.toInt(), 0xFF001001.toInt(), 0xFF001002.toInt(), 0xFF001003.toInt(), + ) + val rect = NormalizedRect(0f, 0f, 0.5f, 0.5f) // source top-left quadrant + for (rotation in FrameRotation.entries) { + val source = frame(4, 4, rotation, pixels.copyOf()) + val result = PixelSlotImageExtractor.extract(source, slot(rect)) + + assertTrue("rotation $rotation must succeed", result.isSuccess) + val image = result.getOrThrow().payload as PixelArrayFramePayload + assertEquals("dims[$rotation]", 2, image.width) + assertEquals("dims[$rotation]", 2, image.height) + val content = image.pixels().toList() + assertEquals("upright[$rotation]", expectedQuadrant(rotation), content) + } + } + + private fun expectedQuadrant(rotation: FrameRotation): List { + val a = 0xFF000000.toInt() + val b = 0xFF000001.toInt() + val e = 0xFF000010.toInt() + val f = 0xFF000011.toInt() + // Axis-aligned 2x2 crop of the source top-left quadrant, upright-rotated per frame rotation. + return when (rotation) { + FrameRotation.ROTATION_0 -> listOf(a, b, e, f) + FrameRotation.ROTATION_90 -> listOf(e, f, a, b) + FrameRotation.ROTATION_180 -> listOf(f, e, b, a) + FrameRotation.ROTATION_270 -> listOf(b, a, f, e) + } + } + + @Test fun `a rectangle touching the image edges extracts the edge pixels`() { + val pixels = IntArray(16) { it } + val source = frame(4, 4, FrameRotation.ROTATION_0, pixels) + val result = PixelSlotImageExtractor.extract(source, slot(NormalizedRect(0.5f, 0.5f, 1f, 1f))) + + val image = result.getOrThrow().payload as PixelArrayFramePayload + assertEquals(2, image.width) + assertEquals(2, image.height) + assertEquals(listOf(5, 6, 9, 10), image.pixels().toList()) + } + + @Test fun `the minimum-size rectangle extracts exactly one pixel`() { + val source = frame(100, 100, FrameRotation.ROTATION_0, IntArray(10_000) { it }) + val result = PixelSlotImageExtractor.extract(source, slot(NormalizedRect(0.1f, 0.1f, 0.11f, 0.11f))) + + val image = result.getOrThrow().payload as PixelArrayFramePayload + assertEquals(1, image.width) + assertEquals(1, image.height) + assertEquals(10 * 100 + 10, image.pixels().single()) + } + + @Test fun `a rectangle smaller than one pixel is refused`() { + val source = frame(10, 10, FrameRotation.ROTATION_0) + + val result = PixelSlotImageExtractor.extract(source, slot(NormalizedRect(0f, 0f, 0.05f, 0.05f), "tiny")) + + assertTrue(result.isFailure) + } + + @Test fun `a payload that does not match the source dimensions is refused`() { + val source = OriginalFrame( + id = FrameId("mismatch"), + payload = PixelArrayFramePayload(2, 2, IntArray(4)), + sourceWidth = 4, + sourceHeight = 4, + displayRotation = FrameRotation.ROTATION_0, + rotation = FrameRotation.ROTATION_0, + capturedAtElapsedRealtimeMs = 1, + wallClockTimestampMs = 1, + captureLatencyMs = 1, + ) + + val result = PixelSlotImageExtractor.extract(source, slot(NormalizedRect(0.1f, 0.1f, 0.9f, 0.9f))) + + assertTrue(result.isFailure) + } + + @Test fun `the crop bounds are floored at the origin and ceiled at the far edge`() { + val bounds = slotCropBounds(NormalizedRect(0.25f, 0.2f, 0.6f, 0.7f), 100, 200) + + assertEquals(com.draftlens.vision.geometry.PixelRect(25, 40, 60, 140), bounds) + } +} From 2ba69e245cac899f4a63192b75aea1717e83ffa8 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:29:01 +0000 Subject: [PATCH 02/33] fix: keep the cache-read error state instead of overwriting it --- .../com/draftlens/vision/heroes/FileHeroCatalogRepository.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt index 51bf9b8..60f56c6 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt @@ -27,9 +27,10 @@ class FileHeroCatalogRepository( override val state: StateFlow = _state.asStateFlow() init { + // If the cache cannot be decoded this records an Error state and returns null. cached = readDiskSnapshot() - _state.value = cached?.let { HeroCatalogState.Ready(it.entries.size, it.syncedAtMs) } - ?: HeroCatalogState.NotSynced + cached?.let { _state.value = HeroCatalogState.Ready(it.entries.size, it.syncedAtMs) } + // Otherwise leave whatever readDiskSnapshot put in place (Error), or NotSynced. } override fun snapshot(): CatalogSnapshot? { From 730b44a0b144622cf8d8bbd442e1a418c87eeb8a Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:31:38 +0000 Subject: [PATCH 03/33] fix: correct package paths and parameter order for the recognition wiring --- README.md | 12 +- .../feature/visionlab/VisionLabViewModel.kt | 5 +- .../heroes/AndroidBitmapSlotImageExtractor.kt | 2 +- .../vision/heroes/HeroReferenceImages.kt | 3 +- .../vision/heroes/SlotImageExtractor.kt | 2 +- docs/IMPLEMENTATION_REPORT.md | 18 ++ docs/VISION_ARCHITECTURE.md | 20 +++ docs/VISION_TESTING.md | 13 ++ docs/sprints/SPRINT_1B_HERO_RECOGNITION.md | 158 ++++++++++++++++++ tasks/plan.md | 6 + tasks/todo.md | 8 + 11 files changed, 236 insertions(+), 11 deletions(-) create mode 100644 docs/sprints/SPRINT_1B_HERO_RECOGNITION.md diff --git a/README.md b/README.md index ba5188c..fb802e6 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,11 @@ working capture path into a real **image-bearing** foundation: frames carry pixe is tested, preprocessing validates before it renders, and a Vision Lab lets you place manual slot annotations and save calibration profiles. -**Maintainer-reported device smoke:** the Sprint 0 capture path was used on a physical Android -device and confirmed working. Sprint 1A itself has **not** been verified on a physical device — -see [docs/TESTING_PHYSICAL_DEVICE.md](docs/TESTING_PHYSICAL_DEVICE.md) and -[docs/VISION_TESTING.md](docs/VISION_TESTING.md). +**Maintainer-reported device smoke:** the Sprint 0 capture path and the Sprint 1A Vision +Lab interaction behavior were used on a physical Android device and confirmed working. +Device details, the exact build and test dates were not supplied, and the MediaProjection +lifecycle matrix remains **NOT RUN**. See [docs/TESTING_PHYSICAL_DEVICE.md](docs/TESTING_PHYSICAL_DEVICE.md) +and [docs/VISION_TESTING.md](docs/VISION_TESTING.md). ## What works in Sprint 1A - MediaProjection consent flow (`ActivityResultContracts.CreateScreenCaptureIntent`) @@ -26,6 +27,9 @@ see [docs/TESTING_PHYSICAL_DEVICE.md](docs/TESTING_PHYSICAL_DEVICE.md) and unsupported-schema refusal, delete-as-escape-hatch - Vision Lab tab: frame preview, freeze/resume, explicit processing, interactive annotation canvas, text slot editor, profile controls +- Hero recognition v0 (Sprint 1B, PR #3): hero catalog sync from public static data, + explicit Identify action for the selected slot, ranked candidates with similarity scores, + manual confirm/correct — **accuracy on real game screens is not measured yet** ## What is deliberately NOT here (later milestones) ML hero recognition, recommendation engine, counter/synergy/role scoring, floating overlays, diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt index 20c2b61..31ea9a8 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt @@ -7,7 +7,6 @@ import com.draftlens.vision.heroes.HeroCatalogRepository import com.draftlens.vision.heroes.HeroReference import com.draftlens.vision.heroes.HeroRecognition import com.draftlens.vision.heroes.HeroReferenceImages -import com.draftlens.vision.heroes.HeroSlotPhase import com.draftlens.vision.heroes.SlotImageExtractor import com.draftlens.vision.heroes.SlotImageExtractors import com.draftlens.vision.heroes.pixelsOf @@ -427,7 +426,7 @@ class VisionLabViewModel( is IdentifyComputation.Done -> when (val outcome = computation.outcome) { is com.draftlens.vision.heroes.HeroIdentifyOutcome.Candidates -> updateHeroSlot(slot.id) { - com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.CANDIDATES, candidates = outcome.heroCandidates) + com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.CANDIDATES, candidates = outcome.candidates) } is com.draftlens.vision.heroes.HeroIdentifyOutcome.NoMatch -> updateHeroSlot(slot.id) { @@ -517,7 +516,7 @@ class VisionLabViewModel( val file = heroCatalog.assetFile(entry.id, asset.variant) ?: return@mapNotNull null referenceLoader(file, entry) } - val preview = (image.payload as? com.draftlens.vision.model.BitmapFramePayload)?.bitmap + val preview = (image.payload as? com.draftlens.vision.model.android.BitmapFramePayload)?.bitmap return IdentifyComputation.Done( HeroRecognition.identify(pixels.first, pixels.second, pixels.third, references), preview, diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt index c8d1d2d..d959d19 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt @@ -5,7 +5,7 @@ import android.graphics.Canvas import android.graphics.Matrix import android.os.SystemClock import com.draftlens.vision.annotation.DraftSlot -import com.draftlens.vision.model.BitmapFramePayload +import com.draftlens.vision.model.android.BitmapFramePayload import com.draftlens.vision.model.FrameRotation import com.draftlens.vision.model.FrameId import com.draftlens.vision.model.OriginalFrame diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt index a32ab55..b416e89 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt @@ -1,7 +1,6 @@ package com.draftlens.vision.heroes import android.graphics.BitmapFactory -import com.draftlens.vision.model.BitmapFramePayload import java.io.File /** @@ -11,7 +10,7 @@ import java.io.File object HeroReferenceImages { /** Decode a cached reference asset, downsampled so matching stays cheap. Null when missing/corrupt. */ - fun loadReference(entry: HeroCatalogEntry, file: File): HeroReference? = runCatching { + fun loadReference(file: File, entry: HeroCatalogEntry): HeroReference? = runCatching { val options = BitmapFactory.Options().apply { inJustDecodeBounds = true } diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt index 9920f6b..736bf60 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt @@ -36,7 +36,7 @@ fun pixelsOf(image: SlotImage): Triple? = when (val payload Triple(pixels, payload.width, payload.height) } - is com.draftlens.vision.model.BitmapFramePayload -> { + is com.draftlens.vision.model.android.BitmapFramePayload -> { val bitmap = payload.bitmap if (bitmap.isRecycled) { null diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index 65a6e2b..7bbcb88 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -374,3 +374,21 @@ Note: invocation `./gradlew assembleDebug` is intentionally not claimed locally; - Native-memory reclamation after reference clearing is not guaranteed and is not a strict one-Bitmap bound - OEM/background/store policies and review gates remain for release - Physical-device matrix needs execution before store release + +## Sprint 1B — hero recognition (in progress) + +Branch `sprint-1b-hero-recognition`, PR #3 (draft, **not merged**). See +`docs/sprints/SPRINT_1B_HERO_RECOGNITION.md` for the full report, including: + +- provider-independent hero catalog (Rone Arena primary, MLBBDex cross-check/roles/lanes), + atomic file snapshot with offline browsing, HTTPS-only transport; +- slot extraction from the annotation canvas (all four rotations, edge cases pinned by tests); +- recognition v0 (dHash + 3×3 normalized color descriptor), labeled similarity, explicit no-match, + manual confirmation and correction, per-slot persistence; +- **accuracy on real MLBB screens is not measured**; the no-match threshold is a starting + default, not a calibration; physical-device smoke is pending. +- Only static public hero catalog data and reference images are downloaded. Frames and crops + never leave the device; nothing is written to disk automatically. + +This sprint does not add recommendations, counters, synergy scoring, a floating overlay, +or automatic draft-phase detection, and does not merge PR #3 without review. diff --git a/docs/VISION_ARCHITECTURE.md b/docs/VISION_ARCHITECTURE.md index ca31aab..8159767 100644 --- a/docs/VISION_ARCHITECTURE.md +++ b/docs/VISION_ARCHITECTURE.md @@ -134,3 +134,23 @@ Slot semantics are validated once, in `vision/annotation/SlotValidator`, and use the annotation codec and the calibration codec. A slot the editor would refuse cannot be persisted as calibration data, and a stored document that contains one is reported as `Corrupt` and left untouched until it is deleted. + +## Sprint 1B — hero recognition boundary + +`vision/heroes/` holds the recognition pipeline. It consumes `VisionFrame` inputs (the same +`OriginalFrame`/`SlotImage` contract as the annotation canvas) and never imports capture code. +Everything network-facing goes through `HeroCatalogTransport`; tests use fake transports. + +- **Catalog** (`HeroCatalog*`) is public, static metadata only: hero id/name/images/roles/lanes. + Downloaded only on the user's explicit **Sync catalog** action; cached in the app's private + storage; works offline after sync. +- **Extraction** (`SlotImageExtractor`) maps the slot's normalized source rect to source pixels + and produces an upright in-memory `SlotImage`. Never written to disk. +- **Matching** (`HeroMatcher`) is a deterministic dHash + color descriptor. Output is a ranked + list of candidates or an explicit no-match. The UI turns that into manual confirm/correct — + never an automatic selection. +- **Confirmations** (`HeroRecognitionStore`) persist `slotId → {heroId, heroName, confirmedAtMs}` + in a separate preferences document; it never touches the annotation schema or the calibration + store, and it contains no images. +- **Privacy**: frames and crops stay in memory on-device; only the public static catalog and + its reference images are downloaded; INTERNET is the only new permission. diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index 28a1bc6..644d08d 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -94,3 +94,16 @@ mismatches, oversized output, unsupported formats and render errors all produce `ProcessFrameResult.Failure` with a reason code, and no partial frame is published. The calibration suites assert that an unsupported or corrupt document is reported and left untouched, and that only clearing the document recovers from it. + +## Sprint 1B tests (inventory) + +| Suite | File | Covers | +|---|---|---| +| Hero catalog sync | `heroes/HeroCatalogSyncTest.kt` | pagination to the reported total, mirror fallback, non-zero API code, duplicate-ID collapse, invalid-record skip, MLBBDex cross-check match/unavailable, roles/lanes by slug | +| Hero catalog codec | `heroes/HeroCatalogCodecTest.kt` | snapshot round-trip, schema refusal, blank id/name, non-HTTPS asset URL, unknown fields | +| Hero catalog repository | `heroes/FileHeroCatalogRepositoryTest.kt` | refresh stores snapshot + assets, atomic cache, offline restart, corrupt cache error, asset magic-byte/content-type/size validation, refresh failure keeps cache | +| Slot extraction | `heroes/SlotImageExtractorTest.kt` | portrait/landscape, 0/90/180/270 upright output, edge-touching and minimum-size rectangles, below-one-pixel rejection, payload mismatch, source untouched | +| Matcher | `heroes/HeroMatcherTest.kt` | identical images match, ranking order, no-match threshold, ambiguous gap, corrupt references skipped, hamming/color distance, determinism | +| Recognition orchestration | `heroes/HeroRecognitionTest.kt` | candidates path, no usable references → failure, no-match path | +| Confirmation store | `datastore/HeroRecognitionStoreTest.kt` | confirm/load/replace/clear, corrupt and future documents read as empty | +| ViewModel | `feature/visionlab/VisionLabViewModelTest.kt` | identify without slot → message, identify without catalog → FAILED honestly, synced catalog → candidates, confirm → persisted, clear → removed, catalog name search | diff --git a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md new file mode 100644 index 0000000..773803f --- /dev/null +++ b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md @@ -0,0 +1,158 @@ +# Sprint 1B — Hero Recognition + +**Branch:** `sprint-1b-hero-recognition` · **PR:** #3 (draft, **not merged**) + +Sprint 1B builds the first real, on-device hero-identification pipeline: a hero catalog from +public data, slot-image extraction from the annotation canvas, and a deterministic matching +baseline with a manual confirmation flow. It is about identifying heroes, not about draft +recommendations. + +**What is explicitly not here:** no recommendation or synergy scoring, no counter ranking, no +floating overlay, no automatic draft-phase detection, no OCR of arbitrary pixels beyond the +slot crop, no cloud recognition API, no ML model, no OpenCV. Those arrive after recognition +has been measured on real device samples. + +## New modules + +| Module | File(s) | What it does | +|---|---|---| +| Hero catalog model | `vision/heroes/HeroModels.kt`, `HeroIds.kt` | Provider-independent `HeroId` (slug from the English name), entries, assets, attribution, cross-check result | +| Catalog codec | `vision/heroes/HeroCatalogCodec.kt` | Strict v1 snapshot codec; duplicate-id protection; HTTPS-only asset URLs | +| HTTP transport | `vision/heroes/HeroCatalogTransport.kt`, `AndroidHeroCatalogTransport.kt` | HTTPS-only `HttpURLConnection` behind a small interface; size cap enforced while reading; one retry on I/O/5xx; 429 fails fast; no API key | +| Catalog sync | `vision/heroes/HeroCatalogSync.kt` | Paged `GET /heroes` on Rone Arena (plus mirror fallback), MLBBDex for completeness cross-check and roles/lanes by slug | +| Catalog repository | `vision/heroes/FileHeroCatalogRepository.kt`, `HeroCatalogRepository.kt` | Atomic snapshot (`/hero-catalog/catalog.json`), private image cache (`/hero-assets/`), offline browsing after sync | +| Slot extraction | `vision/heroes/SlotImageExtractor.kt`, `AndroidBitmapSlotImageExtractor.kt` | Axis-aligned crop of the source rect, upright for every rotation; never mutates/recycles the source; nothing written to disk | +| Matcher v0 | `vision/heroes/HeroMatcher.kt`, `HeroRecognition.kt` | dHash perceptual hash + normalized 3×3 mean-RGB descriptor; combined similarity in [0, 1]; ranked top-3; explicit no-match below threshold | +| Reference loader | `vision/heroes/HeroReferenceImages.kt` | Decodes cached reference images; downsampled ≤256 px for cheap matching | +| Recognition store | `datastore/HeroRecognitionStore.kt` | Per-slot confirmed hero (id, name, timestamp) in its own preferences document; corrupt documents read as empty | +| UI | `feature/visionlab/HeroRecognitionCard.kt` | Sync button, Identify action, candidate cards, confirm/correct/clear, unofficial notice and attribution | +| Permissions | `AndroidManifest.xml` | `INTERNET` added — hero catalog download only; capture still requires explicit consent | + +## Data sources and attribution + +| Provider | What is used | License / notice | +|---|---|---| +| **Rone Arena** (`arena.rone.dev`, mirror `arena.fastapicloud.dev/api`) | `GET /api/heroes` (paginated, English) for entries, `hero_id`, head/smallmap portrait URLs | BSD-3-Clause (community-maintained, unofficial). Not affiliated with or endorsed by Moonton. Game names and visual assets remain the property of their respective rights holders | +| **MLBBDex** (`mlbbdex.com`) | `GET /api/v1/heroes` for the completeness cross-check and roles/lanes/specialties by slug (`roles`, `lanes`, `specialties`) | Community wiki data under CC BY-SA (https://mobilelegends.fandom.com); written content MIT. Game visuals remain the property of Shanghai Moonton Technology Co., Ltd. | + +- No written analyses are copied into the app. No counter or build statistics are shown. +- Provider-independent identity is a slug derived from the English name; display names are + never an identity. +- `externalIds` records `roneArenaHeroId` and `mlbbdexSlug` per entry. +- Snapshot attribution and notices are kept in the cached document and shown in the app. +- The unofficial/non-affiliation notice is shown in the recognition card. + +### Re-verified contracts (2026-11-15) + +Rone Arena (primary): +- Base: `https://arena.fastapicloud.dev/api` (primary) / `https://arena.rone.dev/api` (mirror). + No key. Public endpoints only. +- `GET /heroes?size&index&lang` — `data.records[].{data.hero.data.{head,name,smallmap}, hero_id, relation}`, + `data.total`. Records ordered by `hero_id`. +- `GET /heroes/{hero_identifier}` — additional variants `head_big`, `squarehead`, `squareheadbig` + (detail endpoint; not fetched in this sprint — `HEAD` is enough to establish the baseline). +- Error handling: a 503 means maintenance/high traffic (`alternative_endpoint` fallback); + a 429 `UPSTREAM_RATE_LIMITED` means back off, never parallel-hammer. + +MLBBDex (cross-check): +- `GET /api/v1/heroes` — `{data, total, source{name,url,license}}` with `slug`, `name`, + `roles`, `lanes`, `specialties`. Envelope source attribution is credited here and in the app. + +## Catalog behaviour + +- **Manual sync only.** Nothing downloads at startup; the user presses *Sync catalog*. +- **Pagination** with `size=100`, `index=1..`, capped at 20 pages; duplicates by + `hero_id` are collapsed and counted; records missing id/name/both images are skipped and counted. +- **Mirror fallback.** If the primary host fails, the request is retried once against the mirror. +- **Catalog entries actually loaded are reported**, never a hardcoded expected count. + (Measured at sync time; see the sync section in `docs/VISION_TESTING.md` for what CI ran.) +- **Caching.** Snapshot JSON is written to a temp file and renamed into place; an interrupted + sync cannot leave a half snapshot. Reference images go to a private per-app cache with + magic-byte validation, `image/*` content-type check and a 4 MiB cap per image. +- **Offline.** After a successful sync the catalog serves from cache with no network at all. + A corrupt cache is reported as an error and never silently replaced; re-sync is the fix. +- **Asset failure handling.** If one image fails, the hero is still in the catalog; the failure + is reported by name in `assetFailures` and matching simply skips that hero. + +## Slot extraction + +- The slot rectangle (normalized *source* coordinates) maps to whole source pixels + (`NormalizedRect.toPixelRect`, floor origin / ceil far edge / ≥1 px) — the same rounding + rule the annotation canvas uses — and the crop is cropped axis-aligned, then rotated + upright to match what the user sees on screen. +- Aspect ratio is never changed; nothing is stretched. +- The original frame bitmap is never mutated or recycled. +- Extracted crops live in memory only and are never written to disk. +- Tests cover portrait and landscape frames, all four rotations, edge-touching rectangles, + minimum-size rectangles, invalid coordinates, payload mismatch and original-frame ownership. + +## Matching algorithm and why + +Two deterministic descriptors, compared on a fixed grid: + +1. **dHash perceptual hash** (9×8 grayscale, area-average downsample, bit = left pixel + brighter than right). 64 bits, hamming distance, robust to scale and mild recompression. +2. **Normalized 3×3 mean-RGB appearance descriptor** (27 floats). Weaker signal but catches + hue/lighting layout that a hash alone can miss. + +Combined similarity: `0.6 × (1 − hamming/64) + 0.4 × (1 − colorDistance)`, clamped to [0, 1]. + +Why this over anything fancier: + +- No training data, no model download, no inference dependency; runs on any Android device this app targets. +- Reference images (head portraits) are small and near-uniform in composition, which is exactly + where dHash works well. +- It is fully deterministic, so tests are stable and failures are reproducible. + +**This is a heuristic ranking aid, not a probability or a calibrated confidence.** Scores are +labeled as similarity everywhere they appear. The no-match threshold (0.55) is a starting +default, not a calibration — it was **not** tuned on real MLBB screens (none were used). +Two candidates closer together than 0.05 are flagged `ambiguous`. The UI never selects a +hero automatically: the user confirms or corrects every match. + +## Evaluation (honest) + +There is **no accuracy claim on real game screens** in this sprint. The matcher is evaluated +by deterministic algorithm tests only (identical images match, ranking order, threshold +behaviour, ambiguous gap, corrupt references skipped). Those tests prove the mechanism works, +not that 134 MLBB hero portraits are linearly separable on a capture full of effects. + +| Metric | Value | Data | +|---|---|---| +| Algorithm tests (deterministic) | see CI | synthetic fixtures only | +| Top-1 accuracy on real MLBB screens | **not measured** | no samples available | +| Top-3 accuracy on real MLBB screens | **not measured** | no samples available | +| No-match false-rejection rate | **not measured** | no samples available | +| Sample count | 0 | no device images committed | + +Synthetic tests are useful for the algorithm; they do not prove accuracy on real MLBB screens. +`head` and `smallmap` are headshots on flat backgrounds and should be more separable than the +matcher assumes; this needs a device smoke test with manually labeled slots before any claim +is printed in the app. + +Physical-device evaluation is **pending** and must use real app captures with manually +identified hero slots. Device details and the number of correctly identified samples must be +recorded here when run; upload or commit those images is prohibited. + +## UI + +Vision Lab, under the slot list: + +- *Sync catalog* button; sync state shown (`Not synced`, progress while syncing, `Ready: N heroes`, or error with the cached count if offline). +- *Identify hero* on the selected slot; per-slot state (`idle → processing → candidates | no match → confirmed | failed`). +- Candidate cards: cached image, name, similarity, `ambiguous` marker when present; **Confirm** per candidate. +- **No match** state with the best similarity, plus manual name search. +- **Confirmed** view with Clear and Change…; manual name search available everywhere. +- Attribution footer with the unofficial notice; images load from the local cache only. + +Recognition never runs continuously, its button never appears on slot creation, and it never +opens the property menu automatically. Slot creation keeps the maintainer's 1A preference. + +## Acceptance status + +- CI green (this PR): unit tests, strict lint, `assembleDebug`, `assembleAndroidTest` (compiled). +- Physical-device smoke: **pending**. +- Catalog sync on a device: **pending** (verified only through fake transport in unit tests). +- On-device accuracy: **not measured**. + +See `tasks/todo.md` and `docs/IMPLEMENTATION_REPORT.md` for the running checklist. diff --git a/tasks/plan.md b/tasks/plan.md index 3081bc9..d770b88 100644 --- a/tasks/plan.md +++ b/tasks/plan.md @@ -128,3 +128,9 @@ grow without re-architecture. - After Phase 1: `./gradlew test` green, app assembles (empty landing OK). - After Phase 2: unit tests for state machine + sampler green; lint + assemble green. - Final: test + lint + assembleDebug all green; APK exists; manifest grep verified. + +### Sprint 1B — Hero recognition (first pipeline) +- Hero catalog sync (manual refresh, atomic snapshot, offline browsing) wired through a small transport seam. +- Slot extraction from the annotation canvas with rotation coverage. +- Recognition v0 with labeled similarity and explicit no-match; manual confirmation required. +- Draft PR #3; accuracy on real screens not measured; device smoke pending. diff --git a/tasks/todo.md b/tasks/todo.md index 190d765..034c141 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -90,3 +90,11 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** ## Phase 4: Docs + verify - [x] Task 9: README, ARCHITECTURE, SCREEN_CAPTURE, DATA_SOURCES, ADR-001 - [x] Task 10: Tests + lint + assembleDebug green; APK + manifest verified; report written + +## Sprint 1B: Hero catalog, slot extraction and recognition +- [x] Hero catalog module (Rone Arena primary, MLBBDex cross-check, atomic cache, offline) +- [x] Slot image extraction (all rotations, edge cases) +- [x] Recognition v0: dHash + color descriptor, explicit no-match, manual confirm/correct +- [x] Vision Lab card: sync, Identify, candidates, confirmed state, manual correction +- [x] Unit coverage for catalog, cache, extraction, matching, VM flows +- [ ] Physical-device smoke test and on-device accuracy evaluation (pending) From 8d960b81e0fe804a61cba25cf543d57a0170ad80 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:33:37 +0000 Subject: [PATCH 04/33] test: pass the hero collaborators as named arguments --- .../visionlab/VisionLabViewModelTest.kt | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 7a5356c..8603397 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -160,7 +160,7 @@ class VisionLabViewModelTest { @Test fun `clear cancels in flight processing and drops the late result`() = runBlocking { val source = FakeVisionFrameSource(originalFrame(BLOCKED_FRAME_ID)) val preprocessor = BlockingFakePreprocessor() - val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.processFrame() assertTrue("the request never reached the preprocessor", preprocessor.awaitEntered()) @@ -178,7 +178,7 @@ class VisionLabViewModelTest { @Test fun `save profile requires a frozen frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository() - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.saveProfile("Ranked") @@ -197,7 +197,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.updateProfile("p1", "Ranked") @@ -216,7 +216,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.applyProfile("p1") @@ -227,7 +227,7 @@ class VisionLabViewModelTest { @Test fun `a frozen frame satisfies the profile frame requirement`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository() - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.freeze() vm.saveProfile("Ranked") @@ -237,7 +237,7 @@ class VisionLabViewModelTest { @Test fun `live frame is exposed in the ui state`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) assertEquals("live", vm.uiState.value.liveFrame?.id?.value) assertEquals(VisionSourceState.CAPTURING, vm.uiState.value.sourceState) @@ -480,11 +480,14 @@ class VisionLabViewModelTest { private fun slotVm(): VisionLabViewModel = VisionLabViewModel( FakeVisionFrameSource(originalFrame("live")), EchoPreprocessor(), - FakeCalibrationRepository(),, heroCatalog(), heroStore()) + FakeCalibrationRepository(), + heroCatalog = heroCatalog(), + heroRecognitionStore = heroStore(), + ) @Test fun `freeze pins the current frame while live frames continue`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("first")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.freeze() source.emit(originalFrame("second")) @@ -496,7 +499,7 @@ class VisionLabViewModelTest { @Test fun `resume releases the pinned frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("first")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.freeze() vm.resume() @@ -507,7 +510,7 @@ class VisionLabViewModelTest { @Test fun `clear releases the pinned frame and the processed frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("first")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.freeze() vm.processFrame() @@ -521,7 +524,7 @@ class VisionLabViewModelTest { @Test fun `process frame publishes the processed frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.processFrame() val state = withTimeout(10_000) { vm.uiState.first { it.processed != null } } @@ -533,7 +536,7 @@ class VisionLabViewModelTest { @Test fun `stale processing result is discarded after a newer request`() = runBlocking { val source = FakeVisionFrameSource(originalFrame(BLOCKED_FRAME_ID)) val preprocessor = BlockingFakePreprocessor() - val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, preprocessor, FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.processFrame() // The first request must be inside the preprocessor before the second one supersedes it; @@ -554,7 +557,7 @@ class VisionLabViewModelTest { @Test fun `processing failure is reported honestly`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, FailingPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, FailingPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.processFrame() val state = withTimeout(10_000) { vm.uiState.first { it.processStatus == ProcessStatus.FAILED } } @@ -565,7 +568,7 @@ class VisionLabViewModelTest { @Test fun `process frame without a frame reports an error`() = runBlocking { val source = FakeVisionFrameSource(null) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.processFrame() @@ -576,7 +579,7 @@ class VisionLabViewModelTest { @Test fun `save profile is refused when the document schema is unsupported`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.UnsupportedSchema("too new")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.saveProfile("Ranked") @@ -587,7 +590,7 @@ class VisionLabViewModelTest { @Test fun `delete all calibration data stays available when the schema is unsupported`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) val repository = FakeCalibrationRepository(CalibrationLoadResult.UnsupportedSchema("too new")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.deleteAllProfiles() @@ -597,7 +600,7 @@ class VisionLabViewModelTest { @Test fun `save profile stores the current slots and the oriented aspect ratio`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository() - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.1f, 0.1f, 0.2f, 0.2f)) vm.freeze() @@ -621,7 +624,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live", width = 10, height = 10)) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.freeze() vm.applyProfile("p1") @@ -650,7 +653,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.6f, 0.6f, 0.7f, 0.7f)) vm.freeze() @@ -682,7 +685,7 @@ class VisionLabViewModelTest { ) val source = FakeVisionFrameSource(originalFrame("live", width = 16, height = 9)) val repository = FakeCalibrationRepository(CalibrationLoadResult.Success(listOf(profile))) - val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), repository, heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.6f, 0.6f, 0.7f, 0.7f)) vm.freeze() @@ -696,7 +699,7 @@ class VisionLabViewModelTest { @Test fun `slot index that duplicates another slot in the same scope is rejected`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("live")) - val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog(), heroStore()) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.1f, 0.1f, 0.2f, 0.2f)) vm.addSlot(SlotType.BAN, SlotTeam.ALLY, NormalizedRect(0.4f, 0.4f, 0.5f, 0.5f)) From 9055826483a23c84a63ecc7d69e7d91aad9faf72 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:35:57 +0000 Subject: [PATCH 05/33] test: fix property name in the recognition orchestration test --- app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt index 5ffebae..05bed91 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt @@ -43,7 +43,7 @@ class HeroRecognitionTest { references = listOf(miyaReference), ) - val candidates = (outcome as HeroIdentifyOutcome.Candidates).heroCandidates + val candidates = (outcome as HeroIdentifyOutcome.Candidates).candidates assertEquals(1, candidates.size) assertEquals("miya", candidates.single().hero.canonicalName) assertEquals(1f, candidates.single().similarity, 0.001f) From 9573b5285e19c7db7e330f86d9c19ef89aeb80e6 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:40:43 +0000 Subject: [PATCH 06/33] test: fix float-precision expectations and real-time waits in the 1B suite --- .../visionlab/VisionLabViewModelTest.kt | 44 +++++++++++-------- .../com/draftlens/heroes/HeroMatcherTest.kt | 2 +- .../heroes/SlotImageExtractorTest.kt | 13 +++--- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 8603397..590ff69 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -477,6 +477,15 @@ class VisionLabViewModelTest { // endregion + private fun eventually(timeoutMs: Long = 10_000L, condition: () -> Boolean) { + val deadline = System.currentTimeMillis() + timeoutMs + while (System.currentTimeMillis() < deadline) { + if (condition()) return + Thread.sleep(20) + } + org.junit.Assert.fail("condition not met within $timeoutMs ms") + } + private fun slotVm(): VisionLabViewModel = VisionLabViewModel( FakeVisionFrameSource(originalFrame("live")), EchoPreprocessor(), @@ -777,27 +786,28 @@ class VisionLabViewModelTest { vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.3f, 0.3f)) vm.identifySelectedSlot() - val state = withTimeout(5_000) { vm.uiState.first { it.heroRecognition[vm.uiState.value.slots.single().id] != null } } + eventually(5_000) { + vm.uiState.value.heroRecognition[vm.uiState.value.slots.single().id]?.phase == HeroSlotPhase.FAILED + } + val slotId = vm.uiState.value.slots.single().id - assertEquals(HeroSlotPhase.FAILED, state.heroRecognition[state.slots.single().id]!!.phase) + assertEquals(HeroSlotPhase.FAILED, vm.uiState.value.heroRecognition[slotId]!!.phase) assertTrue( - state.heroRecognition[state.slots.single().id]!!.message!!.contains("Sync the hero catalog"), + vm.uiState.value.heroRecognition[slotId]!!.message!!.contains("Sync the hero catalog"), ) } @Test fun `a synced catalog produces ranked candidates for the selected slot`() = runBlocking { val vm = recognitionVm() vm.syncHeroCatalog() - withTimeout(10_000) { vm.uiState.first { it.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } } + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.4f, 0.4f)) val slotId = vm.uiState.value.slots.single().id vm.identifySelectedSlot() - val state = withTimeout(10_000) { - vm.uiState.first { it.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } - } + eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } - val candidates = state.heroRecognition[slotId]!!.candidates + val candidates = vm.uiState.value.heroRecognition[slotId]!!.candidates assertTrue(candidates.isNotEmpty()) assertEquals(2, candidates.size.coerceAtMost(2)) } @@ -805,29 +815,25 @@ class VisionLabViewModelTest { @Test fun `confirming a candidate persists the hero and clearing removes it`() = runBlocking { val vm = recognitionVm() vm.syncHeroCatalog() - withTimeout(10_000) { vm.uiState.first { it.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } } + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.4f, 0.4f)) val slotId = vm.uiState.value.slots.single().id vm.identifySelectedSlot() - withTimeout(10_000) { vm.uiState.first { it.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } } + eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } vm.confirmHero(slotId, "miya") - val confirmed = withTimeout(5_000) { - vm.uiState.first { it.heroRecognition[slotId]?.phase == HeroSlotPhase.CONFIRMED } - } - assertEquals("Miya", confirmed.heroRecognition[slotId]!!.message) + eventually(5_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CONFIRMED } + assertEquals("Miya", vm.uiState.value.heroRecognition[slotId]!!.message) vm.clearHeroConfirmation(slotId) - val cleared = withTimeout(5_000) { - vm.uiState.first { it.heroRecognition[slotId] == null } - } - assertNull(cleared.heroRecognition[slotId]) + eventually(5_000) { vm.uiState.value.heroRecognition[slotId] == null } + assertNull(vm.uiState.value.heroRecognition[slotId]) } @Test fun `the search filters the cached catalog by name`() = runBlocking { val vm = recognitionVm() vm.syncHeroCatalog() - withTimeout(10_000) { vm.uiState.first { it.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } } + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } val results = vm.searchCatalog("mi") diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt index 04c7e9f..3eed156 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt @@ -97,7 +97,7 @@ class HeroMatcherTest { @Test fun `hamming distance counts differing bits`() { assertEquals(0, HeroMatcher.hammingDistance(0b1010, 0b1010)) - assertEquals(2, HeroMatcher.hammingDistance(0b1010, 0b0101)) + assertEquals(4, HeroMatcher.hammingDistance(0b1010, 0b0101)) assertEquals(64, HeroMatcher.hammingDistance(Long.MAX_VALUE, Long.MIN_VALUE + 1)) } diff --git a/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt b/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt index f565fc6..1e82079 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt @@ -99,17 +99,18 @@ class SlotImageExtractorTest { val image = result.getOrThrow().payload as PixelArrayFramePayload assertEquals(2, image.width) assertEquals(2, image.height) - assertEquals(listOf(5, 6, 9, 10), image.pixels().toList()) + assertEquals(listOf(10, 11, 14, 15), image.pixels().toList()) } @Test fun `the minimum-size rectangle extracts exactly one pixel`() { - val source = frame(100, 100, FrameRotation.ROTATION_0, IntArray(10_000) { it }) - val result = PixelSlotImageExtractor.extract(source, slot(NormalizedRect(0.1f, 0.1f, 0.11f, 0.11f))) + // 128x128 keeps 1/128-steps exact in float; 0.125..0.1328125 spans exactly one pixel. + val source = frame(128, 128, FrameRotation.ROTATION_0, IntArray(128 * 128) { it }) + val result = PixelSlotImageExtractor.extract(source, slot(NormalizedRect(0.125f, 0.125f, 0.1328125f, 0.1328125f))) val image = result.getOrThrow().payload as PixelArrayFramePayload assertEquals(1, image.width) assertEquals(1, image.height) - assertEquals(10 * 100 + 10, image.pixels().single()) + assertEquals(16 * 128 + 16, image.pixels().single()) } @Test fun `a rectangle smaller than one pixel is refused`() { @@ -139,8 +140,8 @@ class SlotImageExtractorTest { } @Test fun `the crop bounds are floored at the origin and ceiled at the far edge`() { - val bounds = slotCropBounds(NormalizedRect(0.25f, 0.2f, 0.6f, 0.7f), 100, 200) + val bounds = slotCropBounds(NormalizedRect(0.2f, 0.25f, 0.5f, 0.75f), 100, 200) - assertEquals(com.draftlens.vision.geometry.PixelRect(25, 40, 60, 140), bounds) + assertEquals(com.draftlens.vision.geometry.PixelRect(20, 50, 50, 150), bounds) } } From 7d0c9032a52db8e0f962beea72ceae7120822c88 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:43:34 +0000 Subject: [PATCH 07/33] test: fix hamming vector and sub-pixel rects on the 4x2 synthetic frame --- .../draftlens/feature/visionlab/VisionLabViewModelTest.kt | 6 +++--- app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 590ff69..10daf36 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -783,7 +783,7 @@ class VisionLabViewModelTest { @Test fun `identify without a synced catalog fails honestly instead of guessing`() = runBlocking { val vm = recognitionVm() - vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.3f, 0.3f)) + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) vm.identifySelectedSlot() eventually(5_000) { @@ -801,7 +801,7 @@ class VisionLabViewModelTest { val vm = recognitionVm() vm.syncHeroCatalog() eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } - vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.4f, 0.4f)) + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) val slotId = vm.uiState.value.slots.single().id vm.identifySelectedSlot() @@ -816,7 +816,7 @@ class VisionLabViewModelTest { val vm = recognitionVm() vm.syncHeroCatalog() eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } - vm.createSlot(NormalizedRect(0.1f, 0.1f, 0.4f, 0.4f)) + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) val slotId = vm.uiState.value.slots.single().id vm.identifySelectedSlot() eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt index 3eed156..1d33bc4 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroMatcherTest.kt @@ -98,7 +98,7 @@ class HeroMatcherTest { @Test fun `hamming distance counts differing bits`() { assertEquals(0, HeroMatcher.hammingDistance(0b1010, 0b1010)) assertEquals(4, HeroMatcher.hammingDistance(0b1010, 0b0101)) - assertEquals(64, HeroMatcher.hammingDistance(Long.MAX_VALUE, Long.MIN_VALUE + 1)) + assertEquals(64, HeroMatcher.hammingDistance(Long.MAX_VALUE, Long.MIN_VALUE)) } @Test fun `color distance is zero for identical descriptors`() { From 3398891c095283036ffb9650c620712b4091cfdd Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:47:22 +0000 Subject: [PATCH 08/33] docs: record the Sprint 1B green run and acceptance state --- docs/IMPLEMENTATION_REPORT.md | 7 +++++++ docs/VISION_TESTING.md | 7 +++++++ docs/sprints/SPRINT_1B_HERO_RECOGNITION.md | 9 ++++++++- tasks/todo.md | 1 + 4 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index 7bbcb88..008e963 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -392,3 +392,10 @@ Branch `sprint-1b-hero-recognition`, PR #3 (draft, **not merged**). See This sprint does not add recommendations, counters, synergy scoring, a floating overlay, or automatic draft-phase detection, and does not merge PR #3 without review. + +Sprint 1B green run (code head, 7d0c9032a52db8e0f962beea72ceae7120822c88): +https://github.com/xeriomy/DraftLens/actions/runs/43731484922 — **323 tests (646 executions), +0 failures, 0 errors, 0 skipped**; instrumentation tests compiled, not executed; strict lint; +assembleDebug; artifact `draftlens-debug`, 10 334 760 bytes, digest +`sha256:122433490a5da130390a8a7b7b1f69eb0040551ca0c7fa82f8bc375f9405741f`, APK SHA-256 +`c0c02f70fd5af56ce3f8e3631dfe5c207a77fa294172ceb473c9a549bbf81ce3`. diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index 644d08d..ef5719d 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -107,3 +107,10 @@ untouched, and that only clearing the document recovers from it. | Recognition orchestration | `heroes/HeroRecognitionTest.kt` | candidates path, no usable references → failure, no-match path | | Confirmation store | `datastore/HeroRecognitionStoreTest.kt` | confirm/load/replace/clear, corrupt and future documents read as empty | | ViewModel | `feature/visionlab/VisionLabViewModelTest.kt` | identify without slot → message, identify without catalog → FAILED honestly, synced catalog → candidates, confirm → persisted, clear → removed, catalog name search | + +Sprint 1B green run (code head): +— head SHA `7d0c9032a52db8e0f962beea72ceae7120822c88`, **323 unit tests (646 executions, +debug + release), 0 failures, 0 errors, 0 skipped**. Instrumentation tests compiled, not +executed. The 1B transport is exercised only through fake responses; no test performs a real +network call. The number of catalog entries loaded from the live API is measured at device +sync time, not in CI. diff --git a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md index 773803f..0f7c51b 100644 --- a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md +++ b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md @@ -150,9 +150,16 @@ opens the property menu automatically. Slot creation keeps the maintainer's 1A p ## Acceptance status -- CI green (this PR): unit tests, strict lint, `assembleDebug`, `assembleAndroidTest` (compiled). +- **CI green** (code head `7d0c9032a52db8e0f962beea72ceae7120822c88`, run 43731484922): + **323 unit tests (646 executions, debug + release), 0 failures, 0 errors, 0 skipped**; + `assembleAndroidTest` **compiled, not executed**; strict lint passed; `assembleDebug` passed; + artifact `draftlens-debug`, 10 334 760 bytes, digest + `sha256:122433490a5da130390a8a7b7b1f69eb0040551ca0c7fa82f8bc375f9405741f`. - Physical-device smoke: **pending**. - Catalog sync on a device: **pending** (verified only through fake transport in unit tests). - On-device accuracy: **not measured**. +- **Catalog entries actually loaded:** measured at sync time on the device that syncs; not yet + recorded here. The pagination logic is exercised against fake transports in unit tests; + the real total is whatever the API returns when the user presses *Sync catalog*. See `tasks/todo.md` and `docs/IMPLEMENTATION_REPORT.md` for the running checklist. diff --git a/tasks/todo.md b/tasks/todo.md index 034c141..ecc6f9f 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -97,4 +97,5 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** - [x] Recognition v0: dHash + color descriptor, explicit no-match, manual confirm/correct - [x] Vision Lab card: sync, Identify, candidates, confirmed state, manual correction - [x] Unit coverage for catalog, cache, extraction, matching, VM flows +- [x] CI green: 323 tests (646 executions), strict lint, assembleDebug, assembleAndroidTest compiled (run 43731484922, code head 7d0c903) - [ ] Physical-device smoke test and on-device accuracy evaluation (pending) From b4eac9d856fe049d8d6ae8d232472bd4627c1f4e Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:50:20 +0000 Subject: [PATCH 09/33] test: reset Main defensively in setUp to survive order-dependent leaks --- .../com/draftlens/feature/visionlab/VisionLabViewModelTest.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 10daf36..44cada6 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -47,6 +47,8 @@ import java.util.concurrent.atomic.AtomicBoolean class VisionLabViewModelTest { @Before fun setUp() { + // Defensive: if a previous test's Main leaked (order-dependent), reset it first. + runCatching { Dispatchers.resetMain() } Dispatchers.setMain(UnconfinedTestDispatcher()) } From a8bbead6741e94098da93a5fb148c418cf75103a Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:53:46 +0000 Subject: [PATCH 10/33] docs: record both Sprint 1B green runs --- docs/IMPLEMENTATION_REPORT.md | 12 ++++++------ docs/VISION_TESTING.md | 13 +++++++------ docs/sprints/SPRINT_1B_HERO_RECOGNITION.md | 11 ++++++----- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index 008e963..338fd65 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -393,9 +393,9 @@ Branch `sprint-1b-hero-recognition`, PR #3 (draft, **not merged**). See This sprint does not add recommendations, counters, synergy scoring, a floating overlay, or automatic draft-phase detection, and does not merge PR #3 without review. -Sprint 1B green run (code head, 7d0c9032a52db8e0f962beea72ceae7120822c88): -https://github.com/xeriomy/DraftLens/actions/runs/43731484922 — **323 tests (646 executions), -0 failures, 0 errors, 0 skipped**; instrumentation tests compiled, not executed; strict lint; -assembleDebug; artifact `draftlens-debug`, 10 334 760 bytes, digest -`sha256:122433490a5da130390a8a7b7b1f69eb0040551ca0c7fa82f8bc375f9405741f`, APK SHA-256 -`c0c02f70fd5af56ce3f8e3631dfe5c207a77fa294172ceb473c9a549bbf81ce3`. +Sprint 1B green runs: final branch head `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` — +https://github.com/xeriomy/DraftLens/actions/runs/43732111174 (646 executions, 0 failures); +code head `7d0c9032a52db8e0f962beea72ceae7120822c88` — +https://github.com/xeriomy/DraftLens/actions/runs/43731484922 (646 executions, 0 failures). +Instrumentation tests compiled, not executed; strict lint; assembleDebug; artifact +`draftlens-debug`, digest `sha256:3c977d55d77c5452425ccfd5ea950ef55722d77319149c076d8bd5d115b85cc9`. diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index ef5719d..c2ab9dc 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -108,9 +108,10 @@ untouched, and that only clearing the document recovers from it. | Confirmation store | `datastore/HeroRecognitionStoreTest.kt` | confirm/load/replace/clear, corrupt and future documents read as empty | | ViewModel | `feature/visionlab/VisionLabViewModelTest.kt` | identify without slot → message, identify without catalog → FAILED honestly, synced catalog → candidates, confirm → persisted, clear → removed, catalog name search | -Sprint 1B green run (code head): -— head SHA `7d0c9032a52db8e0f962beea72ceae7120822c88`, **323 unit tests (646 executions, -debug + release), 0 failures, 0 errors, 0 skipped**. Instrumentation tests compiled, not -executed. The 1B transport is exercised only through fake responses; no test performs a real -network call. The number of catalog entries loaded from the live API is measured at device -sync time, not in CI. +Sprint 1B green runs: final branch head `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` — + (646 executions, debug + +release, **0 failures, 0 errors, 0 skipped**); code head `7d0c9032a52db8e0f962beea72ceae7120822c88` — + (same totals). Instrumentation +tests compiled, not executed. The 1B transport is exercised only through fake responses; no +test performs a real network call. The number of catalog entries loaded from the live API is +measured at device sync time, not in CI. diff --git a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md index 0f7c51b..ede8fb8 100644 --- a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md +++ b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md @@ -150,11 +150,12 @@ opens the property menu automatically. Slot creation keeps the maintainer's 1A p ## Acceptance status -- **CI green** (code head `7d0c9032a52db8e0f962beea72ceae7120822c88`, run 43731484922): - **323 unit tests (646 executions, debug + release), 0 failures, 0 errors, 0 skipped**; - `assembleAndroidTest` **compiled, not executed**; strict lint passed; `assembleDebug` passed; - artifact `draftlens-debug`, 10 334 760 bytes, digest - `sha256:122433490a5da130390a8a7b7b1f69eb0040551ca0c7fa82f8bc375f9405741f`. +- **CI green**: final branch head `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` (run 43732111174, + test-only increment on the code head) and code head `7d0c9032a52db8e0f962beea72ceae7120822c88` + (run 43731484922); **323 unit tests (646 executions, debug + release), 0 failures, + 0 errors, 0 skipped**; `assembleAndroidTest` **compiled, not executed**; strict lint passed; + `assembleDebug` passed; artifact `draftlens-debug`, digest + `sha256:3c977d55d77c5452425ccfd5ea950ef55722d77319149c076d8bd5d115b85cc9`. - Physical-device smoke: **pending**. - Catalog sync on a device: **pending** (verified only through fake transport in unit tests). - On-device accuracy: **not measured**. From e58611c25d7d9a40592aa3d81ad3347869771cc3 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 15:59:21 +0000 Subject: [PATCH 11/33] test: class-scoped Main dispatcher to kill the order-dependent race --- .../visionlab/VisionLabViewModelTest.kt | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 44cada6..3309b0d 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -31,14 +31,14 @@ import kotlinx.coroutines.test.UnconfinedTestDispatcher import kotlinx.coroutines.test.resetMain import kotlinx.coroutines.test.setMain import kotlinx.coroutines.withTimeout -import org.junit.After import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertNotEquals import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Assert.assertTrue -import org.junit.Before +import org.junit.BeforeClass +import org.junit.AfterClass import org.junit.Test import java.util.concurrent.CountDownLatch import java.util.concurrent.TimeUnit @@ -46,16 +46,6 @@ import java.util.concurrent.atomic.AtomicBoolean class VisionLabViewModelTest { - @Before fun setUp() { - // Defensive: if a previous test's Main leaked (order-dependent), reset it first. - runCatching { Dispatchers.resetMain() } - Dispatchers.setMain(UnconfinedTestDispatcher()) - } - - @After fun tearDown() { - Dispatchers.resetMain() - } - // region fakes private class FakeVisionFrameSource(initial: OriginalFrame? = null) : VisionFrameSource { @@ -846,9 +836,24 @@ class VisionLabViewModelTest { // endregion - private companion object { + companion object { /** The frame id whose processing call the fake blocks. */ const val BLOCKED_FRAME_ID = "first" + + // One Main for the whole class: per-test set/reset raced with the VM's background + // collectors (the delegate's NonConcurrentlyModifiable throws when a dispatch lands + // mid-mutation). Modify Main once at both ends instead. + @JvmStatic + @BeforeClass + fun setUpClass() { + Dispatchers.setMain(UnconfinedTestDispatcher()) + } + + @JvmStatic + @AfterClass + fun tearDownClass() { + runCatching { Dispatchers.resetMain() } + } } } From 84f3d012e55c629f9b33fd4d7a3370ac0a5d85d5 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 16:03:50 +0000 Subject: [PATCH 12/33] docs: point the Sprint 1B verification at the latest green head --- docs/IMPLEMENTATION_REPORT.md | 6 +++--- docs/VISION_TESTING.md | 9 +++++---- docs/sprints/SPRINT_1B_HERO_RECOGNITION.md | 7 +++---- tasks/todo.md | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index 338fd65..8c504de 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -393,9 +393,9 @@ Branch `sprint-1b-hero-recognition`, PR #3 (draft, **not merged**). See This sprint does not add recommendations, counters, synergy scoring, a floating overlay, or automatic draft-phase detection, and does not merge PR #3 without review. -Sprint 1B green runs: final branch head `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` — -https://github.com/xeriomy/DraftLens/actions/runs/43732111174 (646 executions, 0 failures); +Sprint 1B green runs: latest branch head `e58611c25d7d9a40592aa3d81ad3347869771cc3` — +https://github.com/xeriomy/DraftLens/actions/runs/43732908605 (646 executions, 0 failures); code head `7d0c9032a52db8e0f962beea72ceae7120822c88` — -https://github.com/xeriomy/DraftLens/actions/runs/43731484922 (646 executions, 0 failures). +https://github.com/xeriomy/DraftLens/actions/runs/43731484922 (same totals). Instrumentation tests compiled, not executed; strict lint; assembleDebug; artifact `draftlens-debug`, digest `sha256:3c977d55d77c5452425ccfd5ea950ef55722d77319149c076d8bd5d115b85cc9`. diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index c2ab9dc..2398524 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -108,10 +108,11 @@ untouched, and that only clearing the document recovers from it. | Confirmation store | `datastore/HeroRecognitionStoreTest.kt` | confirm/load/replace/clear, corrupt and future documents read as empty | | ViewModel | `feature/visionlab/VisionLabViewModelTest.kt` | identify without slot → message, identify without catalog → FAILED honestly, synced catalog → candidates, confirm → persisted, clear → removed, catalog name search | -Sprint 1B green runs: final branch head `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` — - (646 executions, debug + -release, **0 failures, 0 errors, 0 skipped**); code head `7d0c9032a52db8e0f962beea72ceae7120822c88` — - (same totals). Instrumentation +Sprint 1B green runs: latest branch head `e58611c25d7d9a40592aa3d81ad3347869771cc3` (class-scoped +test Main, test-only) — + (646 executions, debug + +release, **0 failures, 0 errors, 0 skipped**); prior green heads `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` +(run 43732111174) and `7d0c9032a52db8e0f962beea72ceae7120822c88` (run 43731484922), same totals. Instrumentation tests compiled, not executed. The 1B transport is exercised only through fake responses; no test performs a real network call. The number of catalog entries loaded from the live API is measured at device sync time, not in CI. diff --git a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md index ede8fb8..01b2f10 100644 --- a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md +++ b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md @@ -150,12 +150,11 @@ opens the property menu automatically. Slot creation keeps the maintainer's 1A p ## Acceptance status -- **CI green**: final branch head `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` (run 43732111174, - test-only increment on the code head) and code head `7d0c9032a52db8e0f962beea72ceae7120822c88` - (run 43731484922); **323 unit tests (646 executions, debug + release), 0 failures, +- **CI green**: latest branch head `e58611c25d7d9a40592aa3d81ad3347869771cc3` (run 43732908605, + test-only); **323 unit tests (646 executions, debug + release), 0 failures, 0 errors, 0 skipped**; `assembleAndroidTest` **compiled, not executed**; strict lint passed; `assembleDebug` passed; artifact `draftlens-debug`, digest - `sha256:3c977d55d77c5452425ccfd5ea950ef55722d77319149c076d8bd5d115b85cc9`. + `sha256:ad581e4042bc81fb7519c042c35b68d3b152f0c30cbed5c37816a4c2534de788`. - Physical-device smoke: **pending**. - Catalog sync on a device: **pending** (verified only through fake transport in unit tests). - On-device accuracy: **not measured**. diff --git a/tasks/todo.md b/tasks/todo.md index ecc6f9f..cfa8156 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -97,5 +97,5 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** - [x] Recognition v0: dHash + color descriptor, explicit no-match, manual confirm/correct - [x] Vision Lab card: sync, Identify, candidates, confirmed state, manual correction - [x] Unit coverage for catalog, cache, extraction, matching, VM flows -- [x] CI green: 323 tests (646 executions), strict lint, assembleDebug, assembleAndroidTest compiled (run 43731484922, code head 7d0c903) +- [x] CI green: 323 tests (646 executions), strict lint, assembleDebug, assembleAndroidTest compiled (run 43732908605, head e58611c; code head 7d0c903 run 43731484922) - [ ] Physical-device smoke test and on-device accuracy evaluation (pending) From 2e115b03c8f72cfc12d5809e85626c921d24122e Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:09:54 +0000 Subject: [PATCH 13/33] feat: freeze game frames from the capture notification + review fixes (parts A-H) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Part A: capture-side freeze command (ACTION_FREEZE / ACTION_RESUME_LIVE) routed to the running service; FramePipeline freezes publication while images keep being acquired and closed; the frozen frame is an independently owned copy (FrameSnapshots); holder survives projection stops; notification re-posts LIVE/FROZEN with Freeze/Resume actions; Vision Lab resumes from the UI and from the notification; full-device capture guidance and notification-permission handling added. - Part B: pixel extractor rotation rewritten to the FrameTransform sourceToOriented convention; tests rewritten with asymmetric fixtures, edge crops and hand-computed expectations. - Part C: Android bitmap extractor validates payload dimensions, rotates pixel-exactly, keeps source bitmap valid, clear ownership and try/catch release of temporaries; instrumentation tests for all rotations, edges, mismatches and bitmap snapshot ownership. - Part D: VisualInformationAnalyzer (pure Kotlin) rejects blank, near-uniform and detail-free crops before matching; new distinct InsufficientVisualInformation outcome and INSUFFICIENT_INFO phase. - Part E: manual correction reachable from CANDIDATES, NO_MATCH, FAILED, INSUFFICIENT_INFO and CONFIRMED; Change… opens correction instead of re-running Identify; failed clears no longer report success. - Part F: asset cache freshness via per-asset metadata sidecars binding URL/SHA-256/size/content-type; full serve-time validation incl. structural decode check (Android: real decode); changed URLs force re-download; corrupt cache re-downloads; failed refresh keeps the previous valid asset; HTTP redirects refused everywhere (HeroRedirectPolicy). - Part G: writeAtomically is temp-sibling + sync + atomic move only; no direct-write fallback; failure preserves the previous target and cleans temp files; stale *.tmp sweep. - Part H: reference Bitmap dimensions read before recycling; pixel reads complete before release; exceptions still recycle. --- ...AndroidBitmapExtractionInstrumentedTest.kt | 233 ++++++++++++++++++ ...ReferenceImageValidatorInstrumentedTest.kt | 50 ++++ .../main/kotlin/com/draftlens/DraftLensApp.kt | 4 +- .../com/draftlens/capture/CaptureFreeze.kt | 68 +++++ .../capture/CaptureSessionService.kt | 93 ++++++- .../draftlens/capture/CaptureStateHolder.kt | 26 ++ .../capture/CaptureVisionFrameSource.kt | 23 +- .../com/draftlens/capture/FramePipeline.kt | 98 ++++++-- .../feature/home/DebugCaptureScreen.kt | 92 +++++++ .../feature/visionlab/HeroRecognitionCard.kt | 17 +- .../feature/visionlab/VisionLabScreen.kt | 10 +- .../feature/visionlab/VisionLabUiState.kt | 4 +- .../feature/visionlab/VisionLabViewModel.kt | 29 ++- .../com/draftlens/models/FrameSnapshots.kt | 68 +++++ .../heroes/AndroidBitmapSlotImageExtractor.kt | 84 +++++-- .../heroes/AndroidHeroCatalogTransport.kt | 103 ++++++-- .../heroes/AndroidReferenceImageValidator.kt | 43 ++++ .../heroes/FileHeroCatalogRepository.kt | 188 ++++++++++++-- .../vision/heroes/HeroRecognition.kt | 17 +- .../vision/heroes/HeroReferenceImages.kt | 18 +- .../vision/heroes/ReferenceImageValidator.kt | 123 +++++++++ .../vision/heroes/SlotImageExtractor.kt | 32 ++- .../vision/heroes/VisualInformation.kt | 165 +++++++++++++ .../vision/model/VisionFrameSource.kt | 20 +- app/src/main/res/values/strings.xml | 6 + .../draftlens/capture/CaptureFreezeTest.kt | 46 ++++ .../capture/CaptureStateHolderTest.kt | 49 ++++ .../capture/CaptureVisionFrameSourceTest.kt | 23 ++ .../visionlab/VisionLabViewModelTest.kt | 207 +++++++++++++++- .../heroes/FileHeroCatalogRepositoryTest.kt | 201 ++++++++++++++- .../heroes/HeroRedirectPolicyTest.kt | 46 ++++ .../heroes/SlotImageExtractorTest.kt | 113 +++++++-- .../heroes/StructuralImageValidatorTest.kt | 59 +++++ .../draftlens/heroes/VisualInformationTest.kt | 195 +++++++++++++++ .../draftlens/heroes/WriteAtomicallyTest.kt | 106 ++++++++ .../draftlens/models/FrameSnapshotsTest.kt | 70 ++++++ .../kotlin/com/draftlens/vision/TestFrames.kt | 26 ++ 37 files changed, 2602 insertions(+), 153 deletions(-) create mode 100644 app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt create mode 100644 app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt create mode 100644 app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt create mode 100644 app/src/main/kotlin/com/draftlens/models/FrameSnapshots.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/AndroidReferenceImageValidator.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/ReferenceImageValidator.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/heroes/VisualInformation.kt create mode 100644 app/src/test/kotlin/com/draftlens/capture/CaptureFreezeTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/HeroRedirectPolicyTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/StructuralImageValidatorTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/VisualInformationTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/models/FrameSnapshotsTest.kt diff --git a/app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt b/app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt new file mode 100644 index 0000000..7222c02 --- /dev/null +++ b/app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt @@ -0,0 +1,233 @@ +package com.draftlens + +import android.graphics.Bitmap +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.draftlens.models.FrameSnapshots +import com.draftlens.vision.annotation.DraftSlot +import com.draftlens.vision.annotation.SlotTeam +import com.draftlens.vision.annotation.SlotType +import com.draftlens.vision.geometry.NormalizedRect +import com.draftlens.vision.heroes.AndroidBitmapSlotImageExtractor +import com.draftlens.vision.model.FrameId +import com.draftlens.vision.model.FrameRotation +import com.draftlens.vision.model.OriginalFrame +import com.draftlens.vision.model.SlotImage +import com.draftlens.vision.model.android.BitmapFramePayload +import org.junit.After +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNotSame +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Device-side verification of the Android extractor: exact pixels, ownership and failure + * paths. Run on an emulator or device; without one this class is compiled but not executed. + */ +@RunWith(AndroidJUnit4::class) +class AndroidBitmapExtractionInstrumentedTest { + + private val ownedBitmaps = mutableListOf() + + @After fun tearDown() { + ownedBitmaps.forEach { if (!it.isRecycled) it.recycle() } + ownedBitmaps.clear() + } + + private fun track(bitmap: Bitmap): Bitmap = bitmap.also { ownedBitmaps.add(it) } + + private fun slot(rect: NormalizedRect, id: String = "slot-1") = DraftSlot( + id = id, + type = SlotType.BAN, + team = SlotTeam.ALLY, + index = 1, + rect = rect, + ) + + private fun frame(width: Int, height: Int, rotation: FrameRotation, pixels: IntArray): OriginalFrame { + val bitmap = track(Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)) + bitmap.setPixels(pixels, 0, width, 0, 0, width, height) + return OriginalFrame( + id = FrameId("ft-$width-$height-$rotation"), + payload = BitmapFramePayload(bitmap), + sourceWidth = width, + sourceHeight = height, + displayRotation = FrameRotation.ROTATION_0, + rotation = rotation, + capturedAtElapsedRealtimeMs = 1, + wallClockTimestampMs = 1, + captureLatencyMs = 1, + ) + } + + private fun pixelsOf(image: SlotImage): Pair { + val bitmap = (image.payload as BitmapFramePayload).bitmap + assertFalse(bitmap.isRecycled) + val pixels = IntArray(bitmap.width * bitmap.height) + bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) + return pixels to bitmap.width + } + + // Asymmetric 3x2 source; values chosen so every rotation is distinguishable. + private val asymmetric = intArrayOf(10, 11, 12, 20, 21, 22) + + // 5x3 source used for edge crops. + private val edgeFrame = intArrayOf( + 0, 1, 2, 3, 4, + 10, 11, 12, 13, 14, + 20, 21, 22, 23, 24, + ) + + @Test fun `all four rotations produce the exact expected pixels for a non-square crop`() { + val rect = NormalizedRect(0f, 0f, 1f, 1f) + val expected = mapOf( + FrameRotation.ROTATION_0 to Triple(3, 2, intArrayOf(10, 11, 12, 20, 21, 22)), + FrameRotation.ROTATION_90 to Triple(2, 3, intArrayOf(20, 10, 21, 11, 22, 12)), + FrameRotation.ROTATION_180 to Triple(3, 2, intArrayOf(22, 21, 20, 12, 11, 10)), + FrameRotation.ROTATION_270 to Triple(2, 3, intArrayOf(12, 22, 11, 21, 10, 20)), + ) + for (rotation in FrameRotation.entries) { + val source = frame(3, 2, rotation, asymmetric.copyOf()) + val result = AndroidBitmapSlotImageExtractor.extract(source, slot(rect)) + + assertTrue("rotation $rotation must succeed", result.isSuccess) + val image = result.getOrThrow() + val (pixels, width) = pixelsOf(image) + val (expectedWidth, expectedHeight, expectedPixels) = expected.getValue(rotation) + assertEquals("width[$rotation]", expectedWidth, width) + assertEquals("height[$rotation]", expectedHeight, (image.payload as BitmapFramePayload).bitmap.height) + assertArrayEquals("pixels[$rotation]", expectedPixels, pixels) + } + } + + @Test fun `crops touching the top and bottom edges extract exactly the edge columns`() { + val rect = NormalizedRect(0.2f, 0f, 0.6f, 1f) // left=1px, right=3px of 5 + val expected = mapOf( + FrameRotation.ROTATION_0 to Triple(2, 3, intArrayOf(1, 2, 11, 12, 21, 22)), + FrameRotation.ROTATION_90 to Triple(3, 2, intArrayOf(21, 11, 1, 22, 12, 2)), + FrameRotation.ROTATION_180 to Triple(2, 3, intArrayOf(22, 21, 12, 11, 2, 1)), + FrameRotation.ROTATION_270 to Triple(3, 2, intArrayOf(2, 12, 22, 1, 11, 21)), + ) + for (rotation in FrameRotation.entries) { + val source = frame(5, 3, rotation, edgeFrame.copyOf()) + val result = AndroidBitmapSlotImageExtractor.extract(source, slot(rect)) + + assertTrue("rotation $rotation must succeed", result.isSuccess) + val (pixels, width) = pixelsOf(result.getOrThrow()) + val (expectedWidth, expectedHeight, expectedPixels) = expected.getValue(rotation) + assertEquals("width[$rotation]", expectedWidth, width) + assertEquals("height[$rotation]", expectedHeight, (result.getOrThrow().payload as BitmapFramePayload).bitmap.height) + assertArrayEquals("pixels[$rotation]", expectedPixels, pixels) + } + } + + @Test fun `a crop on the bottom and right edges extracts exactly those pixels`() { + val rect = NormalizedRect(0.6f, 1f / 3f, 1f, 1f) // left=3px, top=1px of 5x3 + val expected = mapOf( + FrameRotation.ROTATION_0 to Triple(2, 2, intArrayOf(13, 14, 23, 24)), + FrameRotation.ROTATION_90 to Triple(2, 2, intArrayOf(23, 13, 24, 14)), + FrameRotation.ROTATION_180 to Triple(2, 2, intArrayOf(24, 23, 14, 13)), + FrameRotation.ROTATION_270 to Triple(2, 2, intArrayOf(14, 24, 13, 23)), + ) + for (rotation in FrameRotation.entries) { + val source = frame(5, 3, rotation, edgeFrame.copyOf()) + val result = AndroidBitmapSlotImageExtractor.extract(source, slot(rect)) + + assertTrue("rotation $rotation must succeed", result.isSuccess) + val (pixels, width) = pixelsOf(result.getOrThrow()) + val (expectedWidth, expectedHeight, expectedPixels) = expected.getValue(rotation) + assertEquals("width[$rotation]", expectedWidth, width) + assertEquals("height[$rotation]", expectedHeight, (result.getOrThrow().payload as BitmapFramePayload).bitmap.height) + assertArrayEquals("pixels[$rotation]", expectedPixels, pixels) + } + } + + @Test fun `a payload whose bitmap dimensions do not match the source is refused`() { + val bitmap = track(Bitmap.createBitmap(2, 2, Bitmap.Config.ARGB_8888)) + val frame = OriginalFrame( + id = FrameId("mismatch"), + payload = BitmapFramePayload(bitmap), + sourceWidth = 4, + sourceHeight = 4, + displayRotation = FrameRotation.ROTATION_0, + rotation = FrameRotation.ROTATION_0, + capturedAtElapsedRealtimeMs = 1, + wallClockTimestampMs = 1, + captureLatencyMs = 1, + ) + + val result = AndroidBitmapSlotImageExtractor.extract(frame, slot(NormalizedRect(0.1f, 0.1f, 0.9f, 0.9f))) + + assertTrue(result.isFailure) + assertTrue(result.exceptionOrNull()!!.message!!.contains("does not match")) + } + + @Test fun `the source bitmap stays valid and unchanged after extraction`() { + val source = frame(4, 3, FrameRotation.ROTATION_90, edgeFrame.copyOf()) + val bitmap = (source.payload as BitmapFramePayload).bitmap + val before = IntArray(edgeFrame.size).also { bitmap.getPixels(it, 0, 4, 0, 0, 4, 3) } + + val result = AndroidBitmapSlotImageExtractor.extract(source, slot(NormalizedRect(0.2f, 0f, 0.6f, 1f))) + + assertTrue(result.isSuccess) + assertFalse("the source bitmap must not be recycled", bitmap.isRecycled) + val after = IntArray(edgeFrame.size).also { bitmap.getPixels(it, 0, 4, 0, 0, 4, 3) } + assertArrayEquals("the source pixels must be unchanged", before, after) + } + + @Test fun `the minimum valid crop produces exactly one pixel`() { + val source = frame(128, 128, FrameRotation.ROTATION_0, IntArray(128 * 128) { it }) + val result = AndroidBitmapSlotImageExtractor.extract( + source, + slot(NormalizedRect(0.125f, 0.125f, 0.1328125f, 0.1328125f)), + ) + + assertTrue(result.isSuccess) + val (pixels, width) = pixelsOf(result.getOrThrow()) + assertEquals(1, width) + assertEquals(1, (result.getOrThrow().payload as BitmapFramePayload).bitmap.height) + assertEquals(16 * 128 + 16, pixels.single()) + } + + @Test fun `a rectangle smaller than one pixel is refused`() { + val source = frame(10, 10, FrameRotation.ROTATION_0, IntArray(100) { 7 }) + + val result = AndroidBitmapSlotImageExtractor.extract(source, slot(NormalizedRect(0f, 0f, 0.05f, 0.05f))) + + assertTrue(result.isFailure) + } + + @Test fun `a recycled source bitmap is refused without a crash`() { + val source = frame(4, 3, FrameRotation.ROTATION_0, edgeFrame.copyOf()) + val bitmap = (source.payload as BitmapFramePayload).bitmap + bitmap.recycle() + + val result = AndroidBitmapSlotImageExtractor.extract(source, slot(NormalizedRect(0f, 0f, 1f, 1f))) + + assertTrue(result.isFailure) + } + + @Test fun `a bitmap frame snapshot shares no pixels with the pipeline copy`() { + val original = frame(4, 2, FrameRotation.ROTATION_0, asymmetric.copyOf()) + val snapshot = FrameSnapshots.copyOf(original) + + assertTrue(snapshot != null) + val sourceBitmap = (original.payload as BitmapFramePayload).bitmap + val snapshotBitmap = (snapshot!!.payload as BitmapFramePayload).bitmap + assertNotSame(sourceBitmap, snapshotBitmap) + ownedBitmaps.add(snapshotBitmap) + assertFalse("the source must not be recycled by snapshotting", sourceBitmap.isRecycled) + assertEquals( + snapshotBitmap.width, + sourceBitmap.width, + ) + sourceBitmap.setPixels(IntArray(8) { -1 }, 0, 4, 0, 0, 4, 2) + val snapshotAfter = IntArray(8).also { snapshotBitmap.getPixels(it, 0, 4, 0, 0, 4, 2) } + assertTrue( + "mutating the live bitmap must not leak into the frozen snapshot", + snapshotAfter.contains(10), + ) + } +} diff --git a/app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt b/app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt new file mode 100644 index 0000000..35f026c --- /dev/null +++ b/app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt @@ -0,0 +1,50 @@ +package com.draftlens + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.draftlens.vision.heroes.AndroidReferenceImageValidator +import com.draftlens.vision.heroes.StructuralImageValidator +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +/** Device-side verification that cached assets are actually decodable, not just well-framed. */ +@RunWith(AndroidJUnit4::class) +class ReferenceImageValidatorInstrumentedTest { + + private val realPng = byteArrayOf( + 0x89.toByte(), 0x50.toByte(), 0x4E.toByte(), 0x47.toByte(), 0x0D.toByte(), 0x0A.toByte(), 0x1A.toByte(), 0x0A.toByte(), + 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x0D.toByte(), 0x49.toByte(), 0x48.toByte(), 0x44.toByte(), 0x52.toByte(), + 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x01.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x01.toByte(), + 0x08.toByte(), 0x02.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x90.toByte(), 0x77.toByte(), 0x53.toByte(), + 0xDE.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x0C.toByte(), 0x49.toByte(), 0x44.toByte(), 0x41.toByte(), + 0x54.toByte(), 0x78.toByte(), 0x9C.toByte(), 0x63.toByte(), 0xF8.toByte(), 0xCF.toByte(), 0xC0.toByte(), 0x00.toByte(), + 0x00.toByte(), 0x03.toByte(), 0x01.toByte(), 0x01.toByte(), 0x00.toByte(), 0xC9.toByte(), 0xFE.toByte(), 0x92.toByte(), + 0xEF.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x49.toByte(), 0x45.toByte(), 0x4E.toByte(), + 0x44.toByte(), 0xAE.toByte(), 0x42.toByte(), 0x60.toByte(), 0x82.toByte(), + ) + + @Test fun `a real png decodes on a device`() { + assertTrue(AndroidReferenceImageValidator.isDecodable(realPng)) + assertTrue(StructuralImageValidator.isDecodable(realPng)) + } + + @Test fun `a truncated png does not decode`() { + val truncated = realPng.copyOf(30) + assertFalse(AndroidReferenceImageValidator.isDecodable(truncated)) + assertFalse(StructuralImageValidator.isDecodable(truncated)) + } + + @Test fun `an html error page is not an image`() { + val html = "502 Bad Gateway".toByteArray() + assertFalse(AndroidReferenceImageValidator.isDecodable(html)) + assertFalse(StructuralImageValidator.isDecodable(html)) + } + + @Test fun `a jpeg that passes the structure check fails the device decode`() { + // Structure check is a first gate (SOI + EOI present); the device adds a real decode. + val fakeJpeg = byteArrayOf(0xFF.toByte(), 0xD8.toByte(), 0x01, 0x02, 0x03, 0xFF.toByte(), 0xD9.toByte()) + assertTrue(StructuralImageValidator.isDecodable(fakeJpeg)) + assertFalse(AndroidReferenceImageValidator.isDecodable(fakeJpeg)) + } +} diff --git a/app/src/main/kotlin/com/draftlens/DraftLensApp.kt b/app/src/main/kotlin/com/draftlens/DraftLensApp.kt index 4247ceb..cd7016e 100644 --- a/app/src/main/kotlin/com/draftlens/DraftLensApp.kt +++ b/app/src/main/kotlin/com/draftlens/DraftLensApp.kt @@ -7,6 +7,7 @@ import com.draftlens.datastore.CalibrationProfileDataStore import com.draftlens.datastore.HeroRecognitionStore import com.draftlens.vision.calibration.CalibrationRepository import com.draftlens.vision.heroes.AndroidHeroCatalogTransport +import com.draftlens.vision.heroes.AndroidReferenceImageValidator import com.draftlens.vision.heroes.FileHeroCatalogRepository import com.draftlens.vision.heroes.HeroCatalogRepository import com.draftlens.vision.model.VisionFrameSource @@ -27,9 +28,10 @@ class DraftLensApp : Application() { /** * The hero catalog: provider-independent model, file snapshot in filesDir and a * private image cache in cacheDir. Only public static catalog data is downloaded. + * Android assets are verified by actually decoding them. */ val heroCatalog: HeroCatalogRepository by lazy { - FileHeroCatalogRepository(filesDir, cacheDir, AndroidHeroCatalogTransport()) + FileHeroCatalogRepository(filesDir, cacheDir, AndroidHeroCatalogTransport(), imageValidator = AndroidReferenceImageValidator) } /** Per-slot hero confirmations. Metadata only, never pixels. */ diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt new file mode 100644 index 0000000..73a1553 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt @@ -0,0 +1,68 @@ +package com.draftlens.capture + +import com.draftlens.models.CapturedFrame + +/** + * Process-wide bridge for the freeze workflow. The capture service installs the live + * pipeline here; the Vision Lab reaches the same commands through [CaptureVisionFrameSource]. + * Both triggers run the exact same code path, so a notification tap and the in-app button + * behave identically. + * + * Never creates a capture session: with no active pipeline the commands are no-ops. + */ +object CaptureFreeze { + + @Volatile + private var pipeline: FramePipeline? = null + + /** Called by the capture service once its pipeline exists. */ + fun attachPipeline(instance: FramePipeline) { + pipeline = instance + } + + /** Called by the capture service during teardown; a stale reference is dropped. */ + fun detachPipeline(instance: FramePipeline) { + synchronized(this) { + if (pipeline === instance) pipeline = null + } + } + + /** True when an active pipeline is installed (i.e. capture is running). */ + fun isCapturing(): Boolean = pipeline != null + + fun freezeCurrentFrame(): FreezeOutcome { + val current = CaptureStateHolder.frozenFrame.value + if (current != null) return FreezeOutcome.AlreadyFrozen + val active = pipeline ?: return FreezeOutcome.NoPipeline + return when (val result = active.freezeAndSnapshot()) { + is FramePipeline.FreezeResult.Frozen -> { + CaptureStateHolder.setFrozenFrame(result.frame) + FreezeOutcome.Frozen(result.frame) + } + + FramePipeline.FreezeResult.AlreadyFrozen -> FreezeOutcome.AlreadyFrozen + FramePipeline.FreezeResult.NoFrame -> FreezeOutcome.NoFrame + } + } + + /** Pins the in-memory frame; resume reveals live frames again. Never starts a session. */ + fun resumeLive() { + pipeline?.resumeCapture() + CaptureStateHolder.clearFrozenFrame() + } +} + +/** What a freeze command achieved. */ +sealed interface FreezeOutcome { + /** A newly pinned frame, independently owned by the holder. */ + data class Frozen(val frame: CapturedFrame) : FreezeOutcome + + /** A frame was already pinned; the existing one is kept. */ + data object AlreadyFrozen : FreezeOutcome + + /** Capture is running but no frame has been captured yet. */ + data object NoFrame : FreezeOutcome + + /** No active capture (e.g. a stale notification action after a stop). */ + data object NoPipeline : FreezeOutcome +} diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt index 0539f61..2367c1d 100644 --- a/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt @@ -56,6 +56,7 @@ class CaptureSessionService : Service() { private var displayListener: DisplayManager.DisplayListener? = null private var captureJob: Job? = null private var frameJob: Job? = null + private var frozenJob: Job? = null private var foregroundPromoted = false private val projectionCallback = object : MediaProjection.Callback() { @@ -129,6 +130,32 @@ class CaptureSessionService : Service() { CaptureStateHolder.setPaused(false) START_NOT_STICKY } + ACTION_FREEZE -> { + // Never re-promotes, never restarts MediaProjection: no-op without an active pipeline. + when (val outcome = CaptureFreeze.freezeCurrentFrame()) { + is FreezeOutcome.Frozen -> + Log.i(TAG, "Freeze command honoured (${outcome.frame.width}x${outcome.frame.height})") + FreezeOutcome.AlreadyFrozen -> + Log.i(TAG, "Freeze command: a frame is already frozen; keeping it") + FreezeOutcome.NoFrame -> + Log.i(TAG, "Freeze command ignored: no frame captured yet") + FreezeOutcome.NoPipeline -> { + Log.i(TAG, "Freeze command ignored: no active capture session") + stopSelf() // do not linger for a stale command + return START_NOT_STICKY + } + } + START_NOT_STICKY + } + ACTION_RESUME_LIVE -> { + CaptureFreeze.resumeLive() + if (CaptureStateHolder.state.value.status != CaptureStatus.CAPTURING && + CaptureStateHolder.state.value.status != CaptureStatus.PAUSED + ) { + stopSelf() // stale action after capture stopped; nothing to resume into + } + START_NOT_STICKY + } else -> START_NOT_STICKY } } @@ -166,6 +193,7 @@ class CaptureSessionService : Service() { val displayRotation = currentDisplayRotation() pipeline.start(captureWidth, captureHeight, dpi, displayRotation) + CaptureFreeze.attachPipeline(pipeline) createVirtualDisplay(proj, captureWidth, captureHeight, dpi) registerDisplayListener() CaptureStateHolder.beginCapture(captureWidth, captureHeight, dpi) @@ -260,6 +288,12 @@ class CaptureSessionService : Service() { } } } + frozenJob?.cancel() + frozenJob = scope.launch { + CaptureStateHolder.frozenFrame.collectLatest { frozen -> + updateNotificationState(frozen != null) + } + } } private fun createVirtualDisplay( @@ -386,6 +420,8 @@ class CaptureSessionService : Service() { captureJob = null frameJob?.cancel() frameJob = null + frozenJob?.cancel() + frozenJob = null try { virtualDisplay?.release() } catch (e: Exception) { Log.w(TAG, "VirtualDisplay release failed", e) } virtualDisplay = null unregisterDisplayListener() @@ -396,6 +432,9 @@ class CaptureSessionService : Service() { Log.w(TAG, "MediaProjection stop failed", e) } projection = null + if (::pipeline.isInitialized) { + CaptureFreeze.detachPipeline(pipeline) + } try { pipeline.stop() } catch (e: Exception) { Log.w(TAG, "Pipeline stop failed", e) } if (releaseForeground) { try { @@ -442,7 +481,31 @@ class CaptureSessionService : Service() { .createNotificationChannel(channel) } - private fun buildNotification(): Notification { + /** Re-posts the ongoing notification with the current LIVE/FROZEN state and its action. */ + private fun updateNotificationState(frozen: Boolean) { + if (!foregroundPromoted) { + Log.d(TAG, "Skipping notification update: service is not promoted") + return + } + if (!areNotificationsEnabled()) { + Log.w(TAG, "Notifications are disabled; the capture actions are not reachable from the shade") + return + } + try { + (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager) + .notify(NOTIFICATION_ID, buildNotification(frozen)) + } catch (e: Exception) { + Log.w(TAG, "Failed to update the capture notification", e) + } + } + + /** Android 13+ runtime-granted; API 24+ areNotificationsEnabled when present, safe otherwise. */ + private fun areNotificationsEnabled(): Boolean = runCatching { + (getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager) + .areNotificationsEnabled() + }.getOrDefault(true) + + private fun buildNotification(frozen: Boolean = CaptureStateHolder.frozenFrame.value != null): Notification { val tapIntent = PendingIntent.getActivity( this, 0, Intent(this, MainActivity::class.java), @@ -453,13 +516,33 @@ class CaptureSessionService : Service() { Intent(this, CaptureSessionService::class.java).setAction(ACTION_STOP), PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE ) - return NotificationCompat.Builder(this, CHANNEL_ID) + val freezeIntent = PendingIntent.getService( + this, 2, + Intent(this, CaptureSessionService::class.java).setAction(ACTION_FREEZE), + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE + ) + val resumeCaptureIntent = PendingIntent.getService( + this, 3, + Intent(this, CaptureSessionService::class.java).setAction(ACTION_RESUME_LIVE), + PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE + ) + val subText = if (frozen) getString(R.string.capture_frozen_subtext) else getString(R.string.capture_live_subtext) + val text = if (frozen) getString(R.string.capture_notification_frozen_text) else getString(R.string.capture_notification_text) + val builder = NotificationCompat.Builder(this, CHANNEL_ID) .setContentTitle(getString(R.string.capture_notification_title)) - .setContentText(getString(R.string.capture_notification_text)) + .setContentText(text) + .setSubText(subText) .setSmallIcon(android.R.drawable.ic_menu_camera) .setContentIntent(tapIntent) - .addAction(android.R.drawable.ic_menu_close_clear_cancel, getString(R.string.capture_stop_action), stopIntent) .setOngoing(true) + .setShowWhen(false) + if (frozen) { + builder.addAction(android.R.drawable.ic_media_play, getString(R.string.capture_resume_capture_action), resumeCaptureIntent) + } else { + builder.addAction(android.R.drawable.ic_menu_camera, getString(R.string.capture_freeze_action), freezeIntent) + } + return builder + .addAction(android.R.drawable.ic_menu_close_clear_cancel, getString(R.string.capture_stop_action), stopIntent) .build() } @@ -468,6 +551,8 @@ class CaptureSessionService : Service() { const val ACTION_STOP = "com.draftlens.action.STOP_CAPTURE" const val ACTION_PAUSE = "com.draftlens.action.PAUSE_CAPTURE" const val ACTION_RESUME = "com.draftlens.action.RESUME_CAPTURE" + const val ACTION_FREEZE = "com.draftlens.action.FREEZE_CAPTURE" + const val ACTION_RESUME_LIVE = "com.draftlens.action.RESUME_LIVE_CAPTURE" const val EXTRA_RESULT_CODE = "extra_result_code" const val EXTRA_DATA = "extra_data" const val EXTRA_INTERVAL_MS = "extra_interval_ms" diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureStateHolder.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureStateHolder.kt index 477da34..de87da5 100644 --- a/app/src/main/kotlin/com/draftlens/capture/CaptureStateHolder.kt +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureStateHolder.kt @@ -25,6 +25,15 @@ object CaptureStateHolder { private val _latestFrame = MutableStateFlow(null) val latestFrame = _latestFrame.asStateFlow() + /** + * The frame pinned by a freeze command, owned independently of the live pipeline. + * At most one instance is kept. Survives MediaProjection stopping (release of the + * VirtualDisplay/ImageReader does not touch it) but not process death; cleared only + * by resume/clear or by starting a new capture session. + */ + private val _frozenFrame = MutableStateFlow(null) + val frozenFrame = _frozenFrame.asStateFlow() + fun setStatus(target: CaptureStatus, error: String? = null) { _state.update { current -> if (CaptureTransitions.canTransition(current.status, target)) { @@ -37,6 +46,8 @@ object CaptureStateHolder { fun beginCapture(width: Int, height: Int, densityDpi: Int) { clearLatestFrame() + // A new session starts with live frames; the previous session's pinned frame is dropped. + _frozenFrame.value = null resetSessionMetrics() _state.update { current -> val base = if (CaptureTransitions.canTransition(current.status, CaptureStatus.CAPTURING)) { @@ -86,6 +97,10 @@ object CaptureStateHolder { /** * Shared terminal cleanup for all capture-ending paths. * + * The pinned frame is deliberately preserved: a projection stopped by Android (or the + * user) must leave an independently owned copy available for editing while the process + * lives. Only a new capture session, an explicit resume, or [reset] clears it. + * * If [error] is supplied and the current state is not already ERROR, transition to ERROR * with that message. An existing ERROR and its message are preserved unless [forceIdle] * is requested by an explicit user Stop action. Normal termination ([error] = null and @@ -106,6 +121,16 @@ object CaptureStateHolder { _latestFrame.value = null } + /** Pins [frame] as the frozen frame, replacing any previous one. */ + fun setFrozenFrame(frame: CapturedFrame) { + _frozenFrame.value = frame + } + + /** Releases the pinned frame; live publication may resume. */ + fun clearFrozenFrame() { + _frozenFrame.value = null + } + /** * Clears session counters, timings, dimensions and permission metadata. * Deliberately preserves status and lastError; use [reset] for a full idle reset. @@ -129,6 +154,7 @@ object CaptureStateHolder { /** Full state reset to a fresh idle state. */ fun reset() { clearLatestFrame() + _frozenFrame.value = null _state.value = CaptureSessionState() } } diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureVisionFrameSource.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureVisionFrameSource.kt index ed84297..6f70465 100644 --- a/app/src/main/kotlin/com/draftlens/capture/CaptureVisionFrameSource.kt +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureVisionFrameSource.kt @@ -18,11 +18,14 @@ import kotlinx.coroutines.launch * Flows are injected so tests can drive the adapter without the capture service. * The current values are mapped eagerly in the constructor so a caller that reads * [frames] or [state] right after construction sees the source's present value. + * + * [freeze]/[resume] delegate to [CaptureFreeze], exactly like the notification action. */ class CaptureVisionFrameSource( captureFrames: StateFlow, captureState: StateFlow, scope: CoroutineScope, + captureFrozen: StateFlow = MutableStateFlow(null), ) : VisionFrameSource { private val _frames = MutableStateFlow(captureFrames.value?.toVisionFrame()) @@ -31,6 +34,9 @@ class CaptureVisionFrameSource( private val _state = MutableStateFlow(captureState.value.status.toVisionState()) override val state: StateFlow = _state.asStateFlow() + private val _frozenFrame = MutableStateFlow(captureFrozen.value?.toVisionFrame()) + override val frozenFrame: StateFlow = _frozenFrame.asStateFlow() + init { scope.launch { captureFrames.collect { _frames.value = it?.toVisionFrame() } @@ -38,12 +44,27 @@ class CaptureVisionFrameSource( scope.launch { captureState.collect { _state.value = it.status.toVisionState() } } + scope.launch { + captureFrozen.collect { _frozenFrame.value = it?.toVisionFrame() } + } } + override fun freeze(): Boolean = when (val outcome = CaptureFreeze.freezeCurrentFrame()) { + is FreezeOutcome.Frozen, FreezeOutcome.AlreadyFrozen -> true + FreezeOutcome.NoFrame, FreezeOutcome.NoPipeline -> false + } + + override fun resume() = CaptureFreeze.resumeLive() + companion object { /** Production adapter over the process-local capture holder. */ fun fromHolder(scope: CoroutineScope): CaptureVisionFrameSource = - CaptureVisionFrameSource(CaptureStateHolder.latestFrame, CaptureStateHolder.state, scope) + CaptureVisionFrameSource( + CaptureStateHolder.latestFrame, + CaptureStateHolder.state, + scope, + CaptureStateHolder.frozenFrame, + ) } } diff --git a/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt b/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt index a9c4a6c..ce33dda 100644 --- a/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt +++ b/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt @@ -7,6 +7,7 @@ import android.os.HandlerThread import android.util.Log import android.view.Surface import com.draftlens.models.CapturedFrame +import com.draftlens.models.FrameSnapshots import com.draftlens.vision.model.FrameId import com.draftlens.vision.model.FrameOrientation import com.draftlens.vision.model.FrameRotation @@ -31,6 +32,29 @@ class FramePipeline( private var handlerThread: HandlerThread? = null private var surface: Surface? = null + /** + * Guards the frozen flag and the latest-frame publication as one unit, so a freeze + * command can never be crossed by a half-published frame. While frozen the images + * arriving at the ImageReader are acquired and closed (the reader stays healthy) but + * no conversion is performed and nothing is published. + */ + private val freezeGate = Any() + + @Volatile + private var frozen = false + + /** Result of a freeze request against this pipeline. */ + sealed interface FreezeResult { + /** A frame was pinned and is ready to store. */ + data class Frozen(val frame: CapturedFrame) : FreezeResult + + /** A frame is already pinned; the caller keeps the existing one. */ + data object AlreadyFrozen : FreezeResult + + /** No captured frame is available to pin yet. */ + data object NoFrame : FreezeResult + } + private val _latestFrame = MutableStateFlow(null) val latestFrame: StateFlow = _latestFrame.asStateFlow() @@ -84,6 +108,10 @@ class FramePipeline( val wallStartedAtMs = System.currentTimeMillis() try { if (!isActive()) return@setOnImageAvailableListener + if (frozen) { + // Frozen: images are acquired and closed in finally; no conversion, no retention. + return@setOnImageAvailableListener + } val sampledAtMs = sampler.currentTimeMs() if (sampler.shouldEmit(sampledAtMs)) { // Reference image only to avoid leaks; copy pixels via Image plane. @@ -93,26 +121,33 @@ class FramePipeline( Log.w(TAG, "Dropped frame: RGBA_8888 payload conversion failed") return@setOnImageAvailableListener } - sampler.markEmitted(sampledAtMs) - val processingLatencyMs = System.currentTimeMillis() - wallStartedAtMs - // One read per frame: both fields below see the same rotation. - val frameDisplayRotation = displayRotation - val visionFrame = OriginalFrame( - id = FrameId.random(), - payload = payload, - sourceWidth = width, - sourceHeight = height, - displayRotation = frameDisplayRotation, - // MediaProjection buffers are display-oriented; never double-rotate pixels. - rotation = FrameOrientation.correctionFor(frameDisplayRotation), - capturedAtElapsedRealtimeMs = sampledAtMs, - wallClockTimestampMs = System.currentTimeMillis(), - captureLatencyMs = processingLatencyMs - ) - _latestFrame.value = CapturedFrame( - frame = visionFrame, - processingLatencyMs = processingLatencyMs - ) + synchronized(freezeGate) { + if (frozen) { + // A freeze command landed during the conversion; the snapshot is authoritative. + Log.d(TAG, "Dropped frame: a freeze command landed during conversion") + } else { + val processingLatencyMs = System.currentTimeMillis() - wallStartedAtMs + // One read per frame: both fields below see the same rotation. + val frameDisplayRotation = displayRotation + val visionFrame = OriginalFrame( + id = FrameId.random(), + payload = payload, + sourceWidth = width, + sourceHeight = height, + displayRotation = frameDisplayRotation, + // MediaProjection buffers are display-oriented; never double-rotate pixels. + rotation = FrameOrientation.correctionFor(frameDisplayRotation), + capturedAtElapsedRealtimeMs = sampledAtMs, + wallClockTimestampMs = System.currentTimeMillis(), + captureLatencyMs = processingLatencyMs, + ) + sampler.markEmitted(sampledAtMs) + _latestFrame.value = CapturedFrame( + frame = visionFrame, + processingLatencyMs = processingLatencyMs + ) + } + } } } catch (e: Exception) { // Never let a bad frame kill the pipeline; next frame recovers. @@ -128,6 +163,29 @@ class FramePipeline( fun surface(): Surface? = surface + /** + * Pins the most recently published valid frame and stops publication. Images keep being + * acquired and closed (the reader and VirtualDisplay stay healthy); no conversion runs + * until [resumeCapture]. The returned copy shares no payload buffer with the pipeline. + * An already-frozen pipeline keeps its existing snapshot. + */ + fun freezeAndSnapshot(): FreezeResult = synchronized(freezeGate) { + if (frozen) return FreezeResult.AlreadyFrozen + val frame = _latestFrame.value ?: return FreezeResult.NoFrame + val copy = FrameSnapshots.copyOf(frame) ?: return FreezeResult.NoFrame + frozen = true + FreezeResult.Frozen(copy) + } + + /** Resumes live publication. No projection or display is recreated. */ + fun resumeCapture() { + synchronized(freezeGate) { frozen = false } + sampler.reset() + } + + /** True while the pipeline is frozen; images flow but nothing is published. */ + val isFrozen: Boolean get() = synchronized(freezeGate) { frozen } + private fun imageToPayload( image: android.media.Image, width: Int, diff --git a/app/src/main/kotlin/com/draftlens/feature/home/DebugCaptureScreen.kt b/app/src/main/kotlin/com/draftlens/feature/home/DebugCaptureScreen.kt index 99a0c6a..58a2665 100644 --- a/app/src/main/kotlin/com/draftlens/feature/home/DebugCaptureScreen.kt +++ b/app/src/main/kotlin/com/draftlens/feature/home/DebugCaptureScreen.kt @@ -22,18 +22,28 @@ import androidx.compose.material3.MaterialTheme import androidx.compose.material3.OutlinedButton import androidx.compose.material3.Slider import androidx.compose.material3.Text +import androidx.compose.material3.TextButton import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.graphics.asImageBitmap import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.unit.dp +import androidx.core.app.NotificationManagerCompat +import androidx.core.content.ContextCompat import androidx.lifecycle.viewmodel.compose.viewModel import com.draftlens.capture.CaptureController import com.draftlens.datastore.AppPreferencesRepository import com.draftlens.models.CaptureStatus +import android.content.Intent +import android.provider.Settings +import android.Manifest +import android.os.Build import com.draftlens.vision.model.android.BitmapFramePayload @Composable @@ -58,6 +68,13 @@ fun DebugCaptureScreen( } } + // Notification permission: the capture notification carries the "Freeze current frame" + // action, so without it the MLBB freeze workflow is unreachable. + var notificationsEnabled by remember { mutableStateOf(areCaptureNotificationsEnabled(activity)) } + val notificationPermission = rememberLauncherForActivityResult( + ActivityResultContracts.RequestPermission() + ) { granted -> notificationsEnabled = granted || areCaptureNotificationsEnabled(activity) } + val requestStartCapture = { vm.onPermissionRequested() launcher.launch(com.draftlens.permissions.ProjectionPermission.createScreenCaptureIntent(activity)) @@ -74,6 +91,21 @@ fun DebugCaptureScreen( Text("DraftLens — Capture Debug (Sprint 0)", color = Color.White, style = MaterialTheme.typography.titleLarge) Text("Detection: Not implemented in Sprint 0", color = Color.Gray) + MobileLegendsCaptureGuide( + notificationsEnabled = notificationsEnabled, + onEnableNotifications = { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + notificationPermission.launch(Manifest.permission.POST_NOTIFICATIONS) + } else { + activity.startActivity( + Intent(Settings.ACTION_APP_NOTIFICATION_SETTINGS) + .putExtra(Settings.EXTRA_APP_PACKAGE, activity.packageName) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK), + ) + } + }, + ) + Card(modifier = Modifier.fillMaxWidth()) { Column(modifier = Modifier.padding(12.dp), verticalArrangement = Arrangement.spacedBy(6.dp)) { Text("Status: ${state.status}") @@ -159,3 +191,63 @@ fun DebugCaptureScreen( } } } + +/** Guidance for the Mobile Legends freeze workflow and missing-notification handling. */ +@Composable +private fun MobileLegendsCaptureGuide( + notificationsEnabled: Boolean, + onEnableNotifications: () -> Unit, +) { + val body = MaterialTheme.typography.bodySmall + Card(modifier = Modifier.fillMaxWidth()) { + Column( + modifier = Modifier.padding(12.dp), + verticalArrangement = Arrangement.spacedBy(2.dp), + ) { + Text("Capturing Mobile Legends: Bang Bang", style = MaterialTheme.typography.titleMedium) + Text("1. Tap Start Capture.", style = body) + Text( + "2. When Android asks what to capture, choose Entire screen / full device display — " + + "not the DraftLens app window. App-window capture cannot see the game, " + + "so the frozen frame would show this app instead of MLBB.", + style = body, + ) + Text( + "3. Open Mobile Legends and wait for the desired draft screen.", + style = body, + ) + Text( + "4. Pull down the notification shade and tap “Freeze current frame” on the " + + "DraftLens capture notification. The latest game frame is kept in memory — no need to leave the game.", + style = body, + ) + Text( + "5. Return to DraftLens. Vision Lab shows the frozen MLBB frame, not this screen. " + + "Tap Resume there, or “Resume live capture” in the notification, to go back to live capture.", + style = body, + ) + if (!notificationsEnabled) { + Text( + "Notifications are disabled: the “Freeze current frame” action is not reachable from the shade while in game.", + style = body, + color = MaterialTheme.colorScheme.error, + ) + TextButton(onClick = onEnableNotifications) { Text("Enable notifications") } + } + } + } +} + +private fun areCaptureNotificationsEnabled(context: android.content.Context): Boolean { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU && + ContextCompat.checkSelfPermission(context, Manifest.permission.POST_NOTIFICATIONS) != + android.content.pm.PackageManager.PERMISSION_GRANTED + ) { + return false + } + return try { + NotificationManagerCompat.from(context).areNotificationsEnabled() + } catch (e: Exception) { + true + } +} diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt index a1dc451..16f8095 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/HeroRecognitionCard.kt @@ -160,6 +160,7 @@ private fun HeroResultArea( ) } TextButton(onClick = vm::identifySelectedSlot) { Text("Try again") } + ManualHeroSearch(slotId = slotId, vm = vm) } HeroSlotPhase.NO_MATCH -> { @@ -174,14 +175,28 @@ private fun HeroResultArea( TextButton(onClick = vm::identifySelectedSlot) { Text("Try again") } } + HeroSlotPhase.INSUFFICIENT_INFO -> { + Text( + state?.message ?: "Not enough visual information. Select a clearer hero crop.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.error, + ) + OutlinedButton(onClick = vm::identifySelectedSlot) { Text("Try again") } + ManualHeroSearch(slotId = slotId, vm = vm) + } + HeroSlotPhase.CONFIRMED -> { + var showCorrection by remember(slotId) { mutableStateOf(false) } Text( "Confirmed hero: ${state?.message}", style = MaterialTheme.typography.bodyLarge, ) Row(horizontalArrangement = Arrangement.spacedBy(8.dp)) { OutlinedButton(onClick = { vm.clearHeroConfirmation(slotId) }) { Text("Clear") } - TextButton(onClick = vm::identifySelectedSlot) { Text("Change…") } + TextButton(onClick = { showCorrection = true }) { Text("Change…") } + } + if (showCorrection) { + ManualHeroSearch(slotId = slotId, vm = vm) } } diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt index d82789e..825a91e 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabScreen.kt @@ -191,10 +191,18 @@ private fun FramePreviewCard( if (state.isFrozen) { "Resume returns to live frames." } else { - "Freeze locks the current frame before annotation." + "Freeze locks the current frame before annotation. While in game, tap " + + "“Freeze current frame” on the capture notification instead." }, style = MaterialTheme.typography.bodySmall, ) + Text( + "Capturing Mobile Legends? When Android asks what to capture, choose Entire screen / " + + "full device display. App-window capture cannot see the game. The capture " + + "notification’s Freeze action needs notifications enabled.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) } } } diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt index b91002b..c1db46a 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabUiState.kt @@ -59,7 +59,7 @@ data class VisionLabUiState( } /** Phase of a slot's hero recognition flow. */ -enum class HeroSlotPhase { IDLE, PROCESSING, CANDIDATES, NO_MATCH, CONFIRMED, FAILED } +enum class HeroSlotPhase { IDLE, PROCESSING, CANDIDATES, NO_MATCH, CONFIRMED, FAILED, INSUFFICIENT_INFO } /** Recognition state of one slot. */ data class HeroSlotUiState( @@ -68,7 +68,7 @@ data class HeroSlotUiState( val candidates: List = emptyList(), /** Best similarity seen in [HeroSlotPhase.NO_MATCH]; null elsewhere. */ val bestSimilarity: Float? = null, - /** User-facing message for [HeroSlotPhase.FAILED]; also used for the confirmed hero's name in [HeroSlotPhase.CONFIRMED]. */ + /** User-facing message for [HeroSlotPhase.FAILED] and [HeroSlotPhase.INSUFFICIENT_INFO]; also used for the confirmed hero's name in [HeroSlotPhase.CONFIRMED]. */ val message: String? = null, /** The confirmed hero id in [HeroSlotPhase.CONFIRMED]. */ val confirmedHeroId: String? = null, diff --git a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt index 31ea9a8..9b31b0c 100644 --- a/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt +++ b/app/src/main/kotlin/com/draftlens/feature/visionlab/VisionLabViewModel.kt @@ -71,6 +71,11 @@ class VisionLabViewModel( viewModelScope.launch { source.frames.collect { frame -> _uiState.update { it.copy(liveFrame = frame) } } } + viewModelScope.launch { + // The pinned frame comes from the capture side (notification or in-app command); + // newer live frames can never replace it here. + source.frozenFrame.collect { frame -> _uiState.update { it.copy(frozenFrame = frame) } } + } viewModelScope.launch { source.state.collect { state -> _uiState.update { it.copy(sourceState = state) } } } @@ -106,15 +111,20 @@ class VisionLabViewModel( // region frame control - /** Pins the frame that is on screen; live frames keep arriving but are not shown. */ + /** + * Pins the newest valid frame through the capture side (same command the notification + * runs). The pinned frame is independently owned and shown until [resume]/[clear]. + * Idempotent: an already-pinned frame is kept. + */ fun freeze() { - val frame = source.frames.value ?: return - _uiState.update { it.copy(frozenFrame = frame) } + if (!source.freeze()) { + report("No captured frame to freeze yet. Start capture and wait for a frame.") + } } /** Releases the pinned frame and shows live frames again. */ fun resume() { - _uiState.update { it.copy(frozenFrame = null) } + source.resume() } /** @@ -125,6 +135,7 @@ class VisionLabViewModel( generation.incrementAndGet() processingJob?.cancel() processingJob = null + source.resume() _uiState.update { it.copy(frozenFrame = null, processed = null, processStatus = ProcessStatus.IDLE, processError = null) } @@ -433,6 +444,10 @@ class VisionLabViewModel( com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.NO_MATCH, bestSimilarity = outcome.bestSimilarity) } + is com.draftlens.vision.heroes.HeroIdentifyOutcome.InsufficientVisualInformation -> updateHeroSlot(slot.id) { + com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.INSUFFICIENT_INFO, message = outcome.reason) + } + is com.draftlens.vision.heroes.HeroIdentifyOutcome.Failed -> updateHeroSlot(slot.id) { com.draftlens.feature.visionlab.HeroSlotUiState(HeroSlotPhase.FAILED, message = outcome.message) } @@ -475,7 +490,11 @@ class VisionLabViewModel( /** Removes a confirmed hero so the slot can be identified again. */ fun clearHeroConfirmation(slotId: String) { viewModelScope.launch { - runCatching { heroRecognitionStore.clear(slotId) } + val result = runCatching { heroRecognitionStore.clear(slotId) } + if (result.isFailure) { + report("The hero confirmation could not be cleared: ${result.exceptionOrNull()?.message}") + return@launch + } _uiState.update { it.copy(heroRecognition = it.heroRecognition - slotId, heroPreview = null) } diff --git a/app/src/main/kotlin/com/draftlens/models/FrameSnapshots.kt b/app/src/main/kotlin/com/draftlens/models/FrameSnapshots.kt new file mode 100644 index 0000000..80b394a --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/models/FrameSnapshots.kt @@ -0,0 +1,68 @@ +package com.draftlens.models + +import android.graphics.Bitmap +import com.draftlens.vision.model.OriginalFrame +import com.draftlens.vision.model.PixelArrayFramePayload +import com.draftlens.vision.model.android.BitmapFramePayload + +/** + * Creates independently owned copies of frames for the freeze workflow. + * + * The copy shares no payload buffer with the source: PixelArray payloads get a fresh + * IntArray, Bitmap payloads a fresh Bitmap. All acquisition metadata (dimensions, + * rotation, display rotation, timestamps, latency) is preserved verbatim; geometry and + * pixel contents are untouched. Copies are not recycled here; consumers may not recycle + * them either. The live pipeline never sees or mutates these copies. + * + * Returns null when the payload cannot be copied (for example a recycled bitmap), so a + * freeze can fail cleanly without touching the source. + */ +object FrameSnapshots { + + fun copyOf(frame: OriginalFrame): OriginalFrame? = when (val payload = frame.payload) { + is PixelArrayFramePayload -> OriginalFrame( + id = frame.id, + payload = PixelArrayFramePayload(payload.width, payload.height, payload.pixels()), + sourceWidth = frame.sourceWidth, + sourceHeight = frame.sourceHeight, + displayRotation = frame.displayRotation, + rotation = frame.rotation, + capturedAtElapsedRealtimeMs = frame.capturedAtElapsedRealtimeMs, + wallClockTimestampMs = frame.wallClockTimestampMs, + captureLatencyMs = frame.captureLatencyMs, + ) + + is BitmapFramePayload -> { + val source = payload.bitmap + if (source.isRecycled) { + null + } else { + val copy = try { + source.copy(Bitmap.Config.ARGB_8888, false) + } catch (e: Exception) { + null + } + if (copy == null) { + null + } else { + OriginalFrame( + id = frame.id, + payload = BitmapFramePayload(copy), + sourceWidth = frame.sourceWidth, + sourceHeight = frame.sourceHeight, + displayRotation = frame.displayRotation, + rotation = frame.rotation, + capturedAtElapsedRealtimeMs = frame.capturedAtElapsedRealtimeMs, + wallClockTimestampMs = frame.wallClockTimestampMs, + captureLatencyMs = frame.captureLatencyMs, + ) + } + } + } + + else -> null + } + + fun copyOf(captured: CapturedFrame): CapturedFrame? = + copyOf(captured.frame)?.let { CapturedFrame(it, captured.processingLatencyMs) } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt index d959d19..f7d1028 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt @@ -1,8 +1,6 @@ package com.draftlens.vision.heroes import android.graphics.Bitmap -import android.graphics.Canvas -import android.graphics.Matrix import android.os.SystemClock import com.draftlens.vision.annotation.DraftSlot import com.draftlens.vision.model.android.BitmapFramePayload @@ -12,7 +10,16 @@ import com.draftlens.vision.model.OriginalFrame import com.draftlens.vision.model.PixelArrayFramePayload import com.draftlens.vision.model.SlotImage -/** Android extractor for Bitmap payloads: axis-aligned crop, then upright rotation. */ +/** + * Android extractor for Bitmap payloads: axis-aligned crop, then upright rotation. + * + * The rotation mapping is the FrameTransform.sourceToOriented() convention applied + * pixel-exactly (the same mapping the JVM extractor uses), never an approximating + * matrix. The source bitmap is never mutated or recycled. + * + * Ownership: the input bitmap is borrowed. The cropped intermediate is recycled in + * every path exactly once; the returned payload owns the (possibly rotated) bitmap. + */ object AndroidBitmapSlotImageExtractor : SlotImageExtractor { override fun extract(frame: OriginalFrame, slot: DraftSlot): Result { @@ -22,6 +29,14 @@ object AndroidBitmapSlotImageExtractor : SlotImageExtractor { if (bitmap.isRecycled) { return Result.failure(IllegalStateException("The frame bitmap has been recycled")) } + if (bitmap.width != frame.sourceWidth || bitmap.height != frame.sourceHeight) { + return Result.failure( + IllegalArgumentException( + "payload bitmap ${bitmap.width}x${bitmap.height} does not match source " + + "${frame.sourceWidth}x${frame.sourceHeight}", + ), + ) + } if (slot.rect.width * frame.sourceWidth < 1f || slot.rect.height * frame.sourceHeight < 1f) { return Result.failure(IllegalArgumentException("The slot rectangle is smaller than one pixel")) } @@ -31,7 +46,13 @@ object AndroidBitmapSlotImageExtractor : SlotImageExtractor { } catch (e: Exception) { return Result.failure(IllegalStateException("The slot could not be cropped: ${e.message}", e)) } - val upright = rotateToUpright(cropped, frame.rotation) + val upright = try { + rotateToUpright(cropped, frame.rotation) + } catch (e: Exception) { + if (!cropped.isRecycled) cropped.recycle() + return Result.failure(IllegalStateException("The slot could not be rotated: ${e.message}", e)) + } + // Ownership transfers: cropped is not needed once upright is built. if (upright !== cropped) cropped.recycle() return Result.success( SlotImage( @@ -45,26 +66,51 @@ object AndroidBitmapSlotImageExtractor : SlotImageExtractor { ) } + /** + * Pixel-exact rotation following FrameTransform.sourceToOriented(): + * ROTATION_90 maps source (sx, sy) to oriented (height - 1 - sy, sx), + * ROTATION_270 maps source (sx, sy) to oriented (sy, width - 1 - sx). + * [source] is not recycled here; the caller owns it. The returned bitmap is owned + * by the caller. On failure no output bitmap leaks. + */ private fun rotateToUpright(source: Bitmap, rotation: FrameRotation): Bitmap { if (rotation == FrameRotation.ROTATION_0) return source + val width = source.width + val height = source.height val swapsAxes = rotation == FrameRotation.ROTATION_90 || rotation == FrameRotation.ROTATION_270 - val outWidth = if (swapsAxes) source.height else source.width - val outHeight = if (swapsAxes) source.width else source.height + val outWidth = if (swapsAxes) height else width + val outHeight = if (swapsAxes) width else height val out = Bitmap.createBitmap(outWidth, outHeight, Bitmap.Config.ARGB_8888) - val matrix = Matrix().apply { - postTranslate(-source.width / 2f, -source.height / 2f) - postRotate( - when (rotation) { - FrameRotation.ROTATION_90 -> 90f - FrameRotation.ROTATION_180 -> 180f - FrameRotation.ROTATION_270 -> 270f - else -> 0f - }, - ) - postTranslate(outWidth / 2f, outHeight / 2f) + try { + val src = IntArray(width * height) + source.getPixels(src, 0, width, 0, 0, width, height) + val dst = IntArray(outWidth * outHeight) + when (rotation) { + FrameRotation.ROTATION_90 -> + for (y in 0 until height) { + for (x in 0 until width) { + dst[x * height + (height - 1 - y)] = src[y * width + x] + } + } + + FrameRotation.ROTATION_180 -> + for (i in src.indices) { + dst[src.size - 1 - i] = src[i] + } + + FrameRotation.ROTATION_270 -> + for (y in 0 until height) { + for (x in 0 until width) { + dst[(width - 1 - x) * height + y] = src[y * width + x] + } + } + } + out.setPixels(dst, 0, outWidth, 0, 0, outWidth, outHeight) + return out + } catch (e: Exception) { + if (!out.isRecycled) out.recycle() + throw e } - Canvas(out).drawBitmap(source, matrix, null) - return out } } diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt index e5e661b..6274ff2 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidHeroCatalogTransport.kt @@ -4,60 +4,129 @@ import java.io.BufferedInputStream import java.net.HttpURLConnection import java.net.URL +/** Redirect policy shared by the transport; each hop is re-validated. */ +object HeroRedirectPolicy { + const val MAX_REDIRECTS = 5 + private const val MAX_URL_LENGTH = 2048 + + /** + * Resolves [location] against [currentUrl] (relative redirects allowed) and enforces the + * only policy this app accepts: HTTPS everywhere. Downgrades to HTTP and any other + * scheme are refused. Returns null when the redirect cannot be honoured safely. + */ + fun resolveTarget(currentUrl: String, location: String?): String? { + if (location.isNullOrBlank()) return null + val target = try { + URL(URL(currentUrl), location).toString() + } catch (e: Exception) { + return null + } + if (target.length > MAX_URL_LENGTH) return null + if (!target.startsWith("https://")) return null + return target + } +} + +/** One attempt's outcome: a terminal result, or a redirect the caller must re-validate. */ +private sealed interface TryOutcome +private class Redirected(val target: String) : TryOutcome +private class Completed(val result: HeroHttpResult) : TryOutcome + /** * HTTPS-only transport over the platform HTTP client. No third-party dependency. - * One retry on I/O errors and 5xx; 429 fails fast with its code. The size cap is - * enforced while reading, so a hostile or broken endpoint cannot exhaust memory. + * Redirects are followed manually and every hop is re-validated by [HeroRedirectPolicy]: + * HTTP or HTTPS-to-HTTP downgrade redirects are refused. One retry on I/O errors and + * 5xx; 429 fails fast with its code. The size cap is enforced while reading, so a + * hostile or broken endpoint cannot exhaust memory. */ class AndroidHeroCatalogTransport( private val connectTimeoutMs: Int = 15_000, private val readTimeoutMs: Int = 30_000, ) : HeroCatalogTransport { + private val redirectCodes = setOf(301, 302, 303, 307, 308) + override fun get(url: String, maxBytes: Long, accept: String?): HeroHttpResult { if (maxBytes <= 0L) return HeroHttpResult.Failure("The size cap must be positive") if (!url.startsWith("https://")) { return HeroHttpResult.Failure("Only HTTPS URLs are allowed: $url") } + var current = url + var redirectBudget = HeroRedirectPolicy.MAX_REDIRECTS + while (true) { + when (val outcome = tryWithRetries(current, maxBytes, accept)) { + is Redirected -> { + if (redirectBudget == 0) { + return HeroHttpResult.Failure("Too many redirects from $url") + } + redirectBudget-- + current = outcome.target + } + + is Completed -> return outcome.result + } + } + } + + /** One hop with the one-retry-on-I-O/5xx policy. Redirects bubble up unretried. */ + private fun tryWithRetries(url: String, maxBytes: Long, accept: String?): TryOutcome { var lastFailure: HeroHttpResult.Failure? = null repeat(2) { attempt -> when (val outcome = attemptOnce(url, maxBytes, accept)) { - is HeroHttpResult.Success -> return outcome - is HeroHttpResult.Failure -> { - lastFailure = outcome - val retryable = outcome.httpCode == null || outcome.httpCode >= 500 - if (!retryable) return outcome // 4xx, including 429: fail fast, do not parallel-hammer - if (attempt == 1) return outcome + is Redirected -> return outcome + is Completed -> when (val result = outcome.result) { + is HeroHttpResult.Success -> return outcome + is HeroHttpResult.Failure -> { + lastFailure = result + val retryable = result.httpCode == null || result.httpCode >= 500 + if (!retryable) return outcome // 4xx, including 429: fail fast + if (attempt == 1) return outcome + } } } } - return lastFailure ?: HeroHttpResult.Failure("The request failed") + return Completed(lastFailure ?: HeroHttpResult.Failure("The request failed")) } - private fun attemptOnce(url: String, maxBytes: Long, accept: String?): HeroHttpResult { + private fun attemptOnce(url: String, maxBytes: Long, accept: String?): TryOutcome { val connection = try { (URL(url).openConnection() as? HttpURLConnection) - ?: return HeroHttpResult.Failure("Only HTTP(S) URLs are supported: $url") + ?: return Completed(HeroHttpResult.Failure("Only HTTP(S) URLs are supported: $url")) } catch (e: Exception) { - return HeroHttpResult.Failure("The URL could not be opened: ${e.message}") + return Completed(HeroHttpResult.Failure("The URL could not be opened: ${e.message}")) } return try { connection.requestMethod = "GET" connection.connectTimeout = connectTimeoutMs connection.readTimeout = readTimeoutMs + // Redirects are handled manually so every hop is re-validated (HTTPS only). + connection.instanceFollowRedirects = false if (accept != null) connection.setRequestProperty("Accept", accept) connection.setRequestProperty("User-Agent", "DraftLens-HeroCatalog/1.0") val code = connection.responseCode - if (code in 200..299) { + if (code in redirectCodes) { + val location = connection.getHeaderField("Location") + val target = HeroRedirectPolicy.resolveTarget(url, location) + if (target != null) { + Redirected(target) + } else { + Completed( + HeroHttpResult.Failure( + "Refusing insecure redirect from $url to $location", + code, + ), + ) + } + } else if (code in 200..299) { val bytes = readCapped(connection.inputStream, maxBytes) - ?: return HeroHttpResult.Failure("The response exceeds $maxBytes bytes", code) + ?: return Completed(HeroHttpResult.Failure("The response exceeds $maxBytes bytes", code)) val contentType = connection.contentType?.substringBefore(';')?.trim()?.lowercase() - HeroHttpResult.Success(HeroHttpResponse(bytes, contentType)) + Completed(HeroHttpResult.Success(HeroHttpResponse(bytes, contentType))) } else { - HeroHttpResult.Failure("HTTP $code from $url", code) + Completed(HeroHttpResult.Failure("HTTP $code from $url", code)) } } catch (e: Exception) { - HeroHttpResult.Failure("The request failed: ${e.message}") + Completed(HeroHttpResult.Failure("The request failed: ${e.message}")) } finally { try { connection.disconnect() diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidReferenceImageValidator.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidReferenceImageValidator.kt new file mode 100644 index 0000000..3c5a4e0 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidReferenceImageValidator.kt @@ -0,0 +1,43 @@ +package com.draftlens.vision.heroes + +import android.graphics.BitmapFactory + +/** + * Decode-based validator for cached reference images on a device: it parses the header, + * then fully decodes (downsampled when huge) to prove the bytes are a readable image. + * The decode is released immediately; nothing is retained. + */ +object AndroidReferenceImageValidator : ReferenceImageValidator { + + override fun isDecodable(bytes: ByteArray): Boolean = try { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(bytes, 0, bytes.size, bounds) + if (bounds.outWidth <= 0 || bounds.outHeight <= 0) { + false + } else { + val decode = BitmapFactory.Options().apply { + inSampleSize = sampleFor(bounds.outWidth, bounds.outHeight) + } + val decoded = BitmapFactory.decodeByteArray(bytes, 0, bytes.size, decode) + if (decoded == null) { + false + } else { + try { + true + } finally { + decoded.recycle() + } + } + } + } catch (e: Exception) { + false + } + + private fun sampleFor(width: Int, height: Int): Int { + var sample = 1 + while (maxOf(width, height) / sample > MAX_DECODE_DIMENSION) sample *= 2 + return sample + } + + private const val MAX_DECODE_DIMENSION = 1024 +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt index 60f56c6..de576fb 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt @@ -3,18 +3,36 @@ package com.draftlens.vision.heroes import kotlinx.coroutines.flow.MutableStateFlow import kotlinx.coroutines.flow.StateFlow import kotlinx.coroutines.flow.asStateFlow +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json import java.io.File +import java.io.FileOutputStream +import java.security.MessageDigest +import java.nio.file.Files +import java.nio.file.StandardCopyOption /** - * File-based catalog: one atomic snapshot JSON plus one file per reference image. - * Writes go to a `.tmp` sibling first and are renamed into place, so an interrupted - * sync never leaves a half-read snapshot behind. + * File-based hero catalog: one snapshot JSON plus one file per reference image and a small + * metadata sidecar per image (`*.meta`) binding it to its source URL and content hash. + * + * Guarantees: + * - Snapshot writes are temp-file-then-atomic-move into place; an interrupted write never + * leaves a half-read or truncated snapshot, and a failed write leaves the previous + * snapshot untouched (there is no direct-write fallback). + * - A cached asset is reused only when it passes full validation: present, within the + * 4 MiB cap, matching sidecar (URL, size, SHA-256), magic bytes and a container + * structure check (plus an actual decode on Android, see `DraftLensApp`). Corrupt or + * stale files are never served; a changed source URL forces a re-download. + * - Refresh keeps the previous valid asset when the download fails; nothing is deleted + * on failure paths, and a stale asset is never exposed under a changed URL. + * - Temp files are created as siblings of the target and swept after an hour on init. */ class FileHeroCatalogRepository( private val dataDir: File, private val cacheDir: File, private val transport: HeroCatalogTransport, private val clock: () -> Long = System::currentTimeMillis, + private val imageValidator: ReferenceImageValidator = StructuralImageValidator, ) : HeroCatalogRepository { private val snapshotFile = File(dataDir, "hero-catalog/catalog.json") @@ -27,6 +45,8 @@ class FileHeroCatalogRepository( override val state: StateFlow = _state.asStateFlow() init { + sweepStaleTempFiles(snapshotFile.parentFile) + sweepStaleTempFiles(assetsDir) // If the cache cannot be decoded this records an Error state and returns null. cached = readDiskSnapshot() cached?.let { _state.value = HeroCatalogState.Ready(it.entries.size, it.syncedAtMs) } @@ -72,7 +92,8 @@ class FileHeroCatalogRepository( override fun assetFile(id: HeroId, variant: HeroAssetVariant): File? { val file = assetFileFor(id, variant) - return file.takeIf { it.exists() && it.length() > 0L } + val expectedUrl = currentAssetUrl(id, variant) ?: return null + return file.takeIf { isAssetFresh(it, expectedUrl) } } // region internals @@ -80,34 +101,93 @@ class FileHeroCatalogRepository( private fun assetFileFor(id: HeroId, variant: HeroAssetVariant): File = File(assetsDir, "${id.value}-${variant.name.lowercase()}.img") + private fun metaFileFor(assetFile: File): File = File(assetsDir, "${assetFile.name}.meta") + + /** The URL the current catalog advertises for this asset; null when unknown. */ + private fun currentAssetUrl(id: HeroId, variant: HeroAssetVariant): String? = + (cached ?: snapshot()) + ?.entries?.firstOrNull { it.id == id } + ?.assets?.firstOrNull { it.variant == variant } + ?.url + + /** + * Full serve-time validation of an existing cache entry against [expectedUrl]. + * A file that fails anything here is invisible to the rest of the app. + */ + private fun isAssetFresh(file: File, expectedUrl: String): Boolean { + if (!file.isFile) return false + val length = file.length() + if (length == 0L || length > MAX_IMAGE_BYTES) return false + val meta = readMeta(metaFileFor(file)) ?: return false + if (meta.url != expectedUrl) return false + if (meta.bytes != length) return false + if (meta.contentType != null && !meta.contentType.startsWith("image/")) return false + val bytes = runCatching { file.readBytes() }.getOrNull() ?: return false + if (sha256Hex(bytes) != meta.sha256) return false + if (!looksLikeImage(bytes)) return false + return imageValidator.isDecodable(bytes) + } + /** Returns the names of heroes whose images failed to download or validate. */ private fun downloadAssets(entries: List): List { val failures = mutableListOf() for (entry in entries) { val asset = entry.assets.firstOrNull { it.variant == HeroAssetVariant.HEAD } ?: continue val target = assetFileFor(entry.id, HeroAssetVariant.HEAD) - if (target.exists() && target.length() > 0L) continue // keep the cache; refresh is about the catalog - val ok = downloadAsset(asset, target) - if (!ok) { - runCatching { target.delete() } + // Unchanged URL and a valid file: keep the cache untouched. + if (isAssetFresh(target, asset.url)) continue + // Changed URL or corrupt cache: re-download. A failure keeps the on-disk files + // untouched; they are served only if they pass validation for the *current* URL. + if (!refreshAsset(asset, target)) { failures += entry.canonicalName } } return failures } - /** HTTPS + image content check via magic bytes, with a size cap enforced by the transport. */ - private fun downloadAsset(asset: HeroAsset, target: File): Boolean { + /** + * Fetches and fully validates the image, then atomically replaces the sidecar metadata + * before the image itself. A stale metadata sidecar never matches a fresh image and + * vice versa, so a crash or failure in between leaves the entry unserved and + * self-healing on the next refresh. The previous image is only ever replaced by a + * fully validated download. + */ + private fun refreshAsset(asset: HeroAsset, target: File): Boolean { + val downloaded = fetchAsset(asset) ?: return false + val meta = HeroAssetMeta( + url = asset.url, + sha256 = sha256Hex(downloaded.bytes), + bytes = downloaded.bytes.size.toLong(), + contentType = downloaded.contentType, + savedAtMs = clock(), + ) + val metaBytes = Json { encodeDefaults = true } + .encodeToString(HeroAssetMeta.serializer(), meta) + .toByteArray(Charsets.UTF_8) + if (!writeAtomically(metaFileFor(target), metaBytes)) return false + if (!writeAtomically(target, downloaded.bytes)) { + // The metadata now describes an image that was not written; drop it so the + // previous pair is not shadowed by a mismatch that never heals. + runCatching { metaFileFor(target).delete() } + return false + } + return true + } + + /** HTTPS + cap + content type + magic bytes + container validation. */ + private fun fetchAsset(asset: HeroAsset): DownloadedAsset? { + if (!asset.url.startsWith("https://")) return null val response = when (val result = transport.get(asset.url, MAX_IMAGE_BYTES, null)) { is HeroHttpResult.Success -> result.response - is HeroHttpResult.Failure -> return false + is HeroHttpResult.Failure -> return null } - if (asset.url.startsWith("https://").not()) return false - if (response.bytes.size > MAX_IMAGE_BYTES) return false + if (response.bytes.size > MAX_IMAGE_BYTES) return null val contentType = response.contentType?.lowercase() - if (contentType != null && !contentType.startsWith("image/")) return false - if (!looksLikeImage(response.bytes)) return false - return writeAtomically(target, response.bytes) + if (contentType != null && !contentType.startsWith("image/")) return null + val bytes = response.bytes + if (!looksLikeImage(bytes)) return null + if (!imageValidator.isDecodable(bytes)) return null + return DownloadedAsset(bytes, contentType) } private fun looksLikeImage(bytes: ByteArray): Boolean { @@ -137,22 +217,78 @@ class FileHeroCatalogRepository( } } - private fun writeAtomically(target: File, bytes: ByteArray): Boolean = runCatching { + private fun readMeta(file: File): HeroAssetMeta? = try { + Json { ignoreUnknownKeys = true }.decodeFromString(HeroAssetMeta.serializer(), file.readText()) + } catch (e: Exception) { + null + } + + private fun sha256Hex(bytes: ByteArray): String = + MessageDigest.getInstance("SHA-256").digest(bytes) + .joinToString(separator = "") { "%02x".format(it) } + + /** + * Writes [bytes] to [target] by writing a sibling temp file in the same directory + * (same filesystem), flushing and syncing it, then atomically moving it into place. + * If the atomic move cannot be completed, the previous [target] is preserved and the + * temp file is deleted. Never writes directly to [target]. + */ + internal fun writeAtomically(target: File, bytes: ByteArray): Boolean { val parent = target.parentFile ?: return false - parent.mkdirs() - val tmp = File(parent, "${target.name}.tmp") - tmp.writeBytes(bytes) - if (!tmp.renameTo(target)) { - // renameTo can fail across file systems; fall back to a copy+delete. - target.writeBytes(bytes) - tmp.delete() + if (!parent.exists() && !parent.mkdirs()) return false + val tmp = try { + File.createTempFile("${target.name}.", ".tmp", parent) + } catch (e: Exception) { + return false } - true - }.getOrDefault(false) + return try { + FileOutputStream(tmp).use { out -> + out.write(bytes) + out.flush() + // Best effort on filesystems that support it; close is the correctness floor. + runCatching { out.fd.sync() } + } + Files.move( + tmp.toPath(), + target.toPath(), + StandardCopyOption.ATOMIC_MOVE, + StandardCopyOption.REPLACE_EXISTING, + ) + true + } catch (e: Exception) { + // Atomic replacement was not possible: keep the previous target untouched. + runCatching { tmp.delete() } + false + } + } + + /** Removes abandoned `*.tmp` siblings older than an hour; never touches other files. */ + private fun sweepStaleTempFiles(dir: File?) { + if (dir == null) return + runCatching { + val cutoff = clock() - TEMP_SWEEP_AGE_MS + dir.listFiles { file -> file.isFile && file.name.endsWith(".tmp") }?.forEach { file -> + if (file.lastModified() < cutoff) file.delete() + } + } + } // endregion private companion object { const val MAX_IMAGE_BYTES = 4L * 1024L * 1024L // 4 MiB per reference image + const val TEMP_SWEEP_AGE_MS = 60L * 60L * 1000L // sweep abandoned temp siblings hourly } } + +/** Sidecar binding one cached reference image to its source and content. */ +@Serializable +internal data class HeroAssetMeta( + val url: String, + val sha256: String, + val bytes: Long, + val contentType: String? = null, + val savedAtMs: Long, +) + +private data class DownloadedAsset(val bytes: ByteArray, val contentType: String?) diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt index cc28583..0e01694 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroRecognition.kt @@ -12,6 +12,13 @@ sealed interface HeroIdentifyOutcome { data class Candidates(val candidates: List) : HeroIdentifyOutcome data class NoMatch(val bestSimilarity: Float) : HeroIdentifyOutcome data class Failed(val message: String) : HeroIdentifyOutcome + + /** + * The crop is too empty or featureless for matching to mean anything: a uniform gray, + * black, white or single-colour image, or a crop smaller than the minimum useful size. + * Distinct from [NoMatch]: no similarity was computed and no candidates exist. + */ + data class InsufficientVisualInformation(val reason: String) : HeroIdentifyOutcome } /** @@ -28,10 +35,18 @@ object HeroRecognition { references: List, config: HeroMatchConfig = HeroMatchConfig(), ): HeroIdentifyOutcome { - val usable = references.filter { it.pixels.size == it.width * it.height && it.width > 0 && it.height > 0 } if (width <= 0 || height <= 0 || pixels.size != width * height) { return HeroIdentifyOutcome.Failed("The extracted image is empty or invalid.") } + // Blank and low-information crops must never produce candidates. This runs before the + // catalog is consulted: there is nothing meaningful to match. + val information = VisualInformationAnalyzer.analyze(pixels, width, height) + information.insufficientReason?.let { reason -> + return HeroIdentifyOutcome.InsufficientVisualInformation( + "Not enough visual information: $reason. Select a clearer hero crop.", + ) + } + val usable = references.filter { it.pixels.size == it.width * it.height && it.width > 0 && it.height > 0 } if (usable.isEmpty()) { return HeroIdentifyOutcome.Failed( "No usable hero reference images are cached. Sync the hero catalog first.", diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt index b416e89..403e3ae 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/HeroReferenceImages.kt @@ -19,10 +19,20 @@ object HeroReferenceImages { val sample = (maxOf(options.outWidth, options.outHeight) / MAX_REFERENCE_DIMENSION).coerceAtLeast(1) val decoded = BitmapFactory.Options().apply { inSampleSize = sample } val bitmap = BitmapFactory.decodeFile(file.absolutePath, decoded) ?: return null - val pixels = IntArray(bitmap.width * bitmap.height) - bitmap.getPixels(pixels, 0, bitmap.width, 0, 0, bitmap.width, bitmap.height) - bitmap.recycle() - HeroReference(entry, pixels, bitmap.width, bitmap.height) + try { + // Read the dimensions before recycling; the reads must complete first. + val width = bitmap.width + val height = bitmap.height + val pixels = IntArray(width * height) + bitmap.getPixels(pixels, 0, width, 0, 0, width, height) + HeroReference(entry, pixels, width, height) + } finally { + try { + bitmap.recycle() + } catch (_: Exception) { + // A bitmap that could not be recycled is a leak, not a correctness failure. + } + } }.getOrNull() private const val MAX_REFERENCE_DIMENSION = 256 diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/ReferenceImageValidator.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/ReferenceImageValidator.kt new file mode 100644 index 0000000..1a87a13 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/ReferenceImageValidator.kt @@ -0,0 +1,123 @@ +package com.draftlens.vision.heroes + +import java.util.zip.CRC32 + +/** + * Verifies that bytes are a decodable image, without pulling Android APIs into the + * catalog cache. Implementations must be deterministic. + */ +fun interface ReferenceImageValidator { + fun isDecodable(bytes: ByteArray): Boolean +} + +/** + * Pure-Kotlin container structure check: signature, chunk framing, per-chunk CRC for PNG, + * trailer markers for JPEG and GIF, RIFF/WEBP chunk framing for WebP. It does not run an + * image decoder; it rejects truncation, corruption, HTML error pages and non-images that + * happen to carry a magic prefix. This is what unit tests exercise on the JVM. + */ +object StructuralImageValidator : ReferenceImageValidator { + + override fun isDecodable(bytes: ByteArray): Boolean = when { + isPng(bytes) -> pngStructureValid(bytes) + isJpeg(bytes) -> jpegStructureValid(bytes) + isGif(bytes) -> gifStructureValid(bytes) + isWebp(bytes) -> webpStructureValid(bytes) + else -> false + } + + private fun isPng(b: ByteArray): Boolean = + b.size >= 8 && + b[0] == 0x89.toByte() && b[1] == 'P'.code.toByte() && + b[2] == 'N'.code.toByte() && b[3] == 'G'.code.toByte() && + b[4] == 0x0D.toByte() && b[5] == 0x0A.toByte() && + b[6] == 0x1A.toByte() && b[7] == 0x0A.toByte() + + private val pngSignature = byteArrayOf( + 0x89.toByte(), 'P'.code.toByte(), 'N'.code.toByte(), 'G'.code.toByte(), + 0x0D.toByte(), 0x0A.toByte(), 0x1A.toByte(), 0x0A.toByte(), + ) + + private fun pngStructureValid(b: ByteArray): Boolean { + var offset = pngSignature.size + val crc = CRC32() + var seenIhdr = false + while (true) { + if (offset + 12 > b.size) return false + val length = readInt32Be(b, offset) + if (length < 0) return false + val typeStart = offset + 4 + val dataStart = typeStart + 4 + val dataEnd = dataStart + length + if (length > b.size - dataStart) return false + if (dataEnd + 4 > b.size) return false + crc.reset() + crc.update(b, typeStart, 4 + length) + val storedCrc = readInt32Be(b, dataEnd) + if (crc.value.toInt() != storedCrc) return false + val type = String(b, typeStart, 4, Charsets.US_ASCII) + when (type) { + "IHDR" -> { + // The first chunk must be IHDR and carry exactly 13 bytes. + if (seenIhdr || offset != pngSignature.size || length != 13) return false + seenIhdr = true + } + + "IEND" -> return seenIhdr + } + offset = dataEnd + 4 + } + } + + private fun isJpeg(b: ByteArray): Boolean = + b.size >= 4 && + b[0] == 0xFF.toByte() && b[1] == 0xD8.toByte() + + private fun jpegStructureValid(b: ByteArray): Boolean { + // Minimal structural sanity: SOI at the front, EOI at the back, no empty body. + if (!isJpeg(b)) return false + val last = b.size - 1 + return b[last - 1] == 0xFF.toByte() && b[last] == 0xD9.toByte() + } + + private fun isGif(b: ByteArray): Boolean { + if (b.size < 6) return false + val header = String(b, 0, 6, Charsets.US_ASCII) + return header == "GIF87a" || header == "GIF89a" + } + + private fun gifStructureValid(b: ByteArray): Boolean { + // Header plus logical screen descriptor; trailer byte 0x3B at the very end. + if (!isGif(b) || b.size < 20) return false + return b[b.size - 1] == 0x3B.toByte() + } + + private fun isWebp(b: ByteArray): Boolean = + b.size >= 16 && + String(b, 0, 4, Charsets.US_ASCII) == "RIFF" && + String(b, 8, 4, Charsets.US_ASCII) == "WEBP" + + private fun webpStructureValid(b: ByteArray): Boolean { + if (!isWebp(b)) return false + val riffSize = readInt32Le(b, 4) + if (riffSize < 4 || riffSize.toLong() > b.size.toLong() - 8) return false + // At least the first chunk must be a 4-character type with a well-formed size. + val chunkTypeStart = 12 + if (chunkTypeStart + 8 > b.size) return false + val chunkSize = readInt32Le(b, chunkTypeStart + 4) + if (chunkSize < 0 || chunkTypeStart.toLong() + 8 + chunkSize > b.size) return false + return true + } + + private fun readInt32Be(b: ByteArray, offset: Int): Int = + ((b[offset].toInt() and 0xFF) shl 24) or + ((b[offset + 1].toInt() and 0xFF) shl 16) or + ((b[offset + 2].toInt() and 0xFF) shl 8) or + (b[offset + 3].toInt() and 0xFF) + + private fun readInt32Le(b: ByteArray, offset: Int): Int = + (b[offset].toInt() and 0xFF) or + ((b[offset + 1].toInt() and 0xFF) shl 8) or + ((b[offset + 2].toInt() and 0xFF) shl 16) or + ((b[offset + 3].toInt() and 0xFF) shl 24) +} diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt index 736bf60..3975d2b 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/SlotImageExtractor.kt @@ -89,7 +89,12 @@ object PixelSlotImageExtractor : SlotImageExtractor { return Result.success(output) } - /** Rotates the axis-aligned crop so the result matches what the canvas shows. */ + /** + * Rotates the axis-aligned crop so the result matches what the canvas shows. + * The mappings follow [FrameTransform.sourceToOriented]: oriented coordinate + * (x, y) receives the source pixel at the inverse point, sampled at pixel centers. + * For example, ROTATION_90 maps source (sx, sy) to oriented (height - 1 - sy, sx). + */ private fun rotateToUpright( pixels: IntArray, width: Int, @@ -98,28 +103,31 @@ object PixelSlotImageExtractor : SlotImageExtractor { ): Triple = when (rotation) { FrameRotation.ROTATION_0 -> Triple(pixels, width, height) FrameRotation.ROTATION_90 -> { - // oriented(xx, yy) = source(col = yy, row = height - 1 - xx) - val out = IntArray(width * height) - for (xx in 0 until height) { - for (yy in 0 until width) { - out[xx * width + yy] = pixels[(height - 1 - xx) * width + yy] + // sourceToOriented: (x, y) -> (height - y, x); oriented is height x width. + // oriented(ox, oy) = source(oy, height - 1 - ox). + val out = IntArray(pixels.size) + for (y in 0 until height) { + for (x in 0 until width) { + out[x * height + (height - 1 - y)] = pixels[y * width + x] } } Triple(out, height, width) } FrameRotation.ROTATION_180 -> { - val out = IntArray(width * height) + // sourceToOriented: (x, y) -> (width - x, height - y); reversed array. + val out = IntArray(pixels.size) for (i in pixels.indices) out[pixels.size - 1 - i] = pixels[i] Triple(out, width, height) } FrameRotation.ROTATION_270 -> { - // oriented(xx, yy) = source(col = width - 1 - yy, row = xx) - val out = IntArray(width * height) - for (xx in 0 until height) { - for (yy in 0 until width) { - out[xx * width + yy] = pixels[xx * width + (width - 1 - yy)] + // sourceToOriented: (x, y) -> (y, width - x); oriented is height x width. + // oriented(ox, oy) = source(width - 1 - oy, ox). + val out = IntArray(pixels.size) + for (y in 0 until height) { + for (x in 0 until width) { + out[(width - 1 - x) * height + y] = pixels[y * width + x] } } Triple(out, height, width) diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/VisualInformation.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/VisualInformation.kt new file mode 100644 index 0000000..ba2a0fb --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/VisualInformation.kt @@ -0,0 +1,165 @@ +package com.draftlens.vision.heroes + +import kotlin.math.abs + +/** + * The outcome of the deterministic visual-information analysis of one crop. + * + * [insufficientReason] is null when the crop carries enough structure for matching to mean + * anything. All signals are deterministic functions of the pixels: no model, no randomness, + * no platform APIs. + * + * **The thresholds are conservative heuristics, not calibrated MLBB recognition thresholds.** + * They were chosen to separate "a blank gray rectangle" (the confirmed product failure) from + * ordinary portraits. They are not an accuracy measurement and will be revisited only after + * real-device evaluation with manually labeled samples. + */ +data class VisualInformationReport( + val width: Int, + val height: Int, + /** Mean luma in [0, 255]. Informational only. */ + val meanLuminance: Float, + /** Variance of per-pixel luma in luma². */ + val luminanceVariance: Float, + /** Variance of the most varied color channel (R, G or B) in channel². */ + val colorVariance: Float, + /** Fraction of horizontal/vertical neighbor pairs whose luma step exceeds the edge threshold, in [0, 1]. */ + val edgeDensity: Float, + /** Raw count of neighbor pairs above the edge threshold. */ + val meaningfulDifferences: Int, + /** Null when the crop is usable; otherwise a short reason it is not. */ + val insufficientReason: String?, +) { + val isSufficient: Boolean get() = insufficientReason == null +} + +object VisualInformationAnalyzer { + + /** Crops below this on either axis carry no reliable detail for the matcher. */ + const val MIN_USEFUL_DIMENSION = 12 + + /** Luma variance below this (std ~1.4/255) means a flat or single-colour image. */ + const val FLAT_LUMINANCE_VARIANCE = 2.0f + + /** Channel variance below this, together with flat luma, confirms a single-colour image. */ + const val FLAT_COLOR_VARIANCE = 2.0f + + /** Below this the image only shows smooth tonality: no texture, no edges, no detail. */ + const val DETAIL_LUMINANCE_VARIANCE = 8.0f + + /** Less than 1% of neighbor pairs stepping at least [EDGE_DELTA] luma units apart means no detail. */ + const val MIN_EDGE_DENSITY = 0.01f + + /** Luma step (0-255) that counts a neighbor pair as an edge. */ + const val EDGE_DELTA = 12 + + private const val LUMA_R = 0.299f + private const val LUMA_G = 0.587f + private const val LUMA_B = 0.114f + + /** + * Analyses [pixels] laid out row-major for a [width] x [height] image. + * Never throws; a dimension/pixel-count mismatch is reported as insufficient. + */ + fun analyze(pixels: IntArray, width: Int, height: Int): VisualInformationReport { + if (width <= 0 || height <= 0 || pixels.size != width * height) { + return VisualInformationReport( + width = width.coerceAtLeast(0), + height = height.coerceAtLeast(0), + meanLuminance = 0f, + luminanceVariance = 0f, + colorVariance = 0f, + edgeDensity = 0f, + meaningfulDifferences = 0, + insufficientReason = "the image dimensions do not match its pixel data", + ) + } + + val count = width * height + var lumaSum = 0.0 + var lumaSumOfSquares = 0.0 + var rSum = 0.0 + var gSum = 0.0 + var bSum = 0.0 + var rSumOfSquares = 0.0 + var gSumOfSquares = 0.0 + var bSumOfSquares = 0.0 + for (i in 0 until count) { + val p = pixels[i] + val r = ((p shr 16) and 0xFF).toDouble() + val g = ((p shr 8) and 0xFF).toDouble() + val b = (p and 0xFF).toDouble() + val l = LUMA_R * r + LUMA_G * g + LUMA_B * b + lumaSum += l + lumaSumOfSquares += l * l + rSum += r + gSum += g + bSum += b + rSumOfSquares += r * r + gSumOfSquares += g * g + bSumOfSquares += b * b + } + val meanLuma = (lumaSum / count).toFloat() + val lumaVariance = varianceOf(lumaSumOfSquares, lumaSum, count) + val colorVariance = maxOf( + varianceOf(rSumOfSquares, rSum, count), + maxOf(varianceOf(gSumOfSquares, gSum, count), varianceOf(bSumOfSquares, bSum, count)), + ) + + // Edge pass keeps only the current and the previous row of luma values. + var meaningfulDifferences = 0 + var previousRow = FloatArray(width) + var currentRow = FloatArray(width) + for (y in 0 until height) { + val rowBase = y * width + for (x in 0 until width) currentRow[x] = lumaOf(pixels[rowBase + x]) + for (x in 0 until width - 1) { + if (abs(currentRow[x] - currentRow[x + 1]) > EDGE_DELTA) meaningfulDifferences++ + } + if (y > 0) { + for (x in 0 until width) { + if (abs(previousRow[x] - currentRow[x]) > EDGE_DELTA) meaningfulDifferences++ + } + } + val swap = previousRow + previousRow = currentRow + currentRow = swap + } + val neighborPairs = (width - 1) * height + width * (height - 1) + val edgeDensity = if (neighborPairs > 0) meaningfulDifferences.toFloat() / neighborPairs else 0f + + val minSide = minOf(width, height) + val reason = when { + minSide < MIN_USEFUL_DIMENSION -> + "smaller than ${MIN_USEFUL_DIMENSION}x$MIN_USEFUL_DIMENSION pixels" + lumaVariance < FLAT_LUMINANCE_VARIANCE && colorVariance < FLAT_COLOR_VARIANCE -> + "near-uniform (flat or single-colour)" + edgeDensity < MIN_EDGE_DENSITY && lumaVariance < DETAIL_LUMINANCE_VARIANCE -> + "no edges or texture (too little detail)" + else -> null + } + + return VisualInformationReport( + width = width, + height = height, + meanLuminance = meanLuma, + luminanceVariance = lumaVariance, + colorVariance = colorVariance, + edgeDensity = edgeDensity, + meaningfulDifferences = meaningfulDifferences, + insufficientReason = reason, + ) + } + + private fun varianceOf(sumOfSquares: Double, sum: Double, count: Int): Float { + val mean = sum / count + return (sumOfSquares / count - mean * mean).coerceAtLeast(0.0).toFloat() + } + + private fun lumaOf(p: Int): Float { + val r = (p shr 16) and 0xFF + val g = (p shr 8) and 0xFF + val b = p and 0xFF + return LUMA_R * r + LUMA_G * g + LUMA_B * b + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/model/VisionFrameSource.kt b/app/src/main/kotlin/com/draftlens/vision/model/VisionFrameSource.kt index 43f0ae8..3ef6f4a 100644 --- a/app/src/main/kotlin/com/draftlens/vision/model/VisionFrameSource.kt +++ b/app/src/main/kotlin/com/draftlens/vision/model/VisionFrameSource.kt @@ -14,9 +14,27 @@ enum class VisionSourceState { IDLE, CAPTURING, PAUSED, ERROR, STOPPED, UNKNOWN * Narrow seam between capture and Vision. * * Vision consumes frames through this interface only; it never imports the capture - * service or the capture state holder. + * service or the capture state holder. The freeze commands are part of the seam: + * they run the capture-side freeze/resume workflow, whose results come back + * through [frozenFrame]. */ interface VisionFrameSource { val frames: StateFlow val state: StateFlow + + /** + * The independently owned frame pinned by a freeze command; null while live. + * The capture side publishes this; the UI renders it in place of live frames. + */ + val frozenFrame: StateFlow + + /** + * Pins the newest valid frame and pauses publication of live frames. Idempotent + * for an already-frozen source; false is returned only when there is nothing to pin + * (no capture running, or no frame captured yet). + */ + fun freeze(): Boolean + + /** Releases the pinned frame and resumes live publication. */ + fun resume() } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 4ab7500..c67b6ac 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -4,5 +4,11 @@ DraftLens is capturing your screen Tap to view debug screen Stop + LIVE + FROZEN + Live preview of what you are sharing + Frame frozen — the Vision Lab shows the pinned frame + Freeze current frame + Resume live capture Not implemented in Sprint 0 diff --git a/app/src/test/kotlin/com/draftlens/capture/CaptureFreezeTest.kt b/app/src/test/kotlin/com/draftlens/capture/CaptureFreezeTest.kt new file mode 100644 index 0000000..33b2e9a --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/capture/CaptureFreezeTest.kt @@ -0,0 +1,46 @@ +package com.draftlens.capture + +import com.draftlens.vision.testCapturedFrame +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Before +import org.junit.Test + +/** + * Command-flow coverage for the parts of the freeze path that are deterministic on the JVM. + * The pipeline itself is Android-only, so the full flow is verified manually and on-device; + * these tests pin the guarantees that do not need a MediaProjection session. + */ +class CaptureFreezeTest { + + @Before fun resetHolder() { + CaptureStateHolder.reset() + } + + @Test fun `freezing with no active capture reports NoPipeline and stores nothing`() { + val outcome = CaptureFreeze.freezeCurrentFrame() + + assertEquals(FreezeOutcome.NoPipeline, outcome) + assertNull(CaptureStateHolder.frozenFrame.value) + } + + @Test fun `resume with no active capture is a safe no-op that drops the pin`() { + CaptureStateHolder.setFrozenFrame(testCapturedFrame()) + + CaptureFreeze.resumeLive() + + assertNull(CaptureStateHolder.frozenFrame.value) + } + + @Test fun `a duplicate freeze keeps the original frame and reports AlreadyFrozen`() { + val pinned = testCapturedFrame() + CaptureStateHolder.setFrozenFrame(pinned) + + // The holder already pins a frame, so the command is a no-op before the pipeline + // would even be consulted. + val outcome = CaptureFreeze.freezeCurrentFrame() + + assertEquals(FreezeOutcome.AlreadyFrozen, outcome) + assertEquals(pinned, CaptureStateHolder.frozenFrame.value) + } +} diff --git a/app/src/test/kotlin/com/draftlens/capture/CaptureStateHolderTest.kt b/app/src/test/kotlin/com/draftlens/capture/CaptureStateHolderTest.kt index 8a883c0..85a3790 100644 --- a/app/src/test/kotlin/com/draftlens/capture/CaptureStateHolderTest.kt +++ b/app/src/test/kotlin/com/draftlens/capture/CaptureStateHolderTest.kt @@ -4,6 +4,7 @@ import com.draftlens.models.CaptureStatus import com.draftlens.vision.model.FrameRotation import com.draftlens.vision.testCapturedFrame import org.junit.Assert.* +import org.junit.After import org.junit.Before import org.junit.Test @@ -13,6 +14,10 @@ class CaptureStateHolderTest { CaptureStateHolder.reset() } + @After fun clearHolder() { + CaptureStateHolder.reset() + } + @Test fun `termination without error returns to IDLE and clears permission`() { CaptureStateHolder.setStatus(CaptureStatus.AWAITING_PERMISSION) CaptureStateHolder.setStatus(CaptureStatus.STARTING) @@ -221,4 +226,48 @@ class CaptureStateHolderTest { assertNull(state.lastFrameWallClockMs) assertNull(CaptureStateHolder.latestFrame.value) } + + @Test fun `a frozen frame survives a capture stop`() { + CaptureStateHolder.beginCapture(1080, 2400, 440) + val frozen = testCapturedFrame() + CaptureStateHolder.setFrozenFrame(frozen) + + CaptureStateHolder.terminateSession(forceIdle = true) + + assertEquals("the independently owned snapshot must survive the stop", frozen, CaptureStateHolder.frozenFrame.value) + } + + @Test fun `a frozen frame survives an error termination`() { + CaptureStateHolder.beginCapture(1080, 2400, 440) + val frozen = testCapturedFrame() + CaptureStateHolder.setFrozenFrame(frozen) + + CaptureStateHolder.terminateSession("consent revoked") + + assertEquals(frozen, CaptureStateHolder.frozenFrame.value) + } + + @Test fun `a new capture session drops the frozen frame`() { + CaptureStateHolder.setFrozenFrame(testCapturedFrame()) + + CaptureStateHolder.beginCapture(1080, 2400, 440) + + assertNull(CaptureStateHolder.frozenFrame.value) + } + + @Test fun `clearing the frozen frame releases it`() { + CaptureStateHolder.setFrozenFrame(testCapturedFrame()) + + CaptureStateHolder.clearFrozenFrame() + + assertNull(CaptureStateHolder.frozenFrame.value) + } + + @Test fun `reset clears the frozen frame`() { + CaptureStateHolder.setFrozenFrame(testCapturedFrame()) + + CaptureStateHolder.reset() + + assertNull(CaptureStateHolder.frozenFrame.value) + } } diff --git a/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt b/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt index 898324c..74437fb 100644 --- a/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt +++ b/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt @@ -94,6 +94,29 @@ class CaptureVisionFrameSourceTest { assertNull(cleared) } + @Test fun `adapter exposes the capture-side frozen frame`() { + val frames = MutableStateFlow(testCapturedFrame()) + val capture = MutableStateFlow(CaptureSessionState(status = CaptureStatus.CAPTURING)) + val frozen = MutableStateFlow(null) + val adapter = CaptureVisionFrameSource(frames, capture, scope, frozen) + + frozen.value = testCapturedFrame() + + val delivered = runBlocking { + withTimeout(AWAIT_TIMEOUT_MS) { adapter.frozenFrame.first { it != null } } + } + assertEquals("test-frame", delivered.id.value) + } + + @Test fun `freezing without an active capture reports false and resume is safe`() { + CaptureStateHolder.reset() + val adapter = harness().adapter + + assertEquals(false, adapter.freeze()) + adapter.resume() + assertNull(adapter.frozenFrame.value) + } + private companion object { const val AWAIT_TIMEOUT_MS = 5_000L } diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 3309b0d..27dbe58 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -53,10 +53,27 @@ class VisionLabViewModelTest { override val frames: StateFlow = _frames.asStateFlow() private val _state = MutableStateFlow(VisionSourceState.CAPTURING) override val state: StateFlow = _state.asStateFlow() + private val _frozenFrame = MutableStateFlow(null) + override val frozenFrame: StateFlow = _frozenFrame.asStateFlow() fun emit(frame: OriginalFrame?) { _frames.value = frame } + + /** Simulates a capture-side freeze command (e.g. from the notification). */ + fun emitFrozen(frame: OriginalFrame?) { + _frozenFrame.value = frame + } + + override fun freeze(): Boolean { + val frame = _frames.value ?: return false + _frozenFrame.value = frame + return true + } + + override fun resume() { + _frozenFrame.value = null + } } private class FakeCalibrationRepository( @@ -137,9 +154,9 @@ class VisionLabViewModelTest { // endregion - private fun originalFrame(id: String, width: Int = 4, height: Int = 2) = OriginalFrame( + private fun originalFrame(id: String, width: Int = 16, height: Int = 16) = OriginalFrame( id = FrameId(id), - payload = PixelArrayFramePayload(width, height, IntArray(width * height) { it }), + payload = PixelArrayFramePayload(width, height, IntArray(width * height) { (it * 137) % 251 }), sourceWidth = width, sourceHeight = height, displayRotation = FrameRotation.ROTATION_0, @@ -509,6 +526,45 @@ class VisionLabViewModelTest { assertEquals("first", vm.uiState.value.displayFrame?.id?.value) } + @Test fun `a capture-side freeze pins the frame while live frames keep arriving`() = runBlocking { + val source = FakeVisionFrameSource(originalFrame("game")) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) + + // Simulates the notification action: the capture side pins the latest game frame. + source.emitFrozen(originalFrame("game")) + // Newer frames (the DraftLens UI itself) must never replace the pinned frame. + source.emit(originalFrame("draftlens-self")) + + assertEquals("game", vm.uiState.value.frozenFrame?.id?.value) + assertEquals("game", vm.uiState.value.displayFrame?.id?.value) + assertEquals("draftlens-self", vm.uiState.value.liveFrame?.id?.value) + assertTrue(vm.uiState.value.isFrozen) + } + + @Test fun `resume from the ui reveals live frames again after a capture-side freeze`() = runBlocking { + val source = FakeVisionFrameSource(originalFrame("game")) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) + + source.emitFrozen(originalFrame("game")) + source.emit(originalFrame("new-live")) + + vm.resume() + + withTimeout(10_000) { vm.uiState.first { it.frozenFrame == null } } + assertEquals("new-live", vm.uiState.value.displayFrame?.id?.value) + assertFalse(vm.uiState.value.isFrozen) + } + + @Test fun `freeze without a captured frame reports a message instead of pinning nothing`() = runBlocking { + val source = FakeVisionFrameSource(null) + val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) + + vm.freeze() + + assertNull(vm.uiState.value.frozenFrame) + assertTrue(vm.uiState.value.message!!.contains("No captured frame")) + } + @Test fun `clear releases the pinned frame and the processed frame`() = runBlocking { val source = FakeVisionFrameSource(originalFrame("first")) val vm = VisionLabViewModel(source, EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = heroCatalog(), heroRecognitionStore = heroStore()) @@ -721,8 +777,7 @@ class VisionLabViewModelTest { routes[url] ?: com.draftlens.vision.heroes.HeroHttpResult.Failure("no route", 404) } - private val fakePng = byteArrayOf(0x89.toByte(), 'P'.code.toByte(), 'N'.code.toByte(), 'G'.code.toByte(), - 0x0D, 0x0A, 0x1A, 0x0A) + ByteArray(64) { 0x01 } + private val fakePng = com.draftlens.vision.testRealPng() private fun gradient(size: Int = 32): IntArray = IntArray(size * size) { i -> @@ -749,19 +804,35 @@ class VisionLabViewModelTest { com.draftlens.vision.heroes.HeroHttpResult.Failure("offline", 503), ) - private fun recognitionVm(referencePixels: IntArray = gradient()): VisionLabViewModel { + private fun recognitionVm( + frame: OriginalFrame? = null, + referencePixels: IntArray = gradient(16), + recognitionStore: com.draftlens.datastore.HeroRecognitionStore = heroStore(), + ): VisionLabViewModel { val repo = com.draftlens.vision.heroes.FileHeroCatalogRepository( java.nio.file.Files.createTempDirectory("hero-catalog").toFile(), java.nio.file.Files.createTempDirectory("hero-assets").toFile(), RoutesTransport(happyRoutes()), ) + // A textured frame matching the injected reference, so matching is deterministic. + val actualFrame = frame ?: OriginalFrame( + id = FrameId("live"), + payload = PixelArrayFramePayload(16, 16, gradient(16)), + sourceWidth = 16, + sourceHeight = 16, + displayRotation = FrameRotation.ROTATION_0, + rotation = FrameRotation.ROTATION_0, + capturedAtElapsedRealtimeMs = 10_000, + wallClockTimestampMs = 1_700_000_000_000, + captureLatencyMs = 4, + ) return VisionLabViewModel( - FakeVisionFrameSource(originalFrame("live")), + FakeVisionFrameSource(actualFrame), EchoPreprocessor(), FakeCalibrationRepository(), heroCatalog = repo, - heroRecognitionStore = heroStore(), - referenceLoader = { _, entry -> com.draftlens.vision.heroes.HeroReference(entry, referencePixels, 32, 32) }, + heroRecognitionStore = recognitionStore, + referenceLoader = { _, entry -> com.draftlens.vision.heroes.HeroReference(entry, referencePixels, 16, 16) }, ) } @@ -834,6 +905,126 @@ class VisionLabViewModelTest { assertTrue(vm.searchCatalog("zzz-nobody").isEmpty()) } + private fun inverseGradient(size: Int = 16): IntArray = + IntArray(size * size) { gradient(size)[it].let { p -> + 0xFF000000.toInt() or + (255 - ((p shr 16) and 0xFF)) or + ((255 - ((p shr 8) and 0xFF)) shl 8) or + (255 - (p and 0xFF)) + } } + + private fun storeAt(file: java.io.File): com.draftlens.datastore.HeroRecognitionStore = + com.draftlens.datastore.HeroRecognitionStore( + androidx.datastore.preferences.core.PreferenceDataStoreFactory.create( + scope = kotlinx.coroutines.CoroutineScope(kotlinx.coroutines.Dispatchers.IO + kotlinx.coroutines.SupervisorJob()), + ) { file }, + ) + + @Test fun `a uniform gray crop produces InsufficientVisualInformation instead of candidates`() = runBlocking { + val blank = OriginalFrame( + id = FrameId("blank"), + payload = PixelArrayFramePayload(16, 16, IntArray(16 * 16) { 0xFF808080.toInt() }), + sourceWidth = 16, + sourceHeight = 16, + displayRotation = FrameRotation.ROTATION_0, + rotation = FrameRotation.ROTATION_0, + capturedAtElapsedRealtimeMs = 10_000, + wallClockTimestampMs = 1_700_000_000_000, + captureLatencyMs = 4, + ) + val vm = recognitionVm(frame = blank) + vm.syncHeroCatalog() + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) + val slotId = vm.uiState.value.slots.single().id + + vm.identifySelectedSlot() + eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.INSUFFICIENT_INFO } + + val slot = vm.uiState.value.heroRecognition[slotId]!! + assertEquals(HeroSlotPhase.INSUFFICIENT_INFO, slot.phase) + assertTrue("candidates must never appear for a blank crop", slot.candidates.isEmpty()) + assertTrue( + "message was: ${slot.message}", + slot.message!!.contains("Not enough visual information"), + ) + } + + @Test fun `a textured crop matching nothing keeps a distinct NoMatch state`() = runBlocking { + val vm = recognitionVm(referencePixels = inverseGradient(16)) + vm.syncHeroCatalog() + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) + val slotId = vm.uiState.value.slots.single().id + + vm.identifySelectedSlot() + eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.NO_MATCH } + + val slot = vm.uiState.value.heroRecognition[slotId]!! + assertEquals(HeroSlotPhase.NO_MATCH, slot.phase) + assertNotNull(slot.bestSimilarity) + } + + @Test fun `a confirmation that cannot be persisted is reported and never shown as successful`() = runBlocking { + val dir = java.nio.file.Files.createTempDirectory("hero-confirmations").toFile() + val store = storeAt(java.io.File(dir, "confirmations.preferences_pb")) + val vm = recognitionVm(recognitionStore = store) + vm.syncHeroCatalog() + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) + val slotId = vm.uiState.value.slots.single().id + vm.identifySelectedSlot() + eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } + + // Sabotage the store: remove the directory the preferences document lives in. + dir.deleteRecursively() + vm.confirmHero(slotId, "miya") + eventually(5_000) { vm.uiState.value.message?.contains("could not be saved") == true } + + assertEquals(HeroSlotPhase.CANDIDATES, vm.uiState.value.heroRecognition[slotId]?.phase) + } + + @Test fun `a confirmation that cannot be cleared is reported and never removed silently`() = runBlocking { + val dir = java.nio.file.Files.createTempDirectory("hero-confirmations").toFile() + val store = storeAt(java.io.File(dir, "confirmations.preferences_pb")) + val vm = recognitionVm(recognitionStore = store) + vm.syncHeroCatalog() + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) + val slotId = vm.uiState.value.slots.single().id + vm.identifySelectedSlot() + eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } + vm.confirmHero(slotId, "miya") + eventually(5_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CONFIRMED } + + // Sabotage the store, then try to clear. + dir.deleteRecursively() + vm.clearHeroConfirmation(slotId) + eventually(5_000) { vm.uiState.value.message?.contains("could not be cleared") == true } + + assertEquals(HeroSlotPhase.CONFIRMED, vm.uiState.value.heroRecognition[slotId]?.phase) + } + + @Test fun `correcting a candidate directly works without clearing first`() = runBlocking { + val vm = recognitionVm() + vm.syncHeroCatalog() + eventually(10_000) { vm.uiState.value.heroCatalog is com.draftlens.vision.heroes.HeroCatalogState.Ready } + vm.createSlot(NormalizedRect(0f, 0f, 1f, 1f)) + val slotId = vm.uiState.value.slots.single().id + vm.identifySelectedSlot() + eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } + vm.confirmHero(slotId, "miya") + eventually(5_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CONFIRMED } + + // Correct the confirmation directly; the user must not clear or re-identify first. + vm.confirmHero(slotId, "layla") + eventually(5_000) { vm.uiState.value.heroRecognition[slotId]?.let { it.phase == HeroSlotPhase.CONFIRMED && it.message == "Layla" } == true } + + val state = vm.uiState.value.heroRecognition[slotId]!! + assertEquals(HeroSlotPhase.CONFIRMED, state.phase) + assertEquals("Layla", state.message) + } + // endregion companion object { diff --git a/app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt b/app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt index 6ceed97..d6e8790 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/FileHeroCatalogRepositoryTest.kt @@ -6,21 +6,18 @@ import com.draftlens.vision.heroes.HeroCatalogState import com.draftlens.vision.heroes.HeroHttpResponse import com.draftlens.vision.heroes.HeroHttpResult import com.draftlens.vision.heroes.HeroId +import com.draftlens.vision.testRealPng import kotlinx.coroutines.runBlocking import org.junit.Assert.assertEquals import org.junit.Assert.assertNotNull import org.junit.Assert.assertNull import org.junit.Assert.assertTrue +import org.junit.Assert.assertFalse import org.junit.Test import java.io.File import java.nio.file.Files -private const val PNG_MAGIC = 0x89.toByte() - -private fun fakePng(): ByteArray { - val header = byteArrayOf(PNG_MAGIC, 'P'.code.toByte(), 'N'.code.toByte(), 'G'.code.toByte(), 0x0D, 0x0A, 0x1A, 0x0A) - return header + ByteArray(64) { 0x01 } -} +private fun fakePng(): ByteArray = testRealPng() private fun listRoute(): String = "https://arena.fastapicloud.dev/api/heroes?size=100&index=1&lang=en" @@ -160,4 +157,196 @@ class FileHeroCatalogRepositoryTest { assertTrue(repo.state.value is HeroCatalogState.Error) assertNull(repo.snapshot()) } + + private class CountingTransport( + var routes: Map, + ) : com.draftlens.vision.heroes.HeroCatalogTransport { + val requested = mutableListOf() + override fun get(url: String, maxBytes: Long, accept: String?): HeroHttpResult { + requested += url + return routes[url] ?: HeroHttpResult.Failure("no route for $url", 404) + } + } + + private fun greenPng(): ByteArray = com.draftlens.vision.testRealGreenPng() + + private fun listBodyWithHeads(miya: String, layla: String): String = """ + {"code":0,"message":"OK","data":{"records":[ + {"data":{"hero":{"data":{"head":"$miya","name":"Miya","smallmap":"https://wsrv.nl/?url=miya.map"}},"hero_id":1}}, + {"data":{"hero":{"data":{"head":"$layla","name":"Layla","smallmap":"https://wsrv.nl/?url=layla.map"}},"hero_id":2}} + ],"total":2}} + """.trimIndent() + + @Test fun `an unchanged catalog does not re-download a valid cached asset`() = runBlocking { + val (data, cache) = tempDirs() + val transport = CountingTransport(syncRoutes()) + val repo = FileHeroCatalogRepository(data, cache, transport) { 1234L } + repo.refresh() + val downloadsAfterFirstSync = transport.requested.count { it.contains(".head") } + assertEquals(2, downloadsAfterFirstSync) + + repo.refresh() + + assertEquals(2, transport.requested.count { it.contains(".head") }) + } + + @Test fun `a changed asset url forces a re-download and the new asset is served`() = runBlocking { + val (data, cache) = tempDirs() + val transport = CountingTransport(syncRoutes()) + val repo = FileHeroCatalogRepository(data, cache, transport) { 1234L } + repo.refresh() + + transport.routes = mapOf( + listRoute() to HeroHttpResult.Success(HeroHttpResponse( + listBodyWithHeads("https://wsrv.nl/?url=miya2.head", "https://wsrv.nl/?url=layla2.head").toByteArray(), + "application/json", + )), + "https://wsrv.nl/?url=miya2.head" to HeroHttpResult.Success(HeroHttpResponse(greenPng(), "image/png")), + "https://wsrv.nl/?url=layla2.head" to HeroHttpResult.Success(HeroHttpResponse(greenPng(), "image/png")), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("offline", 503), + ) + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertTrue(transport.requested.contains("https://wsrv.nl/?url=miya2.head")) + assertEquals( + greenPng().toList(), + repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)!!.readBytes().toList(), + ) + } + + @Test fun `an asset with a changed url is not served from the old cache when the refresh fails`() = runBlocking { + val (data, cache) = tempDirs() + val transport = CountingTransport(syncRoutes()) + val repo = FileHeroCatalogRepository(data, cache, transport) { 1234L } + repo.refresh() + + transport.routes = mapOf( + listRoute() to HeroHttpResult.Success(HeroHttpResponse( + listBodyWithHeads("https://wsrv.nl/?url=miya2.head", "https://wsrv.nl/?url=layla2.head").toByteArray(), + "application/json", + )), + // No route for miya2.head: its download fails. Layla still refreshes fine. + "https://wsrv.nl/?url=layla2.head" to HeroHttpResult.Success(HeroHttpResponse(greenPng(), "image/png")), + "https://mlbbdex.com/api/v1/heroes" to HeroHttpResult.Failure("offline", 503), + ) + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertTrue(result.getOrThrow().assetFailures.contains("Miya")) + // The previous asset must not be exposed under the changed URL. + assertNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `a valid cached asset is still served when the network dies and the url is unchanged`() = runBlocking { + val (data, cache) = tempDirs() + val transport = CountingTransport(syncRoutes()) + val repo = FileHeroCatalogRepository(data, cache, transport) { 1234L } + repo.refresh() + + transport.routes = emptyMap() + assertTrue(repo.refresh().isFailure) + + assertNotNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `a corrupted cached asset is not served`() = runBlocking { + val (data, cache) = tempDirs() + val repo = FileHeroCatalogRepository(data, cache, CountingTransport(syncRoutes())) { 1234L } + repo.refresh() + val file = File(cache, "hero-assets/miya-head.img") + file.writeBytes("tampered".toByteArray()) + + assertNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `a forged url binding in the metadata blocks the asset`() = runBlocking { + val (data, cache) = tempDirs() + val repo = FileHeroCatalogRepository(data, cache, CountingTransport(syncRoutes())) { 1234L } + repo.refresh() + val meta = File(cache, "hero-assets/miya-head.img.meta") + meta.writeText(meta.readText().replace("miya.head", "attacker.head")) + + assertNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `a corrupt cache is re-downloaded on refresh`() = runBlocking { + val (data, cache) = tempDirs() + val transport = CountingTransport(syncRoutes()) + val repo = FileHeroCatalogRepository(data, cache, transport) { 1234L } + repo.refresh() + val file = File(cache, "hero-assets/miya-head.img") + file.writeBytes("tampered".toByteArray()) + + repo.refresh() + + assertEquals(fakePng().toList(), file.readBytes().toList()) + assertEquals(2, transport.requested.count { it == "https://wsrv.nl/?url=miya.head" }) + } + + @Test fun `a missing metadata sidecar forces a re-download`() = runBlocking { + val (data, cache) = tempDirs() + val transport = CountingTransport(syncRoutes()) + val repo = FileHeroCatalogRepository(data, cache, transport) { 1234L } + repo.refresh() + assertTrue(File(cache, "hero-assets/miya-head.img.meta").delete()) + + repo.refresh() + + assertEquals(2, transport.requested.count { it == "https://wsrv.nl/?url=miya.head" }) + } + + @Test fun `an oversized cached asset is rejected at serve time`() = runBlocking { + val (data, cache) = tempDirs() + val repo = FileHeroCatalogRepository(data, cache, CountingTransport(syncRoutes())) { 1234L } + repo.refresh() + val file = File(cache, "hero-assets/miya-head.img") + file.writeBytes(fakePng() + ByteArray(5 * 1024 * 1024)) + + assertNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `a truncated png is rejected at download time`() = runBlocking { + val (data, cache) = tempDirs() + val truncated = fakePng().copyOf(30) + val repo = FileHeroCatalogRepository( + data, cache, CountingTransport(syncRoutes(pngBytes = truncated)), + ) { 1234L } + + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertEquals(listOf("Miya", "Layla"), result.getOrThrow().assetFailures) + assertNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `an html error page with a png signature is rejected`() = runBlocking { + val (data, cache) = tempDirs() + val html = byteArrayOf( + 0x89.toByte(), 'P'.code.toByte(), 'N'.code.toByte(), 'G'.code.toByte(), + 0x0D, 0x0A, 0x1A, 0x0A, + ) + "not an image".toByteArray() + val repo = FileHeroCatalogRepository( + data, cache, CountingTransport(syncRoutes(pngBytes = html)), + ) { 1234L } + + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertEquals(listOf("Miya", "Layla"), result.getOrThrow().assetFailures) + assertNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } + + @Test fun `a missing content type is accepted`() = runBlocking { + val (data, cache) = tempDirs() + val repo = FileHeroCatalogRepository( + data, cache, CountingTransport(syncRoutes(contentType = null)), + ) { 1234L } + + val result = repo.refresh() + + assertTrue(result.isSuccess) + assertTrue(result.getOrThrow().assetFailures.isEmpty()) + assertNotNull(repo.assetFile(HeroId("miya"), HeroAssetVariant.HEAD)) + } } diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroRedirectPolicyTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroRedirectPolicyTest.kt new file mode 100644 index 0000000..695c0d8 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroRedirectPolicyTest.kt @@ -0,0 +1,46 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.HeroRedirectPolicy +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +class HeroRedirectPolicyTest { + + @Test fun `an absolute https redirect target is accepted`() { + assertEquals( + "https://cdn.example.com/hero.png", + HeroRedirectPolicy.resolveTarget("https://a.example.com/", "https://cdn.example.com/hero.png"), + ) + } + + @Test fun `a relative redirect target is resolved against the current url`() { + assertEquals( + "https://a.example.com/dir/asset.png", + HeroRedirectPolicy.resolveTarget("https://a.example.com/dir/list", "asset.png"), + ) + } + + @Test fun `an https to http downgrade redirect is refused`() { + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", "http://a.example.com/hero.png")) + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", "http://127.0.0.1:8080/x")) + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", "http://169.254.169.254/latest")) + } + + @Test fun `non-http schemes are refused`() { + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", "ftp://a.example.com/x")) + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", "file:///etc/passwd")) + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", "javascript:alert(1)")) + } + + @Test fun `missing or blank locations are refused`() { + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", null)) + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", "")) + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", " ")) + } + + @Test fun `an over-long redirect target is refused`() { + val huge = "https://a.example.com/" + "a".repeat(4096) + assertNull(HeroRedirectPolicy.resolveTarget("https://a.example.com/", huge)) + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt b/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt index 1e82079..6cc4a85 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/SlotImageExtractorTest.kt @@ -55,39 +55,104 @@ class SlotImageExtractorTest { assertEquals(2, wide.getOrThrow().payload.height) } - @Test fun `the extracted image matches the oriented appearance for every rotation`() { - // Asymmetric image so orientation mistakes are visible. - val pixels = intArrayOf( - 0xFF000000.toInt(), 0xFF000001.toInt(), 0xFF000002.toInt(), 0xFF000003.toInt(), - 0xFF000010.toInt(), 0xFF000011.toInt(), 0xFF000012.toInt(), 0xFF000013.toInt(), - 0xFF000100.toInt(), 0xFF000101.toInt(), 0xFF000102.toInt(), 0xFF000103.toInt(), - 0xFF001000.toInt(), 0xFF001001.toInt(), 0xFF001002.toInt(), 0xFF001003.toInt(), + /** + * Asymmetric 3x2 source for rotation tests (not square on purpose): + * row 0: 10 11 12 + * row 1: 20 21 22 + * Independently calculated by hand against FrameTransform.sourceToOriented(). + */ + private val asymmetricPixels = intArrayOf(10, 11, 12, 20, 21, 22) + + @Test fun `the extracted image matches the sourceToOriented convention for every rotation`() { + val rect = NormalizedRect(0f, 0f, 1f, 1f) // full frame; rotation is translation-invariant + val expected = mapOf( + FrameRotation.ROTATION_0 to Triple(3, 2, intArrayOf(10, 11, 12, 20, 21, 22)), + FrameRotation.ROTATION_90 to Triple(2, 3, intArrayOf(20, 10, 21, 11, 22, 12)), + FrameRotation.ROTATION_180 to Triple(3, 2, intArrayOf(22, 21, 20, 12, 11, 10)), + FrameRotation.ROTATION_270 to Triple(2, 3, intArrayOf(12, 22, 11, 21, 10, 20)), ) - val rect = NormalizedRect(0f, 0f, 0.5f, 0.5f) // source top-left quadrant for (rotation in FrameRotation.entries) { - val source = frame(4, 4, rotation, pixels.copyOf()) + val source = frame(3, 2, rotation, asymmetricPixels.copyOf()) val result = PixelSlotImageExtractor.extract(source, slot(rect)) assertTrue("rotation $rotation must succeed", result.isSuccess) val image = result.getOrThrow().payload as PixelArrayFramePayload - assertEquals("dims[$rotation]", 2, image.width) - assertEquals("dims[$rotation]", 2, image.height) - val content = image.pixels().toList() - assertEquals("upright[$rotation]", expectedQuadrant(rotation), content) + val (expectedWidth, expectedHeight, expectedPixels) = expected.getValue(rotation) + assertEquals("dims[$rotation]", expectedWidth, image.width) + assertEquals("dims[$rotation]", expectedHeight, image.height) + assertEquals("pixels[$rotation]", expectedPixels.toList(), image.pixels().toList()) } } - private fun expectedQuadrant(rotation: FrameRotation): List { - val a = 0xFF000000.toInt() - val b = 0xFF000001.toInt() - val e = 0xFF000010.toInt() - val f = 0xFF000011.toInt() - // Axis-aligned 2x2 crop of the source top-left quadrant, upright-rotated per frame rotation. - return when (rotation) { - FrameRotation.ROTATION_0 -> listOf(a, b, e, f) - FrameRotation.ROTATION_90 -> listOf(e, f, a, b) - FrameRotation.ROTATION_180 -> listOf(f, e, b, a) - FrameRotation.ROTATION_270 -> listOf(b, a, f, e) + /** + * 5x3 frame; crop covers columns 1..3, all rows (top and bottom edge, half the width): + * row 0: 01 02 + * row 1: 11 12 + * row 2: 21 22 (crop is 2 wide x 3 tall) + * Expected values computed by hand from FrameTransform.sourceToOriented(). + */ + private val edgeFramePixels = intArrayOf( + 0, 1, 2, 3, 4, + 10, 11, 12, 13, 14, + 20, 21, 22, 23, 24, + ) + + @Test fun `a crop touching the top and bottom edges extracts the edge columns exactly`() { + val rect = NormalizedRect(0.2f, 0f, 0.6f, 1f) // left=1px, right=3px of 5 + val expected = mapOf( + FrameRotation.ROTATION_0 to Triple(2, 3, intArrayOf(1, 2, 11, 12, 21, 22)), + FrameRotation.ROTATION_90 to Triple(3, 2, intArrayOf(21, 11, 1, 22, 12, 2)), + FrameRotation.ROTATION_180 to Triple(2, 3, intArrayOf(22, 21, 12, 11, 2, 1)), + FrameRotation.ROTATION_270 to Triple(3, 2, intArrayOf(2, 12, 22, 1, 11, 21)), + ) + for (rotation in FrameRotation.entries) { + val source = frame(5, 3, rotation, edgeFramePixels.copyOf()) + val result = PixelSlotImageExtractor.extract(source, slot(rect)) + + assertTrue("rotation $rotation must succeed", result.isSuccess) + val image = result.getOrThrow().payload as PixelArrayFramePayload + val (expectedWidth, expectedHeight, expectedPixels) = expected.getValue(rotation) + assertEquals("dims[$rotation]", expectedWidth, image.width) + assertEquals("dims[$rotation]", expectedHeight, image.height) + assertEquals("pixels[$rotation]", expectedPixels.toList(), image.pixels().toList()) + } + } + + /** + * Same 5x3 frame; crop covers rows 1..3, columns 3..5 (bottom and right edges): + * row 1: 13 14 + * row 2: 23 24 (crop is 2 wide x 2 tall) + */ + @Test fun `a crop touching the bottom and right edges extracts exactly those pixels`() { + val rect = NormalizedRect(0.6f, 1f / 3f, 1f, 1f) // left=3px, top=1px + val expected = mapOf( + FrameRotation.ROTATION_0 to Triple(2, 2, intArrayOf(13, 14, 23, 24)), + FrameRotation.ROTATION_90 to Triple(2, 2, intArrayOf(23, 13, 24, 14)), + FrameRotation.ROTATION_180 to Triple(2, 2, intArrayOf(24, 23, 14, 13)), + FrameRotation.ROTATION_270 to Triple(2, 2, intArrayOf(14, 24, 13, 23)), + ) + for (rotation in FrameRotation.entries) { + val source = frame(5, 3, rotation, edgeFramePixels.copyOf()) + val result = PixelSlotImageExtractor.extract(source, slot(rect)) + + assertTrue("rotation $rotation must succeed", result.isSuccess) + val image = result.getOrThrow().payload as PixelArrayFramePayload + val (expectedWidth, expectedHeight, expectedPixels) = expected.getValue(rotation) + assertEquals("dims[$rotation]", expectedWidth, image.width) + assertEquals("dims[$rotation]", expectedHeight, image.height) + assertEquals("pixels[$rotation]", expectedPixels.toList(), image.pixels().toList()) + } + } + + @Test fun `extracting does not mutate the source pixel array for any rotation`() { + val rect = NormalizedRect(0.2f, 0f, 0.6f, 1f) + for (rotation in FrameRotation.entries) { + val pixels = edgeFramePixels.copyOf() + val source = frame(5, 3, rotation, pixels) + + PixelSlotImageExtractor.extract(source, slot(rect)) + + assertEquals("source mutated for $rotation", edgeFramePixels.toList(), pixels.toList()) } } diff --git a/app/src/test/kotlin/com/draftlens/heroes/StructuralImageValidatorTest.kt b/app/src/test/kotlin/com/draftlens/heroes/StructuralImageValidatorTest.kt new file mode 100644 index 0000000..74f4a33 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/StructuralImageValidatorTest.kt @@ -0,0 +1,59 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.StructuralImageValidator +import com.draftlens.vision.testRealPng +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class StructuralImageValidatorTest { + + @Test fun `a real png passes`() { + assertTrue(StructuralImageValidator.isDecodable(testRealPng())) + } + + @Test fun `a truncated png fails`() { + assertFalse(StructuralImageValidator.isDecodable(testRealPng().copyOf(30))) + assertFalse(StructuralImageValidator.isDecodable(testRealPng().copyOf(8))) + } + + @Test fun `a png with a corrupted crc byte fails`() { + val broken = testRealPng().copyOf() + broken[20] = (broken[20] + 1).toByte() + + assertFalse(StructuralImageValidator.isDecodable(broken)) + } + + @Test fun `an html document is not an image`() { + assertFalse(StructuralImageValidator.isDecodable("502 Bad Gateway".toByteArray())) + } + + @Test fun `a minimal well-formed gif passes`() { + val bytes = "GIF89a".toByteArray() + ByteArray(14) { 0x01 } + byteArrayOf(0x3B) + assertTrue(StructuralImageValidator.isDecodable(bytes)) + } + + @Test fun `a minimal well-formed webp passes`() { + val bytes = "RIFF".toByteArray() + + byteArrayOf(16, 0, 0, 0) + // little-endian RIFF size + "WEBP".toByteArray() + + "VP8L".toByteArray() + + byteArrayOf(4, 0, 0, 0) + // little-endian chunk size + byteArrayOf(1, 2, 3, 4) + assertTrue(StructuralImageValidator.isDecodable(bytes)) + } + + @Test fun `a webp with an out-of-bounds chunk fails`() { + val bytes = "RIFF".toByteArray() + + byteArrayOf(100, 0, 0, 0) + + "WEBP".toByteArray() + + "VP8L".toByteArray() + + byteArrayOf(99, 0, 0, 0) + + byteArrayOf(1, 2) + assertFalse(StructuralImageValidator.isDecodable(bytes)) + } + + @Test fun `random bytes are not an image`() { + assertFalse(StructuralImageValidator.isDecodable(ByteArray(64) { (it * 67 + 3).toByte() })) + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/VisualInformationTest.kt b/app/src/test/kotlin/com/draftlens/heroes/VisualInformationTest.kt new file mode 100644 index 0000000..f8b9e39 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/VisualInformationTest.kt @@ -0,0 +1,195 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.HeroCatalogEntry +import com.draftlens.vision.heroes.HeroCatalogSource +import com.draftlens.vision.heroes.HeroIdentifyOutcome +import com.draftlens.vision.heroes.HeroId +import com.draftlens.vision.heroes.HeroReference +import com.draftlens.vision.heroes.HeroRecognition +import com.draftlens.vision.heroes.VisualInformationAnalyzer +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +private fun uniform(width: Int, height: Int, value: Int) = IntArray(width * height) { value } + +private fun uniformColor(width: Int, height: Int, r: Int, g: Int, b: Int) = + IntArray(width * height) { 0xFF000000.toInt() or (r shl 16) or (g shl 8) or b } + +private fun nearUniformNoise(width: Int, height: Int, base: Int, amplitude: Int) = + IntArray(width * height) { i -> + val wobble = ((i * 37 + 11) % (2 * amplitude + 1)) - amplitude + 0xFF000000.toInt() or ((base + wobble).coerceIn(0, 255)) + } + +/** Deterministic 0..255 horizontal gradient; the same fixture the matcher tests use. */ +private fun gradient(size: Int, invert: Boolean = false): IntArray = + IntArray(size * size) { i -> + val x = i % size + val y = i / size + val span = (x * 255) / (size - 1) + val v = if (invert) 255 - span else span + 0xFF000000.toInt() or v or (v shl 8) or (v shl 16) + } + +/** High-contrast 4x4-block checkerboard: dense edges, two tones. */ +private fun blockCheckerboard(size: Int, low: Int = 40, high: Int = 210): IntArray = + IntArray(size * size) { i -> + val x = i % size + val y = i / size + val v = if (((x / 4) + (y / 4)) % 2 == 0) low else high + 0xFF000000.toInt() or v or (v shl 8) or (v shl 16) + } + +/** Deterministic pseudo-random detail: many edges, high variance, no pattern reuse. */ +private fun pseudoRandomDetail(size: Int): IntArray = + IntArray(size * size) { i -> + val x = i % size + val y = i / size + val v = (x * 31 + y * 17 + (x * y) % 7 * 29 + y * 13) % 200 + 30 + 0xFF000000.toInt() or v or (v shl 8) or (v shl 16) + } + +private fun reference(id: String, pixels: IntArray, size: Int): HeroReference { + val entry = HeroCatalogEntry( + id = HeroId(id), + canonicalName = id.replaceFirstChar { it.uppercaseChar() }, + aliases = emptyList(), + roles = emptyList(), + lanes = emptyList(), + specialties = emptyList(), + assets = emptyList(), + externalIds = emptyMap(), + source = HeroCatalogSource.RONE_ARENA, + ) + return HeroReference(entry, pixels, size, size) +} + +class VisualInformationTest { + + // ---- Pure analyzer ---- + + @Test fun `uniform gray is rejected`() { + val report = VisualInformationAnalyzer.analyze(uniform(16, 16, 128), 16, 16) + assertNotNull(report.insufficientReason) + assertEquals(0f, report.luminanceVariance, 0f) + assertEquals(0f, report.edgeDensity, 0f) + assertEquals(0, report.meaningfulDifferences) + } + + @Test fun `uniform black and white are rejected`() { + assertNotNull(VisualInformationAnalyzer.analyze(uniform(16, 16, 0), 16, 16).insufficientReason) + assertNotNull(VisualInformationAnalyzer.analyze(uniform(16, 16, 255), 16, 16).insufficientReason) + } + + @Test fun `uniform saturated colors are rejected`() { + assertNotNull(VisualInformationAnalyzer.analyze(uniformColor(16, 16, 200, 30, 30), 16, 16).insufficientReason) + assertNotNull(VisualInformationAnalyzer.analyze(uniformColor(16, 16, 30, 200, 30), 16, 16).insufficientReason) + assertNotNull(VisualInformationAnalyzer.analyze(uniformColor(16, 16, 30, 30, 200), 16, 16).insufficientReason) + } + + @Test fun `near-uniform images with tiny noise are rejected`() { + assertNotNull(VisualInformationAnalyzer.analyze(nearUniformNoise(16, 16, 128, 1), 16, 16).insufficientReason) + assertNotNull(VisualInformationAnalyzer.analyze(nearUniformNoise(16, 16, 128, 3), 16, 16).insufficientReason) + } + + @Test fun `a large blank image is still rejected`() { + assertNotNull(VisualInformationAnalyzer.analyze(uniform(64, 64, 128), 64, 64).insufficientReason) + } + + @Test fun `low-resolution crops are rejected`() { + assertTrue(blockCheckerboard(8).size != 0) + assertNotNull( + "a detailed 8x8 crop is still too small to be useful", + VisualInformationAnalyzer.analyze(blockCheckerboard(8), 8, 8).insufficientReason, + ) + assertNull( + "the same image at 16x16 has enough pixels", + VisualInformationAnalyzer.analyze(blockCheckerboard(16), 16, 16).insufficientReason, + ) + } + + @Test fun `a smooth wide gradient is accepted`() { + assertNull(VisualInformationAnalyzer.analyze(gradient(32), 32, 32).insufficientReason) + } + + @Test fun `a low-contrast gradient is not automatically rejected`() { + // Luma range 110..124 across the image: low contrast but far from uniform. + val lowContrast = IntArray(32 * 32) { i -> + val x = i % 32 + val y = i / 32 + val v = 110 + (x * 14) / 31 + 0xFF000000.toInt() or v or (v shl 8) or (v shl 16) + } + assertNull(VisualInformationAnalyzer.analyze(lowContrast, 32, 32).insufficientReason) + } + + @Test fun `a low-contrast detailed image is accepted`() { + val detailed = IntArray(32 * 32) { i -> + val x = i % 32 + val y = i / 32 + val v = if (((x / 2) + (y / 2)) % 2 == 0) 110 else 120 + 0xFF000000.toInt() or v or (v shl 8) or (v shl 16) + } + assertNull(VisualInformationAnalyzer.analyze(detailed, 32, 32).insufficientReason) + } + + @Test fun `textured synthetic images are accepted`() { + assertNull(VisualInformationAnalyzer.analyze(blockCheckerboard(32), 32, 32).insufficientReason) + } + + @Test fun `realistic synthetic high-detail patterns are accepted`() { + assertNull(VisualInformationAnalyzer.analyze(pseudoRandomDetail(32), 32, 32).insufficientReason) + } + + @Test fun `a dimension mismatch is reported as insufficient, never thrown`() { + val report = VisualInformationAnalyzer.analyze(IntArray(10), 16, 16) + assertNotNull(report.insufficientReason) + } + + // ---- End-to-end through Identify ---- + + @Test fun `a uniform gray crop produces InsufficientVisualInformation, never candidates`() { + val outcome = HeroRecognition.identify( + uniformColor(16, 16, 128, 128, 128), + 16, + 16, + listOf(reference("miya", gradient(16), 16)), + ) + assertTrue( + "expected InsufficientVisualInformation, got $outcome", + outcome is HeroIdentifyOutcome.InsufficientVisualInformation, + ) + } + + @Test fun `a saturated solid-color crop produces InsufficientVisualInformation`() { + val outcome = HeroRecognition.identify( + uniformColor(16, 16, 180, 40, 200), + 16, + 16, + listOf(reference("miya", gradient(16), 16)), + ) + assertTrue(outcome is HeroIdentifyOutcome.InsufficientVisualInformation) + } + + @Test fun `blank crops are insufficient even with an empty reference set`() { + val outcome = HeroRecognition.identify(uniform(16, 16, 0), 16, 16, emptyList()) + assertTrue(outcome is HeroIdentifyOutcome.InsufficientVisualInformation) + } + + @Test fun `NoMatch is kept distinct from InsufficientVisualInformation`() { + val inverse = gradient(16, invert = true) + val outcome = HeroRecognition.identify( + gradient(16), + 16, + 16, + listOf(reference("anti", inverse, 16)), + ) + assertTrue( + "a textured crop with an alien reference must be NoMatch, not Insufficient: $outcome", + outcome is HeroIdentifyOutcome.NoMatch, + ) + } +} diff --git a/app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt b/app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt new file mode 100644 index 0000000..1b019af --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt @@ -0,0 +1,106 @@ +package com.draftlens.heroes + +import com.draftlens.vision.heroes.FileHeroCatalogRepository +import com.draftlens.vision.heroes.HeroCatalogTransport +import com.draftlens.vision.heroes.HeroHttpResult +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import java.io.File +import java.nio.file.Files + +private object EmptyTransport : HeroCatalogTransport { + override fun get(url: String, maxBytes: Long, accept: String?): HeroHttpResult = + HeroHttpResult.Failure("no network in this test", 503) +} + +/** + * Failure-path coverage for the atomic snapshot/asset write. Successful writes are covered + * end-to-end by the catalog tests; these tests focus on "never a partial target, never a + * leftover temp file". + */ +class WriteAtomicallyTest { + + private fun tempDirs(): Pair = + Files.createTempDirectory("hero-catalog").toFile() to Files.createTempDirectory("hero-assets").toFile() + + private fun repo(data: File, cache: File): FileHeroCatalogRepository = + FileHeroCatalogRepository(data, cache, EmptyTransport) { 1234L } + + @Test fun `an atomic write stores the exact bytes`() { + val (data, cache) = tempDirs() + val target = File(data, "snapshot/catalog.json") + + val ok = repo(data, cache).writeAtomically(target, "payload".toByteArray()) + + assertTrue(ok) + assertEquals("payload", target.readText()) + } + + @Test fun `a failed atomic write leaves a directory target untouched and removes the temp file`() { + val (data, cache) = tempDirs() + // The atomic move cannot replace a non-empty/occupied directory; it must fail cleanly. + val target = File(data, "catalog.json").also { it.mkdirs() } + + val ok = repo(data, cache).writeAtomically(target, "new".toByteArray()) + + assertFalse(ok) + assertTrue("the previous target must remain a directory", target.isDirectory) + assertTrue("no temp file may remain", data.listFiles()!!.none { it.name.endsWith(".tmp") }) + } + + @Test fun `a failed atomic write when the parent is a file reports failure without touching the blocker`() { + val (data, cache) = tempDirs() + val blocker = File(data, "blocker").also { it.writeText("file") } + val target = File(blocker, "catalog.json") + + val ok = repo(data, cache).writeAtomically(target, "new".toByteArray()) + + assertFalse(ok) + assertEquals("file", blocker.readText()) + assertTrue("no temp file may remain next to the blocker", data.listFiles()!!.none { it.name.endsWith(".tmp") }) + } + + @Test fun `abandoned temp siblings older than an hour are swept on init`() { + val (data, cache) = tempDirs() + val assetsDir = File(cache, "hero-assets").also { it.mkdirs() } + val now = System.currentTimeMillis() + val stale = File(assetsDir, "miya-head.img.999.tmp").also { it.writeText("x") } + val fresh = File(assetsDir, "layla-head.img.111.tmp").also { it.writeText("x") } + stale.setLastModified(now - 2 * 60 * 60 * 1000) + fresh.setLastModified(now - 60 * 1000) + + repo(data, cache) + + assertFalse("stale temp files are swept", stale.exists()) + assertTrue("fresh temp files are kept", fresh.exists()) + } + + private fun workingCatalogTransport(): HeroCatalogTransport = object : HeroCatalogTransport { + private val body = """ + {"code":0,"message":"OK","data":{"records":[ + {"data":{"hero":{"data":{"head":"https://wsrv.nl/?url=miya.head","name":"Miya","smallmap":"https://wsrv.nl/?url=miya.map"}},"hero_id":1}} + ],"total":1}} + """.trimIndent().toByteArray() + + override fun get(url: String, maxBytes: Long, accept: String?): HeroHttpResult = when { + url.startsWith("https://arena.fastapicloud.dev/api/heroes") -> + HeroHttpResult.Success(com.draftlens.vision.heroes.HeroHttpResponse(body, "application/json")) + else -> HeroHttpResult.Failure("no route for $url", 404) + } + } + + @Test fun `a failed snapshot write leaves no temp files behind`() = kotlinx.coroutines.runBlocking { + val (data, cache) = tempDirs() + // "hero-catalog" is a FILE, so the snapshot sibling temp file cannot be created. + val blocker = File(data, "hero-catalog") + blocker.writeText("blocked") + val repo = FileHeroCatalogRepository(data, cache, workingCatalogTransport()) { 1234L } + + val result = repo.refresh() + + assertTrue("a refresh against a blocked data dir must fail", result.isFailure) + assertTrue("no temp file may remain beside the blocker", data.listFiles()!!.none { it.name.endsWith(".tmp") }) + } +} diff --git a/app/src/test/kotlin/com/draftlens/models/FrameSnapshotsTest.kt b/app/src/test/kotlin/com/draftlens/models/FrameSnapshotsTest.kt new file mode 100644 index 0000000..f6dda8b --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/models/FrameSnapshotsTest.kt @@ -0,0 +1,70 @@ +package com.draftlens.models + +import com.draftlens.vision.model.FrameId +import com.draftlens.vision.model.FrameRotation +import com.draftlens.vision.model.OriginalFrame +import com.draftlens.vision.model.PixelArrayFramePayload +import org.junit.Assert.assertEquals +import org.junit.Test + +private fun frame(width: Int, height: Int, rotation: FrameRotation, pixels: IntArray) = OriginalFrame( + id = FrameId("frame-$rotation"), + payload = PixelArrayFramePayload(width, height, pixels), + sourceWidth = width, + sourceHeight = height, + displayRotation = FrameRotation.ROTATION_90, + rotation = rotation, + capturedAtElapsedRealtimeMs = 11_000, + wallClockTimestampMs = 1_700_000_000_000, + captureLatencyMs = 7, +) + +class FrameSnapshotsTest { + + @Test fun `the copy preserves dimensions rotation and timestamps exactly`() { + val pixels = IntArray(6) { it + 1 } + val source = frame(3, 2, FrameRotation.ROTATION_90, pixels) + + val copy = FrameSnapshots.copyOf(source)!! + + assertEquals(source.sourceWidth, copy.sourceWidth) + assertEquals(source.sourceHeight, copy.sourceHeight) + assertEquals(source.displayRotation, copy.displayRotation) + assertEquals(source.rotation, copy.rotation) + assertEquals(source.capturedAtElapsedRealtimeMs, copy.capturedAtElapsedRealtimeMs) + assertEquals(source.wallClockTimestampMs, copy.wallClockTimestampMs) + assertEquals(source.captureLatencyMs, copy.captureLatencyMs) + assertEquals(source.id, copy.id) + } + + @Test fun `the copy carries equal pixel values to the source`() { + val pixels = IntArray(6) { it + 1 } + val source = frame(3, 2, FrameRotation.ROTATION_0, pixels) + + val copy = FrameSnapshots.copyOf(source)!! + + val sourcePixels = (source.payload as PixelArrayFramePayload).pixels() + val copyPixels = (copy.payload as PixelArrayFramePayload).pixels() + assertEquals(sourcePixels.toList(), copyPixels.toList()) + } + + @Test fun `the copy does not change when the caller mutates the array it was built from`() { + val pixels = IntArray(4) { it + 1 } + val source = frame(2, 2, FrameRotation.ROTATION_0, pixels) + + val copy = FrameSnapshots.copyOf(source)!! + pixels[0] = -1 + + assertEquals(1, (copy.payload as PixelArrayFramePayload).pixels()[0]) + } + + @Test fun `the copy preserves capture latency metadata via the captured frame wrapper`() { + val source = frame(2, 2, FrameRotation.ROTATION_0, intArrayOf(1, 2, 3, 4)) + val captured = CapturedFrame(source, processingLatencyMs = 42) + + val copy = FrameSnapshots.copyOf(captured)!! + + assertEquals(42L, copy.processingLatencyMs) + assertEquals(1, (copy.frame.payload as PixelArrayFramePayload).pixels().first()) + } +} diff --git a/app/src/test/kotlin/com/draftlens/vision/TestFrames.kt b/app/src/test/kotlin/com/draftlens/vision/TestFrames.kt index 082f460..ba00a30 100644 --- a/app/src/test/kotlin/com/draftlens/vision/TestFrames.kt +++ b/app/src/test/kotlin/com/draftlens/vision/TestFrames.kt @@ -20,3 +20,29 @@ fun testCapturedFrame(): CapturedFrame = CapturedFrame( ), processingLatencyMs = 12, ) + +/** A real, minimal, decodable 1x1 red PNG (69 bytes). */ +fun testRealPng(): ByteArray = byteArrayOf( + 0x89.toByte(), 0x50.toByte(), 0x4E.toByte(), 0x47.toByte(), 0x0D.toByte(), 0x0A.toByte(), 0x1A.toByte(), 0x0A.toByte(), + 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x0D.toByte(), 0x49.toByte(), 0x48.toByte(), 0x44.toByte(), 0x52.toByte(), + 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x01.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x01.toByte(), + 0x08.toByte(), 0x02.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x90.toByte(), 0x77.toByte(), 0x53.toByte(), + 0xDE.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x0C.toByte(), 0x49.toByte(), 0x44.toByte(), 0x41.toByte(), + 0x54.toByte(), 0x78.toByte(), 0x9C.toByte(), 0x63.toByte(), 0xF8.toByte(), 0xCF.toByte(), 0xC0.toByte(), 0x00.toByte(), + 0x00.toByte(), 0x03.toByte(), 0x01.toByte(), 0x01.toByte(), 0x00.toByte(), 0xC9.toByte(), 0xFE.toByte(), 0x92.toByte(), + 0xEF.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x49.toByte(), 0x45.toByte(), 0x4E.toByte(), + 0x44.toByte(), 0xAE.toByte(), 0x42.toByte(), 0x60.toByte(), 0x82.toByte(), +) + +/** A real, minimal, decodable 1x1 green PNG (69 bytes); a distinct asset for cache tests. */ +fun testRealGreenPng(): ByteArray = byteArrayOf( + 0x89.toByte(), 0x50.toByte(), 0x4E.toByte(), 0x47.toByte(), 0x0D.toByte(), 0x0A.toByte(), 0x1A.toByte(), 0x0A.toByte(), + 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x0D.toByte(), 0x49.toByte(), 0x48.toByte(), 0x44.toByte(), 0x52.toByte(), + 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x01.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x01.toByte(), + 0x08.toByte(), 0x02.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x90.toByte(), 0x77.toByte(), 0x53.toByte(), + 0xDE.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x0C.toByte(), 0x49.toByte(), 0x44.toByte(), 0x41.toByte(), + 0x54.toByte(), 0x78.toByte(), 0x9C.toByte(), 0x63.toByte(), 0x60.toByte(), 0xF8.toByte(), 0xCF.toByte(), 0x00.toByte(), + 0x00.toByte(), 0x02.toByte(), 0x02.toByte(), 0x01.toByte(), 0x00.toByte(), 0x7B.toByte(), 0x09.toByte(), 0x81.toByte(), + 0x78.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x00.toByte(), 0x49.toByte(), 0x45.toByte(), 0x4E.toByte(), + 0x44.toByte(), 0xAE.toByte(), 0x42.toByte(), 0x60.toByte(), 0x82.toByte(), +) From 044ddd891ac3a5072bb87834f820ed11fc4e7c46 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:10:20 +0000 Subject: [PATCH 14/33] ci: run the instrumented suite on an emulator The instrumentation tests are deterministic (bitmap extraction, image validation, manifest checks) and need no MediaProjection session. The physical capture path stays in the manual checklist. --- .github/workflows/build.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2399d39..3e5b116 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,3 +73,37 @@ jobs: grep -E "android.permission.FOREGROUND_SERVICE(_MEDIA_PROJECTION)?" /tmp/draftlens-permissions.txt grep -E "android:foregroundServiceType.*0x00000020" /tmp/draftlens-manifest.txt grep "CaptureSessionService" /tmp/draftlens-manifest.txt + + instrumented: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '17' + cache: gradle + + - name: Install SDK packages + run: | + SDK_ROOT="${ANDROID_HOME:-/usr/local/lib/android/sdk}" + MANAGER="$(find "$SDK_ROOT/cmdline-tools" -name sdkmanager -type f | sort -V | tail -n 1)" + yes | "$MANAGER" --sdk_root="$SDK_ROOT" --licenses + "$MANAGER" --sdk_root="$SDK_ROOT" "platforms;android-35" "build-tools;35.0.0" "platform-tools" + + - name: Make gradlew executable + run: chmod +x ./gradlew || ls -la + + # The instrumented suite covers the Android bitmap extractor, the image + # validator and the capture manifest checks. It does not need MediaProjection: + # the physical capture path stays in docs/TESTING_PHYSICAL_DEVICE.md. + - name: Run instrumentation tests + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 30 + target: default + arch: x86_64 + profile: pixel + script: ./gradlew connectedDebugAndroidTest --stacktrace From 24fe770be6bb2ed89320f0e6b683714d22906cc5 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:20:15 +0000 Subject: [PATCH 15/33] docs: freeze workflow, review fixes and reporting skeleton Duplicate capture_notification_text string removed; README, Vision Lab, screen capture, Vision architecture, Sprint 1B review-fix doc, plan and todo updated to the new behavior. --- README.md | 29 ++- app/src/main/res/values/strings.xml | 3 +- docs/SCREEN_CAPTURE.md | 29 ++- docs/VISION_ARCHITECTURE.md | 64 +++-- docs/VISION_LAB.md | 8 +- docs/sprints/SPRINT_1B_HERO_RECOGNITION.md | 285 ++++++++++++++------- tasks/plan.md | 41 +++ tasks/todo.md | 18 ++ 8 files changed, 356 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index fb802e6..4e8dc3d 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,10 @@ and [docs/VISION_TESTING.md](docs/VISION_TESTING.md). - Hero recognition v0 (Sprint 1B, PR #3): hero catalog sync from public static data, explicit Identify action for the selected slot, ranked candidates with similarity scores, manual confirm/correct — **accuracy on real game screens is not measured yet** +- Freeze the game frame from the notification (Sprint 1B): tap **Freeze current frame** on the + capture notification while you are in MLBB — no need to leave the game; Vision Lab shows the + pinned game frame, not the DraftLens screen; **Resume live capture** from the notification or + in the Vision Lab ## What is deliberately NOT here (later milestones) ML hero recognition, recommendation engine, counter/synergy/role scoring, floating overlays, @@ -53,14 +57,23 @@ build of record.** See `docs/VISION_TESTING.md` for exactly what ran. 1. Install the APK from the `draftlens-debug` CI artifact on a physical device (the emulator validates compilation, not MediaProjection reliability). 2. Open the app; the **Debug Capture** tab is first. -3. Tap **Start Capture** → grant the system screen-capture consent dialog. -4. MLBB (or any app) — the debug screen shows live frames. -5. Switch to the **Vision Lab** tab. Capture keeps running while you are there. -6. Freeze a frame, drag out a slot rectangle, then edit it in the text editor below the - canvas. Press **Process Frame** to run the preprocessing engine on the selected frame. -7. Save the annotations as a calibration profile; apply it later on a matching frame. -8. **Stop Capture**, or the notification's **Stop**, or revoke consent in Settings to - exercise teardown. +3. Tap **Start Capture** → grant the system screen-capture consent dialog. When Android asks + what to capture, choose **Entire screen / full device display** — app-window capture shows + this app, not Mobile Legends, and the notification freeze workflow would be pointless. +4. Open Mobile Legends and wait for the draft screen you want to annotate. +5. Pull down the notification shade and tap **Freeze current frame** on the DraftLens capture + notification (notifications must be enabled — the app checks and tells you). +6. Switch back to **DraftLens → Vision Lab**. Capture kept running; the Vision Lab shows the + pinned MLBB frame instead of the DraftLens screen. +7. Draw a slot rectangle, then run **Identify hero** on the frozen frame. Candidates are + similarity scores, not probabilities — confirm or correct manually (the manual search is + available in every recognition state). +8. Tap **Resume live capture** in the notification, or **Resume** in the Vision Lab, to show + live frames again. No second capture session is created. +9. Press **Process Frame** to run the preprocessing engine on the selected frame. +10. Save the annotations as a calibration profile; apply it later on a matching frame. +11. **Stop Capture**, or the notification's **Stop**, or revoke consent in Settings to + exercise teardown. ## Docs - `docs/ARCHITECTURE.md` — module layout, state machine, data flow, Vision boundary diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index c67b6ac..df1e2b8 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2,11 +2,10 @@ DraftLens DraftLens capture DraftLens is capturing your screen - Tap to view debug screen + Live preview of what you are sharing Stop LIVE FROZEN - Live preview of what you are sharing Frame frozen — the Vision Lab shows the pinned frame Freeze current frame Resume live capture diff --git a/docs/SCREEN_CAPTURE.md b/docs/SCREEN_CAPTURE.md index ec9ec0e..16f9421 100644 --- a/docs/SCREEN_CAPTURE.md +++ b/docs/SCREEN_CAPTURE.md @@ -1,4 +1,4 @@ -# Screen Capture — DraftLens Sprint 0.1.3 +# Screen Capture — DraftLens Sprint 0.1.3 + 1B ## Mechanism `MediaProjectionManager.createScreenCaptureIntent()` → user consent → `ServiceCompat.startForeground()` @@ -6,9 +6,24 @@ with `ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION` → `getMediaProject `createVirtualDisplay()` → `ImageReader` surface. Consent acquisition stays in Activity; `CaptureSessionService` performs foreground promotion, projection creation, VirtualDisplay and all frame work. +## Freezing the game frame from the notification (Sprint 1B) +The ongoing capture notification carries an action (**Freeze current frame**) implemented as +an explicit `PendingIntent.getService` back to the running service — no Activity launch, no +consent re-request, no second projection or VirtualDisplay. The service snapshots the latest +validated frame into an independently owned copy (`FrameSnapshots`), keeps the projection +valid, and re-posts the notification as FROZEN with **Resume live capture**. Images arriving +while frozen are acquired and closed but not converted or published. The freeze can equally be +triggered from the Vision Lab button; both run `CaptureFreeze.freezeCurrentFrame()`. After an +Android-initiated stop the pinned frame survives in process memory for editing, but not +process death. + ## Android version notes (compile/target SDK 35) - Android 13+ `POST_NOTIFICATIONS` may be denied; capture still proceeds with a valid ongoing - foreground notification that Android may hide. In-app status and Stop control remain available. + foreground notification that Android may hide. In-app status and Stop control remain + available. The **Freeze current frame** notification action is not reachable while the + notification is hidden, so the app checks the permission (and requests it on Android 13+) + and says so on the capture screen; without it, freezing is done from the Vision Lab + (or with the notification enabled first). - **Android 14+**: service calls `startForeground` with the `mediaProjection` type **before** `getMediaProjection()`; the consent token is single-session and dies with the service. `MediaProjection.Callback.onStop()` (revocation) and `onCapturedContentResize()` (rotation) @@ -16,8 +31,10 @@ with `ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION` → `getMediaProject `resize()` with the replacement `ImageReader` surface—no second consent prompt. - **Capture dimensions:** API 30+ uses `WindowManager.maximumWindowMetrics`; lower APIs retain the legacy `defaultDisplay.getSize()` path. -- **Android 12+**: partial screen capture (share one app) is possible in the system dialog; - DraftLens requests full-screen capture; behavior is identical when partial is chosen. +- **Android 12+**: partial screen capture (share one app) is possible in the system dialog. + DraftLens requests full-screen capture; choosing app-window capture for the Mobile Legends + workflow shows DraftLens itself, not the game — choose **Entire screen / full device + display**. The app tells the user this in both capture screens. - **minSdk 26**: `ImageReader`, `VirtualDisplay`, and foreground-service APIs used are all available since API 21–26; no version-branched capture code is needed. @@ -40,6 +57,10 @@ termination and task removal all use the same terminate path: - release VirtualDisplay → MediaProjection → ImageReader/Surface → HandlerThread → notification - if no ERROR already exists, transition to IDLE +Stopping does **not** clear the pinned (frozen) frame: it survives the projection session in +process memory for editing, and only disappears with process death, an explicit Resume/Clear, +or a new capture session. + Any exception in the acquisition path is caught per frame so one bad frame never kills the session. ## Limitations (honest list) diff --git a/docs/VISION_ARCHITECTURE.md b/docs/VISION_ARCHITECTURE.md index 8159767..5337841 100644 --- a/docs/VISION_ARCHITECTURE.md +++ b/docs/VISION_ARCHITECTURE.md @@ -1,28 +1,30 @@ -# Vision Architecture — Sprint 1A +# Vision Architecture — Sprints 1A + 1B ## What this layer is Sprint 1A gives the app a real image-bearing pipeline. Frames carry pixels, geometry is tested in pure Kotlin, preprocessing validates before it renders, and a Vision Lab lets a -user place manual slot annotations and save calibration profiles. +user place manual slot annotations and save calibration profiles. Sprint 1B adds the hero +recognition pipeline on top — still offline, still manual-confirmation only. -**What it is not:** there is no recognizer. No OCR, no hero identification, no automatic -ban/pick detection, no confidence values, no ML model. The Vision Lab prints -"Detection: not implemented". Manual slots are calibration data, never detections. +**What it is not:** there is no recognizer that selects a hero by itself. No OCR, no automatic +ban/pick detection, no calibrated confidence values, no ML model. Manual slots are calibration +data, never detections. ## Package map ```text com.draftlens capture/ - CaptureSessionService foreground service, MediaProjection owner + CaptureSessionService foreground service, MediaProjection owner, notification actions CaptureController start/stop/pause entry points - FramePipeline ImageReader -> Bitmap, latest-frame-only + CaptureFreeze freeze/resume command bridge (service commands + in-app button) + FramePipeline ImageReader -> Bitmap, latest-frame-only, freeze-aware FrameSampler monotonic sampling - CaptureStateHolder process-local state holder + CaptureStateHolder process-local state holder (live + frozen frame) CaptureVisionFrameSource adapter: capture -> VisionFrameSource (the only bridge) models/ - CaptureSessionState, CaptureTransitions, CapturedFrame + CaptureSessionState, CaptureTransitions, CapturedFrame, FrameSnapshots datastore/ AppPreferencesRepository sampling interval CalibrationProfileDataStore one JSON document under `profiles_json` @@ -53,11 +55,16 @@ seam: interface VisionFrameSource { val frames: StateFlow val state: StateFlow + val frozenFrame: StateFlow + fun freeze(): Boolean + fun resume() } ``` `CaptureVisionFrameSource` (in `capture/`) is the only file that knows about both sides. -It maps `CaptureStatus` to `VisionSourceState`: +It maps `CaptureStatus` to `VisionSourceState` and delegates the freeze commands to +`CaptureFreeze`, the same bridge the service's notification action uses — in-app and +notification freezes are one code path. | CaptureStatus | VisionSourceState | |---|---| @@ -81,8 +88,23 @@ MediaProjection -> VirtualDisplay -> ImageReader -> capture HandlerThread -> Compose overlay and inspectors ``` -Capture keeps running when the Vision Lab tab is not visible: the service owns the -projection and the process-local holder owns the latest frame. The Vision Lab only reads. +The freeze workflow is a side channel on the same pipeline — never a second capture system: + +```text +"Freeze current frame" (notification) or in-app Freeze + -> CaptureSessionService ACTION_FREEZE (or source.freeze()) + -> CaptureFreeze.freezeCurrentFrame() + -> FramePipeline freezes publication; copies the latest published frame + -> CaptureStateHolder.frozenFrame (independently owned copy, at most one) + -> CaptureVisionFrameSource.frozenFrame -> VisionLabViewModel -> Vision Lab canvas + -> notification re-posted as FROZEN with a "Resume live capture" action +``` + +While frozen, the ImageReader keeps acquiring and closing incoming `Image`s (the reader and +VirtualDisplay stay healthy) but performs no conversion or publication, so new frames can +never replace the pinned one. Capture keeps running when the Vision Lab tab is not visible: +the service owns the projection and the process-local holder owns the latest frame. The +Vision Lab only reads. ## Processing rules @@ -102,7 +124,11 @@ projection and the process-local holder owns the latest frame. The Vision Lab on - No path calls `Bitmap.recycle()`. References are dropped; native memory is reclaimed by GC. - `BitmapOwnership.SHARED_READ_ONLY` is descriptive, not a synchronized lifetime system: consumers may draw the bitmap but must not mutate or recycle it. -- The Vision Lab pins exactly one extra reference while a frame is frozen. *Clear* releases it. +- A frozen frame is an **independently owned copy** (`FrameSnapshots`): pixels are copied and + metadata (dimensions, rotation, display rotation, timestamps, latency) is preserved + verbatim; the live pipeline never shares a payload buffer with it. The copy survives the + projection stopping (for editing while the process lives) but not process death. A new + capture session drops it. `Clear` releases it. - Calibration profiles store normalized rectangles, labels and metadata only — never pixels, bitmaps, transform ids or screenshots. @@ -143,11 +169,17 @@ Everything network-facing goes through `HeroCatalogTransport`; tests use fake tr - **Catalog** (`HeroCatalog*`) is public, static metadata only: hero id/name/images/roles/lanes. Downloaded only on the user's explicit **Sync catalog** action; cached in the app's private - storage; works offline after sync. + storage; works offline after sync. The cache binds each asset to its source URL and a + SHA-256 of the bytes, validates before reuse, and flushes/syncs snapshots atomically. - **Extraction** (`SlotImageExtractor`) maps the slot's normalized source rect to source pixels and produces an upright in-memory `SlotImage`. Never written to disk. -- **Matching** (`HeroMatcher`) is a deterministic dHash + color descriptor. Output is a ranked - list of candidates or an explicit no-match. The UI turns that into manual confirm/correct — +- **Visual information** (`VisualInformationAnalyzer`) runs before matching and refuses + blank/near-uniform/detail-free crops as `InsufficientVisualInformation`; it is a pure + Kotlin, deterministic gate, with conservative uncalibrated heuristics — never presented as + a recognition accuracy. +- **Matching** (`HeroMatcher`) is a deterministic dHash + color descriptor, an uncalibrated + prototype ranking baseline. Output is a ranked list of candidates or an explicit no-match; + neither says the hero exists in the crop. The UI turns that into manual confirm/correct — never an automatic selection. - **Confirmations** (`HeroRecognitionStore`) persist `slotId → {heroId, heroName, confirmedAtMs}` in a separate preferences document; it never touches the annotation schema or the calibration diff --git a/docs/VISION_LAB.md b/docs/VISION_LAB.md index edc450a..3634f3d 100644 --- a/docs/VISION_LAB.md +++ b/docs/VISION_LAB.md @@ -13,7 +13,9 @@ Shows the frozen frame when one is pinned, otherwise the latest live frame. - Source state (`CAPTURING`, `PAUSED`, `IDLE`, `STOPPED`, `ERROR`, `UNKNOWN`) - A `LIVE` / `FROZEN` badge - *Freeze* / *Resume* — *Freeze* locks the current frame before annotation; *Resume* returns to - live frames. + live frames. Freezing is also possible from the capture notification (**Freeze current + frame**) while you are in another app; the pinned frame is independently owned and newer + live frames never replace it. - *Process Frame* — processes the pinned frame if there is one, otherwise the latest live frame at the moment of the press. It resizes and prepares the frame only; no recognition exists yet. @@ -164,8 +166,8 @@ The button is a compact icon in the corner of the canvas card. | Control | What it does | |---|---| -| **Freeze** | locks the current frame before annotation | -| **Resume** | returns to live frames | +| **Freeze** | locks the current frame before annotation (also available as **Freeze current frame** in the capture notification while in another app) | +| **Resume** | returns to live frames (also as **Resume live capture** in the notification) | | **Slot** | a manually marked region where a ban/pick hero portrait is expected | | **Process Frame** | resizes and prepares the frame only; no recognition exists yet | | **Calibration profile** | saves and restores normalized slot coordinates for matching screen layouts | diff --git a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md index 01b2f10..a54800b 100644 --- a/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md +++ b/docs/sprints/SPRINT_1B_HERO_RECOGNITION.md @@ -8,9 +8,66 @@ baseline with a manual confirmation flow. It is about identifying heroes, not ab recommendations. **What is explicitly not here:** no recommendation or synergy scoring, no counter ranking, no -floating overlay, no automatic draft-phase detection, no OCR of arbitrary pixels beyond the -slot crop, no cloud recognition API, no ML model, no OpenCV. Those arrive after recognition -has been measured on real device samples. +floating overlay for MLBB itself, no automatic draft-phase detection, no OCR of arbitrary +pixels beyond the slot crop, no cloud recognition API, no ML model, no OpenCV. Those arrive +after recognition has been measured on real device samples. + +## Review-fix pass (this revision) + +This branch now also contains the fixes for the PR #3 review findings: + +| Part | Finding | Resolution | +|---|---|---| +| A | No way to freeze the game frame while inside MLBB | Capture-side freeze via an explicit `ACTION_FREEZE` / `ACTION_RESUME_LIVE` command routed to the running service; notification gains a **Freeze current frame** action; an independently owned snapshot is pinned in the process-local holder; Vision Lab shows the pinned frame; a **Resume live capture** notification action and an in-app **Resume** both exist | +| B | 90°/270° pixel rotation used incorrect indexing | `PixelSlotImageExtractor` rewritten to the exact mappings established by `FrameTransform.sourceToOriented()`; tests now use asymmetric non-square images and hand-computed expectations | +| C | Android bitmap extractor: unverified Bitmap/rotation contract | Payload dimensions validated against `OriginalFrame.sourceWidth/sourceHeight`; rotation rewritten to pixel-exact `sourceToOriented` mapping; source bitmap never mutated/recycled; temporaries recycled in every path; instrumented tests added | +| D | Blank gray crop produced three hero candidates at ~0.73 | New pure-Kotlin `VisualInformationAnalyzer` gate before matching; new distinct `InsufficientVisualInformation` outcome and `INSUFFICIENT_INFO` UI state; blank crops never produce candidates | +| E | Manual correction hidden in some states | Manual search reachable from CANDIDATES, NO_MATCH, FAILED, INSUFFICIENT_INFO and CONFIRMED; **Change…** now opens correction instead of re-running Identify; confirmation can be corrected without clearing first; failed clears report failure instead of showing success | +| F | Cache freshness: non-empty file was always reused | Per-asset `*.meta` sidecar binds the cached image to URL, SHA-256, size and content type; serve-time validation (size, hash, magic bytes, container structure, content type, plus a real decode on Android); changed URLs force re-download; failed refreshes keep the previous asset and never expose it under a changed URL; HTTPS-only redirect policy re-validates every hop | +| G | `renameTo` failure fell back to a direct write, destroying snapshots | `writeAtomically()` is now temp-sibling + flush/sync + `ATOMIC_MOVE` only; failure preserves the previous target and cleans the temp file; stale `*.tmp` siblings swept on init | +| H | `loadReference` recycled the bitmap, then read its width/height | Dimensions are read before recycling; pixel reads complete before release; exceptional paths recycle too | +| I | Matcher honesty | Matcher stays labelled a prototype ranking baseline; threshold/weights marked uncalibrated; evaluation section added below | +| J | Scope | Sprint 1A behavior preserved (see the preservation section) | +| K | Reporting | See "Evaluation and verification" and `docs/IMPLEMENTATION_REPORT.md` | + +## Capturing Mobile Legends: the freeze workflow + +1. Open DraftLens → **Debug Capture**. When Android asks what to capture, choose + **Entire screen / full device display**. App-window capture shows the DraftLens screen, not + the game, and cannot support this workflow. (The app says this on screen too.) +2. Open Mobile Legends and wait for the draft screen you want to annotate. +3. Pull down the notification shade (without leaving the game) and tap + **Freeze current frame** on the ongoing DraftLens capture notification. +4. DraftLens pins an independently owned copy of the latest validated game frame in memory. + The MediaProjection session, VirtualDisplay and ImageReader stay running; images are still + acquired and closed, but no conversion or publication work is performed while frozen. +5. Return to DraftLens. Vision Lab shows the pinned MLBB frame, not newer frames showing + DraftLens itself. Draw slot rectangles and run **Identify hero** on that frame. +6. Tap **Resume** (Vision Lab) or **Resume live capture** (notification) to see live frames + again. No second projection or VirtualDisplay is ever created for this. + +Properties of the implementation: + +- The notification action is an explicit component with an immutable `PendingIntent` + (`PendingIntent.getService` → `CaptureSessionService`). No Activity is launched. +- The action never re-promotes the service, never restarts MediaProjection, never requests + consent again and never creates a VirtualDisplay. With no active capture session it is a + no-op (a stale command re-creating the service stops itself immediately). +- Only one frozen frame is kept; a new capture session drops it, and resuming replaces live + publication. Freezing is idempotent (a duplicate tap keeps the first pinned frame). +- The frozen snapshot is independent of the live pipeline's payloads: pixels are copied, the + source is never mutated or recycled, and the frozen frame is displayed while newer live + frames keep arriving (they never overwrite it). +- A projection stopped by Android releases the VirtualDisplay, ImageReader and projection, + but preserves the pinned frame in memory for editing while the process lives. It does not + survive process death. +- **Notifications:** the ongoing foreground-service notification stays visible with an + **Stop** action and, depending on state, **Freeze current frame** (LIVE) or + **Resume live capture** (FROZEN). Actions appear in the expanded notification. + If the user disabled notifications (API 33+), the capture can still run but the actions are + not reachable from the shade; the app checks and shows this on the capture screen and in + the Vision Lab guidance. +- Nothing is written to disk or uploaded by the freeze or the capture path. ## New modules @@ -18,15 +75,20 @@ has been measured on real device samples. |---|---|---| | Hero catalog model | `vision/heroes/HeroModels.kt`, `HeroIds.kt` | Provider-independent `HeroId` (slug from the English name), entries, assets, attribution, cross-check result | | Catalog codec | `vision/heroes/HeroCatalogCodec.kt` | Strict v1 snapshot codec; duplicate-id protection; HTTPS-only asset URLs | -| HTTP transport | `vision/heroes/HeroCatalogTransport.kt`, `AndroidHeroCatalogTransport.kt` | HTTPS-only `HttpURLConnection` behind a small interface; size cap enforced while reading; one retry on I/O/5xx; 429 fails fast; no API key | +| HTTP transport | `vision/heroes/HeroCatalogTransport.kt`, `AndroidHeroCatalogTransport.kt` | HTTPS-only `HttpURLConnection` behind a small interface; size cap enforced while reading; one retry on I/O/5xx; 429 fails fast; manual redirects re-validated per hop (HTTPS only); no API key | +| Redirect policy | `vision/heroes/HeroRedirectPolicy.kt` | Every redirect hop resolved and constrained to HTTPS; relative redirects resolved; HTTP downgrades and non-HTTP schemes refused; unit tested | | Catalog sync | `vision/heroes/HeroCatalogSync.kt` | Paged `GET /heroes` on Rone Arena (plus mirror fallback), MLBBDex for completeness cross-check and roles/lanes by slug | -| Catalog repository | `vision/heroes/FileHeroCatalogRepository.kt`, `HeroCatalogRepository.kt` | Atomic snapshot (`/hero-catalog/catalog.json`), private image cache (`/hero-assets/`), offline browsing after sync | -| Slot extraction | `vision/heroes/SlotImageExtractor.kt`, `AndroidBitmapSlotImageExtractor.kt` | Axis-aligned crop of the source rect, upright for every rotation; never mutates/recycles the source; nothing written to disk | -| Matcher v0 | `vision/heroes/HeroMatcher.kt`, `HeroRecognition.kt` | dHash perceptual hash + normalized 3×3 mean-RGB descriptor; combined similarity in [0, 1]; ranked top-3; explicit no-match below threshold | -| Reference loader | `vision/heroes/HeroReferenceImages.kt` | Decodes cached reference images; downsampled ≤256 px for cheap matching | +| Catalog repository | `vision/heroes/FileHeroCatalogRepository.kt`, `HeroCatalogRepository.kt` | Temp-sibling + flush/sync + `ATOMIC_MOVE` snapshot write (no direct-write fallback); private image cache (`/hero-assets/`) with per-asset `*.meta` sidecars binding URL, SHA-256, size and content type; serve-time revalidation; offline browsing after sync | +| Image validation | `vision/heroes/ReferenceImageValidator.kt`, `AndroidReferenceImageValidator.kt` | Two-layer validation: a pure-Kotlin container structure check (signature, chunks, CRCs, trailers) used on all platforms, plus an on-device `BitmapFactory` decode check wired in `DraftLensApp`; JVM tests exercise the structure check, instrumentation tests exercise both | +| Slot extraction | `vision/heroes/SlotImageExtractor.kt`, `AndroidBitmapSlotImageExtractor.kt` | Axis-aligned crop of the source rect, upright for every rotation via the `FrameTransform.sourceToOriented` mapping (pixel-exact on both paths); never mutates/recycles the source; temporaries released exactly once; nothing written to disk | +| Visual information | `vision/heroes/VisualInformation.kt` | Deterministic, Android-free analysis of a crop (luma/color variance, edge density, meaningful differences, minimum dimensions); rejects blank/near-uniform/detail-free crops before matching | +| Matcher v0 | `vision/heroes/HeroMatcher.kt`, `HeroRecognition.kt` | dHash perceptual hash + normalized 3×3 mean-RGB descriptor; combined similarity in [0, 1]; ranked top-3; explicit no-match below threshold; explicit insufficient-information outcome | +| Reference loader | `vision/heroes/HeroReferenceImages.kt` | Decodes cached reference images; dimensions read before recycling; pixel reads complete before release; downsampled ≤256 px for cheap matching | +| Frame snapshots | `models/FrameSnapshots.kt` | Independently owned copies of frames for the freeze workflow (pixel copy, metadata preserved verbatim) | +| Freeze bridge | `capture/CaptureFreeze.kt`, `capture/FramePipeline.kt`, `capture/CaptureStateHolder.kt` | The freeze command path: service/in-app triggers → `CaptureFreeze` → pipeline publication gate → `CaptureStateHolder.frozenFrame` → Vision layer | | Recognition store | `datastore/HeroRecognitionStore.kt` | Per-slot confirmed hero (id, name, timestamp) in its own preferences document; corrupt documents read as empty | -| UI | `feature/visionlab/HeroRecognitionCard.kt` | Sync button, Identify action, candidate cards, confirm/correct/clear, unofficial notice and attribution | -| Permissions | `AndroidManifest.xml` | `INTERNET` added — hero catalog download only; capture still requires explicit consent | +| UI | `feature/visionlab/HeroRecognitionCard.kt` | Sync button, Identify action, candidates, INSUFFICIENT_INFO / NO_MATCH / FAILED / CONFIRMED states, manual search everywhere | +| Permissions | `AndroidManifest.xml` | `INTERNET` (catalog) and `POST_NOTIFICATIONS` (freeze action reachability) declared; capture still requires explicit consent; `mediaProjection` foreground-service type unchanged | ## Data sources and attribution @@ -49,8 +111,6 @@ Rone Arena (primary): No key. Public endpoints only. - `GET /heroes?size&index&lang` — `data.records[].{data.hero.data.{head,name,smallmap}, hero_id, relation}`, `data.total`. Records ordered by `hero_id`. -- `GET /heroes/{hero_identifier}` — additional variants `head_big`, `squarehead`, `squareheadbig` - (detail endpoint; not fetched in this sprint — `HEAD` is enough to establish the baseline). - Error handling: a 503 means maintenance/high traffic (`alternative_endpoint` fallback); a 429 `UPSTREAM_RATE_LIMITED` means back off, never parallel-hammer. @@ -65,14 +125,33 @@ MLBBDex (cross-check): `hero_id` are collapsed and counted; records missing id/name/both images are skipped and counted. - **Mirror fallback.** If the primary host fails, the request is retried once against the mirror. - **Catalog entries actually loaded are reported**, never a hardcoded expected count. - (Measured at sync time; see the sync section in `docs/VISION_TESTING.md` for what CI ran.) -- **Caching.** Snapshot JSON is written to a temp file and renamed into place; an interrupted - sync cannot leave a half snapshot. Reference images go to a private per-app cache with - magic-byte validation, `image/*` content-type check and a 4 MiB cap per image. +- **Caching (freshness enforced).** Per reference image, a small sidecar + (`/.img.meta`) records the source URL, SHA-256 of the bytes, size, content + type and a save timestamp. An asset is reused only if everything matches: file present, + 4 MiB cap, sidecar URL equals the URL the current catalog advertises, size and SHA-256 + match, magic bytes valid, container parses (PNG CRCs / JPEG & GIF trailers / WebP RIFF + framing) and — on Android — the image actually decodes. Anything else is re-downloaded. +- **Changed URLs.** When the catalog starts advertising a different URL for the same hero + asset, the old file is never exposed as the new asset; the refresh re-downloads it, and + the old one is not served even if the refresh fails. +- **Failed refreshes.** A refresh failure never deletes the previous asset and never serves + it under a changed URL. With an unchanged URL and a valid file on disk, the cache is + simply kept; with a corrupt file on disk, that hero reports an `assetFailures` entry and + matching skips it. +- **Redirects.** The transport follows redirects manually and re-validates every hop: + HTTPS only, relative redirects resolved, non-HTTP schemes and over-long targets refused. + Downgrades are never accepted. +- **Snapshot atomicity.** `writeAtomically()` writes to a `*.tmp` sibling in the same + directory, flushes and syncs where supported, then performs an `ATOMIC_MOVE` into place. + If atomic replacement cannot be completed (e.g. `EISDIR`, cross-device), the previous + snapshot is preserved, the temp file is deleted and the write fails loudly. There is no + direct-write fallback. Abandoned `*.tmp` siblings older than an hour are swept on init. - **Offline.** After a successful sync the catalog serves from cache with no network at all. - A corrupt cache is reported as an error and never silently replaced; re-sync is the fix. -- **Asset failure handling.** If one image fails, the hero is still in the catalog; the failure - is reported by name in `assetFailures` and matching simply skips that hero. +- **Asset failure handling.** If one image fails, the hero is still in the catalog; the + failure is reported by name in `assetFailures` and matching simply skips that hero. +- **Privacy.** The cache lives in the app's private dirs. No frames, crops or confirmation + data ever leave the device; no runtime permissions beyond INTERNET/POST_NOTIFICATIONS/ + the capture ones are used. ## Slot extraction @@ -80,86 +159,116 @@ MLBBDex (cross-check): (`NormalizedRect.toPixelRect`, floor origin / ceil far edge / ≥1 px) — the same rounding rule the annotation canvas uses — and the crop is cropped axis-aligned, then rotated upright to match what the user sees on screen. +- The rotation mappings follow `FrameTransform.sourceToOriented()` exactly (oriented pixel + `(x, y)` ← source pixel at the inverse point), on both the JVM extractor and the Android + one, so Java and Android paths produce byte-identical outputs for the same crop. - Aspect ratio is never changed; nothing is stretched. -- The original frame bitmap is never mutated or recycled. +- The original frame bitmap is never mutated or recycled. The Android path validates the + Bitmap dimensions against the frame's `sourceWidth/sourceHeight` before cropping. +- Ownership: the crop creates a fresh bitmap; the rotation creates at most one more; every + temporary is recycled exactly once; failures recycle their temporaries and return `null`/ + `Result.failure` instead of leaking them. - Extracted crops live in memory only and are never written to disk. -- Tests cover portrait and landscape frames, all four rotations, edge-touching rectangles, - minimum-size rectangles, invalid coordinates, payload mismatch and original-frame ownership. - -## Matching algorithm and why - -Two deterministic descriptors, compared on a fixed grid: - -1. **dHash perceptual hash** (9×8 grayscale, area-average downsample, bit = left pixel - brighter than right). 64 bits, hamming distance, robust to scale and mild recompression. -2. **Normalized 3×3 mean-RGB appearance descriptor** (27 floats). Weaker signal but catches - hue/lighting layout that a hash alone can miss. - -Combined similarity: `0.6 × (1 − hamming/64) + 0.4 × (1 − colorDistance)`, clamped to [0, 1]. - -Why this over anything fancier: - -- No training data, no model download, no inference dependency; runs on any Android device this app targets. -- Reference images (head portraits) are small and near-uniform in composition, which is exactly - where dHash works well. -- It is fully deterministic, so tests are stable and failures are reproducible. - -**This is a heuristic ranking aid, not a probability or a calibrated confidence.** Scores are -labeled as similarity everywhere they appear. The no-match threshold (0.55) is a starting -default, not a calibration — it was **not** tuned on real MLBB screens (none were used). -Two candidates closer together than 0.05 are flagged `ambiguous`. The UI never selects a -hero automatically: the user confirms or corrects every match. - -## Evaluation (honest) - -There is **no accuracy claim on real game screens** in this sprint. The matcher is evaluated -by deterministic algorithm tests only (identical images match, ranking order, threshold -behaviour, ambiguous gap, corrupt references skipped). Those tests prove the mechanism works, -not that 134 MLBB hero portraits are linearly separable on a capture full of effects. - -| Metric | Value | Data | +- Tests cover portrait and landscape frames, all four rotations with non-square asymmetric + fixtures, crops touching every edge, minimum-size rectangles, below-one-pixel rectangles, + invalid coordinates, payload mismatch, recycled-bitmap refusal, source immutability, and + the Android path end-to-end on an instrumented device. + +## Recognition v0 — what it is and is not + +**This is a prototype ranking baseline, not production-grade hero recognition.** + +- Two deterministic descriptors: a **dHash** perceptual hash (9×8 grayscale, area-average + downsample, bit = left pixel brighter than right) and a normalized **3×3 mean-RGB** + descriptor (27 floats), combined as `0.6 × (1 − hamming/64) + 0.4 × (1 − colorDistance)`, + clamped to [0, 1]. +- **The combined number is a heuristic similarity, not a probability.** It must never be read + as a likelihood, a confidence, or a "73% chance". The no-match threshold (0.55) and the + ambiguities gap (0.05) are **uncalibrated starting defaults** chosen so the pipeline is + exercisable; they were **not** tuned on real MLBB screens. +- **Before matching** a crop goes through the visual-information gate (`VisualInformationAnalyzer`): + a near-uniform gray/black/white/solid-color crop, a detail-free crop, or one smaller than + 12×12 is reported as `InsufficientVisualInformation` and no candidate is generated. The + thresholds (luma variance < 2.0 and channel variance < 2.0 for "near-uniform"; < 8.0 with + under 1% edge density for "too little detail") are **conservative heuristics, not calibrated + MLBB thresholds**. Low-contrast-but-detailed crops are deliberately *not* rejected. + Real-screen calibration of these heuristics is pending device testing. +- `InsufficientVisualInformation` is deliberately distinct from `NoMatch`: the first says the + crop carries no usable signal, the second says the signal was present but unrecognizable. +- Recognition **never** assigns a hero by itself. The user confirms or corrects every outcome + (candidates, no-match, insufficient-information, failed, or already-confirmed) through the + manual name search; the confirmation flow is explicit everywhere, and a failed persistence + operation is reported, not hidden. +- dHash works best on near-uniform headshots; it does not understand hero identity, and it + can rank wrong heroes confidently. **Accuracy on real MLBB screens is not measured.** + +### Evaluation methodology (synthetic fixtures only) + +| What | How it is measured | Status | |---|---|---| -| Algorithm tests (deterministic) | see CI | synthetic fixtures only | -| Top-1 accuracy on real MLBB screens | **not measured** | no samples available | -| Top-3 accuracy on real MLBB screens | **not measured** | no samples available | -| No-match false-rejection rate | **not measured** | no samples available | -| Sample count | 0 | no device images committed | - -Synthetic tests are useful for the algorithm; they do not prove accuracy on real MLBB screens. -`head` and `smallmap` are headshots on flat backgrounds and should be more separable than the -matcher assumes; this needs a device smoke test with manually labeled slots before any claim -is printed in the app. - -Physical-device evaluation is **pending** and must use real app captures with manually -identified hero slots. Device details and the number of correctly identified samples must be -recorded here when run; upload or commit those images is prohibited. +| Algorithm correctness | Deterministic tests: identical images match, ranking order, threshold behaviour, ambiguous gap, corrupt references skipped, hamming/color math, round trips. | CI (see `docs/VISION_TESTING.md`) | +| Blank-image rejection | Deterministic fixtures: uniform gray/black/white, saturated solids, near-uniform with tiny noise, low-resolution crops. | CI; `VisualInformationTest` covers all of them and the `InsufficientVisualInformation` outcome | +| Top-1 / Top-3 accuracy | **Method when run:** manually identify heroes in slot crops on real device captures, run Identify, record the rank of the correct hero among returned candidates, aggregate. Requires consented captures with manually verified labels; none exist yet. | **Not run** | +| False-match rate | **Method when run:** feed known-non-matching hero crops (and blank UI regions) and count how often any candidate clears the threshold. Blank/low-information crops are now rejected before matching; whether *detailed but wrong* crops still false-match is unmeasured. | **Not run** | +| Top-1/Top-3 on synthetic fixtures | Identical fixtures match (similarity ≈ 1); the inverse-gradient fixture yields NoMatch. No synthetic benchmark suite beyond these deterministic cases exists; no synthetic numbers are claimed as accuracy. | CI only | +| Real-screen status | No MLBB device session has been run. | **Pending** | + +### Known limitations + +- The matcher does not understand hero identity; two similar portraits can both rank highly. +- The threshold and weights are uncalibrated; 0.55 may be far too low or too high on real + captures (compression, UI effects, partial crops, anti-aliasing). +- The visual-information heuristics may reject genuinely flat but correct crops, or admit + genuinely empty but noisy ones. They are not a substitute for calibration. +- Reference images are `HEAD` portraits; full-body or in-game appearance usually differs. +- Java and Android extraction paths are intended to be pixel-identical, on the strength of + the shared mapping and the fixture tests, but have not been compared on two devices. +- Nothing here is a measurement of recognition quality on MLBB content. ## UI Vision Lab, under the slot list: - *Sync catalog* button; sync state shown (`Not synced`, progress while syncing, `Ready: N heroes`, or error with the cached count if offline). -- *Identify hero* on the selected slot; per-slot state (`idle → processing → candidates | no match → confirmed | failed`). +- *Identify hero* on the selected slot; per-slot state (`idle → processing → candidates | no match | insufficient information | confirmed | failed`). - Candidate cards: cached image, name, similarity, `ambiguous` marker when present; **Confirm** per candidate. -- **No match** state with the best similarity, plus manual name search. -- **Confirmed** view with Clear and Change…; manual name search available everywhere. +- **No match** and **Insufficient information** states both offer manual name search. +- **Confirmed** view with Clear and **Change…**; **Change…** opens the manual correction + search (it does not re-run Identify). Corrections persist per slot without clearing first. +- Manual name search is available in every recognition state (candidates, no-match, + insufficient information, failed, confirmed), plus the share note for saved confirmations. - Attribution footer with the unofficial notice; images load from the local cache only. - -Recognition never runs continuously, its button never appears on slot creation, and it never -opens the property menu automatically. Slot creation keeps the maintainer's 1A preference. +- Capture guidance: choosing **Entire screen / full device display** when Android asks is + required for the notification freeze workflow; the Debug Capture screen and the Vision Lab + both say so. + +## Preservation of existing behavior (Part J) + +- Compact red X delete control at the top-right: unchanged. +- Long-press creation selects the slot without opening the property editor: unchanged. +- 450 ms long-press and early-drag cancellation: unchanged. +- Paste source/anchor semantics: unchanged. +- Context menu and selection synchronization: unchanged. +- Property-editor validation feedback: unchanged. +- Fullscreen editor without zoom/fit controls: unchanged. +- Gesture isolation from page scrolling: unchanged. +- Portrait/landscape rendering and the Vision Lab interaction model: unchanged. +- MediaProjection lifecycle: unchanged except the required freeze/resume integration; the + `mediaProjection` foreground-service type, `POST_NOTIFICATIONS` and capture + permissions are preserved; no overlay, accessibility, touch automation or memory reading. +- No hero counters, recommendations, synergy scoring, overlays, packet inspection or cloud + recognition were added. ## Acceptance status -- **CI green**: latest branch head `e58611c25d7d9a40592aa3d81ad3347869771cc3` (run 43732908605, - test-only); **323 unit tests (646 executions, debug + release), 0 failures, - 0 errors, 0 skipped**; `assembleAndroidTest` **compiled, not executed**; strict lint passed; - `assembleDebug` passed; artifact `draftlens-debug`, digest - `sha256:ad581e4042bc81fb7519c042c35b68d3b152f0c30cbed5c37816a4c2534de788`. -- Physical-device smoke: **pending**. -- Catalog sync on a device: **pending** (verified only through fake transport in unit tests). -- On-device accuracy: **not measured**. -- **Catalog entries actually loaded:** measured at sync time on the device that syncs; not yet - recorded here. The pagination logic is exercised against fake transports in unit tests; - the real total is whatever the API returns when the user presses *Sync catalog*. - -See `tasks/todo.md` and `docs/IMPLEMENTATION_REPORT.md` for the running checklist. +- **CI green**: see the final CI section of `docs/VISION_TESTING.md` / + `docs/IMPLEMENTATION_REPORT.md` for the exact head SHA and run URL after this revision. +- Unit tests: see the CI section (totals vary as the suite grows; the required gate is + 0 failures / 0 errors / 0 skipped for both debug and release variants). +- Instrumentation tests: executed on an emulator in CI if the `instrumented` job is green; + otherwise compiled but not executed — see `docs/VISION_TESTING.md` for the honest status. +- Physical-device smoke: **pending** (full matrix in `docs/TESTING_PHYSICAL_DEVICE.md`). +- Catalog sync on a device: **pending** (verified only through fake transports in unit tests). +- On-device accuracy: **not measured**; real-screen threshold calibration is **pending**. +- Catalog entries actually loaded: measured at sync time on the device that syncs; not yet + recorded here. diff --git a/tasks/plan.md b/tasks/plan.md index d770b88..8ea39e2 100644 --- a/tasks/plan.md +++ b/tasks/plan.md @@ -134,3 +134,44 @@ grow without re-architecture. - Slot extraction from the annotation canvas with rotation coverage. - Recognition v0 with labeled similarity and explicit no-match; manual confirmation required. - Draft PR #3; accuracy on real screens not measured; device smoke pending. + +## Sprint 1B — Review fixes and game-frame freeze (this revision) +Objective: fix the PR #3 review findings (freeze from the notification; rotation bugs; +blank-image rejection; manual correction everywhere; cache freshness; atomic persistence; +reference-loader ownership) and wire the freeze workflow through the capture architecture. +No recommendation features, no overlays, no automation, PR stays draft and unmerged. + +### Work items in order +- [x] Freeze a game frame from the notification (`CaptureFreeze`, `ACTION_FREEZE` / + `ACTION_RESUME_LIVE`, `FramePipeline` freeze gate, independently owned snapshot via + `FrameSnapshots`, FROZEN notification with Resume action, in-app Resume unchanged) +- [x] Full-device-capture guidance; POST_NOTIFICATIONS handling; freeze actions in the + expanded notification; no Activity launched by the command +- [x] Part B: pixel extractor rotation fixed to the `FrameTransform.sourceToOriented()` + convention; tests rewritten with asymmetric non-square fixtures and hand-computed + expected values +- [x] Part C: Android extractor validates Bitmap dimensions, pixel-exact rotation, source + bitmap preserved, every temporary released exactly once; instrumentation tests added +- [x] Part D: `VisualInformationAnalyzer` (variance, edge density, meaningful differences, + minimum dimensions) gates matching with a distinct `InsufficientVisualInformation` + outcome and `INSUFFICIENT_INFO` UI state; blank crops never produce candidates +- [x] Part E: manual correction reachable from CANDIDATES, NO_MATCH, FAILED, + INSUFFICIENT_INFO and CONFIRMED; Change… opens correction instead of Identify; + failed persistence reported, never shown as success +- [x] Part F: cache freshness — per-asset `*.meta` sidecar (URL, SHA-256, size, content + type), serve-time validation (hash, magic bytes, container structure, Android decode), + URL change forces re-download, failed refresh keeps previous asset, HTTP downgrades + refused everywhere +- [x] Part G: snapshot/asset writes are temp-sibling + flush/sync + `ATOMIC_MOVE` only; + no direct-write fallback; failures preserve the previous target; `*.tmp` swept +- [x] Part H: reference loader reads dimensions before recycling; pixels read before release; + exceptional paths recycle too +- [ ] CI green on the final head (unit tests, lint, assembleDebug, assembleAndroidTest, + instrumented job on the emulator) +- [ ] Physical-device smoke of the notification freeze workflow; on-device accuracy + evaluation; real-screen threshold and heuristic calibration + +### Sprint 1B verification +See `docs/VISION_TESTING.md` for the exact CI runs, test totals and what was and was not +executed. The authoring host remains aarch64, so the CI is the build of record. +Physical-device verification of the freeze workflow is pending. diff --git a/tasks/todo.md b/tasks/todo.md index cfa8156..8147a7a 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -98,4 +98,22 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** - [x] Vision Lab card: sync, Identify, candidates, confirmed state, manual correction - [x] Unit coverage for catalog, cache, extraction, matching, VM flows - [x] CI green: 323 tests (646 executions), strict lint, assembleDebug, assembleAndroidTest compiled (run 43732908605, head e58611c; code head 7d0c903 run 43731484922) + +## Sprint 1B review fixes + game-frame freeze (PR #3 follow-up) +- [x] Part A: freeze/resume from the notification through the capture architecture + (explicit action to the running service; independently owned snapshot; images stay + acquired+closed while frozen; one frozen frame; notification LIVE/FROZEN; in-app Resume; + full-device capture guidance; POST_NOTIFICATIONS handling) +- [x] Part B: pixel rotation fixed to the FrameTransform convention; asymmetric non-square + tests with independently calculated expectations +- [x] Part C: Android bitmap extractor validated and made pixel-exact; instrumentation tests + for all rotations, edge crops, mismatches, source validity and snapshot ownership +- [x] Part D: visual-information gate (uniform/near-uniform/detail-free crops rejected); + distinct InsufficientVisualInformation outcome and UI state +- [x] Part E: manual correction everywhere; failed clears reported honestly +- [x] Part F: cache freshness (URL-bound sidecar metadata; hash/structure/decode validation; + URL change forces re-download; failed refresh keeps previous asset; HTTPS-only redirects) +- [x] Part G: atomic snapshot/asset writes; failure preserves previous snapshot; temp files swept +- [x] Part H: reference Bitmap dims read before recycling +- [ ] CI green on the final head; instrumented job result recorded honestly - [ ] Physical-device smoke test and on-device accuracy evaluation (pending) From 0e9551557ebeeb72a2a68eefd4bb80250945eb3c Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:24:55 +0000 Subject: [PATCH 16/33] fix: exhaustive when in Android bitmap extractor rotation --- .../vision/heroes/AndroidBitmapSlotImageExtractor.kt | 3 +++ docs/TESTING_PHYSICAL_DEVICE.md | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt index f7d1028..70a6691 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/AndroidBitmapSlotImageExtractor.kt @@ -86,6 +86,9 @@ object AndroidBitmapSlotImageExtractor : SlotImageExtractor { source.getPixels(src, 0, width, 0, 0, width, height) val dst = IntArray(outWidth * outHeight) when (rotation) { + // ROTATION_0 returned early above. + FrameRotation.ROTATION_0 -> Unit + FrameRotation.ROTATION_90 -> for (y in 0 until height) { for (x in 0 until width) { diff --git a/docs/TESTING_PHYSICAL_DEVICE.md b/docs/TESTING_PHYSICAL_DEVICE.md index 95b2fca..f740d9a 100644 --- a/docs/TESTING_PHYSICAL_DEVICE.md +++ b/docs/TESTING_PHYSICAL_DEVICE.md @@ -32,6 +32,15 @@ Status values: **PASS**, **FAIL**, **PARTIAL**, **NOT RUN**. Record the commit S | 11 | Sustained capture memory/latency | Memory bounded; frame processing latency reported separately from detection latency | NOT RUN | | 12 | DraftLens foreground vs MLBB foreground | Selected capture target is clear; preview corresponds to chosen scope | NOT RUN | | 13 | Screenshots in app | System screenshot path works normally | NOT RUN | +| 14 | Full-device capture for the MLBB freeze workflow | When Android asks what to capture, **Entire screen / full device display** is chosen; live preview shows MLBB content | NOT RUN | +| 15 | Freeze from the notification while in MLBB | Tap **Freeze current frame** on the capture notification without leaving the game; the pinned frame survives returning to DraftLens; Vision Lab shows the MLBB frame, not the DraftLens UI | NOT RUN | +| 16 | Newer live frames cannot replace the frozen frame | While frozen, DraftLens keeps acquiring/closing images but publishes none; Vision Lab keeps showing the MLBB frame | NOT RUN | +| 17 | Resume from Vision Lab and from the notification | **Resume** (Vision Lab) or **Resume live capture** (notification) reveals live frames without a new projection or VirtualDisplay; notification action flips back to LIVE | NOT RUN | +| 18 | Freeze with no frame captured yet | **Freeze current frame** is a no-op (logged); nothing is pinned; no crash, no new session | NOT RUN | +| 19 | Duplicate taps of Freeze / Resume | Idempotent: the first pinned frame is kept; notification stays FROZEN | NOT RUN | +| 20 | Notifications disabled | Freeze action unreachable from the shade; the app warns and offers to enable them; in-app Freeze/Resume still work | NOT RUN | +| 21 | Stop with a frame frozen | Projection and resources are released; the frozen frame remains available in Vision Lab until Resume/Clear or a new capture session; it does not survive process death | NOT RUN | +| 22 | Freeze during low frame availability | No frame → the action logs and keeps state; first valid frame freezes on the next tap | NOT RUN | ## Evidence to capture each run - `git rev-parse --short HEAD` @@ -42,4 +51,5 @@ Status values: **PASS**, **FAIL**, **PARTIAL**, **NOT RUN**. Record the commit S - For failures: full exception and steps immediately before it ## CI scope -CI compiles instrumentation tests but does not execute them. JVM/unit tests, lint, APK assembly and merged-manifest inspection are not substitutes for this checklist. +CI runs JVM unit tests, lint, APK assembly and merged-manifest inspection. An `instrumented` +job runs the instrumentation suite on an emulator. Neither substitutes for this checklist. From 7ce688bf6768087f60c2e602668ecb213c844e38 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:29:09 +0000 Subject: [PATCH 17/33] fix: keep the repository clock last so trailing lambdas bind to it imageValidator is a SAM; as the last parameter it stole the { 1234L } clock lambdas. Also assert the frozen delivery non-null in the adapter test. --- .../com/draftlens/vision/heroes/FileHeroCatalogRepository.kt | 4 +++- .../com/draftlens/capture/CaptureVisionFrameSourceTest.kt | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt index de576fb..02382de 100644 --- a/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt +++ b/app/src/main/kotlin/com/draftlens/vision/heroes/FileHeroCatalogRepository.kt @@ -31,8 +31,10 @@ class FileHeroCatalogRepository( private val dataDir: File, private val cacheDir: File, private val transport: HeroCatalogTransport, - private val clock: () -> Long = System::currentTimeMillis, + // Trailing lambdas bind to the last parameter, so the clock must come last; + // imageValidator is a SAM too — a lambda would otherwise land on it. private val imageValidator: ReferenceImageValidator = StructuralImageValidator, + private val clock: () -> Long = System::currentTimeMillis, ) : HeroCatalogRepository { private val snapshotFile = File(dataDir, "hero-catalog/catalog.json") diff --git a/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt b/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt index 74437fb..5b6a30d 100644 --- a/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt +++ b/app/src/test/kotlin/com/draftlens/capture/CaptureVisionFrameSourceTest.kt @@ -104,7 +104,7 @@ class CaptureVisionFrameSourceTest { val delivered = runBlocking { withTimeout(AWAIT_TIMEOUT_MS) { adapter.frozenFrame.first { it != null } } - } + }!! assertEquals("test-frame", delivered.id.value) } From 860c1c4c479e82d8836208bb7d21fe32f5fb3ab7 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:34:36 +0000 Subject: [PATCH 18/33] fix: test failures from the review-fix pass - a uniform crop is now InsufficientVisualInformation; the old black-crop case belongs to the new gate, so the no-match test uses an inverse gradient; a uniform-crop test pins the distinct outcome - sabotage the confirmations store with a file at its directory path (the DataStore recreates missing dirs, plain deletion was not enough) - the temp-sweep test needs a wall-clock, not the fixed 1234L clock --- .../visionlab/VisionLabViewModelTest.kt | 7 ++++-- .../draftlens/heroes/HeroRecognitionTest.kt | 23 +++++++++++++++++-- .../draftlens/heroes/WriteAtomicallyTest.kt | 6 ++--- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index 27dbe58..b45c38c 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -976,8 +976,10 @@ class VisionLabViewModelTest { vm.identifySelectedSlot() eventually(10_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CANDIDATES } - // Sabotage the store: remove the directory the preferences document lives in. + // Sabotage the store: the directory where the document lives becomes a file, + // so the next write cannot succeed no matter what the store recreates. dir.deleteRecursively() + java.io.File(dir).writeText("file") vm.confirmHero(slotId, "miya") eventually(5_000) { vm.uiState.value.message?.contains("could not be saved") == true } @@ -997,8 +999,9 @@ class VisionLabViewModelTest { vm.confirmHero(slotId, "miya") eventually(5_000) { vm.uiState.value.heroRecognition[slotId]?.phase == HeroSlotPhase.CONFIRMED } - // Sabotage the store, then try to clear. + // Sabotage the store the same way, then try to clear. dir.deleteRecursively() + java.io.File(dir).writeText("file") vm.clearHeroConfirmation(slotId) eventually(5_000) { vm.uiState.value.message?.contains("could not be cleared") == true } diff --git a/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt b/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt index 05bed91..712065e 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/HeroRecognitionTest.kt @@ -62,9 +62,9 @@ class HeroRecognitionTest { } @Test fun `evidence below the threshold reports no match instead of a wrong hero`() { - val black = IntArray(32 * 32) { 0xFF000000.toInt() } + val inverse = inverseGradient(32, 32) val outcome = HeroRecognition.identify( - pixels = black, + pixels = inverse, width = 32, height = 32, references = listOf(miyaReference), @@ -73,4 +73,23 @@ class HeroRecognitionTest { assertTrue(outcome is HeroIdentifyOutcome.NoMatch) } + + @Test fun `a uniform crop is insufficient information, not a wrong hero nor a no match`() { + val black = IntArray(32 * 32) { 0xFF000000.toInt() } + val outcome = HeroRecognition.identify( + pixels = black, + width = 32, + height = 32, + references = listOf(miyaReference), + ) + + assertTrue(outcome is HeroIdentifyOutcome.InsufficientVisualInformation) + } + + private fun inverseGradient(width: Int, height: Int): IntArray = + IntArray(width * height) { i -> + val x = i % width + val v = 255 - (x * 255 / width) + 0xFF000000.toInt() or v or (v shl 8) or (v shl 16) + } } diff --git a/app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt b/app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt index 1b019af..104061a 100644 --- a/app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt +++ b/app/src/test/kotlin/com/draftlens/heroes/WriteAtomicallyTest.kt @@ -25,8 +25,8 @@ class WriteAtomicallyTest { private fun tempDirs(): Pair = Files.createTempDirectory("hero-catalog").toFile() to Files.createTempDirectory("hero-assets").toFile() - private fun repo(data: File, cache: File): FileHeroCatalogRepository = - FileHeroCatalogRepository(data, cache, EmptyTransport) { 1234L } + private fun repo(data: File, cache: File, now: Long = 1234L): FileHeroCatalogRepository = + FileHeroCatalogRepository(data, cache, EmptyTransport) { now } @Test fun `an atomic write stores the exact bytes`() { val (data, cache) = tempDirs() @@ -71,7 +71,7 @@ class WriteAtomicallyTest { stale.setLastModified(now - 2 * 60 * 60 * 1000) fresh.setLastModified(now - 60 * 1000) - repo(data, cache) + repo(data, cache, now = now) assertFalse("stale temp files are swept", stale.exists()) assertTrue("fresh temp files are kept", fresh.exists()) From f2cae4a8222c2ac635d1d89e29230b9373753cdc Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:39:57 +0000 Subject: [PATCH 19/33] fix: File(File) does not exist; use the path for the sabotage file --- .../com/draftlens/feature/visionlab/VisionLabViewModelTest.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt index b45c38c..aab0d1b 100644 --- a/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt +++ b/app/src/test/kotlin/com/draftlens/feature/visionlab/VisionLabViewModelTest.kt @@ -979,7 +979,7 @@ class VisionLabViewModelTest { // Sabotage the store: the directory where the document lives becomes a file, // so the next write cannot succeed no matter what the store recreates. dir.deleteRecursively() - java.io.File(dir).writeText("file") + java.io.File(dir.path).writeText("file") vm.confirmHero(slotId, "miya") eventually(5_000) { vm.uiState.value.message?.contains("could not be saved") == true } @@ -1001,7 +1001,7 @@ class VisionLabViewModelTest { // Sabotage the store the same way, then try to clear. dir.deleteRecursively() - java.io.File(dir).writeText("file") + java.io.File(dir.path).writeText("file") vm.clearHeroConfirmation(slotId) eventually(5_000) { vm.uiState.value.message?.contains("could not be cleared") == true } From 5884d62a4c62228eba93b3c94197f47cb6e7f615 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:45:45 +0000 Subject: [PATCH 20/33] fix: dex disallows spaces in SimpleName; camelCase the instrumented tests --- .../AndroidBitmapExtractionInstrumentedTest.kt | 18 +++++++++--------- .../ReferenceImageValidatorInstrumentedTest.kt | 8 ++++---- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt b/app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt index 7222c02..3c46461 100644 --- a/app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt +++ b/app/src/androidTest/kotlin/com/draftlens/AndroidBitmapExtractionInstrumentedTest.kt @@ -80,7 +80,7 @@ class AndroidBitmapExtractionInstrumentedTest { 20, 21, 22, 23, 24, ) - @Test fun `all four rotations produce the exact expected pixels for a non-square crop`() { + @Test fun allFourRotationsProduceExactExpectedPixels() { val rect = NormalizedRect(0f, 0f, 1f, 1f) val expected = mapOf( FrameRotation.ROTATION_0 to Triple(3, 2, intArrayOf(10, 11, 12, 20, 21, 22)), @@ -102,7 +102,7 @@ class AndroidBitmapExtractionInstrumentedTest { } } - @Test fun `crops touching the top and bottom edges extract exactly the edge columns`() { + @Test fun topBottomEdgeCropsExtractExactlyThosePixels() { val rect = NormalizedRect(0.2f, 0f, 0.6f, 1f) // left=1px, right=3px of 5 val expected = mapOf( FrameRotation.ROTATION_0 to Triple(2, 3, intArrayOf(1, 2, 11, 12, 21, 22)), @@ -123,7 +123,7 @@ class AndroidBitmapExtractionInstrumentedTest { } } - @Test fun `a crop on the bottom and right edges extracts exactly those pixels`() { + @Test fun bottomRightEdgeCropExtractsExactlyThosePixels() { val rect = NormalizedRect(0.6f, 1f / 3f, 1f, 1f) // left=3px, top=1px of 5x3 val expected = mapOf( FrameRotation.ROTATION_0 to Triple(2, 2, intArrayOf(13, 14, 23, 24)), @@ -144,7 +144,7 @@ class AndroidBitmapExtractionInstrumentedTest { } } - @Test fun `a payload whose bitmap dimensions do not match the source is refused`() { + @Test fun payloadDimensionMismatchIsRefused() { val bitmap = track(Bitmap.createBitmap(2, 2, Bitmap.Config.ARGB_8888)) val frame = OriginalFrame( id = FrameId("mismatch"), @@ -164,7 +164,7 @@ class AndroidBitmapExtractionInstrumentedTest { assertTrue(result.exceptionOrNull()!!.message!!.contains("does not match")) } - @Test fun `the source bitmap stays valid and unchanged after extraction`() { + @Test fun sourceBitmapStaysValidAfterExtraction() { val source = frame(4, 3, FrameRotation.ROTATION_90, edgeFrame.copyOf()) val bitmap = (source.payload as BitmapFramePayload).bitmap val before = IntArray(edgeFrame.size).also { bitmap.getPixels(it, 0, 4, 0, 0, 4, 3) } @@ -177,7 +177,7 @@ class AndroidBitmapExtractionInstrumentedTest { assertArrayEquals("the source pixels must be unchanged", before, after) } - @Test fun `the minimum valid crop produces exactly one pixel`() { + @Test fun minimumValidCropProducesOnePixel() { val source = frame(128, 128, FrameRotation.ROTATION_0, IntArray(128 * 128) { it }) val result = AndroidBitmapSlotImageExtractor.extract( source, @@ -191,7 +191,7 @@ class AndroidBitmapExtractionInstrumentedTest { assertEquals(16 * 128 + 16, pixels.single()) } - @Test fun `a rectangle smaller than one pixel is refused`() { + @Test fun subPixelRectangleIsRefused() { val source = frame(10, 10, FrameRotation.ROTATION_0, IntArray(100) { 7 }) val result = AndroidBitmapSlotImageExtractor.extract(source, slot(NormalizedRect(0f, 0f, 0.05f, 0.05f))) @@ -199,7 +199,7 @@ class AndroidBitmapExtractionInstrumentedTest { assertTrue(result.isFailure) } - @Test fun `a recycled source bitmap is refused without a crash`() { + @Test fun recycledSourceBitmapIsRefused() { val source = frame(4, 3, FrameRotation.ROTATION_0, edgeFrame.copyOf()) val bitmap = (source.payload as BitmapFramePayload).bitmap bitmap.recycle() @@ -209,7 +209,7 @@ class AndroidBitmapExtractionInstrumentedTest { assertTrue(result.isFailure) } - @Test fun `a bitmap frame snapshot shares no pixels with the pipeline copy`() { + @Test fun bitmapSnapshotSharesNoPixelsWithPipeline() { val original = frame(4, 2, FrameRotation.ROTATION_0, asymmetric.copyOf()) val snapshot = FrameSnapshots.copyOf(original) diff --git a/app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt b/app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt index 35f026c..116ccf3 100644 --- a/app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt +++ b/app/src/androidTest/kotlin/com/draftlens/ReferenceImageValidatorInstrumentedTest.kt @@ -24,24 +24,24 @@ class ReferenceImageValidatorInstrumentedTest { 0x44.toByte(), 0xAE.toByte(), 0x42.toByte(), 0x60.toByte(), 0x82.toByte(), ) - @Test fun `a real png decodes on a device`() { + @Test fun realPngDecodesOnDevice() { assertTrue(AndroidReferenceImageValidator.isDecodable(realPng)) assertTrue(StructuralImageValidator.isDecodable(realPng)) } - @Test fun `a truncated png does not decode`() { + @Test fun truncatedPngDoesNotDecode() { val truncated = realPng.copyOf(30) assertFalse(AndroidReferenceImageValidator.isDecodable(truncated)) assertFalse(StructuralImageValidator.isDecodable(truncated)) } - @Test fun `an html error page is not an image`() { + @Test fun htmlErrorPageIsNotAnImage() { val html = "502 Bad Gateway".toByteArray() assertFalse(AndroidReferenceImageValidator.isDecodable(html)) assertFalse(StructuralImageValidator.isDecodable(html)) } - @Test fun `a jpeg that passes the structure check fails the device decode`() { + @Test fun jpegPassingStructureCheckFailsDeviceDecode() { // Structure check is a first gate (SOI + EOI present); the device adds a real decode. val fakeJpeg = byteArrayOf(0xFF.toByte(), 0xD8.toByte(), 0x01, 0x02, 0x03, 0xFF.toByte(), 0xD9.toByte()) assertTrue(StructuralImageValidator.isDecodable(fakeJpeg)) From f3ea76983b88a591956564e37003b6fb7e6c95ca Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 17:55:27 +0000 Subject: [PATCH 21/33] ci: try the aosp_atd image (runner has no KVM) --- .github/workflows/build.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e5b116..ba97c38 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -97,13 +97,15 @@ jobs: run: chmod +x ./gradlew || ls -la # The instrumented suite covers the Android bitmap extractor, the image - # validator and the capture manifest checks. It does not need MediaProjection: - # the physical capture path stays in docs/TESTING_PHYSICAL_DEVICE.md. + # validator and the capture manifest checks. It does not need MediaProjection. + # The CI runner has no KVM access, so this uses the x86_64 ATD image (fast + # software-bootable) with a long boot timeout. - name: Run instrumentation tests uses: reactivecircus/android-emulator-runner@v2 with: api-level: 30 - target: default + target: aosp_atd arch: x86_64 profile: pixel + emulator-boot-timeout: 900 script: ./gradlew connectedDebugAndroidTest --stacktrace From 7a839d1422ee5ba53f65a78ef746d323091d3b40 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 18:08:42 +0000 Subject: [PATCH 22/33] ci: drop the emulator job; document instrumentation as compile-only The runner has no KVM: the default API-30 image does not boot in time and aosp_atd is reported as an unknown API level. Instrumentation tests compile but do not run; physical-device verification is pending. --- .github/workflows/build.yml | 36 -------- docs/IMPLEMENTATION_REPORT.md | 140 +++++++++++++++++++++++++------ docs/TESTING_PHYSICAL_DEVICE.md | 5 +- docs/VISION_TESTING.md | 141 ++++++++++++++++++++++---------- tasks/plan.md | 6 +- tasks/todo.md | 2 +- 6 files changed, 219 insertions(+), 111 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ba97c38..2399d39 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,39 +73,3 @@ jobs: grep -E "android.permission.FOREGROUND_SERVICE(_MEDIA_PROJECTION)?" /tmp/draftlens-permissions.txt grep -E "android:foregroundServiceType.*0x00000020" /tmp/draftlens-manifest.txt grep "CaptureSessionService" /tmp/draftlens-manifest.txt - - instrumented: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - name: Set up JDK 17 - uses: actions/setup-java@v4 - with: - distribution: temurin - java-version: '17' - cache: gradle - - - name: Install SDK packages - run: | - SDK_ROOT="${ANDROID_HOME:-/usr/local/lib/android/sdk}" - MANAGER="$(find "$SDK_ROOT/cmdline-tools" -name sdkmanager -type f | sort -V | tail -n 1)" - yes | "$MANAGER" --sdk_root="$SDK_ROOT" --licenses - "$MANAGER" --sdk_root="$SDK_ROOT" "platforms;android-35" "build-tools;35.0.0" "platform-tools" - - - name: Make gradlew executable - run: chmod +x ./gradlew || ls -la - - # The instrumented suite covers the Android bitmap extractor, the image - # validator and the capture manifest checks. It does not need MediaProjection. - # The CI runner has no KVM access, so this uses the x86_64 ATD image (fast - # software-bootable) with a long boot timeout. - - name: Run instrumentation tests - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 30 - target: aosp_atd - arch: x86_64 - profile: pixel - emulator-boot-timeout: 900 - script: ./gradlew connectedDebugAndroidTest --stacktrace diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index 8c504de..b8c6477 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -375,27 +375,119 @@ Note: invocation `./gradlew assembleDebug` is intentionally not claimed locally; - OEM/background/store policies and review gates remain for release - Physical-device matrix needs execution before store release -## Sprint 1B — hero recognition (in progress) - -Branch `sprint-1b-hero-recognition`, PR #3 (draft, **not merged**). See -`docs/sprints/SPRINT_1B_HERO_RECOGNITION.md` for the full report, including: - -- provider-independent hero catalog (Rone Arena primary, MLBBDex cross-check/roles/lanes), - atomic file snapshot with offline browsing, HTTPS-only transport; -- slot extraction from the annotation canvas (all four rotations, edge cases pinned by tests); -- recognition v0 (dHash + 3×3 normalized color descriptor), labeled similarity, explicit no-match, - manual confirmation and correction, per-slot persistence; -- **accuracy on real MLBB screens is not measured**; the no-match threshold is a starting - default, not a calibration; physical-device smoke is pending. -- Only static public hero catalog data and reference images are downloaded. Frames and crops - never leave the device; nothing is written to disk automatically. - -This sprint does not add recommendations, counters, synergy scoring, a floating overlay, -or automatic draft-phase detection, and does not merge PR #3 without review. - -Sprint 1B green runs: latest branch head `e58611c25d7d9a40592aa3d81ad3347869771cc3` — -https://github.com/xeriomy/DraftLens/actions/runs/43732908605 (646 executions, 0 failures); -code head `7d0c9032a52db8e0f962beea72ceae7120822c88` — -https://github.com/xeriomy/DraftLens/actions/runs/43731484922 (same totals). -Instrumentation tests compiled, not executed; strict lint; assembleDebug; artifact -`draftlens-debug`, digest `sha256:3c977d55d77c5452425ccfd5ea950ef55722d77319149c076d8bd5d115b85cc9`. +## Sprint 1B — review fixes and game-frame freeze (current) + +Branch `sprint-1b-hero-recognition`, PR #3 (draft, **not merged**). The full report is +`docs/sprints/SPRINT_1B_HERO_RECOGNITION.md`; this section records what changed for the +review-fix pass and how it was verified. + +### Changes made + +- **Part A — Freeze from the notification.** New `CaptureFreeze` bridge (service command + + in-app button are one code path). The ongoing capture notification now exposes + **Freeze current frame** (LIVE state) / **Resume live capture** (FROZEN state) plus + **Stop**. The command goes to the running service via an explicit, immutable + `PendingIntent.getService` (`ACTION_FREEZE` / `ACTION_RESUME_LIVE`); it never launches an + Activity, never re-promotes the service, never re-requests consent and never creates a + VirtualDisplay. `FramePipeline` gates publication on a `freezeGate` lock shared with the + image listener: while frozen, images are still acquired and closed (the reader stays + healthy) but not converted or published; a freeze that lands mid-conversion wins and the + in-flight frame is dropped. `FrameSnapshots.copyOf` copies pixels (bitmap `copy()` or + `IntArray.copyOf`) preserving dimensions, rotation, display rotation, timestamps and the + capture-latency metadata; the live pipeline never shares a payload buffer with a + snapshot; only one frozen frame is kept; a new capture session drops it; an Android-initiated + stop keeps it in process memory for editing (not across process death). The Vision Lab + in-app Freeze/Resume went through the same path. Guidance for full-device capture is in + the capture screen, the Vision Lab card and the README; `POST_NOTIFICATIONS` is checked + and surfaced (request on Android 13+, settings deep-link otherwise); the notification is + re-posted whenever the frozen state changes. +- **Part B — Pixel rotation fixed.** `PixelSlotImageExtractor` now follows + `FrameTransform.sourceToOriented()` exactly; tests rewritten with asymmetric non-square + fixtures and hand-computed expected pixel orders for all four rotations, crops touching + every edge, minimum-size and refused rectangles, and source-immutability checks. +- **Part C — Android extractor.** Validates Bitmap dimensions against + `OriginalFrame.sourceWidth/sourceHeight`; rotation rewritten to pixel-exact mappings; + source Bitmap never mutated/recycled; every temporary created in `extract()` is recycled + exactly once; failures recycle their temporaries. Instrumented tests added (all four + rotations, non-square edge crops, payload mismatch, source validity, minimum sizes, + recycled-bitmap refusal, bitmap snapshot ownership). +- **Part D — Blank rejection.** New pure-Kotlin `VisualInformationAnalyzer` (luma/color + variance, edge density, meaningful differences, minimum dimension 12×12) runs before + matching. Deterministic, no Android APIs. New distinct outcome + `InsufficientVisualInformation` and `INSUFFICIENT_INFO` UI state with + "Not enough visual information. Select a clearer hero crop." — uniform/near-uniform/ + detail-free crops never produce candidates. Thresholds are conservative heuristics, + documented as uncalibrated for MLBB. +- **Part E — Manual correction everywhere.** Manual name search and confirmation are now + reachable from CANDIDATES, NO_MATCH, FAILED, INSUFFICIENT_INFO and CONFIRMED. + **Change…** in CONFIRMED opens the manual correction search instead of re-running + Identify; a candidate can be corrected without clearing first. Failed persistence + operations are reported and never shown as success (confirm already reported; clear did + not — fixed with tests). +- **Part F — Cache freshness.** Per-asset `*.meta` sidecar records source URL, SHA-256, + size, content type. Serve-time validation: size cap 4 MiB, sidecar URL == catalog URL, + size and SHA-256 match, magic bytes, container structure (PNG chunks + CRCs, JPEG/GIF + trailers, WebP RIFF), content type, plus a real `BitmapFactory` decode on Android + (`AndroidReferenceImageValidator`, wired in `DraftLensApp`). Changed URLs force + re-download; the old asset is never served under a changed URL; a failed refresh keeps + the previous asset (unchanged URL) and never exposes it as the new one. Redirects are + followed manually with per-hop HTTPS re-validation (`HeroRedirectPolicy`); HTTP + downgrades and non-HTTP schemes refused. Snapshot writes flush/sync and `ATOMIC_MOVE`. +- **Part G — Atomic persistence.** `writeAtomically()` is now: temp sibling in the same + directory → flush + `FileDescriptor.sync()` (best effort) → `Files.move(ATOMIC_MOVE, + REPLACE_EXISTING)` → on any failure the temp file is deleted and the previous target is + preserved. No direct-write fallback. Stale `*.tmp` siblings older than an hour are swept + on init. Failure-path tests added. +- **Part H — Reference loader.** Dimensions are read before recycling; pixel reads + complete before release; exceptional paths recycle. +- **Part I — Matcher honesty.** The matcher remains clearly labelled a prototype ranking + baseline; weights and the 0.55 threshold are marked uncalibrated; an honest evaluation + section (synthetic fixtures, blank rejection, top-1/top-3 methodology, false-match + methodology, real-screen status, known limitations) is in the Sprint 1B report. No + production-accuracy or probability claims anywhere. +- **Part J — Scope.** Sprint 1A behavior preserved as-is (compact X delete control, 450 ms + long-press, early-drag cancellation, paste semantics, fullscreen without zoom/fit, gesture + isolation, portrait/landscape rendering, MediaProjection lifecycle apart from the freeze + integration). +- **Part K — Verification.** See the exact run below. Instrumented tests run on an + emulator job when available. + +### Verification actually performed (this revision) + +- **Unit tests, debug and release: 792 executions, 0 failures, 0 errors, 0 skipped** — + , + head SHA `f3ea76983b88a591956564e37003b6fb7e6c95ca`. +- strict `lint` (abortOnError = true) — green; `assembleDebug` — green; public artifact + `draftlens-debug` 10 362 310 bytes, digest + `sha256:55919144b62e5ca128c72cb6fb404a22f9fa6c5762ecb1180400f70ea6734826`. +- `assembleAndroidTest` — instrumentation sources compile; **the suite was not executed**. + The CI runner has no KVM access: the default API-30 image did not boot within the + timeout, and the `aosp_atd` image boots but Gradle reports it as an "Unknown API Level" + and skips it. The emulator job was removed rather than left flaky; instrumented tests + must run on a physical device or a KVM-enabled emulator. Nothing here is claimed as + executed. + +### Claims NOT made + +- No production-grade recognition, no calibrated confidence, no MLBB accuracy claim, no + accuracy-based-on-synthetic-fixtures claim. The threshold and weights are uncalibrated. +- No physical-device freeze verification yet (rows 14–22 in + `docs/TESTING_PHYSICAL_DEVICE.md`). Device model, install source, test date and logs are + not recorded; the MediaProjection lifecycle matrix remains NOT RUN. +- The instrumentation suite does not and cannot simulate the consent UI; the notification + action end-to-end (tap in the shade while the game is foreground) needs a device. +- No screenshots or captured frames are committed; none were uploaded anywhere. + +### Remaining acceptance criteria + +- Physical-device smoke of the freeze workflow (notification tap while MLBB is foreground; + Vision Lab displays the pinned frame; Resume returns to live frames; stop keeps the pinned + frame for editing). +- On-device accuracy evaluation with manually labeled samples; threshold and + visual-information-heuristic calibration against real captures. +- Catalog entries loaded at device sync time recorded here when a device sync happens. +- PR #3 stays Draft and unmerged; do not merge without this verification. + +Historical green runs from before the review-fix pass (kept as history; do not cite these +as the current verification): `e58611c` — run 43732908605 (646 executions); `7d0c903` — run +43731484922 (same totals). diff --git a/docs/TESTING_PHYSICAL_DEVICE.md b/docs/TESTING_PHYSICAL_DEVICE.md index f740d9a..96918c0 100644 --- a/docs/TESTING_PHYSICAL_DEVICE.md +++ b/docs/TESTING_PHYSICAL_DEVICE.md @@ -51,5 +51,6 @@ Status values: **PASS**, **FAIL**, **PARTIAL**, **NOT RUN**. Record the commit S - For failures: full exception and steps immediately before it ## CI scope -CI runs JVM unit tests, lint, APK assembly and merged-manifest inspection. An `instrumented` -job runs the instrumentation suite on an emulator. Neither substitutes for this checklist. +CI runs JVM unit tests, lint, APK assembly and merged-manifest inspection; instrumentation +tests compile but do not execute (no KVM on the runner — see `docs/VISION_TESTING.md`). +Neither substitutes for this checklist. diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index 2398524..990487b 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -2,34 +2,13 @@ ## Where the tests run -GitHub Actions is the build of record for this sprint. - -The authoring host is **aarch64**, and AGP 8.6.1 publishes only an **x86_64** `aapt2` — the -binary that compiles and links resources. No Gradle task that touches resources can run -there; the failure is `Cannot run program ".../aapt2": error=2`. Installing an x86_64 SDK, -emulating it, or writing a second build system would each cost more than they are worth, so -every verification below ran in CI on `ubuntu-latest` (x86_64) with JDK 17, the same -`./gradlew` tasks a developer would run locally. - -Workflow: `.github/workflows/build.yml`, triggered by `pull_request`. - -```sh -./gradlew test --stacktrace # JVM unit tests -./gradlew assembleAndroidTest --stacktrace # instrumentation sources compile -./gradlew lint --stacktrace # strict lint, abortOnError = true -./gradlew assembleDebug --stacktrace # debug APK -``` - -The run also prints the unit-test totals and uploads the APK as the `draftlens-debug` -artifact, then re-reads the packaged manifest to confirm the foreground-service permissions -and the `mediaProjection` service type survived the build. - -Latest green run: -— head SHA `5ffb0c193cb84da77fb49984c81e219178c7ef3f`, **269 unit tests, 0 failures, 0 errors, -0 skipped**, executed in both the debug and the release unit-test variants (538 test -executions in total). +GitHub Actions is the build of record for this sprint. The authoring host is aarch64 and +AGP 8.6.1's `aapt2` is x86_64-only, so nothing that touches resources can be assembled +there. The current, authoritative verification record for this branch is the +**Sprint 1B review-fix pass** section below (run `43742933328`); the older Sprint 1A/1B run +records further down remain as history. -The APK artifact digest is reported in the implementation report and in the pull request. +See the sections below for the exact commands CI runs. ## Test inventory @@ -95,24 +74,96 @@ mismatches, oversized output, unsupported formats and render errors all produce calibration suites assert that an unsupported or corrupt document is reported and left untouched, and that only clearing the document recovers from it. -## Sprint 1B tests (inventory) +## Sprint 1B review-fix pass (this revision) + +GitHub Actions is the build of record. The authoring host is **aarch64**, and AGP 8.6.1 +publishes only an **x86_64** `aapt2`, so nothing that touches resources can be assembled +locally; every verification below ran in CI on `ubuntu-latest` (x86_64) with JDK 17. + +Workflow: `.github/workflows/build.yml` (one `build` job), triggered by `pull_request` +(and pushes to `main`/`master`). + +Latest green run: +— head SHA `f3ea76983b88a591956564e37003b6fb7e6c95ca`: + +- **Unit tests: 792 executions (debug + release), 0 failures, 0 errors, 0 skipped.** +- `assembleAndroidTest` — instrumentation sources compile. +- strict `lint` (`abortOnError = true`) — green. +- `assembleDebug` — green; artifact `draftlens-debug`, 10 362 310 bytes, + digest `sha256:55919144b62e5ca128c72cb6fb404a22f9fa6c5762ecb1180400f70ea6734826`. + +The instrumentation suite is **compiled but not executed**. The CI runner has no KVM +access: the default API-30 image does not boot within the runner's timeout, and the +`aosp_atd` image boots but is reported to Gradle as an "Unknown API Level" and is skipped +(`Skipping device 'emulator-5554' ... No compatible devices connected`). Instrumented +tests must be run on a physical device or an emulator with KVM (see +`docs/TESTING_PHYSICAL_DEVICE.md`); they are not claimed as executed anywhere. + +The run also re-reads the packaged manifest to confirm the foreground-service permissions +and the `mediaProjection` service type survived the build. + +## Test inventory (Sprint 1B additions) | Suite | File | Covers | |---|---|---| -| Hero catalog sync | `heroes/HeroCatalogSyncTest.kt` | pagination to the reported total, mirror fallback, non-zero API code, duplicate-ID collapse, invalid-record skip, MLBBDex cross-check match/unavailable, roles/lanes by slug | -| Hero catalog codec | `heroes/HeroCatalogCodecTest.kt` | snapshot round-trip, schema refusal, blank id/name, non-HTTPS asset URL, unknown fields | -| Hero catalog repository | `heroes/FileHeroCatalogRepositoryTest.kt` | refresh stores snapshot + assets, atomic cache, offline restart, corrupt cache error, asset magic-byte/content-type/size validation, refresh failure keeps cache | -| Slot extraction | `heroes/SlotImageExtractorTest.kt` | portrait/landscape, 0/90/180/270 upright output, edge-touching and minimum-size rectangles, below-one-pixel rejection, payload mismatch, source untouched | -| Matcher | `heroes/HeroMatcherTest.kt` | identical images match, ranking order, no-match threshold, ambiguous gap, corrupt references skipped, hamming/color distance, determinism | -| Recognition orchestration | `heroes/HeroRecognitionTest.kt` | candidates path, no usable references → failure, no-match path | -| Confirmation store | `datastore/HeroRecognitionStoreTest.kt` | confirm/load/replace/clear, corrupt and future documents read as empty | -| ViewModel | `feature/visionlab/VisionLabViewModelTest.kt` | identify without slot → message, identify without catalog → FAILED honestly, synced catalog → candidates, confirm → persisted, clear → removed, catalog name search | - -Sprint 1B green runs: latest branch head `e58611c25d7d9a40592aa3d81ad3347869771cc3` (class-scoped -test Main, test-only) — - (646 executions, debug + -release, **0 failures, 0 errors, 0 skipped**); prior green heads `b4eac9d856fe049d8d6ae8d232472bd4627c1f4e` -(run 43732111174) and `7d0c9032a52db8e0f962beea72ceae7120822c88` (run 43731484922), same totals. Instrumentation -tests compiled, not executed. The 1B transport is exercised only through fake responses; no -test performs a real network call. The number of catalog entries loaded from the live API is -measured at device sync time, not in CI. +| Pixel extraction | `heroes/SlotImageExtractorTest.kt` | asymmetric 3x2/5x3 fixtures, exact pixel order for all four rotations (hand-computed), edge-touching crops, minimum-size and refused crops, source immutability | +| Matcher | `heroes/HeroMatcherTest.kt` | identical images match, ranking, no-match threshold, ambiguous gap, corrupt references skipped, determinism | +| Recognition orchestration | `heroes/HeroRecognitionTest.kt` | candidates path, unusable references → failure, below-threshold → no-match (inverse-gradient fixture), uniform crop → `InsufficientVisualInformation` | +| Visual information | `heroes/VisualInformationTest.kt` | uniform gray/black/white, saturated solids, near-uniform noise, low-resolution crops, smooth & low-contrast gradients (not rejected), textured & pseudo-random detail (accepted), dimension mismatch, end-to-end blank → `InsufficientVisualInformation`, `NoMatch` kept distinct | +| Catalog sync | `heroes/HeroCatalogSyncTest.kt` | unchanged behaviour (pagination, mirror, duplicates, cross-check) | +| Catalog repository | `heroes/FileHeroCatalogRepositoryTest.kt` | snapshot + assets, offline restart, **unchanged URL → no re-download**, **URL change → re-download & new asset served**, **URL change + failed refresh → old asset not exposed**, **corrupt file → not served + re-downloaded**, metadata forgery rejected, missing/corrupt sidecar → re-download, oversized cached file rejected, truncated/HTML-as-PNG rejected, null content-type accepted, oversized download rejected, fresh assets served via network outage | +| Atomic writes | `heroes/WriteAtomicallyTest.kt` | exact-bytes write, failure leaves a directory target untouched and removes the temp file, blocked parent reports failure without touching it, hourly `*.tmp` sweep on init, no temp leftovers on a failed refresh | +| Redirect policy | `heroes/HeroRedirectPolicyTest.kt` | HTTPS targets accepted (absolute/relative), HTTPS→HTTP downgrade refused, non-HTTP schemes refused, missing/blank/over-long targets refused | +| Structural validator | `heroes/StructuralImageValidatorTest.kt` | real PNG passes, truncation/CRC corruption/HTML fail, GIF/WebP framing checks | +| Freeze flow (JVM) | `capture/CaptureFreezeTest.kt` | freeze without an active capture → `NoPipeline`; resume drops the pin; a duplicate freeze keeps the original frame | +| Freeze flow (JVM) | `capture/CaptureStateHolderTest.kt` | pinned frame survives `terminateSession` (stop and error), new session drops it, clear drops it, reset drops it | +| Freeze flow (JVM) | `models/FrameSnapshotsTest.kt` | copy preserves dimensions/rotation/timestamps/id, pixel values equal, caller-array mutation does not affect the copy, capture latency wrapper preserved | +| Vision source | `capture/CaptureVisionFrameSourceTest.kt` | adapter publishes capture-side frozen frames; freeze with no capture → false; resume is a safe no-op | +| ViewModel | `feature/visionlab/VisionLabViewModelTest.kt` | **capture-side freeze shows the pinned frame while newer live frames keep arriving**, resume reveals live frames again, freeze with no frame reports honestly, uniform-gray Frame produces `INSUFFICIENT_INFO` instead of candidates, inverse-gradient → `NO_MATCH`, confirmation/correct of candidates works, direct correction without clearing first, **confirm/clear persistence failure reported, never shown as success** | +| Instrumentation | `androidTest/.../AndroidBitmapExtractionInstrumentedTest.kt` | all four rotations pixel-exact (asymmetric non-square fixtures), edge crops, payload/source dimension mismatch refused, source Bitmap valid after extraction, minimum-size and refused crops, recycled Bitmap refused, bitmap snapshot copies pixels independently | +| Instrumentation | `androidTest/.../ReferenceImageValidatorInstrumentedTest.kt` | real PNG decodes on device, truncated PNG and HTML do not decode, JPEG structure-vs-decode layering | +| Instrumentation | `androidTest/.../CaptureLifecycleInstrumentedTest.kt` | manifest declares the service, `mediaProjection` foreground type, required permissions | + +## Fixture policy + +- **Synthetic images only.** No real game screenshots, no captured frames, no image assets are + committed. Fixtures are generated pixel arrays or the built-in 1x1 red/green PNGs + (`com.draftlens.vision.testRealPng()` / `testRealGreenPng()`), which are real, decodable + containers with correct CRCs. +- The calibration store test writes to a temporary directory under the system temp folder and + removes it afterwards. + +## What is not covered + +- **Instrumentation tests: compiled, not executed.** `assembleAndroidTest` proves the + instrumentation sources compile; nothing ran them. The CI runner has no KVM access: the + default API-30 image did not boot within the runner's timeout, and the `aosp_atd` image + boots but is skipped as "Unknown API Level" by Gradle. Two consecutive attempts failed + for these reasons, so the emulator job was removed rather than left flaky. Instrumented + tests must be executed on a physical device or a KVM-enabled emulator before the PR is + marked verified; none of their results are claimed as executed here. +- **Physical-device smoke of the freeze workflow: pending.** The manual lifecycle matrix + in `docs/TESTING_PHYSICAL_DEVICE.md` (rows 14–22) covers the notification freeze, the + FROZEN/Resume cycle, duplicate taps, disabled notifications and stop-with-frozen-frame. + None of it has been run on a device. +- **MediaProjection lifecycle matrix remains NOT RUN** (rows 2–11). +- **On-device recognition accuracy: not measured.** The no-match threshold and the matcher + weights are uncalibrated; the visual-information heuristics are conservative starting + points, not MLBB-calibrated thresholds. Real-screen evaluation and calibration remain + pending; see `docs/sprints/SPRINT_1B_HERO_RECOGNITION.md` § Known limitations. +- **Catalog entries actually loaded** are measured at device sync time; not recorded here. +- Compose screens are not UI-tested (no compose-ui-test dependency exists in this project); + the interaction model is covered by the interaction state-machine tests. +- The service-level notification action (`ACTION_FREEZE` delivery through + `onStartCommand`) is exercised only through the JVM-level `CaptureFreeze`/`CaptureStateHolder` + paths and manual device verification; no instrumented test taps the system notification. + +## Honest failure behaviour + +The suites assert that failures are reported rather than hidden: invalid dimensions, payload +mismatches, oversized output, unsupported formats and render errors produce a +`ProcessFrameResult.Failure` with a reason code; unsupported/corrupt calibration documents +are reported and untouched; a failed cache download never replaces a good asset and never +serves a stale one under a changed URL; a failed persistence operation is reported, not +hidden behind a success state. The instrumentation suite covers the Android-only paths +(bitmap extraction, decode validation, manifest). diff --git a/tasks/plan.md b/tasks/plan.md index 8ea39e2..8564523 100644 --- a/tasks/plan.md +++ b/tasks/plan.md @@ -166,9 +166,9 @@ No recommendation features, no overlays, no automation, PR stays draft and unmer no direct-write fallback; failures preserve the previous target; `*.tmp` swept - [x] Part H: reference loader reads dimensions before recycling; pixels read before release; exceptional paths recycle too -- [ ] CI green on the final head (unit tests, lint, assembleDebug, assembleAndroidTest, - instrumented job on the emulator) -- [ ] Physical-device smoke of the notification freeze workflow; on-device accuracy +- [ ] CI green on the final head; instrumentation suite remains compiled-not-executed + (no KVM on the runner) — needs a device or KVM-enabled emulator +- [ ] Physical-device smoke of the freeze workflow; on-device accuracy evaluation; real-screen threshold and heuristic calibration ### Sprint 1B verification diff --git a/tasks/todo.md b/tasks/todo.md index 8147a7a..47d4e4d 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -115,5 +115,5 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** URL change forces re-download; failed refresh keeps previous asset; HTTPS-only redirects) - [x] Part G: atomic snapshot/asset writes; failure preserves previous snapshot; temp files swept - [x] Part H: reference Bitmap dims read before recycling -- [ ] CI green on the final head; instrumented job result recorded honestly +- [ ] CI green on the final head (instrumentation compiled, not executed — no KVM on the runner) - [ ] Physical-device smoke test and on-device accuracy evaluation (pending) From 96d9a97e732d3d7ec84f60bb80c5d375ec7f3dac Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 18:13:49 +0000 Subject: [PATCH 23/33] docs: point the Sprint 1B verification at the latest green head (7a839d1) --- docs/IMPLEMENTATION_REPORT.md | 8 ++++---- docs/VISION_TESTING.md | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index b8c6477..56162b3 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -455,11 +455,11 @@ review-fix pass and how it was verified. ### Verification actually performed (this revision) - **Unit tests, debug and release: 792 executions, 0 failures, 0 errors, 0 skipped** — - , - head SHA `f3ea76983b88a591956564e37003b6fb7e6c95ca`. + , + head SHA `7a839d1422ee5ba53f65a78ef746d323091d3b40`. - strict `lint` (abortOnError = true) — green; `assembleDebug` — green; public artifact - `draftlens-debug` 10 362 310 bytes, digest - `sha256:55919144b62e5ca128c72cb6fb404a22f9fa6c5762ecb1180400f70ea6734826`. + `draftlens-debug` 10 362 309 bytes, digest + `sha256:53c07492517fdfc2837ebb179393e526d6b397db2146228aa853eb4914221677`. - `assembleAndroidTest` — instrumentation sources compile; **the suite was not executed**. The CI runner has no KVM access: the default API-30 image did not boot within the timeout, and the `aosp_atd` image boots but Gradle reports it as an "Unknown API Level" diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index 990487b..631632e 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -83,14 +83,14 @@ locally; every verification below ran in CI on `ubuntu-latest` (x86_64) with JDK Workflow: `.github/workflows/build.yml` (one `build` job), triggered by `pull_request` (and pushes to `main`/`master`). -Latest green run: -— head SHA `f3ea76983b88a591956564e37003b6fb7e6c95ca`: +Latest green run: +— head SHA `7a839d1422ee5ba53f65a78ef746d323091d3b40`: - **Unit tests: 792 executions (debug + release), 0 failures, 0 errors, 0 skipped.** - `assembleAndroidTest` — instrumentation sources compile. - strict `lint` (`abortOnError = true`) — green. -- `assembleDebug` — green; artifact `draftlens-debug`, 10 362 310 bytes, - digest `sha256:55919144b62e5ca128c72cb6fb404a22f9fa6c5762ecb1180400f70ea6734826`. +- `assembleDebug` — green; artifact `draftlens-debug`, 10 362 309 bytes, + digest `sha256:53c07492517fdfc2837ebb179393e526d6b397db2146228aa853eb4914221677`. The instrumentation suite is **compiled but not executed**. The CI runner has no KVM access: the default API-30 image does not boot within the runner's timeout, and the From 4ff2a341dcda50c7ac44c3340dfeb1d19ee90541 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 18:18:55 +0000 Subject: [PATCH 24/33] docs: point the verification record at the PR description --- docs/IMPLEMENTATION_REPORT.md | 2 ++ docs/VISION_TESTING.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index 56162b3..6f80ac9 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -454,6 +454,8 @@ review-fix pass and how it was verified. ### Verification actually performed (this revision) +Docs-only commits after the code freeze do not change the build; the exact final head and its CI run are recorded in PR #3's description. + - **Unit tests, debug and release: 792 executions, 0 failures, 0 errors, 0 skipped** — , head SHA `7a839d1422ee5ba53f65a78ef746d323091d3b40`. diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index 631632e..495a049 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -83,6 +83,8 @@ locally; every verification below ran in CI on `ubuntu-latest` (x86_64) with JDK Workflow: `.github/workflows/build.yml` (one `build` job), triggered by `pull_request` (and pushes to `main`/`master`). +Docs-only commits after the code freeze do not change the build; the exact final head and its CI run are recorded in PR #3's description. + Latest green run: — head SHA `7a839d1422ee5ba53f65a78ef746d323091d3b40`: From e89c5e6cf50a96f5d61572202f4c003d2b7123da Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 18:19:18 +0000 Subject: [PATCH 25/33] chore: serialize pipeline attach/detach; drop an unused predicate --- app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt index 73a1553..aa8162d 100644 --- a/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt @@ -17,7 +17,7 @@ object CaptureFreeze { /** Called by the capture service once its pipeline exists. */ fun attachPipeline(instance: FramePipeline) { - pipeline = instance + synchronized(this) { pipeline = instance } } /** Called by the capture service during teardown; a stale reference is dropped. */ @@ -27,9 +27,6 @@ object CaptureFreeze { } } - /** True when an active pipeline is installed (i.e. capture is running). */ - fun isCapturing(): Boolean = pipeline != null - fun freezeCurrentFrame(): FreezeOutcome { val current = CaptureStateHolder.frozenFrame.value if (current != null) return FreezeOutcome.AlreadyFrozen From 1c7397b925dd2683b3d763e552d610d2e464ecb7 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 18:24:56 +0000 Subject: [PATCH 26/33] docs: drop the stale run pointer for the review-fix section --- docs/VISION_TESTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index 495a049..67779e4 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -5,7 +5,7 @@ GitHub Actions is the build of record for this sprint. The authoring host is aarch64 and AGP 8.6.1's `aapt2` is x86_64-only, so nothing that touches resources can be assembled there. The current, authoritative verification record for this branch is the -**Sprint 1B review-fix pass** section below (run `43742933328`); the older Sprint 1A/1B run +**Sprint 1B review-fix pass** section below; the older Sprint 1A/1B run records further down remain as history. See the sections below for the exact commands CI runs. From 864d83e329369571bdb60724237e6eaeb243b16f Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 18:30:02 +0000 Subject: [PATCH 27/33] docs: record the final green head in the sprint task lists --- tasks/plan.md | 2 +- tasks/todo.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/plan.md b/tasks/plan.md index 8564523..d2c54e3 100644 --- a/tasks/plan.md +++ b/tasks/plan.md @@ -166,7 +166,7 @@ No recommendation features, no overlays, no automation, PR stays draft and unmer no direct-write fallback; failures preserve the previous target; `*.tmp` swept - [x] Part H: reference loader reads dimensions before recycling; pixels read before release; exceptional paths recycle too -- [ ] CI green on the final head; instrumentation suite remains compiled-not-executed +- [x] CI green on the final head (run 43746648882, head 1c7397b925dd2683b3d763e552d610d2e464ecb7; instrumentation compiled, not executed — no KVM on the runner) (no KVM on the runner) — needs a device or KVM-enabled emulator - [ ] Physical-device smoke of the freeze workflow; on-device accuracy evaluation; real-screen threshold and heuristic calibration diff --git a/tasks/todo.md b/tasks/todo.md index 47d4e4d..d07cdbf 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -115,5 +115,5 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** URL change forces re-download; failed refresh keeps previous asset; HTTPS-only redirects) - [x] Part G: atomic snapshot/asset writes; failure preserves previous snapshot; temp files swept - [x] Part H: reference Bitmap dims read before recycling -- [ ] CI green on the final head (instrumentation compiled, not executed — no KVM on the runner) +- [x] CI green on the final head (run 43746648882, head 1c7397b925dd2683b3d763e552d610d2e464ecb7; instrumentation compiled, not executed — no KVM on the runner) - [ ] Physical-device smoke test and on-device accuracy evaluation (pending) From 6b883ef8392d4f3c2ddaae6d1bcb95ce147966f1 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 18:35:29 +0000 Subject: [PATCH 28/33] fix: a stale freeze action must not leave an idle service behind The action now stops the service whenever no pipeline is attached, regardless of whether a frozen frame was preserved from a previous session. freezeCurrentFrame keeps the holder-first shortcut so the AlreadyFrozen path stays testable on the JVM. --- app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt | 3 +++ .../kotlin/com/draftlens/capture/CaptureSessionService.kt | 6 +++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt index aa8162d..3519160 100644 --- a/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureFreeze.kt @@ -27,6 +27,9 @@ object CaptureFreeze { } } + /** True when a pipeline is installed (i.e. capture is running). */ + fun hasActivePipeline(): Boolean = pipeline != null + fun freezeCurrentFrame(): FreezeOutcome { val current = CaptureStateHolder.frozenFrame.value if (current != null) return FreezeOutcome.AlreadyFrozen diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt index 2367c1d..f5eea61 100644 --- a/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt @@ -132,6 +132,11 @@ class CaptureSessionService : Service() { } ACTION_FREEZE -> { // Never re-promotes, never restarts MediaProjection: no-op without an active pipeline. + if (!CaptureFreeze.hasActivePipeline()) { + Log.i(TAG, "Freeze command ignored: no active capture session") + stopSelf() // do not linger for a stale command + return START_NOT_STICKY + } when (val outcome = CaptureFreeze.freezeCurrentFrame()) { is FreezeOutcome.Frozen -> Log.i(TAG, "Freeze command honoured (${outcome.frame.width}x${outcome.frame.height})") @@ -142,7 +147,6 @@ class CaptureSessionService : Service() { FreezeOutcome.NoPipeline -> { Log.i(TAG, "Freeze command ignored: no active capture session") stopSelf() // do not linger for a stale command - return START_NOT_STICKY } } START_NOT_STICKY From e6d1de8cd6f17236d796c85f601d8dc1e6780b98 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 23:02:49 +0000 Subject: [PATCH 29/33] fix: landscape MLBB capture shows the whole game (PR #3 review pass) Root causes addressed: - The VirtualDisplay kept the session-start dimensions forever: rotation relied on onCapturedContentResize (API 34+) and, failing that, on no signal at all; nothing re-derived the capture size from the display. Now onCapturedContentResize AND a DisplayManager fallback both funnel into one idempotent resize path re-creating the ImageReader/Surface at the reported size. - Late frames from a released reader could publish into the replacement reader's stream; every generation is now numbered and stale callbacks are dropped at the listener and at publish time. - image.cropRect was ignored; letterboxed/partial content produced shifted garbage. The conversion now honors cropRect (offset crops become tightly packed images of the content region), padded rows and arbitrary pixel strides, via a pure, JVM-tested RgbaPlaneCopy. - No diagnostics existed to tell acquisition-side black from conversion- side black. Structured key=value logs now cover dimensions, resize events, plane layout, cropRect, rowStride, frame generation and the preprocessor in/out dimensions; contents are never logged. Freeze keeps working: the pinned frame is never overwritten, resize preserves the frozen snapshot, Resume continues the current generation. --- .../FramePipelineInstrumentedTest.kt | 81 ++++++ .../main/kotlin/com/draftlens/DraftLensApp.kt | 7 + .../capture/CaptureSessionService.kt | 76 ++++-- .../com/draftlens/capture/FramePipeline.kt | 102 +++++-- .../draftlens/capture/GenerationCounter.kt | 21 ++ .../com/draftlens/capture/RgbaPlaneCopy.kt | 119 ++++++++ .../preprocessing/AndroidFramePreprocessor.kt | 5 + .../PixelArrayFramePreprocessor.kt | 5 + .../vision/preprocessing/PreprocessLog.kt | 12 + .../capture/DisplayChangePolicyTest.kt | 23 ++ .../capture/GenerationCounterTest.kt | 39 +++ .../draftlens/capture/RgbaPlaneCopyTest.kt | 258 ++++++++++++++++++ docs/SCREEN_CAPTURE.md | 81 +++++- docs/TESTING_PHYSICAL_DEVICE.md | 14 +- tasks/plan.md | 9 + tasks/todo.md | 11 +- 16 files changed, 817 insertions(+), 46 deletions(-) create mode 100644 app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt create mode 100644 app/src/main/kotlin/com/draftlens/capture/GenerationCounter.kt create mode 100644 app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt create mode 100644 app/src/main/kotlin/com/draftlens/vision/preprocessing/PreprocessLog.kt create mode 100644 app/src/test/kotlin/com/draftlens/capture/GenerationCounterTest.kt create mode 100644 app/src/test/kotlin/com/draftlens/capture/RgbaPlaneCopyTest.kt diff --git a/app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt b/app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt new file mode 100644 index 0000000..e9e389d --- /dev/null +++ b/app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt @@ -0,0 +1,81 @@ +package com.draftlens + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.draftlens.capture.FramePipeline +import com.draftlens.capture.FrameSampler +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNotSame +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith + +/** + * Lifecycle invariants for the capture pipeline on a device or emulator: generation + * bumps, replacement surfaces and the frozen snapshot. No pixels are pushed, so this + * needs no MediaProjection consent. Compiled in CI; execution requires a device/emulator + * (see docs/TESTING_PHYSICAL_DEVICE.md). + */ +@RunWith(AndroidJUnit4::class) +class FramePipelineInstrumentedTest { + + @Test fun `starting a pipeline creates a surface sized to the request`() { + val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) + try { + pipeline.start(1080, 2400, 440) + + assertNotNull(pipeline.surface()) + assertEquals(1080, pipeline.displayWidth) + assertEquals(2400, pipeline.displayHeight) + } finally { + pipeline.stop() + } + } + + @Test fun `a resize replaces the surface and bumps the generation`() { + val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) + try { + pipeline.start(1080, 2400, 440) + val before = pipeline.surface() + val generationBefore = pipeline.peekGeneration() + + val replacement = pipeline.prepareResize(2400, 1080, 440, + com.draftlens.vision.model.FrameRotation.ROTATION_0) + + assertNotNull(replacement) + replacement!!.closePrevious() + assertEquals(2400, pipeline.displayWidth) + assertEquals(1080, pipeline.displayHeight) + assertNotSame("resize must install a replacement surface", before, pipeline.surface()) + assertTrue("resize must advance the generation", pipeline.peekGeneration() > generationBefore) + } finally { + pipeline.stop() + } + } + + @Test fun `freezing with no frames published reports NoFrame and keeps the pipeline alive`() { + val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) + try { + pipeline.start(2400, 1080, 440) + + val outcome = pipeline.freezeAndSnapshot() + + assertEquals(FramePipeline.FreezeResult.NoFrame, outcome) + assertNotNull("the reader must survive a freeze request with no frames", pipeline.surface()) + } finally { + pipeline.stop() + } + } + + @Test fun `stopping the pipeline releases its surface`() { + val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) + pipeline.start(1080, 2400, 440) + + pipeline.stop() + + assertNull(pipeline.surface()) + assertEquals(0, pipeline.displayWidth) + assertEquals(0, pipeline.displayHeight) + } +} diff --git a/app/src/main/kotlin/com/draftlens/DraftLensApp.kt b/app/src/main/kotlin/com/draftlens/DraftLensApp.kt index cd7016e..6d7fec5 100644 --- a/app/src/main/kotlin/com/draftlens/DraftLensApp.kt +++ b/app/src/main/kotlin/com/draftlens/DraftLensApp.kt @@ -16,6 +16,13 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob class DraftLensApp : Application() { + override fun onCreate() { + super.onCreate() + com.draftlens.vision.preprocessing.PreprocessLog.sink = { message -> + android.util.Log.i("VisionPreprocessor", message) + } + } + val preferences: AppPreferencesRepository by lazy { AppPreferencesRepository(this) } diff --git a/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt b/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt index f5eea61..0e7da44 100644 --- a/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt +++ b/app/src/main/kotlin/com/draftlens/capture/CaptureSessionService.kt @@ -58,6 +58,8 @@ class CaptureSessionService : Service() { private var frameJob: Job? = null private var frozenJob: Job? = null private var foregroundPromoted = false + /** Increments once per capture session; tags every structured log line. */ + private var sessionCounter = 0 private val projectionCallback = object : MediaProjection.Callback() { override fun onStop() { @@ -66,7 +68,9 @@ class CaptureSessionService : Service() { } override fun onCapturedContentResize(width: Int, height: Int) { - recreateDisplay(width, height, resources.configuration.densityDpi) + // The captured-content dimensions are authoritative; they are not the same as + // the display-window metrics the session started with (that is the landscape bug). + recreateDisplay(width, height, source = "captured-content-resize") } } @@ -171,6 +175,7 @@ class CaptureSessionService : Service() { return } + sessionCounter += 1 sampler.setIntervalMs(intervalMs) releaseCaptureResources(releaseForeground = false) @@ -195,6 +200,7 @@ class CaptureSessionService : Service() { val (captureWidth, captureHeight) = captureDimensions(wm) val dpi = resources.configuration.densityDpi val displayRotation = currentDisplayRotation() + Log.i(TAG, "session=$sessionCounter capture-start: dimensions=${captureWidth}x${captureHeight} dpi=$dpi rotation=$displayRotation source=windowMetrics") pipeline.start(captureWidth, captureHeight, dpi, displayRotation) CaptureFreeze.attachPipeline(pipeline) @@ -318,21 +324,34 @@ class CaptureSessionService : Service() { null, null ) ?: error("createVirtualDisplay() returned null") - Log.i(TAG, "VirtualDisplay created (${width}x${height}, ${densityDpi}dpi)") + Log.i(TAG, "session=$sessionCounter virtual-display created: ${width}x${height} dpi=$densityDpi (auto-mirror of the default display)") } - private fun recreateDisplay(width: Int, height: Int, densityDpi: Int) { + /** + * Applies new captured-content dimensions to the VirtualDisplay/ImageReader pair. + * [source] identifies which signal requested it ("captured-content-resize" — the + * authoritative callback, or "display-listener" — the rotation/size fallback); both + * funnel here so the handling is identical and idempotent. + */ + private fun recreateDisplay(width: Int, height: Int, source: String) { + val densityDpi = resources.configuration.densityDpi val display = virtualDisplay ?: run { - Log.e(TAG, "Resize requested but VirtualDisplay is missing") + Log.e(TAG, "session=$sessionCounter resize requested ($source) but VirtualDisplay is missing") failProjectionSession("VirtualDisplay disappeared during resize", null) return } - if (CaptureStateHolder.state.value.status != CaptureStatus.CAPTURING) return + if (CaptureStateHolder.state.value.status != CaptureStatus.CAPTURING) { + Log.i(TAG, "session=$sessionCounter ignoring $source ${width}x${height}: capture is not in CAPTURING state") + return + } val displayRotation = currentDisplayRotation() + val currentWidth = CaptureStateHolder.state.value.displayWidth + val currentHeight = CaptureStateHolder.state.value.displayHeight + Log.i(TAG, "session=$sessionCounter $source: ${currentWidth}x${currentHeight} -> ${width}x${height} dpi=$densityDpi rotation=$displayRotation") val change = DisplayChangePolicy.classify( - CaptureStateHolder.state.value.displayWidth, - CaptureStateHolder.state.value.displayHeight, + currentWidth, + currentHeight, width, height, ) @@ -340,19 +359,26 @@ class CaptureSessionService : Service() { is DisplayChange.RotationOnly -> { // The capture size is unchanged: only the rotation metadata needs updating. updateDisplayRotation(displayRotation) - Log.i(TAG, "Display rotation updated to $displayRotation without resizing") + Log.i(TAG, "session=$sessionCounter rotation-only update: rotation=$displayRotation without resizing") } is DisplayChange.Resize -> { try { - val replacement = pipeline.prepareResize(width, height, densityDpi, displayRotation) ?: return - // Caller owns cleanup after detach; VirtualDisplay must not retain a released Surface. + val replacement = pipeline.prepareResize(width, height, densityDpi, displayRotation) + if (replacement == null) { + Log.i(TAG, "session=$sessionCounter resize no-op: dimensions already match") + return + } + // Detach first: the VirtualDisplay must not retain a released Surface. + Log.i(TAG, "session=$sessionCounter surface-detach from VirtualDisplay (previous generation)") display.setSurface(null) + Log.i(TAG, "session=$sessionCounter releasing previous ImageReader/surface/thread") replacement.closePrevious() + Log.i(TAG, "session=$sessionCounter surface-attach new reader generation to VirtualDisplay") display.setSurface(replacement.surface) display.resize(width, height, densityDpi) CaptureStateHolder.updateDisplay(width, height, densityDpi) - Log.i(TAG, "VirtualDisplay resized (${width}x${height}, ${densityDpi}dpi)") + Log.i(TAG, "session=$sessionCounter virtual-display resized: ${width}x${height} dpi=$densityDpi generation=${pipeline.peekGeneration()}") } catch (e: Exception) { failProjectionSession("Failed to resize VirtualDisplay", e) } @@ -372,6 +398,11 @@ class CaptureSessionService : Service() { /** * Watches the default display so an orientation change is picked up even when the * captured content keeps its size and MediaProjection reports no resize. + * + * Sizing itself is re-checked here as well: on API levels below the captured-content + * callback (or whenever a rotation re-lays the display), the VirtualDisplay would + * otherwise keep the session-start dimensions forever (the whole-screen landscape + * bug). Both paths funnel into [recreateDisplay], which is idempotent. */ private fun registerDisplayListener() { val manager = getSystemService(Context.DISPLAY_SERVICE) as? DisplayManager ?: return @@ -381,15 +412,24 @@ class CaptureSessionService : Service() { override fun onDisplayChanged(displayId: Int) { if (displayId == Display.DEFAULT_DISPLAY) { + Log.i(TAG, "session=$sessionCounter default-display changed") updateDisplayRotation(currentDisplayRotation()) + val wm = getSystemService(Context.WINDOW_SERVICE) as WindowManager + val (width, height) = captureDimensions(wm) + val current = CaptureStateHolder.state.value + if (width != current.displayWidth || height != current.displayHeight) { + Log.i(TAG, "session=$sessionCounter display metrics changed ${current.displayWidth}x${current.displayHeight} -> ${width}x${height}; resyncing capture size") + recreateDisplay(width, height, source = "display-listener") + } } } } try { manager.registerDisplayListener(listener, null) displayListener = listener + Log.i(TAG, "session=$sessionCounter display listener registered") } catch (e: Exception) { - Log.w(TAG, "Display listener registration failed", e) + Log.w(TAG, "session=$sessionCounter display listener registration failed", e) } } @@ -399,8 +439,9 @@ class CaptureSessionService : Service() { try { (getSystemService(Context.DISPLAY_SERVICE) as? DisplayManager) ?.unregisterDisplayListener(listener) + Log.i(TAG, "session=$sessionCounter display listener unregistered") } catch (e: Exception) { - Log.w(TAG, "Display listener unregister failed", e) + Log.w(TAG, "session=$sessionCounter display listener unregister failed", e) } } @@ -420,26 +461,29 @@ class CaptureSessionService : Service() { } private fun releaseCaptureResources(releaseForeground: Boolean) { + Log.i(TAG, "session=$sessionCounter release: captureJob/frameJob/frozenJob cancelled=${captureJob != null || frameJob != null || frozenJob != null}") captureJob?.cancel() captureJob = null frameJob?.cancel() frameJob = null frozenJob?.cancel() frozenJob = null - try { virtualDisplay?.release() } catch (e: Exception) { Log.w(TAG, "VirtualDisplay release failed", e) } + try { virtualDisplay?.release() } catch (e: Exception) { Log.w(TAG, "session=$sessionCounter VirtualDisplay release failed", e) } + if (virtualDisplay != null) Log.i(TAG, "session=$sessionCounter virtual-display released") virtualDisplay = null unregisterDisplayListener() try { projection?.unregisterCallback(projectionCallback) projection?.stop() + Log.i(TAG, "session=$sessionCounter media-projection released") } catch (e: Exception) { - Log.w(TAG, "MediaProjection stop failed", e) + Log.w(TAG, "session=$sessionCounter MediaProjection stop failed", e) } projection = null if (::pipeline.isInitialized) { CaptureFreeze.detachPipeline(pipeline) } - try { pipeline.stop() } catch (e: Exception) { Log.w(TAG, "Pipeline stop failed", e) } + try { pipeline.stop() } catch (e: Exception) { Log.w(TAG, "session=$sessionCounter Pipeline stop failed", e) } if (releaseForeground) { try { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { diff --git a/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt b/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt index ce33dda..d4fe1bf 100644 --- a/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt +++ b/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt @@ -43,6 +43,16 @@ class FramePipeline( @Volatile private var frozen = false + /** + * Bumped every time a new reader/surface generation is created (start/resize). + * Images captured with an older generation are dropped, so a late callback from a + * released reader can never publish into the replacement reader's frame stream. + */ + private val generation = GenerationCounter() + + /** Current capture generation; exposed for structured logs only. */ + internal fun peekGeneration(): Long = generation.current() + /** Result of a freeze request against this pipeline. */ sealed interface FreezeResult { /** A frame was pinned and is ready to store. */ @@ -97,6 +107,8 @@ class FramePipeline( displayRotation = initialDisplayRotation handlerThread = HandlerThread("draftlens-capture").also { it.start() } val handler = Handler(handlerThread!!.looper) + val generationAtStart = generation.bump() + Log.i(TAG, "pipeline start: width=$width height=$height density=$densityDpi rotation=$initialDisplayRotation generation=$generationAtStart") imageReader = ImageReader.newInstance(width, height, android.graphics.PixelFormat.RGBA_8888, 3).also { reader -> reader.setOnImageAvailableListener({ reader -> @@ -108,23 +120,30 @@ class FramePipeline( val wallStartedAtMs = System.currentTimeMillis() try { if (!isActive()) return@setOnImageAvailableListener + if (!generation.isCurrent(generationAtStart)) { + // A newer reader/surface generation owns the pipeline; drop late images. + return@setOnImageAvailableListener + } if (frozen) { // Frozen: images are acquired and closed in finally; no conversion, no retention. return@setOnImageAvailableListener } val sampledAtMs = sampler.currentTimeMs() if (sampler.shouldEmit(sampledAtMs)) { - // Reference image only to avoid leaks; copy pixels via Image plane. + // Reference image only to avoid leaks; pixels are copied via the plane. // Sprint 0 keeps this synchronous and in-memory; no disk, no upload. - val payload = imageToPayload(image, width, height) + val payload = imageToPayload(image, generationAtStart) if (payload == null) { - Log.w(TAG, "Dropped frame: RGBA_8888 payload conversion failed") + Log.w(TAG, "generation=$generationAtStart dropped frame: payload conversion failed") return@setOnImageAvailableListener } synchronized(freezeGate) { if (frozen) { // A freeze command landed during the conversion; the snapshot is authoritative. - Log.d(TAG, "Dropped frame: a freeze command landed during conversion") + Log.d(TAG, "generation=$generationAtStart dropped frame: a freeze command landed during conversion") + } else if (!generation.isCurrent(generationAtStart)) { + // An even newer generation took over while converting. + Log.d(TAG, "generation=$generationAtStart dropped frame: superseded by generation=${generation.current()} during conversion") } else { val processingLatencyMs = System.currentTimeMillis() - wallStartedAtMs // One read per frame: both fields below see the same rotation. @@ -132,8 +151,8 @@ class FramePipeline( val visionFrame = OriginalFrame( id = FrameId.random(), payload = payload, - sourceWidth = width, - sourceHeight = height, + sourceWidth = payload.width, + sourceHeight = payload.height, displayRotation = frameDisplayRotation, // MediaProjection buffers are display-oriented; never double-rotate pixels. rotation = FrameOrientation.correctionFor(frameDisplayRotation), @@ -146,6 +165,7 @@ class FramePipeline( frame = visionFrame, processingLatencyMs = processingLatencyMs ) + Log.d(TAG, "generation=$generationAtStart frame stored: source=${payload.width}x${payload.height} rotation=$frameDisplayRotation processingMs=$processingLatencyMs") } } } @@ -174,6 +194,7 @@ class FramePipeline( val frame = _latestFrame.value ?: return FreezeResult.NoFrame val copy = FrameSnapshots.copyOf(frame) ?: return FreezeResult.NoFrame frozen = true + Log.i(TAG, "generation=${generation.current()} frame pinned: ${frame.sourceWidth}x${frame.sourceHeight} id=${frame.id.value}") FreezeResult.Frozen(copy) } @@ -181,36 +202,70 @@ class FramePipeline( fun resumeCapture() { synchronized(freezeGate) { frozen = false } sampler.reset() + Log.i(TAG, "generation=${generation.current()} resumed live capture") } /** True while the pipeline is frozen; images flow but nothing is published. */ val isFrozen: Boolean get() = synchronized(freezeGate) { frozen } - private fun imageToPayload( - image: android.media.Image, - width: Int, - height: Int - ): BitmapFramePayload? { - // Direct plane copy for RGBA_8888; null drops the frame. + /** + * Copies one Image's pixels into a new ARGB_8888 bitmap, honoring the plane layout, + * padded rows and the image's cropRect (full-frame is the common case). Never logs or + * stores image contents. Returns null when the plane or crop is malformed so the + * frame is dropped instead of producing a shifted or partial image. + */ + private fun imageToPayload(image: android.media.Image, generation: Long): BitmapFramePayload? { return try { val planes = image.planes - if (planes.isEmpty()) return null - val buffer = planes[0].buffer - val pixelStride = planes[0].pixelStride - val rowStride = planes[0].rowStride - val rowPadding = rowStride - pixelStride * width - val bmp = Bitmap.createBitmap(width + (rowPadding / pixelStride), height, Bitmap.Config.ARGB_8888) - bmp.copyPixelsFromBuffer(buffer) - val cropped = if (rowPadding == 0) bmp else Bitmap.createBitmap(bmp, 0, 0, width, height).also { - if (it !== bmp) bmp.recycle() + if (planes.size != 1) { + Log.w(TAG, "generation=$generation dropped frame: ${planes.size} planes (expected 1 for RGBA_8888)") + return null } - BitmapFramePayload(cropped) + val plane = planes[0] + val crop = image.cropRect + val layout = RgbaPlaneCopy.layout( + imageWidth = image.width, + imageHeight = image.height, + rowStride = plane.rowStride, + pixelStride = plane.pixelStride, + cropLeft = crop.left, + cropTop = crop.top, + cropRight = crop.right, + cropBottom = crop.bottom, + ) + if (layout == null) { + Log.w( + TAG, + "generation=$generation dropped frame: malformed layout " + + "image=${image.width}x${image.height} " + + "crop=${crop.left},${crop.top}..${crop.right},${crop.bottom} " + + "rowStride=${plane.rowStride} pixelStride=${plane.pixelStride}", + ) + return null + } + val argb = RgbaPlaneCopy.copyArgb(plane.buffer, layout) + if (argb == null) { + Log.w(TAG, "generation=$generation dropped frame: plane copy failed for layout=$layout") + return null + } + Log.d( + TAG, + "generation=$generation plane: image=${layout.imageWidth}x${layout.imageHeight} " + + "crop=${layout.cropLeft},${layout.cropTop} ${layout.cropWidth}x${layout.cropHeight} " + + "rowStride=${layout.rowStride} pixelStride=${layout.pixelStride} " + + "paddingBytes=${layout.rowPaddingBytes} fullCrop=${layout.isFullCrop} bufferBytes=${plane.buffer.remaining()}", + ) + val bitmap = Bitmap.createBitmap(argb, layout.cropWidth, layout.cropHeight, Bitmap.Config.ARGB_8888) + Log.d(TAG, "generation=$generation bitmap: ${bitmap.width}x${bitmap.height}") + BitmapFramePayload(bitmap) } catch (e: Exception) { + Log.w(TAG, "generation=$generation dropped frame: conversion threw", e) null } } fun stop() { + Log.i(TAG, "pipeline stop: generation=${generation.current()} releasing surface/reader/thread") stopInternal(release = true) _latestFrame.value = null } @@ -241,6 +296,8 @@ class FramePipeline( val previousWidth = displayWidth val previousHeight = displayHeight + Log.i(TAG, "pipeline resize: detaching previous generation=${generation.current()} (${previousWidth}x${previousHeight}) for replacement ${newWidth}x${newHeight}") + // Detach the old resources from the pipeline so start() can build the replacement. surface = null imageReader = null @@ -267,6 +324,7 @@ class FramePipeline( return SurfaceSwitch(replacement) { if (closed) return@SurfaceSwitch closed = true + Log.i(TAG, "pipeline resize: releasing previous surface/reader/thread (generation ${generation.current() - 1} superseded)") runCatching { previousSurface?.release() } runCatching { previousReader?.close() } runCatching { previousThread?.quitSafely() } diff --git a/app/src/main/kotlin/com/draftlens/capture/GenerationCounter.kt b/app/src/main/kotlin/com/draftlens/capture/GenerationCounter.kt new file mode 100644 index 0000000..e407585 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/capture/GenerationCounter.kt @@ -0,0 +1,21 @@ +package com.draftlens.capture + +import java.util.concurrent.atomic.AtomicLong + +/** + * Monotonic generation marker for capture sessions and reader generations. + * + * Every time the pipeline builds a new ImageReader generation it calls [bump]; callbacks + * and publish operations captured with an older generation are rejected by [isCurrent]. + * This is what keeps late images from a released reader from overwriting frames from the + * replacement reader. + */ +internal class GenerationCounter { + private val generation = AtomicLong(0) + + fun bump(): Long = generation.incrementAndGet() + + fun current(): Long = generation.get() + + fun isCurrent(captured: Long): Boolean = captured == generation.get() +} diff --git a/app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt b/app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt new file mode 100644 index 0000000..5c1a5c3 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt @@ -0,0 +1,119 @@ +package com.draftlens.capture + +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Pure conversion between an RGBA_8888 image plane and tightly packed ARGB pixels. + * + * Handles every layout MediaProjection can deliver in practice without stretching, + * inventing or discarding pixels: + * - padded rows (`rowStride > imageWidth * pixelStride`), + * - `cropRect` offsets and partial content (letterboxed single-app capture), + * - non-square landscape/portrait sizes, + * - any `pixelStride >= 1` (RGBA_8888 is always 4; the math is generic), + * - little-endian and big-endian plane buffers. + * + * Byte order per pixel follows the buffer's own order: little-endian reads R,G,B,A; + * big-endian reads A,R,G,B. Never mutates or repositions the source buffer. Returns null + * on malformed input so the frame is dropped instead of producing a partial image. + */ +internal object RgbaPlaneCopy { + + /** RGBA_8888 stores 4 bytes per pixel. */ + const val RGBA_PIXEL_BYTES = 4 + + /** A validated plane layout, used for the conversion and for structured logs. */ + data class PlaneLayout( + val imageWidth: Int, + val imageHeight: Int, + val rowStride: Int, + val pixelStride: Int, + val rowPaddingBytes: Int, + val cropLeft: Int, + val cropTop: Int, + val cropWidth: Int, + val cropHeight: Int, + ) { + val rowPaddingPixels: Int get() = rowPaddingBytes / pixelStride + + /** True when the crop covers the whole image. */ + val isFullCrop: Boolean + get() = cropLeft == 0 && cropTop == 0 && + cropWidth == imageWidth && cropHeight == imageHeight + } + + /** + * Validates [imageWidth] x [imageHeight] against [rowStride]/[pixelStride] and a + * crop rectangle given as `[left, top, right, bottom)`. Returns null on any mismatch. + */ + fun layout( + imageWidth: Int, + imageHeight: Int, + rowStride: Int, + pixelStride: Int, + cropLeft: Int, + cropTop: Int, + cropRight: Int, + cropBottom: Int, + ): PlaneLayout? { + if (imageWidth <= 0 || imageHeight <= 0) return null + if (pixelStride <= 0) return null + if (rowStride < imageWidth.toLong() * pixelStride) return null + if (rowStride.toLong() * imageHeight > Int.MAX_VALUE) return null + if (cropRight <= cropLeft || cropBottom <= cropTop) return null + if (cropLeft < 0 || cropTop < 0 || cropRight > imageWidth || cropBottom > imageHeight) return null + return PlaneLayout( + imageWidth = imageWidth, + imageHeight = imageHeight, + rowStride = rowStride, + pixelStride = pixelStride, + rowPaddingBytes = rowStride - imageWidth * pixelStride, + cropLeft = cropLeft, + cropTop = cropTop, + cropWidth = cropRight - cropLeft, + cropHeight = cropBottom - cropTop, + ) + } + + /** + * Copies the crop region out of [buffer] into a tightly packed ARGB_8888 pixel array, + * row by row. Padding bytes are skipped; pixels outside the crop are never read. + * The result is laid out row-major with [PlaneLayout.cropWidth] columns. + */ + fun copyArgb(buffer: ByteBuffer, layout: PlaneLayout): IntArray? { + if (layout.cropWidth <= 0 || layout.cropHeight <= 0) return null + val needsRead = (layout.cropTop.toLong() + layout.cropHeight) * layout.rowStride + if (buffer.remaining().toLong() < needsRead) return null + val bigEndian = buffer.order() == ByteOrder.BIG_ENDIAN + val out = IntArray(layout.cropWidth * layout.cropHeight) + val stride = layout.rowStride + val pixelStride = layout.pixelStride + val cropLeftBytes = layout.cropLeft * pixelStride + for (y in 0 until layout.cropHeight) { + val rowStart = (layout.cropTop + y) * stride + cropLeftBytes + val outRow = y * layout.cropWidth + for (x in 0 until layout.cropWidth) { + val address = rowStart + x * pixelStride + val r: Int + val g: Int + val b: Int + val a: Int + if (bigEndian) { + a = buffer.get(address).toInt() and 0xFF + r = buffer.get(address + 1).toInt() and 0xFF + g = buffer.get(address + 2).toInt() and 0xFF + b = buffer.get(address + 3).toInt() and 0xFF + } else { + r = buffer.get(address).toInt() and 0xFF + g = buffer.get(address + 1).toInt() and 0xFF + b = buffer.get(address + 2).toInt() and 0xFF + a = buffer.get(address + 3).toInt() and 0xFF + } + // Merge into the ARGB int this codebase uses (0xAARRGGBB). + out[outRow + x] = (a shl 24) or (r shl 16) or (g shl 8) or b + } + } + return out + } +} diff --git a/app/src/main/kotlin/com/draftlens/vision/preprocessing/AndroidFramePreprocessor.kt b/app/src/main/kotlin/com/draftlens/vision/preprocessing/AndroidFramePreprocessor.kt index 5462bf1..9256aa4 100644 --- a/app/src/main/kotlin/com/draftlens/vision/preprocessing/AndroidFramePreprocessor.kt +++ b/app/src/main/kotlin/com/draftlens/vision/preprocessing/AndroidFramePreprocessor.kt @@ -23,6 +23,11 @@ object AndroidFramePreprocessor : FramePreprocessor { is PreprocessPlanResult.Failure -> return ProcessFrameResult.Failure(result.reason) is PreprocessPlanResult.Success -> result.plan } + PreprocessLog.write( + "preprocess(Bitmap): source=${input.sourceWidth}x${input.sourceHeight} " + + "rotation=${plan.rotation} crop=${plan.crop} output=${plan.outputWidth}x${plan.outputHeight} " + + "scale=${plan.transform.scale}", + ) val payload = input.payload if (payload !is BitmapFramePayload) { return ProcessFrameResult.Failure( diff --git a/app/src/main/kotlin/com/draftlens/vision/preprocessing/PixelArrayFramePreprocessor.kt b/app/src/main/kotlin/com/draftlens/vision/preprocessing/PixelArrayFramePreprocessor.kt index 44d445b..fda5390 100644 --- a/app/src/main/kotlin/com/draftlens/vision/preprocessing/PixelArrayFramePreprocessor.kt +++ b/app/src/main/kotlin/com/draftlens/vision/preprocessing/PixelArrayFramePreprocessor.kt @@ -27,6 +27,11 @@ object PixelArrayFramePreprocessor : FramePreprocessor { is PreprocessPlanResult.Failure -> return ProcessFrameResult.Failure(result.reason) is PreprocessPlanResult.Success -> result.plan } + PreprocessLog.write( + "preprocess(PixelArray): source=${input.sourceWidth}x${input.sourceHeight} " + + "rotation=${plan.rotation} crop=${plan.crop} output=${plan.outputWidth}x${plan.outputHeight} " + + "scale=${plan.transform.scale}", + ) val payload = input.payload if (payload !is PixelArrayFramePayload) { return ProcessFrameResult.Failure( diff --git a/app/src/main/kotlin/com/draftlens/vision/preprocessing/PreprocessLog.kt b/app/src/main/kotlin/com/draftlens/vision/preprocessing/PreprocessLog.kt new file mode 100644 index 0000000..02374e0 --- /dev/null +++ b/app/src/main/kotlin/com/draftlens/vision/preprocessing/PreprocessLog.kt @@ -0,0 +1,12 @@ +package com.draftlens.vision.preprocessing + +/** + * Diagnostics seam for the preprocessing path. Production wires [sink] to `android.util.Log` + * in `DraftLensApp`; unit tests keep the default no-op, so the pure preprocessors stay + * platform-free and testable. Never logs pixels, only dimensions and geometry. + */ +internal object PreprocessLog { + var sink: (String) -> Unit = {} + + fun write(message: String) = sink(message) +} diff --git a/app/src/test/kotlin/com/draftlens/capture/DisplayChangePolicyTest.kt b/app/src/test/kotlin/com/draftlens/capture/DisplayChangePolicyTest.kt index b28b60a..aa5d4f9 100644 --- a/app/src/test/kotlin/com/draftlens/capture/DisplayChangePolicyTest.kt +++ b/app/src/test/kotlin/com/draftlens/capture/DisplayChangePolicyTest.kt @@ -26,6 +26,29 @@ class DisplayChangePolicyTest { DisplayChangePolicy.classify(1080, 1920, 1200, 1920), ) } + + @Test fun `a portrait-to-landscape rotation always swaps dimensions and needs a resize`() { + assertEquals( + DisplayChange.Resize(2400, 1080), + DisplayChangePolicy.classify(1080, 2400, 2400, 1080), + ) + } + + @Test fun `a landscape-to-portrait rotation always swaps dimensions and needs a resize`() { + assertEquals( + DisplayChange.Resize(1080, 2400), + DisplayChangePolicy.classify(2400, 1080, 1080, 2400), + ) + } + + @Test fun `repeated identical resize callbacks are a rotation-only update`() { + // The same callback can fire several times for one display change; the classification + // must be idempotent so the pipeline is not needlessly recreated. + assertEquals( + DisplayChange.RotationOnly, + DisplayChangePolicy.classify(2400, 1080, 2400, 1080), + ) + } } class FramePipelineRotationTest { diff --git a/app/src/test/kotlin/com/draftlens/capture/GenerationCounterTest.kt b/app/src/test/kotlin/com/draftlens/capture/GenerationCounterTest.kt new file mode 100644 index 0000000..7ace0b0 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/capture/GenerationCounterTest.kt @@ -0,0 +1,39 @@ +package com.draftlens.capture + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class GenerationCounterTest { + + @Test fun `a captured generation goes stale exactly when a new reader generation starts`() { + val counter = GenerationCounter() + val first = counter.bump() + + assertTrue(counter.isCurrent(first)) + + counter.bump() + + assertFalse("the first generation is stale after the second bump", counter.isCurrent(first)) + assertTrue(counter.isCurrent(counter.current())) + } + + @Test fun `generations are monotonic and never reused`() { + val counter = GenerationCounter() + val generations = (1..5).map { counter.bump() } + + assertEquals((1L..5L).toList(), generations) + assertEquals(5L, counter.current()) + } + + @Test fun `freezing and resuming does not change the generation`() { + // The frozen snapshot keeps showing the latest published frame; resuming continues + // the same reader generation, so the counter must not advance by itself. + val counter = GenerationCounter() + val gen = counter.bump() + + assertEquals(gen, counter.current()) + assertTrue(counter.isCurrent(gen)) + } +} diff --git a/app/src/test/kotlin/com/draftlens/capture/RgbaPlaneCopyTest.kt b/app/src/test/kotlin/com/draftlens/capture/RgbaPlaneCopyTest.kt new file mode 100644 index 0000000..411d746 --- /dev/null +++ b/app/src/test/kotlin/com/draftlens/capture/RgbaPlaneCopyTest.kt @@ -0,0 +1,258 @@ +package com.draftlens.capture + +import org.junit.Assert.assertArrayEquals +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test +import java.nio.ByteBuffer +import java.nio.ByteOrder + +private fun planeLayoutOf( + width: Int, + height: Int, + rowStride: Int = width * 4, + pixelStride: Int = 4, + cropLeft: Int = 0, + cropTop: Int = 0, + cropRight: Int = width, + cropBottom: Int = height, +): RgbaPlaneCopy.PlaneLayout = + RgbaPlaneCopy.layout(width, height, rowStride, pixelStride, cropLeft, cropTop, cropRight, cropBottom)!! + +/** Deterministic, asymmetric pixel values so transposition/row-offset bugs are visible. */ +private fun asymmetricPixel(x: Int, y: Int): Int { + val r = (x * 37 + 11) and 0xFF + val g = (y * 91 + 7) and 0xFF + val b = ((x * 13) xor (y * 29)) and 0xFF + return 0xFF000000.toInt() or (r shl 16) or (g shl 8) or b +} + +private fun pixels(width: Int, height: Int): IntArray = + IntArray(width * height) { i -> asymmetricPixel(i % width, i / width) } + +private fun bufferFor( + pixels: IntArray, + width: Int, + height: Int, + rowStride: Int = width * 4, + pixelStride: Int = 4, + bigEndian: Boolean = false, +): ByteBuffer { + val bigEndianMode = if (bigEndian) ByteOrder.BIG_ENDIAN else ByteOrder.LITTLE_ENDIAN + val buffer = ByteBuffer.allocate(rowStride * height) + buffer.order(bigEndianMode) + for (y in 0 until height) { + for (x in 0 until width) { + val p = pixels[y * width + x] + val offset = y * rowStride + x * pixelStride + if (bigEndianMode == ByteOrder.BIG_ENDIAN) { + buffer.put(offset, ((p shr 24) and 0xFF).toByte()) // A + buffer.put(offset + 1, ((p shr 16) and 0xFF).toByte()) // R + buffer.put(offset + 2, ((p shr 8) and 0xFF).toByte()) // G + buffer.put(offset + 3, (p and 0xFF).toByte()) // B + } else { + buffer.put(offset, ((p shr 16) and 0xFF).toByte()) // R + buffer.put(offset + 1, ((p shr 8) and 0xFF).toByte()) // G + buffer.put(offset + 2, (p and 0xFF).toByte()) // B + buffer.put(offset + 3, ((p shr 24) and 0xFF).toByte()) // A + } + } + } + return buffer +} + +private fun subImage(pixels: IntArray, frameWidth: Int, left: Int, top: Int, width: Int, height: Int): IntArray { + val out = IntArray(width * height) + for (y in 0 until height) { + for (x in 0 until width) { + out[y * width + x] = pixels[(top + y) * frameWidth + (left + x)] + } + } + return out +} + +class RgbaPlaneCopyTest { + + @Test fun `a landscape 2400x1080 frame converts pixel-exactly`() { + val scaled = IntArray(2400 * 1080) { i -> asymmetricPixel(i % 2400, i / 2400) } + val buffer = bufferFor(scaled, 2400, 1080) + + val layout = planeLayoutOf(2400, 1080) + val out = RgbaPlaneCopy.copyArgb(buffer, layout)!! + + assertEquals(scaled.size, out.size) + assertArrayEquals(scaled, out) + } + + @Test fun `a portrait 1080x2400 frame converts pixel-exactly`() { + val scaled = IntArray(1080 * 2400) { i -> asymmetricPixel(i % 1080, i / 1080) } + val buffer = bufferFor(scaled, 1080, 2400) + + val out = RgbaPlaneCopy.copyArgb(buffer, planeLayoutOf(1080, 2400))!! + + assertArrayEquals(scaled, out) + } + + @Test fun `a non-square asymmetric frame converts pixel-exactly`() { + val source = pixels(5, 3) + val buffer = bufferFor(source, 5, 3) + + val out = RgbaPlaneCopy.copyArgb(buffer, planeLayoutOf(5, 3))!! + + assertArrayEquals(source, out) + } + + @Test fun `padded rows are skipped row by row`() { + val source = pixels(17, 9) + val width = 17 + val rowStride = width * 4 + 64 // 16 pad pixels per row + val buffer = bufferFor(source, width, 9, rowStride = rowStride) + + val out = RgbaPlaneCopy.copyArgb(buffer, planeLayoutOf(width, 9, rowStride = rowStride))!! + + assertArrayEquals(source, out) + } + + @Test fun `row padding of one pixel per row is skipped`() { + val source = pixels(7, 4) + val buffer = bufferFor(source, 7, 4, rowStride = 7 * 4 + 4) // one 4-byte pixel of padding + + val out = RgbaPlaneCopy.copyArgb(buffer, planeLayoutOf(7, 4, rowStride = 32))!! + + assertArrayEquals(source, out) + } + + @Test fun `an offset cropRect reads only the crop region`() { + val source = pixels(9, 6) + val buffer = bufferFor(source, 9, 6) + + // Crop columns 2..6 and rows 1..4 of the 9x6 frame. + val layout = planeLayoutOf(9, 6, cropLeft = 2, cropTop = 1, cropRight = 6, cropBottom = 4) + val out = RgbaPlaneCopy.copyArgb(buffer, layout)!! + + assertArrayEquals(subImage(source, 9, left = 2, top = 1, width = 4, height = 3), out) + assertEquals(4, layout.cropWidth) + assertEquals(3, layout.cropHeight) + } + + @Test fun `an offset cropRect with padded rows is still read exactly`() { + val source = pixels(11, 7) + val rowStride = 11 * 4 + 128 + val buffer = bufferFor(source, 11, 7, rowStride = rowStride) + + val layout = planeLayoutOf(11, 7, rowStride = rowStride, cropLeft = 3, cropTop = 2, cropRight = 9, cropBottom = 6) + val out = RgbaPlaneCopy.copyArgb(buffer, layout)!! + + assertArrayEquals(subImage(source, 11, left = 3, top = 2, width = 6, height = 4), out) + } + + @Test fun `a crop touching the right and bottom edges reads exactly those pixels`() { + val source = pixels(6, 4) + val buffer = bufferFor(source, 6, 4) + + val layout = planeLayoutOf(6, 4, cropLeft = 4, cropTop = 2, cropRight = 6, cropBottom = 4) + val out = RgbaPlaneCopy.copyArgb(buffer, layout)!! + + assertArrayEquals(subImage(source, 6, left = 4, top = 2, width = 2, height = 2), out) + } + + @Test fun `a crop smaller than one pixel is refused`() { + val buffer = bufferFor(pixels(4, 4), 4, 4) + + assertNull(RgbaPlaneCopy.layout(4, 4, 16, 4, 1, 1, 1, 2)) + assertNull(RgbaPlaneCopy.layout(4, 4, 16, 4, 2, 2, 2, 2)) + } + + @Test fun `a crop outside the image bounds is refused`() { + assertNull(RgbaPlaneCopy.layout(4, 4, 16, 4, -1, 0, 2, 2)) + assertNull(RgbaPlaneCopy.layout(4, 4, 16, 4, 0, -1, 2, 2)) + assertNull(RgbaPlaneCopy.layout(4, 4, 16, 4, 3, 3, 5, 4)) + assertNull(RgbaPlaneCopy.layout(4, 4, 16, 4, 0, 0, 4, 5)) + } + + @Test fun `a row stride smaller than the image row is refused`() { + assertNull(RgbaPlaneCopy.layout(10, 4, 8 * 4, 4, 0, 0, 10, 4)) + assertNull(RgbaPlaneCopy.layout(10, 4, 0, 4, 0, 0, 10, 4)) + } + + @Test fun `pixelStride variations are supported`() { + // RGBA_8888 is 4 bytes per pixel; 8 exercises a per-pixel padding variant. + val source = pixels(4, 2) + for (pixelStride in listOf(4, 8)) { + val rowStride = 4 * pixelStride + 2 * pixelStride // two pad elements + val buffer = bufferFor(source, 4, 2, rowStride = rowStride, pixelStride = pixelStride) + val layout = planeLayoutOf(4, 2, rowStride = rowStride, pixelStride = pixelStride) + val out = RgbaPlaneCopy.copyArgb(buffer, layout)!! + assertArrayEquals("pixelStride=$pixelStride", source, out) + } + } + + @Test fun `a big-endian buffer is read with its own byte order`() { + val source = pixels(5, 3) + val buffer = bufferFor(source, 5, 3, bigEndian = true) + + val out = RgbaPlaneCopy.copyArgb(buffer, planeLayoutOf(5, 3))!! + + assertArrayEquals(source, out) + } + + @Test fun `the caller buffer keeps its position and remaining bytes`() { + val source = pixels(6, 2) + val buffer = bufferFor(source, 6, 2) + buffer.position(4) + val remainingBefore = buffer.remaining() + + val out = RgbaPlaneCopy.copyArgb(buffer, planeLayoutOf(6, 2))!! + + assertArrayEquals(source, out) + assertEquals(remainingBefore, buffer.remaining()) + assertEquals(4, buffer.position()) + } + + @Test fun `a buffer smaller than the declared plane is refused`() { + val source = pixels(4, 1) + val buffer = ByteBuffer.allocate(source.size) // too small for rowStride * height + val layout = planeLayoutOf(4, 1) + + assertNull(RgbaPlaneCopy.copyArgb(buffer, layout)) + } + + @Test fun `a zero-sized image is refused`() { + assertNull(RgbaPlaneCopy.layout(0, 2, 0, 4, 0, 0, 0, 2)) + assertNull(RgbaPlaneCopy.layout(2, 0, 8, 4, 0, 0, 2, 0)) + } + + @Test fun `asymmetric pixel values survive a full crop at a different scale`() { + // The "captured only part of the game" layout: a landscape game letterboxed + // inside a portrait-sized buffer. The converter must return exactly the + // letterboxed region, not the buffer with garbage around it. + val game = pixels(24, 8) + val bufferWidth = 24 + val bufferHeight = 32 + val frame = IntArray(bufferWidth * bufferHeight) { i -> + // letterbox: rows 12..19 hold the game + val y = i / bufferWidth + if (y in 12 until 20) game[(y - 12) * bufferWidth + (i % bufferWidth)] else 0xFF111111.toInt() + } + val buffer = bufferFor(frame, bufferWidth, bufferHeight) + + val layout = planeLayoutOf(bufferWidth, bufferHeight, cropLeft = 0, cropTop = 12, cropRight = 24, cropBottom = 20) + val out = RgbaPlaneCopy.copyArgb(buffer, layout)!! + + assertArrayEquals(game, out) + assertNotEquals(frame.size, out.size) + } + + @Test fun `the full-crop case equals a direct read of the same plane`() { + val source = pixels(13, 5) + val buffer = bufferFor(source, 13, 5) + + val layout = planeLayoutOf(13, 5) + val out = RgbaPlaneCopy.copyArgb(buffer, layout)!! + + assertTrue(layout.isFullCrop) + assertArrayEquals(source, out) + } +} diff --git a/docs/SCREEN_CAPTURE.md b/docs/SCREEN_CAPTURE.md index 16f9421..407f0b5 100644 --- a/docs/SCREEN_CAPTURE.md +++ b/docs/SCREEN_CAPTURE.md @@ -27,8 +27,21 @@ process death. - **Android 14+**: service calls `startForeground` with the `mediaProjection` type **before** `getMediaProjection()`; the consent token is single-session and dies with the service. `MediaProjection.Callback.onStop()` (revocation) and `onCapturedContentResize()` (rotation) - are both handled. Rotation keeps the existing VirtualDisplay and calls `setSurface()` then - `resize()` with the replacement `ImageReader` surface—no second consent prompt. + are both handled. A resize signal dispatches to one authoritative path: the VirtualDisplay + keeps running, but the ImageReader/Surface generation is replaced and re-attached; + no second consent prompt is requested. +- **Orientation changes without the resize callback** (pre-API-34 devices, and any API level + where the callback does not fire) are handled by a `DisplayManager` fallback: when the + default display's metrics change, the capture size is re-derived from the current window + metrics and the same resize path runs. Both paths are idempotent and deduped by + `DisplayChangePolicy`. +- **Single-app capture** (sharing one window instead of the full screen): the captured + window's size is known only through `onCapturedContentResize` (API 34+). Until the callback + fires, the VirtualDisplay uses the full-screen metrics; when it fires, capture is resized + to the window's content size. Below API 34 single-app size changes cannot be tracked and + are not tracked; use full-screen capture for the MLBB workflow. +- **onCapturedContentVisibilityChanged is not handled**: the app neither reacts to nor + depends on single-app visibility transitions; size changes flow through the resize path. - **Capture dimensions:** API 30+ uses `WindowManager.maximumWindowMetrics`; lower APIs retain the legacy `defaultDisplay.getSize()` path. - **Android 12+**: partial screen capture (share one app) is possible in the system dialog. @@ -38,6 +51,62 @@ process death. - **minSdk 26**: `ImageReader`, `VirtualDisplay`, and foreground-service APIs used are all available since API 21–26; no version-branched capture code is needed. +## Landscape capture (MLBB) + +Full-screen captures mirror the display. If the device rotates from portrait to landscape +while a capture is running, the mirror's VirtualDisplay must change size with it. Three +inputs cover every supported API level, and all three funnel into one idempotent resize +path (`CaptureSessionService.recreateDisplay` → `FramePipeline.prepareResize`): + +1. `MediaProjection.Callback.onCapturedContentResize(width, height)` — the authoritative + content dimensions (API 34+). Both the buffer and the VirtualDisplay move to those dims. +2. `DisplayManager.DisplayListener.onDisplayChanged` — re-derives the size from the current + window metrics; the standard path on pre-34 devices. +3. `Resources.configuration` changes — the resize path itself is configuration-independent. + +While the capture size is stale (typically before the first callback after a rotation), +letterboxed or partial frames can be delivered. The conversion honors `image.cropRect`, so +such a frame shows the full game at its content size instead of garbage, until the resize +completes (typically the next frame). Nothing is ever stretched, padded with invented pixels, +or hidden by invented crops. + +**Reader generations.** Every ImageReader/Surface generation is numbered. Images captured +with an older generation are dropped at the listener and again at publish time, so a late +frame from a released reader can never overwrite the replacement reader's frames. A freeze +snapshot taken during a resize pins the newest *valid* published frame; resuming continues +with the current generation. Stale frames never reach the pinned slot either. + +### Diagnosing the landscape bug with logs + +All capture diagnostics are structured `key=value` lines under the tags +`CaptureSessionService`, `FramePipeline` and `VisionPreprocessor`. To collect them: + +```sh +adb logcat -c && adb logcat -v time -s CaptureSessionService FramePipeline VisionPreprocessor +``` + +Look for, in order: + +1. `session=N capture-start: dimensions=WxH dpi=D rotation=R source=windowMetrics` — the + size chosen at session start. If it is `1080x2400` while the game is landscape, this is + the session-start case, and the resize must follow (see 3). +2. `session=N default-display changed` and/or `captured-content-resize: CxC -> WxH` — which + signal fired and what dimensions it carried. If neither line appears on rotation, the + platform delivered no signal; the capture stays at the session-start size forever. +3. `session=N surface-detach ... surface-attach ... virtual-display resized: WxH dpi=D generation=G` + — the resize sequence completed; generation G is attached to the VirtualDisplay. + `pipeline resize: detaching previous generation=G-1 ...` shows the old generation detaching first. +4. `generation=G plane: image=WxH crop=.. rowStride=R pixelStride=4 paddingBytes=P fullCrop=true` + — one frame parsed straight out of the plane. Compare `rowStride` with `image width x 4` + and `crop` with the game's content rect. A mismatch here means the black region originates + in the acquired image (acquisition-side), not in the conversion. +5. `generation=G frame stored: source=WxH rotation=...` and later + `preprocess(Bitmap): source=... output=WxH` — the frame left the capture pipeline and the + preprocessing step; if (5) disagrees with (4), the black region is introduced in + conversion or transformation, not acquisition. + +Image contents are never logged, written to disk or uploaded. + ## Sampling Default 500 ms (2 FPS), configurable 334–2000 ms via DataStore, hard-capped at ~3 FPS (`FrameSampler`). `ImageReader(maxImages=3)` bounds backpressure; the listener drops (does not @@ -64,9 +133,11 @@ or a new capture session. Any exception in the acquisition path is caught per frame so one bad frame never kills the session. ## Limitations (honest list) -- Frame→Bitmap conversion uses the RGBA_8888 plane path; padded-stride rows are cropped. - If a device reports a non-RGBA ImageReader format the frame is dropped (counted, not faked). +- Frame→Bitmap conversion honors the RGBA_8888 plane layout, padded rows and `image.cropRect`; + offset crops produce tightly packed images of the content region. A malformed plane drops + the frame instead of producing a partial or shifted one. - The ViewModel mirrors service state through explicit calls (Sprint 0 uses a single activity); a Binder/AIDL API is the documented upgrade if the service must survive longer. - Emulator testing proves compile + logic, **not** MediaProjection behavior on real devices. - Physical-device runbook: `CaptureLifecycleInstrumentedTest` class comment. + Physical-device runbook: `CaptureLifecycleInstrumentedTest` class comment and rows 14–33 + of `docs/TESTING_PHYSICAL_DEVICE.md`. diff --git a/docs/TESTING_PHYSICAL_DEVICE.md b/docs/TESTING_PHYSICAL_DEVICE.md index 96918c0..3c02171 100644 --- a/docs/TESTING_PHYSICAL_DEVICE.md +++ b/docs/TESTING_PHYSICAL_DEVICE.md @@ -41,12 +41,24 @@ Status values: **PASS**, **FAIL**, **PARTIAL**, **NOT RUN**. Record the commit S | 20 | Notifications disabled | Freeze action unreachable from the shade; the app warns and offers to enable them; in-app Freeze/Resume still work | NOT RUN | | 21 | Stop with a frame frozen | Projection and resources are released; the frozen frame remains available in Vision Lab until Resume/Clear or a new capture session; it does not survive process death | NOT RUN | | 22 | Freeze during low frame availability | No frame → the action logs and keeps state; first valid frame freezes on the next tap | NOT RUN | +| 23 | Landscape game capture end-to-end | MLBB in landscape: frame shows the *entire* game, no black region; verify both session-start-in-landscape and portrait→landscape rotation; also landscape→portrait→landscape twice | NOT RUN | +| 24 | Portrait app capture after landscape game | Switch back to a portrait app; frame again shows the whole portrait app without black regions | NOT RUN | +| 25 | Resize with freeze active | Freeze from the notification while in MLBB; rotate device; frozen frame still shows the pinned game frame; no stale-frame overwrite; Resume continues the *current* generation | NOT RUN | +| 26 | cropRect/letterbox handling | Session starts in portrait, game opens landscape before any resize lands: frame is letterboxed-but-complete (no garbage), replaced by the full-size frame after the resize | NOT RUN | +| 27 | Stride-heavy device | On a device/GPU whose ImageReader reports `rowStride > width * 4`: landscape frame is pixel-shifted-free, no skewed rows; compare a full-screen region against a hardware screenshot | NOT RUN | +| 28 | Resize storm | Rotate device repeatedly portrait↔landscape 6+ times: one generation settles per change; no crash, no stale-dimension frame, notification still shows LIVE | NOT RUN | +| 29 | Resize while capture is paused | Pause during landscape→portrait; the resize still applies and is logged; resume shows the resized frame | NOT RUN | +| 30 | Freeze after resize | After a portrait→landscape resize, tap Freeze: pinned frame is the landscape frame at the new dimensions, complete | NOT RUN | +| 31 | Mismatch diagnosis | Intentionally start capture with the game in landscape, then force-stop rotation events via lock/unlock: collect `CaptureSessionService`/`FramePipeline` logcat and verify dimensions, cropRect, rowStride and generation lines; confirm no frame mixes generations | NOT RUN | +| 32 | Single-app share resize (API 34+) | Share only the MLBB window: capture resizes to the window's content size via `onCapturedContentResize`; the app logs the resize; releasing the app restores the full share | NOT RUN | ## Evidence to capture each run - `git rev-parse --short HEAD` - Device model and Android version (`adb shell getprop ro.product.model`, `adb shell getprop ro.build.version.release`) - DraftLens version/APK artifact run ID -- Logcat lines tagged `CaptureSessionService` +- Logcat lines tagged `CaptureSessionService`, `FramePipeline`, `VisionPreprocessor` + (`adb logcat -v time -s CaptureSessionService FramePipeline VisionPreprocessor`) for the + capture dimensions, resize events, plane layout and frame-generation lines - Initial resolution, frame count after 10 seconds, preview visible Y/N - For failures: full exception and steps immediately before it diff --git a/tasks/plan.md b/tasks/plan.md index d2c54e3..d4c8c0a 100644 --- a/tasks/plan.md +++ b/tasks/plan.md @@ -150,6 +150,15 @@ No recommendation features, no overlays, no automation, PR stays draft and unmer - [x] Part B: pixel extractor rotation fixed to the `FrameTransform.sourceToOriented()` convention; tests rewritten with asymmetric non-square fixtures and hand-computed expected values +- [x] Landscape capture fix (MLBB portrait→landscape rotation bug): + - authoritative `onCapturedContentResize` signal feeds one idempotent resize path; + - `DisplayManager` fallback re-derives the capture size on any display change; + - VirtualDisplay/ImageReader re-created at the reported size; + - capture generations reject stale reader callbacks at listener and publish time; + - plane conversion honors `image.cropRect` and padded rows (`RgbaPlaneCopy`, JVM-tested + with asymmetric fixtures, letterboxed crops and stride variants); + - structured diagnostics in `CaptureSessionService`/`FramePipeline`/`VisionPreprocessor`; + - physical-device verification pending (rows 23–32 of `docs/TESTING_PHYSICAL_DEVICE.md`) - [x] Part C: Android extractor validates Bitmap dimensions, pixel-exact rotation, source bitmap preserved, every temporary released exactly once; instrumentation tests added - [x] Part D: `VisualInformationAnalyzer` (variance, edge density, meaningful differences, diff --git a/tasks/todo.md b/tasks/todo.md index d07cdbf..201401e 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -115,5 +115,12 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** URL change forces re-download; failed refresh keeps previous asset; HTTPS-only redirects) - [x] Part G: atomic snapshot/asset writes; failure preserves previous snapshot; temp files swept - [x] Part H: reference Bitmap dims read before recycling -- [x] CI green on the final head (run 43746648882, head 1c7397b925dd2683b3d763e552d610d2e464ecb7; instrumentation compiled, not executed — no KVM on the runner) -- [ ] Physical-device smoke test and on-device accuracy evaluation (pending) +- [x] Landscape capture fix (MLBB): authoritative captured-content resize via + `onCapturedContentResize` + `DisplayManager` fallback to current window metrics; + VirtualDisplay/ImageReader re-created at the reported size; generation numbers reject + stale reader callbacks; plane conversion honors `image.cropRect` and padded rows; + structured diagnostics in `CaptureSessionService`/`FramePipeline`/`VisionPreprocessor` +- [ ] CI green on the new head for this revision (see PR description for the latest run) +- [ ] Physical-device smoke of the freeze workflow; on-device accuracy evaluation; + threshold/heuristic calibration (pending) +- [ ] Physical-device verification of the landscape capture fix in MLBB (rows 23–32) (pending) From 2597fdb4e51be907ce334d690bf34fc573bef2e7 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 23:05:17 +0000 Subject: [PATCH 30/33] fix: CapturedFrame carries no sourceWidth/sourceHeight of its own --- app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt b/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt index d4fe1bf..5a938bd 100644 --- a/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt +++ b/app/src/main/kotlin/com/draftlens/capture/FramePipeline.kt @@ -194,7 +194,7 @@ class FramePipeline( val frame = _latestFrame.value ?: return FreezeResult.NoFrame val copy = FrameSnapshots.copyOf(frame) ?: return FreezeResult.NoFrame frozen = true - Log.i(TAG, "generation=${generation.current()} frame pinned: ${frame.sourceWidth}x${frame.sourceHeight} id=${frame.id.value}") + Log.i(TAG, "generation=${generation.current()} frame pinned: ${frame.width}x${frame.height} id=${frame.frame.id.value}") FreezeResult.Frozen(copy) } From 856418bf655202303689779ee719c067d3d41c57 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 23:08:07 +0000 Subject: [PATCH 31/33] fix: check the buffer limit, not remaining, for positioned plane buffers --- app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt b/app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt index 5c1a5c3..874f0a4 100644 --- a/app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt +++ b/app/src/main/kotlin/com/draftlens/capture/RgbaPlaneCopy.kt @@ -84,7 +84,8 @@ internal object RgbaPlaneCopy { fun copyArgb(buffer: ByteBuffer, layout: PlaneLayout): IntArray? { if (layout.cropWidth <= 0 || layout.cropHeight <= 0) return null val needsRead = (layout.cropTop.toLong() + layout.cropHeight) * layout.rowStride - if (buffer.remaining().toLong() < needsRead) return null + // Absolute reads do not depend on the buffer's position; the limit is the real end. + if (buffer.limit().toLong() < needsRead) return null val bigEndian = buffer.order() == ByteOrder.BIG_ENDIAN val out = IntArray(layout.cropWidth * layout.cropHeight) val stride = layout.rowStride From 5252016cd15c824009368e831ed499f24209980d Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 23:10:53 +0000 Subject: [PATCH 32/33] fix: dex disallows spaces in SimpleName; camelCase the pipeline tests --- .../kotlin/com/draftlens/FramePipelineInstrumentedTest.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt b/app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt index e9e389d..6909ebe 100644 --- a/app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt +++ b/app/src/androidTest/kotlin/com/draftlens/FramePipelineInstrumentedTest.kt @@ -20,7 +20,7 @@ import org.junit.runner.RunWith @RunWith(AndroidJUnit4::class) class FramePipelineInstrumentedTest { - @Test fun `starting a pipeline creates a surface sized to the request`() { + @Test fun startingAPipelineCreatesASurfaceSizedToTheRequest() { val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) try { pipeline.start(1080, 2400, 440) @@ -33,7 +33,7 @@ class FramePipelineInstrumentedTest { } } - @Test fun `a resize replaces the surface and bumps the generation`() { + @Test fun aResizeReplacesTheSurfaceAndBumpsTheGeneration() { val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) try { pipeline.start(1080, 2400, 440) @@ -54,7 +54,7 @@ class FramePipelineInstrumentedTest { } } - @Test fun `freezing with no frames published reports NoFrame and keeps the pipeline alive`() { + @Test fun freezeWithNoFramesReportsNoFrameAndKeepsPipelineAlive() { val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) try { pipeline.start(2400, 1080, 440) @@ -68,7 +68,7 @@ class FramePipelineInstrumentedTest { } } - @Test fun `stopping the pipeline releases its surface`() { + @Test fun stoppingThePipelineReleasesItsSurface() { val pipeline = FramePipeline(FrameSampler(FrameSampler.DEFAULT_INTERVAL_MS)) pipeline.start(1080, 2400, 440) From 9ca1c362d5615f371381a5105370201805f6aeb6 Mon Sep 17 00:00:00 2001 From: Xerio Date: Sun, 15 Nov 2026 23:16:45 +0000 Subject: [PATCH 33/33] docs: landscape capture bugfix report, diagnostics and verification matrix --- docs/IMPLEMENTATION_REPORT.md | 69 +++++++++++++++++++++++++++++++++++ docs/VISION_TESTING.md | 16 +++++--- tasks/todo.md | 2 +- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/docs/IMPLEMENTATION_REPORT.md b/docs/IMPLEMENTATION_REPORT.md index 6f80ac9..a6e3c91 100644 --- a/docs/IMPLEMENTATION_REPORT.md +++ b/docs/IMPLEMENTATION_REPORT.md @@ -493,3 +493,72 @@ Docs-only commits after the code freeze do not change the build; the exact final Historical green runs from before the review-fix pass (kept as history; do not cite these as the current verification): `e58611c` — run 43732908605 (646 executions); `7d0c903` — run 43731484922 (same totals). + +## Critical fix — landscape capture shows only part of the game (device-reported) + +Reported by the maintainer on a physical phone: a normal screenshot captures MLBB correctly, +but the MediaProjection preview/capture shows only part of the game with the rest black; +portrait apps capture fine. (PR #3, still Draft and unmerged.) + +### Root cause analysis + +The visible path from Android's capture-resize callback to the frame on screen was inspected: +`CaptureSessionService` → `FramePipeline` → `ImageReader` → `imageToPayload` → +`PixelArrayFramePreprocessor`/`AndroidFramePreprocessor` → Vision Lab. + +Three defects combined, any one sufficient to produce the symptom: + +1. **The capture size was fixed at session start and never re-derived.** The only resize + trigger was `MediaProjection.Callback.onCapturedContentResize` (API 34+), so on pre-34 + devices — and on API 34+ whenever the callback does not fire for whole-screen captures — + rotating to landscape left the VirtualDisplay at the portrait session-start size forever. + The mirrored landscape content was clipped/letterboxed into the stale buffer and the rest + stayed black. The `DisplayManager` listener only updated rotation *metadata*, never size. +2. **Late frames from a released reader could publish into the replacement reader's stream** + (no generation guard) — stale frames could land on top of fresh ones after a resize. +3. **`image.cropRect` was ignored** in the plane conversion, so letterboxed/partial content + (e.g. before the first resize after a rotation) produced shifted garbage rather than the + full game. + +What was NOT the cause: the game does not block screen capture (a hardware screenshot works); +the matcher/Vision Lab are unaffected (this bug is in the capture pipeline only). + +### Fix + +- The captured-content resize callback now feeds a shared, authoritative resize path + (`recreateDisplay`) that re-creates the ImageReader/Surface generation at the *reported* + dimensions; `DisplayManager` changes re-derive the size from the current window metrics + and funnel into the same path — covered on all API levels ≥ 26; both paths are idempotent + and deduped by `DisplayChangePolicy`. +- Every reader/surface generation is numbered (`GenerationCounter`); images captured with a + stale generation are dropped at the listener and again at publish time, so a late frame + from a released reader can never reach the UI or the frozen snapshot. +- The plane conversion is rewritten into a pure, JVM-tested `RgbaPlaneCopy`: it honors + padded rows (`rowStride > width * pixelStride`), `image.cropRect` (letterboxed/offset + content produces a tightly packed image of the content), arbitrary `pixelStride >= 1` + and both byte orders; it never mutates the caller's buffer and drops malformed frames + instead of faking pixels. Bitmap dimensions used by the vision pipeline equal the + payload dimensions exactly. +- Structured `key=value` diagnostics cover every stage (`session=` id, `generation=`, + image dims, cropRect, rowStride, pixelStride, padding, bitmap dims, preprocess in/out). + No image contents are logged. See `docs/SCREEN_CAPTURE.md` → "Diagnosing the landscape bug". +- Freeze continues to work unchanged: the pinned frame is never overwritten by newer frames; + a resize during freeze pins the newest *valid* frame; Resume continues with the current + generation; the frozen frame is an independently owned copy with no shared payload buffer. + +### Evidence + +- CI at head `5252016cd15c824009368e831ed499f24209980d` (run 43773398149): **840 unit tests + (debug+release), 0 failures / 0 errors / 0 skipped**; strict lint green; `assembleDebug` + green; artifact `draftlens-debug`, digest + `sha256:c46580570a85054f225bd4d927350b153991f98956b1e3ee9a877d5ed9c1b3f1`. +- Letterbox/stride/offset behavior verified by deterministic JVM tests + (`RgbaPlaneCopyTest` — 22 cases) and the orientation swap in `DisplayChangePolicyTest`. +- Instrumented tests compiled, not executed (no KVM on the CI runner); the new + `FramePipelineInstrumentedTest` covers lifecycle invariants and must run on a device. +- On-device verification is **pending**: rows 23–32 of `docs/TESTING_PHYSICAL_DEVICE.md` + (portrait→landscape, resize storm, freeze-during-rotate, letterbox transient, API-34+ + single-app resize). **The fix is not claimed to work on a device until those rows are run.** +- If anything still shows black after this fix, the new logs pin down whether acquisition + (`generation=... plane:` line) or conversion (`... frame stored: ... bitmap:` lines) is + the source — see `docs/SCREEN_CAPTURE.md` → "Diagnosing the landscape bug with logs". diff --git a/docs/VISION_TESTING.md b/docs/VISION_TESTING.md index 67779e4..8012c59 100644 --- a/docs/VISION_TESTING.md +++ b/docs/VISION_TESTING.md @@ -40,6 +40,11 @@ See the sections below for the exact commands CI runs. | Models | `models/CaptureSessionStateTest.kt`, `permissions/ProjectionPermissionTest.kt` | state machine transitions, consent intent | | Detection stubs | `detection/DetectionStubsTest.kt` | the Sprint 0 seams still report `NotImplemented` and never fake a result | +| Plane/stride conversion | `capture/RgbaPlaneCopyTest.kt` | landscape 2400x1080 & portrait 1080x2400 exact pixels, non-square frames, padded rows (full-pixel pads and one-pixel pads), offset & edge-touching `cropRect`s, `cropRect` smaller than the frame (letterbox), pixelStride 4/8, big-endian buffers, caller-buffer position independence, malformed inputs refused, full-crop equals direct read | +| Generation counter | `capture/GenerationCounterTest.kt` | generations monotonic, stale after bump, frozen resume does not advance it | +| Display policy | `capture/DisplayChangePolicyTest.kt` | portrait↔landscape always a size-swap resize, identical resize callbacks idempotent | +| Instrumentation | `androidTest/.../FramePipelineInstrumentedTest.kt` | pipeline starts/resizes with new surface and generation bump, freeze with no frames keeps the reader alive, stop releases the surface | + ## Fixture policy - **Synthetic images only.** No real game screenshots, no captured frames, no image assets are @@ -85,14 +90,15 @@ Workflow: `.github/workflows/build.yml` (one `build` job), triggered by `pull_re Docs-only commits after the code freeze do not change the build; the exact final head and its CI run are recorded in PR #3's description. -Latest green run: -— head SHA `7a839d1422ee5ba53f65a78ef746d323091d3b40`: +Latest green run: +— head SHA `5252016cd15c824009368e831ed499f24209980d` (docs-only commits after the code freeze +are not part of the 840-tests green state; PR #3's description records the final head). -- **Unit tests: 792 executions (debug + release), 0 failures, 0 errors, 0 skipped.** +- **Unit tests: 840 executions (debug + release), 0 failures, 0 errors, 0 skipped.** - `assembleAndroidTest` — instrumentation sources compile. - strict `lint` (`abortOnError = true`) — green. -- `assembleDebug` — green; artifact `draftlens-debug`, 10 362 309 bytes, - digest `sha256:53c07492517fdfc2837ebb179393e526d6b397db2146228aa853eb4914221677`. +- `assembleDebug` — green; artifact `draftlens-debug`, 10 369 786 bytes, + digest `sha256:c46580570a85054f225bd4d927350b153991f98956b1e3ee9a877d5ed9c1b3f1`. The instrumentation suite is **compiled but not executed**. The CI runner has no KVM access: the default API-30 image does not boot within the runner's timeout, and the diff --git a/tasks/todo.md b/tasks/todo.md index 201401e..f20bfcb 100644 --- a/tasks/todo.md +++ b/tasks/todo.md @@ -120,7 +120,7 @@ PASS** (device details not supplied). The MediaProjection lifecycle matrix is ** VirtualDisplay/ImageReader re-created at the reported size; generation numbers reject stale reader callbacks; plane conversion honors `image.cropRect` and padded rows; structured diagnostics in `CaptureSessionService`/`FramePipeline`/`VisionPreprocessor` -- [ ] CI green on the new head for this revision (see PR description for the latest run) +- [x] CI green on the new head for this revision (run 43773398149, head 5252016cd15c824009368e831ed499f24209980d; 840 unit tests; instrumentation compiled, not executed — no KVM on the runner) - [ ] Physical-device smoke of the freeze workflow; on-device accuracy evaluation; threshold/heuristic calibration (pending) - [ ] Physical-device verification of the landscape capture fix in MLBB (rows 23–32) (pending)