package com.oliver.kidio.backend.ingestion import kotlinx.serialization.json.* import java.net.URI import java.net.URLEncoder import java.net.http.HttpClient import java.net.http.HttpRequest import java.net.http.HttpResponse import java.time.Duration data class ZumOerItem( val id: String, val title: String, val subject: String, val grade: Int? = null, val downloadUrl: String, val license: String = "CC BY-SA 4.0", val library: String? = null, val rawJsonContent: String? = null, ) class ZumApiClient( private val httpClient: HttpClient = HttpClient.newBuilder() .connectTimeout(Duration.ofSeconds(10)) .followRedirects(HttpClient.Redirect.NORMAL) .build(), ) { private val json = Json { ignoreUnknownKeys = true isLenient = true } companion object { fun mapSubject(raw: String): String { val lower = raw.trim().lowercase() return when { lower.contains("mathe") -> "Mathematik" lower.contains("deutsch") || lower.contains("german") -> "Deutsch" lower.contains("sach") || lower.contains("hsu") || lower.contains("natur") || lower.contains("heimat") -> "Sachunterricht" lower.contains("englisch") || lower.contains("english") -> "Englisch" lower.contains("bio") -> "Biologie" lower.contains("geo") || lower.contains("erdkunde") -> "Geographie" lower.contains("gesch") -> "Geschichte" lower.contains("chemie") -> "Chemie" lower.contains("physik") -> "Physik" lower.contains("musik") -> "Musik" lower.contains("kunst") -> "Kunst" lower.contains("informatik") -> "Informatik" else -> raw.trim().replaceFirstChar { if (it.isLowerCase()) it.titlecase() else it.toString() } } } fun mapGradeToAgeRange(grade: Int?): Pair { return when (grade) { 1 -> 6 to 7 2 -> 7 to 8 3 -> 8 to 9 4 -> 9 to 10 5 -> 10 to 11 6 -> 11 to 12 7 -> 12 to 13 8 -> 13 to 14 9 -> 14 to 15 10 -> 15 to 16 11, 12, 13 -> 16 to 18 else -> 6 to 12 } } } /** * Downloads an .h5p package file from a remote URL. */ fun downloadH5P(downloadUrl: String): ByteArray? { return try { val request = HttpRequest.newBuilder() .uri(URI.create(downloadUrl)) .timeout(Duration.ofSeconds(15)) .header("User-Agent", "Mozilla/5.0 (compatible; KidioBot/1.0; +https://kidio.oliver.com)") .GET() .build() val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray()) if (response.statusCode() == 200) { response.body() } else { null } } catch (e: Exception) { println("WARNUNG: Download fehlgeschlagen für $downloadUrl: ${e.message}") null } } /** * Fetches live OER items from apps.zum.de via search and Drupal settings extraction. * Skips items whose IDs are already present in existingIds. * Falls back to curated items if remote catalog is unreachable or returns 0 matching interactive items. */ fun fetchLiveOerItems( subject: String? = null, limit: Int = 50, maxPages: Int = 15, existingIds: Set = emptySet(), ): List { val results = mutableListOf() val seenAppIds = mutableSetOf() println("[ZUM-Apps Live] Starte Live-Abfrage auf apps.zum.de (Fach: ${subject ?: "Alle Fächer"}, Limit: $limit)...") for (page in 0 until maxPages) { if (results.size >= limit) break val searchUrl = if (subject.isNullOrBlank()) { "https://apps.zum.de/apps?page=$page" } else { val encoded = URLEncoder.encode(subject, "UTF-8") "https://apps.zum.de/apps?keys=$encoded&page=$page" } val html = fetchString(searchUrl) if (html == null) { println("[ZUM-Apps Live] Warnung: Seite $page konnte nicht geladen werden ($searchUrl)") continue } // Filtert bereits verarbeitete oder in DB vorhandene Apps heraus val appIds = Regex("""href=["']/apps/(\d+)["']""") .findAll(html) .map { it.groupValues[1] } .distinct() .filter { appId -> val fullId = "zum-$appId" !seenAppIds.contains(appId) && !existingIds.contains(fullId) && !existingIds.contains(appId) } .toList() if (appIds.isEmpty()) { println("[ZUM-Apps Live] Seite $page enthält keine neuen Aufgaben. Blättere weiter...") continue } for (appId in appIds) { if (results.size >= limit) break seenAppIds.add(appId) val item = fetchAppDetails(appId, subject) if (item != null) { results.add(item) val lib = item.library ?: "H5P" val gr = item.grade ?: "k.A." println( "[ZUM-Apps Live] Neue App #$appId: \"${item.title}\" ($lib) [${item.subject}, Kl. $gr]", ) } } } if (results.isEmpty()) { println("[ZUM-Apps Live] Keine neuen Live-Inhalte gefunden oder Seite nicht erreichbar. Nutze kuratierte OER-Fallback-Daten.") return getCuratedOerItems(subject = subject, limit = limit) } println("[ZUM-Apps Live] ${results.size} neue interaktive Aufgaben erfolgreich von apps.zum.de geladen.") return results } private fun fetchAppDetails( appId: String, fallbackSubject: String?, ): ZumOerItem? { val appUrl = "https://apps.zum.de/apps/$appId" val html = fetchString(appUrl) ?: return null val settingsMatch = Regex("""data-drupal-selector="drupal-settings-json">([^<]+)<""").find(html) ?: return null val jsonStr = settingsMatch.groupValues[1] val drupalSettings = try { json.parseToJsonElement(jsonStr).jsonObject } catch (_: Exception) { return null } val h5pContents = drupalSettings["h5p"]?.jsonObject ?.get("H5PIntegration")?.jsonObject ?.get("contents")?.jsonObject ?: return null val contentObj = h5pContents["cid-$appId"]?.jsonObject ?: h5pContents.values.firstOrNull()?.jsonObject ?: return null val library = contentObj["library"]?.jsonPrimitive?.contentOrNull ?: "" if (!library.contains("MultiChoice", ignoreCase = true) && !library.contains("TrueFalse", ignoreCase = true) && !library.contains("QuestionSet", ignoreCase = true) ) { return null } val jsonContent = contentObj["jsonContent"]?.jsonPrimitive?.contentOrNull ?: return null val metadata = contentObj["metadata"]?.jsonObject val title = metadata?.get("title")?.jsonPrimitive?.contentOrNull ?: Regex("""(.*?)(?:\|.*)?""").find(html)?.groupValues?.get(1)?.trim() ?: "ZUM App #$appId" val rawLicense = metadata?.get("license")?.jsonPrimitive?.contentOrNull val license = when { rawLicense == "U" || rawLicense == null || rawLicense.isBlank() -> "CC BY-SA 4.0" rawLicense.contains("CC", ignoreCase = true) -> rawLicense else -> "CC BY-SA 4.0 ($rawLicense)" } val exportUrl = contentObj["exportUrl"]?.jsonPrimitive?.contentOrNull ?: appUrl // Breitgefächerter Abruf aller HTML-Linktexte (Kategorien, Schulstufen, Tags) val tags = Regex("""]*>([^<]+)""") .findAll(html) .map { it.groupValues[1].trim() } .filter { it.isNotBlank() && it.length < 60 } .toList() val subject = inferSubject(title, tags, fallbackSubject) val grade = inferGrade(title, html, tags) return ZumOerItem( id = "zum-$appId", title = title, subject = subject, grade = grade, downloadUrl = exportUrl, license = license, library = library, rawJsonContent = jsonContent, ) } private fun inferSubject( title: String, tags: List, fallbackSubject: String?, ): String { for (tag in tags) { val mapped = mapSubject(tag) if (mapped != tag || tag.equals("Mathematik", ignoreCase = true) || tag.equals("Deutsch", ignoreCase = true) || tag.equals("Biologie", ignoreCase = true) || tag.equals("Englisch", ignoreCase = true) ) { return mapped } } if (!fallbackSubject.isNullOrBlank()) { return mapSubject(fallbackSubject) } return mapSubject(title) } private fun inferGrade( title: String, html: String, tags: List, ): Int? { val primaryText = (tags + title).joinToString(" ") // 1. Muster: "3. Klasse", "3. Schuljahr", "3. Kl." Regex("""\b(\d+)\.\s*(?:Klasse|Schuljahr|Kl)\b""", RegexOption.IGNORE_CASE) .find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it } // 2. Muster: "Klasse 3", "Kl. 3", "Schuljahr 3" Regex("""\b(?:Klasse|Schuljahr|Kl\.)\s*(\d+)\b""", RegexOption.IGNORE_CASE) .find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it } // 3. Muster: "Klasse 3-4", "Klasse 3/4", "3.-4. Klasse" Regex("""\b(?:Klasse|Kl\.)?\s*(\d+)\s*[-/]\s*(\d+)\b""", RegexOption.IGNORE_CASE) .find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it } // 4. Fallback: Gesamten HTML-Text der Seite nach Klassenangaben absuchen Regex("""\b(\d+)\.\s*(?:Klasse|Schuljahr|Kl)\b""", RegexOption.IGNORE_CASE) .find(html)?.groupValues?.get(1)?.toIntOrNull()?.let { return it } Regex("""\b(?:Klasse|Schuljahr|Kl\.)\s*(\d+)\b""", RegexOption.IGNORE_CASE) .find(html)?.groupValues?.get(1)?.toIntOrNull()?.let { return it } // 5. Fallback: Schulstufen-Schlüsselwörter val combinedLower = (primaryText + " " + html).lowercase() if (combinedLower.contains("primarstufe") || combinedLower.contains("grundschule")) return 3 val isSek1 = combinedLower.contains("sekundarstufe i") || combinedLower.contains("sek i") || combinedLower.contains("realschule") || combinedLower.contains("hauptschule") if (isSek1) return 6 val isSek2 = combinedLower.contains("sekundarstufe ii") || combinedLower.contains("sek ii") || combinedLower.contains("gymnasium") || combinedLower.contains("abitur") if (isSek2) return 10 return null } private fun fetchString(url: String): String? { return try { val request = HttpRequest.newBuilder() .uri(URI.create(url)) .timeout(Duration.ofSeconds(12)) .header("User-Agent", "Mozilla/5.0 (compatible; KidioBot/1.0; +https://kidio.oliver.com)") .header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") .header("Accept-Language", "de-DE,de;q=0.9,en;q=0.8") .GET() .build() val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString(Charsets.UTF_8)) if (response.statusCode() == 200) { response.body() } else { null } } catch (e: Exception) { null } } /** * Returns curated ZUM/OER sample metadata items, filterable by subject. */ fun getCuratedOerItems( subject: String? = null, limit: Int = 20, ): List { val allItems = listOf( ZumOerItem( id = "zum-math-001", title = "Grundrechenarten: Multiplikation 3. Klasse", subject = "Mathematik", grade = 3, downloadUrl = "https://apps.zum.de/api/h5p/math-001.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

Was ist das Ergebnis von 6 × 8?

", "answers": [ {"text": "42", "correct": false}, {"text": "48", "correct": true}, {"text": "54", "correct": false}, {"text": "56", "correct": false} ] } """.trimIndent(), ), ZumOerItem( id = "zum-math-002", title = "Zahlenraum bis 100: Addition", subject = "Mathematik", grade = 2, downloadUrl = "https://apps.zum.de/api/h5p/math-002.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

Berechne: 34 + 28

", "answers": [ {"text": "52", "correct": false}, {"text": "62", "correct": true}, {"text": "64", "correct": false} ] } """.trimIndent(), ), ZumOerItem( id = "zum-math-003", title = "Geometrie: Formen erkennen", subject = "Mathematik", grade = 4, downloadUrl = "https://apps.zum.de/api/h5p/math-003.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

Ein Quadrat hat vier gleich lange Seiten und vier rechte Winkel.

", "correct": true } """.trimIndent(), ), ZumOerItem( id = "zum-de-001", title = "Wortarten: Nomen erkennen", subject = "Deutsch", grade = 2, downloadUrl = "https://apps.zum.de/api/h5p/de-001.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

Welches der folgenden Wörter ist ein Nomen (Namenwort)?

", "answers": [ {"text": "schnell", "correct": false}, {"text": "laufen", "correct": false}, {"text": "Schule", "correct": true}, {"text": "schön", "correct": false} ] } """.trimIndent(), ), ZumOerItem( id = "zum-de-002", title = "Rechtschreibung: Doppel-s oder ss", subject = "Deutsch", grade = 4, downloadUrl = "https://apps.zum.de/api/h5p/de-002.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

Wie wird das Wort richtig geschrieben? „Der Flu... ist tief.“

", "answers": [ {"text": "Flus", "correct": false}, {"text": "Fluss", "correct": true}, {"text": "Fluß", "correct": false} ] } """.trimIndent(), ), ZumOerItem( id = "zum-sach-001", title = "Natur & Tiere: Winterschlaf", subject = "Sachunterricht", grade = 3, downloadUrl = "https://apps.zum.de/api/h5p/sach-001.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

Welches dieser Tiere hält einen echten Winterschlaf?

", "answers": [ {"text": "Igel", "correct": true}, {"text": "Eichhörnchen", "correct": false}, {"text": "Amsel", "correct": false}, {"text": "Fuchs", "correct": false} ] } """.trimIndent(), ), ZumOerItem( id = "zum-sach-002", title = "Planetensystem: Sonnennähe", subject = "Sachunterricht", grade = 4, downloadUrl = "https://apps.zum.de/api/h5p/sach-002.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

Welcher Planet ist der Sonne am nächsten?

", "answers": [ {"text": "Venus", "correct": false}, {"text": "Mars", "correct": false}, {"text": "Merkur", "correct": true}, {"text": "Jupiter", "correct": false} ] } """.trimIndent(), ), ZumOerItem( id = "zum-eng-001", title = "English Vocabulary: Colors & Animals", subject = "Englisch", grade = 4, downloadUrl = "https://apps.zum.de/api/h5p/eng-001.h5p", license = "CC BY-SA 4.0", rawJsonContent = """ { "question": "

What is the English word for „Hund“?

", "answers": [ {"text": "Cat", "correct": false}, {"text": "Dog", "correct": true}, {"text": "Bird", "correct": false}, {"text": "Rabbit", "correct": false} ] } """.trimIndent(), ), ) val normalizedSubject = subject?.let { mapSubject(it) } val filtered = if (normalizedSubject.isNullOrBlank()) { allItems } else { allItems.filter { it.subject.equals(normalizedSubject, ignoreCase = true) } } return filtered.take(limit) } }