118677d960
All checks were successful
CI Pipeline / lint-and-test (pull_request) Successful in 4m3s
- extracted data classes (ZumOerItem, ParsedH5PTask) - moved hardcoded fallback OER tasks into a dedicated JSON resource file - removed hardcoded tasks from InMemoryKidioRepository
389 lines
14 KiB
Kotlin
389 lines
14 KiB
Kotlin
package com.oliver.kidio.backend.ingestion
|
|
|
|
import kotlinx.serialization.json.*
|
|
import java.net.URI
|
|
import java.net.URLEncoder
|
|
import java.net.http.HttpClient
|
|
import java.net.http.HttpRequest
|
|
import java.net.http.HttpResponse
|
|
import java.time.Duration
|
|
|
|
class ZumApiClient(
|
|
private val httpClient: HttpClient =
|
|
HttpClient.newBuilder()
|
|
.connectTimeout(Duration.ofSeconds(10))
|
|
.followRedirects(HttpClient.Redirect.NORMAL)
|
|
.build(),
|
|
) {
|
|
private val json =
|
|
Json {
|
|
ignoreUnknownKeys = true
|
|
isLenient = true
|
|
}
|
|
|
|
companion object {
|
|
fun mapSubject(raw: String): String {
|
|
val lower = raw.trim().lowercase()
|
|
return when {
|
|
lower.contains("mathe") -> "Mathematik"
|
|
lower.contains("deutsch") || lower.contains("german") -> "Deutsch"
|
|
lower.contains("sach") || lower.contains("hsu") || lower.contains("natur") || lower.contains("heimat") -> "Sachunterricht"
|
|
lower.contains("englisch") || lower.contains("english") -> "Englisch"
|
|
lower.contains("bio") -> "Biologie"
|
|
lower.contains("geo") || lower.contains("erdkunde") -> "Geographie"
|
|
lower.contains("gesch") -> "Geschichte"
|
|
lower.contains("chemie") -> "Chemie"
|
|
lower.contains("physik") -> "Physik"
|
|
lower.contains("musik") -> "Musik"
|
|
lower.contains("kunst") -> "Kunst"
|
|
lower.contains("informatik") -> "Informatik"
|
|
else -> raw.trim().replaceFirstChar { if (it.isLowerCase()) it.titlecase() else it.toString() }
|
|
}
|
|
}
|
|
|
|
fun mapGradeToAgeRange(grade: Int?): Pair<Int, Int> {
|
|
return when (grade) {
|
|
1 -> 6 to 7
|
|
2 -> 7 to 8
|
|
3 -> 8 to 9
|
|
4 -> 9 to 10
|
|
5 -> 10 to 11
|
|
6 -> 11 to 12
|
|
7 -> 12 to 13
|
|
8 -> 13 to 14
|
|
9 -> 14 to 15
|
|
10 -> 15 to 16
|
|
11, 12, 13 -> 16 to 18
|
|
else -> 6 to 12
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Downloads an .h5p package file from a remote URL.
|
|
*/
|
|
fun downloadH5P(downloadUrl: String): ByteArray? {
|
|
return try {
|
|
val request =
|
|
HttpRequest.newBuilder()
|
|
.uri(URI.create(downloadUrl))
|
|
.timeout(Duration.ofSeconds(15))
|
|
.header("User-Agent", "Mozilla/5.0 (compatible; KidioBot/1.0; +https://kidio.oliver.com)")
|
|
.GET()
|
|
.build()
|
|
|
|
val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray())
|
|
if (response.statusCode() == 200) {
|
|
response.body()
|
|
} else {
|
|
null
|
|
}
|
|
} catch (e: Exception) {
|
|
println("WARNUNG: Download fehlgeschlagen für $downloadUrl: ${e.message}")
|
|
null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetches live OER items from apps.zum.de via search and Drupal settings extraction.
|
|
* Skips items whose IDs are already present in existingIds.
|
|
* Falls back to curated items if remote catalog is unreachable or returns 0 matching interactive items.
|
|
*/
|
|
fun fetchLiveOerItems(
|
|
subject: String? = null,
|
|
limit: Int = 50,
|
|
maxPages: Int = 15,
|
|
existingIds: Set<String> = emptySet(),
|
|
): List<ZumOerItem> {
|
|
val results = mutableListOf<ZumOerItem>()
|
|
val seenAppIds = mutableSetOf<String>()
|
|
|
|
println("[ZUM-Apps Live] Starte Live-Abfrage auf apps.zum.de (Fach: ${subject ?: "Alle Fächer"}, Limit: $limit)...")
|
|
|
|
for (page in 0 until maxPages) {
|
|
if (results.size >= limit) break
|
|
processPage(page, subject, limit, existingIds, seenAppIds, results)
|
|
}
|
|
|
|
if (results.isEmpty()) {
|
|
println("[ZUM-Apps Live] Keine neuen Live-Inhalte gefunden oder Seite nicht erreichbar. Nutze kuratierte OER-Fallback-Daten.")
|
|
return getCuratedOerItems(subject = subject, limit = limit)
|
|
}
|
|
|
|
println("[ZUM-Apps Live] ${results.size} neue interaktive Aufgaben erfolgreich von apps.zum.de geladen.")
|
|
return results
|
|
}
|
|
|
|
private fun processPage(
|
|
page: Int,
|
|
subject: String?,
|
|
limit: Int,
|
|
existingIds: Set<String>,
|
|
seenAppIds: MutableSet<String>,
|
|
results: MutableList<ZumOerItem>,
|
|
) {
|
|
val searchUrl = buildSearchUrl(subject, page)
|
|
val html = fetchString(searchUrl)
|
|
|
|
if (html == null) {
|
|
println("[ZUM-Apps Live] Warnung: Seite $page konnte nicht geladen werden ($searchUrl)")
|
|
return
|
|
}
|
|
|
|
val appIds = extractNewAppIds(html, existingIds, seenAppIds)
|
|
|
|
if (appIds.isEmpty()) {
|
|
println("[ZUM-Apps Live] Seite $page enthält keine neuen Aufgaben. Blättere weiter...")
|
|
return
|
|
}
|
|
|
|
for (appId in appIds) {
|
|
if (results.size >= limit) break
|
|
seenAppIds.add(appId)
|
|
|
|
val item = fetchAppDetails(appId, subject)
|
|
if (item != null) {
|
|
results.add(item)
|
|
logNewItem(appId, item)
|
|
}
|
|
}
|
|
}
|
|
|
|
private fun buildSearchUrl(
|
|
subject: String?,
|
|
page: Int,
|
|
): String {
|
|
return if (subject.isNullOrBlank()) {
|
|
"https://apps.zum.de/apps?page=$page"
|
|
} else {
|
|
val encoded = URLEncoder.encode(subject, "UTF-8")
|
|
"https://apps.zum.de/apps?keys=$encoded&page=$page"
|
|
}
|
|
}
|
|
|
|
private fun extractNewAppIds(
|
|
html: String,
|
|
existingIds: Set<String>,
|
|
seenAppIds: Set<String>,
|
|
): List<String> {
|
|
return Regex("""href=["']/apps/(\d+)["']""")
|
|
.findAll(html)
|
|
.map { it.groupValues[1] }
|
|
.distinct()
|
|
.filter { appId ->
|
|
val fullId = "zum-$appId"
|
|
!seenAppIds.contains(appId) && !existingIds.contains(fullId) && !existingIds.contains(appId)
|
|
}
|
|
.toList()
|
|
}
|
|
|
|
private fun logNewItem(
|
|
appId: String,
|
|
item: ZumOerItem,
|
|
) {
|
|
val lib = item.library ?: "H5P"
|
|
val gr = item.grade ?: "k.A."
|
|
println("[ZUM-Apps Live] Neue App #$appId: \"${item.title}\" ($lib) [${item.subject}, Kl. $gr]")
|
|
}
|
|
|
|
private fun fetchAppDetails(
|
|
appId: String,
|
|
fallbackSubject: String?,
|
|
): ZumOerItem? {
|
|
val appUrl = "https://apps.zum.de/apps/$appId"
|
|
val html = fetchString(appUrl) ?: return null
|
|
|
|
val settingsMatch = Regex("""data-drupal-selector="drupal-settings-json">([^<]+)<""").find(html) ?: return null
|
|
val jsonStr = settingsMatch.groupValues[1]
|
|
|
|
val drupalSettings =
|
|
try {
|
|
json.parseToJsonElement(jsonStr).jsonObject
|
|
} catch (_: Exception) {
|
|
return null
|
|
}
|
|
|
|
val h5pContents =
|
|
drupalSettings["h5p"]?.jsonObject
|
|
?.get("H5PIntegration")?.jsonObject
|
|
?.get("contents")?.jsonObject
|
|
?: return null
|
|
|
|
val contentObj =
|
|
h5pContents["cid-$appId"]?.jsonObject
|
|
?: h5pContents.values.firstOrNull()?.jsonObject
|
|
?: return null
|
|
|
|
val library = contentObj["library"]?.jsonPrimitive?.contentOrNull ?: ""
|
|
if (!library.contains("MultiChoice", ignoreCase = true) &&
|
|
!library.contains("TrueFalse", ignoreCase = true) &&
|
|
!library.contains("QuestionSet", ignoreCase = true)
|
|
) {
|
|
return null
|
|
}
|
|
|
|
val jsonContent = contentObj["jsonContent"]?.jsonPrimitive?.contentOrNull ?: return null
|
|
val metadata = contentObj["metadata"]?.jsonObject
|
|
val title =
|
|
metadata?.get("title")?.jsonPrimitive?.contentOrNull
|
|
?: Regex("""<title>(.*?)(?:\|.*)?</title>""").find(html)?.groupValues?.get(1)?.trim()
|
|
?: "ZUM App #$appId"
|
|
|
|
val rawLicense = metadata?.get("license")?.jsonPrimitive?.contentOrNull
|
|
val license =
|
|
when {
|
|
rawLicense == "U" || rawLicense == null || rawLicense.isBlank() -> DEFAULT_LICENSE
|
|
rawLicense.contains("CC", ignoreCase = true) -> rawLicense
|
|
else -> "CC BY-SA 4.0 ($rawLicense)"
|
|
}
|
|
|
|
val exportUrl = contentObj["exportUrl"]?.jsonPrimitive?.contentOrNull ?: appUrl
|
|
|
|
// Breitgefächerter Abruf aller HTML-Linktexte (Kategorien, Schulstufen, Tags)
|
|
val tags =
|
|
Regex("""<a[^>]*>([^<]+)</a>""")
|
|
.findAll(html)
|
|
.map { it.groupValues[1].trim() }
|
|
.filter { it.isNotBlank() && it.length < 60 }
|
|
.toList()
|
|
|
|
val subject = inferSubject(title, tags, fallbackSubject)
|
|
val grade = inferGrade(title, html, tags)
|
|
|
|
return ZumOerItem(
|
|
id = "zum-$appId",
|
|
title = title,
|
|
subject = subject,
|
|
grade = grade,
|
|
downloadUrl = exportUrl,
|
|
license = license,
|
|
library = library,
|
|
rawJsonContent = jsonContent,
|
|
)
|
|
}
|
|
|
|
private fun inferSubject(
|
|
title: String,
|
|
tags: List<String>,
|
|
fallbackSubject: String?,
|
|
): String {
|
|
for (tag in tags) {
|
|
val mapped = mapSubject(tag)
|
|
if (mapped != tag ||
|
|
tag.equals("Mathematik", ignoreCase = true) ||
|
|
tag.equals("Deutsch", ignoreCase = true) ||
|
|
tag.equals("Biologie", ignoreCase = true) ||
|
|
tag.equals("Englisch", ignoreCase = true)
|
|
) {
|
|
return mapped
|
|
}
|
|
}
|
|
if (!fallbackSubject.isNullOrBlank()) {
|
|
return mapSubject(fallbackSubject)
|
|
}
|
|
return mapSubject(title)
|
|
}
|
|
|
|
private fun inferGrade(
|
|
title: String,
|
|
html: String,
|
|
tags: List<String>,
|
|
): Int? {
|
|
val primaryText = (tags + title).joinToString(" ")
|
|
|
|
// 1. Muster: "3. Klasse", "3. Schuljahr", "3. Kl."
|
|
Regex("""\b(\d+)\.\s*(?:Klasse|Schuljahr|Kl)\b""", RegexOption.IGNORE_CASE)
|
|
.find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 2. Muster: "Klasse 3", "Kl. 3", "Schuljahr 3"
|
|
Regex("""\b(?:Klasse|Schuljahr|Kl\.)\s*(\d+)\b""", RegexOption.IGNORE_CASE)
|
|
.find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 3. Muster: "Klasse 3-4", "Klasse 3/4", "3.-4. Klasse"
|
|
Regex("""\b(?:Klasse|Kl\.)?\s*(\d+)\s*[-/]\s*(\d+)\b""", RegexOption.IGNORE_CASE)
|
|
.find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 4. Fallback: Gesamten HTML-Text der Seite nach Klassenangaben absuchen
|
|
Regex("""\b(\d+)\.\s*(?:Klasse|Schuljahr|Kl)\b""", RegexOption.IGNORE_CASE)
|
|
.find(html)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
Regex("""\b(?:Klasse|Schuljahr|Kl\.)\s*(\d+)\b""", RegexOption.IGNORE_CASE)
|
|
.find(html)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 5. Fallback: Schulstufen-Schlüsselwörter
|
|
val combinedLower = (primaryText + " " + html).lowercase()
|
|
if (combinedLower.contains("primarstufe") || combinedLower.contains("grundschule")) return 3
|
|
val isSek1 =
|
|
combinedLower.contains("sekundarstufe i") ||
|
|
combinedLower.contains("sek i") ||
|
|
combinedLower.contains("realschule") ||
|
|
combinedLower.contains("hauptschule")
|
|
if (isSek1) return 6
|
|
|
|
val isSek2 =
|
|
combinedLower.contains("sekundarstufe ii") ||
|
|
combinedLower.contains("sek ii") ||
|
|
combinedLower.contains("gymnasium") ||
|
|
combinedLower.contains("abitur")
|
|
if (isSek2) return 10
|
|
|
|
return null
|
|
}
|
|
|
|
private fun fetchString(url: String): String? {
|
|
return try {
|
|
val request =
|
|
HttpRequest.newBuilder()
|
|
.uri(URI.create(url))
|
|
.timeout(Duration.ofSeconds(12))
|
|
.header("User-Agent", "Mozilla/5.0 (compatible; KidioBot/1.0; +https://kidio.oliver.com)")
|
|
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
|
.header("Accept-Language", "de-DE,de;q=0.9,en;q=0.8")
|
|
.GET()
|
|
.build()
|
|
|
|
val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString(Charsets.UTF_8))
|
|
if (response.statusCode() == 200) {
|
|
response.body()
|
|
} else {
|
|
null
|
|
}
|
|
} catch (e: Exception) {
|
|
null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns curated ZUM/OER sample metadata items from resources, filterable by subject.
|
|
*/
|
|
fun getCuratedOerItems(
|
|
subject: String? = null,
|
|
limit: Int = 20,
|
|
): List<ZumOerItem> {
|
|
val allItems: List<ZumOerItem> =
|
|
try {
|
|
val inputStream = this::class.java.getResourceAsStream("/fallback_oer_items.json")
|
|
if (inputStream != null) {
|
|
val jsonText = inputStream.bufferedReader().use { it.readText() }
|
|
json.decodeFromString(jsonText)
|
|
} else {
|
|
println("WARNUNG: fallback_oer_items.json wurde nicht gefunden!")
|
|
emptyList()
|
|
}
|
|
} catch (e: Exception) {
|
|
println("WARNUNG: Fehler beim Laden der Fallback-Daten: ${e.message}")
|
|
emptyList()
|
|
}
|
|
|
|
val normalizedSubject = subject?.let { mapSubject(it) }
|
|
val filtered =
|
|
if (normalizedSubject.isNullOrBlank()) {
|
|
allItems
|
|
} else {
|
|
allItems.filter { it.subject.equals(normalizedSubject, ignoreCase = true) }
|
|
}
|
|
|
|
return filtered.take(limit)
|
|
}
|
|
}
|