b082df962f
All checks were successful
CI Pipeline / lint-and-test (pull_request) Successful in 2m38s
515 lines
20 KiB
Kotlin
515 lines
20 KiB
Kotlin
package com.oliver.kidio.backend.ingestion
|
|
|
|
import kotlinx.serialization.json.*
|
|
import java.net.URI
|
|
import java.net.URLEncoder
|
|
import java.net.http.HttpClient
|
|
import java.net.http.HttpRequest
|
|
import java.net.http.HttpResponse
|
|
import java.time.Duration
|
|
|
|
data class ZumOerItem(
|
|
val id: String,
|
|
val title: String,
|
|
val subject: String,
|
|
val grade: Int? = null,
|
|
val downloadUrl: String,
|
|
val license: String = "CC BY-SA 4.0",
|
|
val library: String? = null,
|
|
val rawJsonContent: String? = null,
|
|
)
|
|
|
|
class ZumApiClient(
|
|
private val httpClient: HttpClient =
|
|
HttpClient.newBuilder()
|
|
.connectTimeout(Duration.ofSeconds(10))
|
|
.followRedirects(HttpClient.Redirect.NORMAL)
|
|
.build(),
|
|
) {
|
|
private val json =
|
|
Json {
|
|
ignoreUnknownKeys = true
|
|
isLenient = true
|
|
}
|
|
|
|
companion object {
|
|
fun mapSubject(raw: String): String {
|
|
val lower = raw.trim().lowercase()
|
|
return when {
|
|
lower.contains("mathe") -> "Mathematik"
|
|
lower.contains("deutsch") || lower.contains("german") -> "Deutsch"
|
|
lower.contains("sach") || lower.contains("hsu") || lower.contains("natur") || lower.contains("heimat") -> "Sachunterricht"
|
|
lower.contains("englisch") || lower.contains("english") -> "Englisch"
|
|
lower.contains("bio") -> "Biologie"
|
|
lower.contains("geo") || lower.contains("erdkunde") -> "Geographie"
|
|
lower.contains("gesch") -> "Geschichte"
|
|
lower.contains("chemie") -> "Chemie"
|
|
lower.contains("physik") -> "Physik"
|
|
lower.contains("musik") -> "Musik"
|
|
lower.contains("kunst") -> "Kunst"
|
|
lower.contains("informatik") -> "Informatik"
|
|
else -> raw.trim().replaceFirstChar { if (it.isLowerCase()) it.titlecase() else it.toString() }
|
|
}
|
|
}
|
|
|
|
fun mapGradeToAgeRange(grade: Int?): Pair<Int, Int> {
|
|
return when (grade) {
|
|
1 -> 6 to 7
|
|
2 -> 7 to 8
|
|
3 -> 8 to 9
|
|
4 -> 9 to 10
|
|
5 -> 10 to 11
|
|
6 -> 11 to 12
|
|
7 -> 12 to 13
|
|
8 -> 13 to 14
|
|
9 -> 14 to 15
|
|
10 -> 15 to 16
|
|
11, 12, 13 -> 16 to 18
|
|
else -> 6 to 12
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Downloads an .h5p package file from a remote URL.
|
|
*/
|
|
fun downloadH5P(downloadUrl: String): ByteArray? {
|
|
return try {
|
|
val request =
|
|
HttpRequest.newBuilder()
|
|
.uri(URI.create(downloadUrl))
|
|
.timeout(Duration.ofSeconds(15))
|
|
.header("User-Agent", "Mozilla/5.0 (compatible; KidioBot/1.0; +https://kidio.oliver.com)")
|
|
.GET()
|
|
.build()
|
|
|
|
val response = httpClient.send(request, HttpResponse.BodyHandlers.ofByteArray())
|
|
if (response.statusCode() == 200) {
|
|
response.body()
|
|
} else {
|
|
null
|
|
}
|
|
} catch (e: Exception) {
|
|
println("WARNUNG: Download fehlgeschlagen für $downloadUrl: ${e.message}")
|
|
null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Fetches live OER items from apps.zum.de via search and Drupal settings extraction.
|
|
* Skips items whose IDs are already present in existingIds.
|
|
* Falls back to curated items if remote catalog is unreachable or returns 0 matching interactive items.
|
|
*/
|
|
fun fetchLiveOerItems(
|
|
subject: String? = null,
|
|
limit: Int = 50,
|
|
maxPages: Int = 15,
|
|
existingIds: Set<String> = emptySet(),
|
|
): List<ZumOerItem> {
|
|
val results = mutableListOf<ZumOerItem>()
|
|
val seenAppIds = mutableSetOf<String>()
|
|
|
|
println("[ZUM-Apps Live] Starte Live-Abfrage auf apps.zum.de (Fach: ${subject ?: "Alle Fächer"}, Limit: $limit)...")
|
|
|
|
for (page in 0 until maxPages) {
|
|
if (results.size >= limit) break
|
|
|
|
val searchUrl =
|
|
if (subject.isNullOrBlank()) {
|
|
"https://apps.zum.de/apps?page=$page"
|
|
} else {
|
|
val encoded = URLEncoder.encode(subject, "UTF-8")
|
|
"https://apps.zum.de/apps?keys=$encoded&page=$page"
|
|
}
|
|
|
|
val html = fetchString(searchUrl)
|
|
if (html == null) {
|
|
println("[ZUM-Apps Live] Warnung: Seite $page konnte nicht geladen werden ($searchUrl)")
|
|
continue
|
|
}
|
|
|
|
// Filtert bereits verarbeitete oder in DB vorhandene Apps heraus
|
|
val appIds =
|
|
Regex("""href=["']/apps/(\d+)["']""")
|
|
.findAll(html)
|
|
.map { it.groupValues[1] }
|
|
.distinct()
|
|
.filter { appId ->
|
|
val fullId = "zum-$appId"
|
|
!seenAppIds.contains(appId) && !existingIds.contains(fullId) && !existingIds.contains(appId)
|
|
}
|
|
.toList()
|
|
|
|
if (appIds.isEmpty()) {
|
|
println("[ZUM-Apps Live] Seite $page enthält keine neuen Aufgaben. Blättere weiter...")
|
|
continue
|
|
}
|
|
|
|
for (appId in appIds) {
|
|
if (results.size >= limit) break
|
|
seenAppIds.add(appId)
|
|
|
|
val item = fetchAppDetails(appId, subject)
|
|
if (item != null) {
|
|
results.add(item)
|
|
val lib = item.library ?: "H5P"
|
|
val gr = item.grade ?: "k.A."
|
|
println(
|
|
"[ZUM-Apps Live] Neue App #$appId: \"${item.title}\" ($lib) [${item.subject}, Kl. $gr]",
|
|
)
|
|
}
|
|
}
|
|
}
|
|
|
|
if (results.isEmpty()) {
|
|
println("[ZUM-Apps Live] Keine neuen Live-Inhalte gefunden oder Seite nicht erreichbar. Nutze kuratierte OER-Fallback-Daten.")
|
|
return getCuratedOerItems(subject = subject, limit = limit)
|
|
}
|
|
|
|
println("[ZUM-Apps Live] ${results.size} neue interaktive Aufgaben erfolgreich von apps.zum.de geladen.")
|
|
return results
|
|
}
|
|
|
|
private fun fetchAppDetails(
|
|
appId: String,
|
|
fallbackSubject: String?,
|
|
): ZumOerItem? {
|
|
val appUrl = "https://apps.zum.de/apps/$appId"
|
|
val html = fetchString(appUrl) ?: return null
|
|
|
|
val settingsMatch = Regex("""data-drupal-selector="drupal-settings-json">([^<]+)<""").find(html) ?: return null
|
|
val jsonStr = settingsMatch.groupValues[1]
|
|
|
|
val drupalSettings =
|
|
try {
|
|
json.parseToJsonElement(jsonStr).jsonObject
|
|
} catch (_: Exception) {
|
|
return null
|
|
}
|
|
|
|
val h5pContents =
|
|
drupalSettings["h5p"]?.jsonObject
|
|
?.get("H5PIntegration")?.jsonObject
|
|
?.get("contents")?.jsonObject
|
|
?: return null
|
|
|
|
val contentObj =
|
|
h5pContents["cid-$appId"]?.jsonObject
|
|
?: h5pContents.values.firstOrNull()?.jsonObject
|
|
?: return null
|
|
|
|
val library = contentObj["library"]?.jsonPrimitive?.contentOrNull ?: ""
|
|
if (!library.contains("MultiChoice", ignoreCase = true) &&
|
|
!library.contains("TrueFalse", ignoreCase = true) &&
|
|
!library.contains("QuestionSet", ignoreCase = true)
|
|
) {
|
|
return null
|
|
}
|
|
|
|
val jsonContent = contentObj["jsonContent"]?.jsonPrimitive?.contentOrNull ?: return null
|
|
val metadata = contentObj["metadata"]?.jsonObject
|
|
val title =
|
|
metadata?.get("title")?.jsonPrimitive?.contentOrNull
|
|
?: Regex("""<title>(.*?)(?:\|.*)?</title>""").find(html)?.groupValues?.get(1)?.trim()
|
|
?: "ZUM App #$appId"
|
|
|
|
val rawLicense = metadata?.get("license")?.jsonPrimitive?.contentOrNull
|
|
val license =
|
|
when {
|
|
rawLicense == "U" || rawLicense == null || rawLicense.isBlank() -> "CC BY-SA 4.0"
|
|
rawLicense.contains("CC", ignoreCase = true) -> rawLicense
|
|
else -> "CC BY-SA 4.0 ($rawLicense)"
|
|
}
|
|
|
|
val exportUrl = contentObj["exportUrl"]?.jsonPrimitive?.contentOrNull ?: appUrl
|
|
|
|
// Breitgefächerter Abruf aller HTML-Linktexte (Kategorien, Schulstufen, Tags)
|
|
val tags =
|
|
Regex("""<a[^>]*>([^<]+)</a>""")
|
|
.findAll(html)
|
|
.map { it.groupValues[1].trim() }
|
|
.filter { it.isNotBlank() && it.length < 60 }
|
|
.toList()
|
|
|
|
val subject = inferSubject(title, tags, fallbackSubject)
|
|
val grade = inferGrade(title, html, tags)
|
|
|
|
return ZumOerItem(
|
|
id = "zum-$appId",
|
|
title = title,
|
|
subject = subject,
|
|
grade = grade,
|
|
downloadUrl = exportUrl,
|
|
license = license,
|
|
library = library,
|
|
rawJsonContent = jsonContent,
|
|
)
|
|
}
|
|
|
|
private fun inferSubject(
|
|
title: String,
|
|
tags: List<String>,
|
|
fallbackSubject: String?,
|
|
): String {
|
|
for (tag in tags) {
|
|
val mapped = mapSubject(tag)
|
|
if (mapped != tag ||
|
|
tag.equals("Mathematik", ignoreCase = true) ||
|
|
tag.equals("Deutsch", ignoreCase = true) ||
|
|
tag.equals("Biologie", ignoreCase = true) ||
|
|
tag.equals("Englisch", ignoreCase = true)
|
|
) {
|
|
return mapped
|
|
}
|
|
}
|
|
if (!fallbackSubject.isNullOrBlank()) {
|
|
return mapSubject(fallbackSubject)
|
|
}
|
|
return mapSubject(title)
|
|
}
|
|
|
|
private fun inferGrade(
|
|
title: String,
|
|
html: String,
|
|
tags: List<String>,
|
|
): Int? {
|
|
val primaryText = (tags + title).joinToString(" ")
|
|
|
|
// 1. Muster: "3. Klasse", "3. Schuljahr", "3. Kl."
|
|
Regex("""\b(\d+)\.\s*(?:Klasse|Schuljahr|Kl)\b""", RegexOption.IGNORE_CASE)
|
|
.find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 2. Muster: "Klasse 3", "Kl. 3", "Schuljahr 3"
|
|
Regex("""\b(?:Klasse|Schuljahr|Kl\.)\s*(\d+)\b""", RegexOption.IGNORE_CASE)
|
|
.find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 3. Muster: "Klasse 3-4", "Klasse 3/4", "3.-4. Klasse"
|
|
Regex("""\b(?:Klasse|Kl\.)?\s*(\d+)\s*[-/]\s*(\d+)\b""", RegexOption.IGNORE_CASE)
|
|
.find(primaryText)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 4. Fallback: Gesamten HTML-Text der Seite nach Klassenangaben absuchen
|
|
Regex("""\b(\d+)\.\s*(?:Klasse|Schuljahr|Kl)\b""", RegexOption.IGNORE_CASE)
|
|
.find(html)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
Regex("""\b(?:Klasse|Schuljahr|Kl\.)\s*(\d+)\b""", RegexOption.IGNORE_CASE)
|
|
.find(html)?.groupValues?.get(1)?.toIntOrNull()?.let { return it }
|
|
|
|
// 5. Fallback: Schulstufen-Schlüsselwörter
|
|
val combinedLower = (primaryText + " " + html).lowercase()
|
|
if (combinedLower.contains("primarstufe") || combinedLower.contains("grundschule")) return 3
|
|
val isSek1 =
|
|
combinedLower.contains("sekundarstufe i") ||
|
|
combinedLower.contains("sek i") ||
|
|
combinedLower.contains("realschule") ||
|
|
combinedLower.contains("hauptschule")
|
|
if (isSek1) return 6
|
|
|
|
val isSek2 =
|
|
combinedLower.contains("sekundarstufe ii") ||
|
|
combinedLower.contains("sek ii") ||
|
|
combinedLower.contains("gymnasium") ||
|
|
combinedLower.contains("abitur")
|
|
if (isSek2) return 10
|
|
|
|
return null
|
|
}
|
|
|
|
private fun fetchString(url: String): String? {
|
|
return try {
|
|
val request =
|
|
HttpRequest.newBuilder()
|
|
.uri(URI.create(url))
|
|
.timeout(Duration.ofSeconds(12))
|
|
.header("User-Agent", "Mozilla/5.0 (compatible; KidioBot/1.0; +https://kidio.oliver.com)")
|
|
.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
|
|
.header("Accept-Language", "de-DE,de;q=0.9,en;q=0.8")
|
|
.GET()
|
|
.build()
|
|
|
|
val response = httpClient.send(request, HttpResponse.BodyHandlers.ofString(Charsets.UTF_8))
|
|
if (response.statusCode() == 200) {
|
|
response.body()
|
|
} else {
|
|
null
|
|
}
|
|
} catch (e: Exception) {
|
|
null
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Returns curated ZUM/OER sample metadata items, filterable by subject.
|
|
*/
|
|
fun getCuratedOerItems(
|
|
subject: String? = null,
|
|
limit: Int = 20,
|
|
): List<ZumOerItem> {
|
|
val allItems =
|
|
listOf(
|
|
ZumOerItem(
|
|
id = "zum-math-001",
|
|
title = "Grundrechenarten: Multiplikation 3. Klasse",
|
|
subject = "Mathematik",
|
|
grade = 3,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/math-001.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>Was ist das Ergebnis von <strong>6 × 8</strong>?</p>",
|
|
"answers": [
|
|
{"text": "42", "correct": false},
|
|
{"text": "48", "correct": true},
|
|
{"text": "54", "correct": false},
|
|
{"text": "56", "correct": false}
|
|
]
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
ZumOerItem(
|
|
id = "zum-math-002",
|
|
title = "Zahlenraum bis 100: Addition",
|
|
subject = "Mathematik",
|
|
grade = 2,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/math-002.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>Berechne: 34 + 28</p>",
|
|
"answers": [
|
|
{"text": "52", "correct": false},
|
|
{"text": "62", "correct": true},
|
|
{"text": "64", "correct": false}
|
|
]
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
ZumOerItem(
|
|
id = "zum-math-003",
|
|
title = "Geometrie: Formen erkennen",
|
|
subject = "Mathematik",
|
|
grade = 4,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/math-003.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>Ein Quadrat hat vier gleich lange Seiten und vier rechte Winkel.</p>",
|
|
"correct": true
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
ZumOerItem(
|
|
id = "zum-de-001",
|
|
title = "Wortarten: Nomen erkennen",
|
|
subject = "Deutsch",
|
|
grade = 2,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/de-001.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>Welches der folgenden Wörter ist ein <strong>Nomen (Namenwort)</strong>?</p>",
|
|
"answers": [
|
|
{"text": "schnell", "correct": false},
|
|
{"text": "laufen", "correct": false},
|
|
{"text": "Schule", "correct": true},
|
|
{"text": "schön", "correct": false}
|
|
]
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
ZumOerItem(
|
|
id = "zum-de-002",
|
|
title = "Rechtschreibung: Doppel-s oder ss",
|
|
subject = "Deutsch",
|
|
grade = 4,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/de-002.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>Wie wird das Wort richtig geschrieben? „Der Flu... ist tief.“</p>",
|
|
"answers": [
|
|
{"text": "Flus", "correct": false},
|
|
{"text": "Fluss", "correct": true},
|
|
{"text": "Fluß", "correct": false}
|
|
]
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
ZumOerItem(
|
|
id = "zum-sach-001",
|
|
title = "Natur & Tiere: Winterschlaf",
|
|
subject = "Sachunterricht",
|
|
grade = 3,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/sach-001.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>Welches dieser Tiere hält einen echten <strong>Winterschlaf</strong>?</p>",
|
|
"answers": [
|
|
{"text": "Igel", "correct": true},
|
|
{"text": "Eichhörnchen", "correct": false},
|
|
{"text": "Amsel", "correct": false},
|
|
{"text": "Fuchs", "correct": false}
|
|
]
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
ZumOerItem(
|
|
id = "zum-sach-002",
|
|
title = "Planetensystem: Sonnennähe",
|
|
subject = "Sachunterricht",
|
|
grade = 4,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/sach-002.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>Welcher Planet ist der Sonne am nächsten?</p>",
|
|
"answers": [
|
|
{"text": "Venus", "correct": false},
|
|
{"text": "Mars", "correct": false},
|
|
{"text": "Merkur", "correct": true},
|
|
{"text": "Jupiter", "correct": false}
|
|
]
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
ZumOerItem(
|
|
id = "zum-eng-001",
|
|
title = "English Vocabulary: Colors & Animals",
|
|
subject = "Englisch",
|
|
grade = 4,
|
|
downloadUrl = "https://apps.zum.de/api/h5p/eng-001.h5p",
|
|
license = "CC BY-SA 4.0",
|
|
rawJsonContent =
|
|
"""
|
|
{
|
|
"question": "<p>What is the English word for „Hund“?</p>",
|
|
"answers": [
|
|
{"text": "Cat", "correct": false},
|
|
{"text": "Dog", "correct": true},
|
|
{"text": "Bird", "correct": false},
|
|
{"text": "Rabbit", "correct": false}
|
|
]
|
|
}
|
|
""".trimIndent(),
|
|
),
|
|
)
|
|
|
|
val normalizedSubject = subject?.let { mapSubject(it) }
|
|
val filtered =
|
|
if (normalizedSubject.isNullOrBlank()) {
|
|
allItems
|
|
} else {
|
|
allItems.filter { it.subject.equals(normalizedSubject, ignoreCase = true) }
|
|
}
|
|
|
|
return filtered.take(limit)
|
|
}
|
|
}
|