launcher: wizard bar follows the copied megabytes, saves table shows 4 rows at the minimum size, mods tab keeps focus and caret when it is rebuilt, take-over notice expires; geom_export previews in game space; apptest checks every menu step with longer holds, loads slot 2, retries an early crash once and records load

tools/apptest.py

@@ -14,14 +14,17 @@ Commands (NAME first; --display only for 'start'/'run', remembered afterwards):
   apptest.py apply NAME [ids] [--opt id.k=v]... apply mods (none = original) via the app CLI
   apptest.py unlock NAME SLOT WORLD CHAPTER     unlock up to WORLD/CHAPTER, map cursor there
   apptest.py start NAME --display N             Xvfb :N + private pulse + game (app --launch)
-  apptest.py tolevel NAME [--world W]           main menu -> slot 3 -> map -> level -> gameplay
+  apptest.py tolevel NAME [--world W] [--slot S] main menu -> slot S (3) -> map -> level -> gameplay
   apptest.py keys NAME Up:1.5 x sleep=1 a+x:0.5 shot:name ...   key sequence (hold s, default 0.2)
   apptest.py shot NAME out.png [--small]        screenshot (1920x1080 PNG, --small = 960x540 JPEG)
   apptest.py ocr NAME [WxH+X+Y]                 German OCR of the screen (or a crop)
   apptest.py status NAME                        game/window/crash/audio state (exit 3 = crashed)
   apptest.py stop NAME                          kill game (wineserver -k), Xvfb and pulse of NAME
   apptest.py run NAME --display N --mods a,b [--opt id.k=v]... --level W C [--seq "..."] --out DIR [--keep]
-                                                all in one; exit 0 ok, 3 crash, 4 timeout, 2 usage
+                [--no-retry]                    all in one; exit 0 ok, 3 crash, 4 timeout, 2 usage.
+                                                A crash before the main menu or a missed save slot screen
+                                                is retried once (report.json "retries", out/retry1_*);
+                                                --no-retry turns that off
 
 Options of every command: --root DIR, --jar FILE (default: newest build/libs/*-all.jar),
 --iso FILE (default ~/Downloads/GROOVE_DK.iso). Python: `from apptest import Session`.
@@ -34,6 +37,7 @@ Safety: never uses display :0, never touches ~/.wine-lama, kills only PIDs it st
 import fcntl
 import json
 import os
+import re
 import shutil
 import signal
 import subprocess
@@ -51,6 +55,10 @@ SLOT_CROP = "1600x280+160+740"        # save-slot text "Gespeichertes Spiel N /
 MENU_CROP = "700x300+1150+650"        # main menu items (bottom right)
 
 
+# world -> word the save slot shows for it (OCR, lower case); only the verified ones
+SLOT_WORLD_NAMES = {1: "dorf", 9: "testlevel"}
+
+
 class Crash(Exception):
     pass
 
@@ -517,8 +525,8 @@ class Session:
         raise Timeout("timeout (%d s) waiting for %s" % (timeout, what))
 
     # ------------------------------------------------------------ navigation
-    def tolevel(self, world=None, skip_flyby=True):
-        """Main menu -> "Spiel starten" -> slot 3 (rightmost) -> map (cursor from --unlock ... --cursor)
+    def tolevel(self, world=None, skip_flyby=True, on_menu=None, slot=3):
+        """Main menu -> "Spiel starten" -> save slot (default 3) -> map (cursor from --unlock ... --cursor)
         -> Enter -> skip the fly-by -> gameplay. Returns timings."""
         T = {}
         t0 = tx = time.time()
@@ -528,10 +536,14 @@ class Session:
                 raise Crash(c)
             if fuzzy_in("spiel starten", self.ocr(MENU_CROP, threshold=75, psm=6, scale="100%")):
                 log("main menu")
+                if on_menu:
+                    on_menu()
                 time.sleep(0.5)
-                self.key("Return")
+                self.key("Return", 0.6)
                 break
             if fuzzy_in("gespeichertes", self.slot_text()):
+                if on_menu:
+                    on_menu()
                 break
             if time.time() - tx > 40:
                 log("no menu yet -> Return (skips logo/demo)")
@@ -541,21 +553,15 @@ class Session:
                 raise Timeout("main menu not reached")
             time.sleep(1)
         T["menu"] = round(time.time() - t0, 1)
-        self.wait_for(lambda: fuzzy_in("gespeichertes", self.slot_text()), 20, "save slot screen")
-        time.sleep(0.4)
-        for _ in range(2):          # 3-slot slider without wrap: slot 3 is the rightmost
-            self.key("Right", 0.3)
-            time.sleep(0.3)
-        for i in range(3):
-            time.sleep(0.6)
-            t = self.slot_text()
-            if fuzzy_in("kapitel", t) and not fuzzy_in("leer", t):
-                log("slot: %s" % t.split(" | ")[0])
-                break
-            self.key("Right", 0.3)
-        else:
-            log("WARNING: slot 3 text not recognised (%s)" % t)
-        self.key("Return")
+        self._to_slot_screen()
+        t = self._select_slot(slot)
+        if not self._slot_ok(t):
+            # an empty slot starts a new game in the village: never the level that was asked for
+            raise Timeout("save slot %d is empty (%s) - unlock it first" % (slot, t))
+        name = SLOT_WORLD_NAMES.get(world)
+        if name and not fuzzy_in(name, t):
+            raise Timeout("save slot %d holds another world (%s), expected %r" % (slot, t, name))
+        self._open_slot()
         T["slot"] = round(time.time() - t0, 1)
         if world == 9:
             self.wait_for(lambda: self.classify() == "map", 30, "world 9 map")
@@ -563,10 +569,11 @@ class Session:
         else:
             self.wait_for(lambda: self.classify() not in ("black", "none"), 30, "map")
             self.wait_stable(still=1.5, timeout=20)
-        self.key("Return")
+        map_thumb = self.thumb()
+        self.key("Return", 0.6)
         t_load = time.time()
         T["map"] = round(t_load - t0, 1)
-        skipped, good, states = False, 0, []
+        skipped, good, states, again = False, 0, [], 0
         while time.time() - t_load < 150:
             c = self.crashed()
             if c:
@@ -575,7 +582,12 @@ class Session:
             if not states or states[-1] != st:
                 states.append(st)
                 log("state: %s (%.1f s after level start)" % (st, time.time() - t_load))
-            if st == "letterbox" and skip_flyby and not skipped:
+            if "black" not in states and "letterbox" not in states and time.time() - t_load > 8 + 8 * again \
+                    and again < 2 and self.diff(self.thumb(), map_thumb) < 6:
+                again += 1      # still on the map: the Return was missed
+                log("level did not start -> Return again")
+                self.key("Return", 0.6 + 0.3 * again)
+            elif st == "letterbox" and skip_flyby and not skipped:
                 time.sleep(0.5)
                 self.key("Return")
                 skipped = True
@@ -596,6 +608,79 @@ class Session:
         log("gameplay after %.1f s, states %s" % (T["gameplay"], states))
         return T
 
+    def _to_slot_screen(self, timeout=45):
+        """Waits for the save slot screen; a Return the menu missed (still fading in, or a key press shorter
+        than one frame on a loaded machine) is repeated, held a little longer each time."""
+        t0 = tr = time.time()
+        holds = [0.8, 1.0, 1.2]
+        while time.time() - t0 < timeout:
+            c = self.crashed()
+            if c:
+                raise Crash(c)
+            if fuzzy_in("gespeichertes", self.slot_text()):
+                return
+            if time.time() - tr > 5 and fuzzy_in("spiel starten", self.ocr(MENU_CROP, threshold=75, psm=6,
+                                                                          scale="100%")):
+                hold = holds.pop(0) if holds else 1.2
+                log("still in the main menu -> Return again (held %.1f s)" % hold)
+                self.key("Return", hold)
+                tr = time.time()
+            time.sleep(0.7)
+        raise Timeout("timeout (%d s) waiting for save slot screen" % timeout)
+
+    @staticmethod
+    def _slot_ok(text):
+        return fuzzy_in("kapitel", text) and not fuzzy_in("leer", text)
+
+    @staticmethod
+    def _slot_no(text):
+        """Slot number from "gespeichertes spiel N" (OCR text), or None."""
+        m = re.search(r"spiel\s*([123])\b", text)
+        return int(m.group(1)) if m else None
+
+    def _select_slot(self, slot=3, timeout=240):
+        """Turns the save slot carousel (it opens on slot 2) to `slot` and checks its text after every step.
+        The carousel wraps (3 -> 1) and ignores keys while it turns, so Right is pressed once per step and
+        only again when the text shows another slot. A step that did not move is repeated with a longer
+        hold (0.3/0.6/1.0/1.5 s). On a busy machine (load ~30, several games) the carousel ignored
+        Left/Right for minutes while Return/Escape worked: `run` therefore uses slot 2, which needs no
+        turning."""
+        self.wait_stable(still=0.8, timeout=6)
+        holds = [0.3, 0.6, 1.0, 1.5]
+        t0 = time.time()
+        t = self.slot_text()
+        while time.time() - t0 < timeout:
+            if self._slot_no(t) == slot:
+                log("slot: %s" % t.split(" | ")[0])
+                return t
+            if not fuzzy_in("gespeichertes", t):
+                raise Timeout("save slot screen lost while selecting slot %d (%s)" % (slot, t))
+            before = self._slot_no(t)
+            self.key("Right", holds[0])
+            t1 = time.time()
+            while time.time() - t1 < 8:      # the carousel turns, then the new text shows
+                time.sleep(0.6)
+                t = self.slot_text()
+                if self._slot_no(t) not in (None, before):
+                    break
+            else:
+                if len(holds) > 1:
+                    holds.pop(0)
+                log("slot carousel did not move (slot %s) -> Right again, held %.1f s" % (before, holds[0]))
+        raise Timeout("save slot %d not selected after %d s (%s)" % (slot, timeout, t))
+
+    def _open_slot(self):
+        """Return on the slot until the slot screen is gone (a short press can be missed under load)."""
+        for hold in (0.6, 0.9, 1.2):
+            self.key("Return", hold)
+            t1 = time.time()
+            while time.time() - t1 < 6:
+                time.sleep(0.7)
+                if not fuzzy_in("gespeichertes", self.slot_text()):
+                    return
+            log("still on the save slot screen -> Return again")
+        raise Timeout("save slot did not open")
+
     def status(self):
         s = {"display": self.display, "data": self.data, "game_pids": self.game_pids(),
              "window": self.find_window() if self.display and self.state.get("xvfb_pid") else None,
@@ -606,18 +691,53 @@ class Session:
         return s
 
 
-def run(sess, mods, opts, level, seq, out, keep=False):
+def start_to_level(sess, level, out, report, retry=True):
+    """start + tolevel; a crash before the main menu (Wine page faults at start-up, not caused by mods) or a
+    menu that was not recognised is retried once with a fresh game start. Each retry is listed in
+    report["retries"] and logged; the failed attempt's screenshot/log go to out/retry1_*."""
+    for attempt in (1, 2):
+        phase = "start"
+        try:
+            sess.start()
+            if level:
+                phase = "menu"
+                report["tolevel"] = sess.tolevel(world=level[0], slot=2,
+                                                 on_menu=lambda: report.update(menu_reached=True))
+            return
+        except (Crash, Timeout) as e:
+            early = isinstance(e, Crash) and not report.get("menu_reached") or \
+                isinstance(e, Timeout) and "save slot" in str(e)
+            if not retry or attempt == 2 or not early:
+                raise
+            log("WARNING: %s during %s -> stopping and starting the game once more" % (e, phase))
+            try:
+                sess.shot(os.path.join(out, "retry1_failure.jpg"), small=True)
+            except SystemExit:
+                pass
+            gl = os.path.join(sess.data, "logs", "game.log")
+            if os.path.exists(gl):
+                shutil.copy(gl, os.path.join(out, "retry1_game.log"))
+            report.setdefault("retries", []).append({"attempt": attempt, "phase": phase, "error": str(e)})
+            report.pop("menu_reached", None)
+            sess.stop()
+
+
+def run(sess, mods, opts, level, seq, out, keep=False, retry=True):
     os.makedirs(out, exist_ok=True)
-    report = {"mods": mods, "opts": opts, "level": level}
+    # a busy machine (several agents' games) slows the game to a few frames per second: the usual cause of
+    # missed key presses; recorded to tell flaky runs apart
+    report = {"mods": mods, "opts": opts, "level": level, "load_avg": [round(x, 1) for x in os.getloadavg()],
+              "cpus": os.cpu_count()}
     code = 0
     try:
         sess.setup()
         report["apply"] = sess.apply(mods, opts).strip().splitlines()[-3:]
         if level:
+            # slot 2 is where the carousel opens (no turning, see _select_slot); slot 3 gets the same
+            # progress for mods whose "save slot" option defaults to 3
+            sess.unlock(2, level[0], level[1])
             sess.unlock(3, level[0], level[1])
-        sess.start()
-        if level:
-            report["tolevel"] = sess.tolevel(world=level[0])
+        start_to_level(sess, level, out, report, retry)
         sess.shot(os.path.join(out, "00_start.png"))
         if seq:
             report["shots"] = sess.keys(seq, outdir=out)
@@ -647,7 +767,8 @@ def run(sess, mods, opts, level, seq, out, keep=False):
         json.dump(report, open(os.path.join(out, "report.json"), "w"), indent=1)
         if not keep:
             sess.stop()
-    log("result: %s" % (report.get("error") or "OK"))
+    retried = "" if not report.get("retries") else " (after a retry: %s)" % report["retries"][0]["error"]
+    log("result: %s%s" % (report.get("error") or "OK", retried))
     return code
 
 
@@ -660,7 +781,8 @@ def main(argv):
     i = 0
     while i < len(rest):
         a = rest[i]
-        if a in ("--display", "--root", "--jar", "--iso", "--mods", "--world", "--seq", "--out") and i + 1 < len(rest):
+        if a in ("--display", "--root", "--jar", "--iso", "--mods", "--world", "--slot", "--seq", "--out") \
+                and i + 1 < len(rest):
             kw[a[2:]] = rest[i + 1]
             i += 2
         elif a == "--opt" and i + 1 < len(rest):
@@ -685,7 +807,8 @@ def main(argv):
         elif cmd == "start":
             s.start()
         elif cmd == "tolevel":
-            print(json.dumps(s.tolevel(world=int(kw["world"]) if "world" in kw else None)))
+            print(json.dumps(s.tolevel(world=int(kw["world"]) if "world" in kw else None,
+                                       slot=int(kw.get("slot", 3)))))
         elif cmd == "keys":
             print(s.keys(pos, outdir=kw.get("out")))
         elif cmd == "shot":
@@ -703,7 +826,8 @@ def main(argv):
                 print("run needs --display N and --out DIR")
                 return 2
             mods = [x for x in kw.get("mods", "").split(",") if x]
-            return run(s, mods, opts, kw.get("level"), kw.get("seq"), kw["out"], keep="--keep" in pos)
+            return run(s, mods, opts, kw.get("level"), kw.get("seq"), kw["out"], keep="--keep" in pos,
+                       retry="--no-retry" not in pos)
         else:
             print(__doc__)
             return 2

tools/geom_export.py

@@ -9,7 +9,8 @@ Writes to outdir:
   page_NN.png             texture pages (256x256, from TEXT)
   collision.obj           (--collision) collision polygons, grouped by material
   objects.obj             (--objects) static STPC object models side by side
-  preview_top.png, preview_persp.png  textured software renderings for checking
+  preview_top.png, preview_persp.png  textured software renderings for checking (game space,
+                          the same way round as in the game)
 
 Coordinates: OBJ file space = game space (x, y, z_game): right-handed, y up, z > 0 in the grid, as
 Blender shows the level the same way round as the game (geomlib.to_file). --export-space writes
@@ -101,6 +102,11 @@ def write_mtl(path, tris, texs, npages):
 
 # ------------------------------------------------------------ software renderer
 
+def game_space_tris(tris):
+    """world_tris (export space) -> game space for the right-handed preview camera."""
+    return [([G.to_file(p) for p in P], *rest) for P, *rest in tris]
+
+
 def render(tris, pages, cam, size, path, texs):
     Wd, Hd = size
     eye, target, fov = cam
@@ -237,10 +243,13 @@ def main():
         for k, im in enumerate(imgs):
             Image.fromarray(im).save(os.path.join(a.outdir, f'page_{k:02d}.png'))
         size = tuple(int(s) for s in a.size.split('x'))
+        # the renderer's camera is right-handed: game space shows the level as the game does
+        # (export space would come out mirrored); also with --export-space
+        tris = game_space_tris(tris)
         P = np.array([p for t in tris for p in t[0]])
         lo, hi = P.min(0), P.max(0); c = (lo + hi) / 2
         ext = max(hi[0] - lo[0], hi[2] - lo[2])
-        # top view (almost vertical, north = -z at the top)
+        # top view (almost vertical, game -z at the top)
         render(tris, imgs, ((c[0], hi[1] + ext * 1.2, c[2] + 0.01), (c[0], lo[1], c[2]), 50), size,
                os.path.join(a.outdir, 'preview_top.png'), texs)
         render(tris, imgs, ((c[0] + ext * 0.35, hi[1] + ext * 0.45, hi[2] + ext * 0.45), tuple(c), 60), size,

tools/test_apptest.py

@@ -0,0 +1,86 @@
+"""Offline tests of apptest's retry/slot logic (no game needed): python3 -m unittest tools/test_apptest.py"""
+import os
+import sys
+import tempfile
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import apptest as A  # noqa: E402
+
+
+class FakeSession:
+    def __init__(self, outcomes):
+        self.outcomes = list(outcomes)       # per start: None ok, or exception raised by start/tolevel
+        self.data = tempfile.mkdtemp()
+        self.starts = self.stops = 0
+
+    def start(self):
+        self.starts += 1
+        o = self.outcomes.pop(0)
+        if isinstance(o, A.Crash) and "start" in str(o):
+            raise o
+        self._next = o
+
+    def tolevel(self, world=None, slot=3, on_menu=None):
+        o = self._next
+        if isinstance(o, A.Crash) and "level" in str(o):
+            on_menu()
+            raise o
+        if o:
+            raise o
+        on_menu()
+        return {"slot": slot}
+
+    def shot(self, path, small=False):
+        open(path, "w").close()
+
+    def stop(self):
+        self.stops += 1
+
+
+class StartRetryTest(unittest.TestCase):
+    def setUp(self):
+        self.out = tempfile.mkdtemp()
+
+    def test_early_crash_is_retried_once_and_reported(self):
+        s, rep = FakeSession([A.Crash("start: page fault"), None]), {}
+        A.start_to_level(s, (1, 1), self.out, rep)
+        self.assertEqual(2, s.starts)
+        self.assertEqual([{"attempt": 1, "phase": "start", "error": "start: page fault"}], rep["retries"])
+        self.assertEqual({"slot": 2}, rep["tolevel"])
+        self.assertTrue(os.path.exists(os.path.join(self.out, "retry1_failure.jpg")))
+
+    def test_missed_slot_screen_is_retried(self):
+        s, rep = FakeSession([A.Timeout("timeout (45 s) waiting for save slot screen"), None]), {}
+        A.start_to_level(s, (1, 1), self.out, rep)
+        self.assertEqual("menu", rep["retries"][0]["phase"])
+
+    def test_crash_after_the_menu_is_not_retried(self):
+        s, rep = FakeSession([A.Crash("level: page fault"), None]), {}
+        with self.assertRaises(A.Crash):
+            A.start_to_level(s, (1, 1), self.out, rep)
+        self.assertEqual(1, s.starts)
+        self.assertNotIn("retries", rep)
+
+    def test_second_failure_and_no_retry_raise(self):
+        s = FakeSession([A.Crash("start: a"), A.Crash("start: b")])
+        with self.assertRaises(A.Crash):
+            A.start_to_level(s, None, self.out, {})
+        self.assertEqual(2, s.starts)
+        s = FakeSession([A.Crash("start: a"), None])
+        with self.assertRaises(A.Crash):
+            A.start_to_level(s, None, self.out, {}, retry=False)
+
+
+class SlotTextTest(unittest.TestCase):
+    def test_slot_number_and_content(self):
+        S = A.Session
+        self.assertEqual(2, S._slot_no(". gespeichertes spiel 2 u leer | gespeichertes spiel 2 u leer"))
+        self.assertEqual(3, S._slot_no("gespeichertes spiel 3 dorf kapitel 1"))
+        self.assertIsNone(S._slot_no("eingabe neues spiel starten"))
+        self.assertTrue(S._slot_ok("gespeichertes spiel 2 u testlevel kapitel 1 goldmünzen"))
+        self.assertFalse(S._slot_ok("gespeichertes spiel 2 u leer"))
+
+
+if __name__ == "__main__":
+    unittest.main()