reduce harsh dithering: constrained error diffusion in Lab space

Colors the map palette cannot represent (light sky blue, sunset tones) were
approximated by mixing far-apart palette colors, producing a high-contrast
checkerboard in the sky and stray speckles elsewhere. Diffuse the error in
CIELAB and only allow palette colors nearly as close to the original as the
best match, so the dither blends between neighbouring colors only.

src/main/java/eu/mhsl/minecraft/pixelpics/utils/MapColorPalette.java

@@ -74,6 +74,12 @@ public final class MapColorPalette {
         return bestPos;
     }
 
+    /** CIELAB of the palette color at {@code pos} (do not modify). */
+    public static double[] lab(int pos) {
+        ensure();
+        return lab[pos];
+    }
+
     public static byte mapIndex(int pos) {
         return indices[pos];
     }
@@ -82,7 +88,7 @@ public final class MapColorPalette {
         return rgb[pos];
     }
 
-    private static double[] rgbToLab(int r, int g, int b) {
+    public static double[] rgbToLab(int r, int g, int b) {
         double rl = ColorUtil.toLinear(r), gl = ColorUtil.toLinear(g), bl = ColorUtil.toLinear(b);
         // sRGB -> XYZ (D65)
         double x = rl * 0.4124 + gl * 0.3576 + bl * 0.1805;

src/main/java/eu/mhsl/minecraft/pixelpics/utils/MapImageDither.java

@@ -3,10 +3,16 @@ package eu.mhsl.minecraft.pixelpics.utils;
 import java.awt.image.BufferedImage;
 
 /**
- * Quantizes a rendered image to Minecraft map color indices using dampened Floyd–Steinberg error
- * diffusion with perceptual (CIELAB) nearest-color matching. The result is a {@code 128*128} byte
+ * Quantizes a rendered image to Minecraft map color indices. The result is a {@code 128*128} byte
  * array of palette indices, ready to blit onto a {@link org.bukkit.map.MapCanvas}.
  *
+ * <p>Uses dampened Floyd–Steinberg error diffusion in CIELAB space with <em>constrained</em>
+ * candidates: a pixel may only take a palette color that is nearly as close to its original color
+ * as the best match. Colors the map palette cannot represent well (e.g. light sky blue, sunset
+ * tones) would otherwise be approximated by mixing far-apart colors, producing a harsh,
+ * high-contrast checkerboard; the constraint keeps the dither between neighbouring colors only,
+ * so flat areas stay calm while gradients still blend.
+ *
  * <p>This is the expensive step; computing it once and caching the byte array lets map re-rendering
  * after a restart be a cheap copy instead of a full re-quantization.
  */
@@ -14,8 +20,17 @@ public final class MapImageDither {
 
     public static final int SIZE = 128;
 
-    /** Fraction of quantization error diffused. Low values keep gradients smooth without visible noise. */
-    private static final float DITHER_STRENGTH = 0.35f;
+    /** Fraction of quantization error diffused. */
+    private static final double DITHER_STRENGTH = 0.6;
+
+    /**
+     * A candidate is allowed when its Lab distance to the original pixel is at most
+     * {@code best * CANDIDATE_RATIO + CANDIDATE_SLACK}, where {@code best} is the distance of the
+     * nearest palette color.
+     */
+    private static final double CANDIDATE_RATIO = 1.1;
+
+    private static final double CANDIDATE_SLACK = 1.5;
 
     private MapImageDither() {}
 
@@ -23,51 +38,69 @@ public final class MapImageDither {
     public static byte[] dither(BufferedImage image) {
         int w = Math.min(SIZE, image.getWidth());
         int h = Math.min(SIZE, image.getHeight());
+        int paletteSize = MapColorPalette.size();
         byte[] out = new byte[SIZE * SIZE];
 
-        float[][] buf = new float[h][w * 3];
+        double[][] original = new double[w * h][];
+        double[][] buf = new double[w * h][];
         for (int y = 0; y < h; y++) {
             for (int x = 0; x < w; x++) {
                 int argb = image.getRGB(x, y);
-                buf[y][x * 3] = (argb >> 16) & 0xFF;
-                buf[y][x * 3 + 1] = (argb >> 8) & 0xFF;
-                buf[y][x * 3 + 2] = argb & 0xFF;
+                double[] lab = MapColorPalette.rgbToLab((argb >> 16) & 0xFF, (argb >> 8) & 0xFF, argb & 0xFF);
+                original[y * w + x] = lab;
+                buf[y * w + x] = lab.clone();
             }
         }
 
         for (int y = 0; y < h; y++) {
             for (int x = 0; x < w; x++) {
-                int i = x * 3;
-                int r = clamp(buf[y][i]);
-                int g = clamp(buf[y][i + 1]);
-                int b = clamp(buf[y][i + 2]);
+                int i = y * w + x;
+                double[] orig = original[i];
+                double[] want = buf[i];
 
-                int pos = MapColorPalette.nearestPos(r, g, b);
-                out[y * SIZE + x] = MapColorPalette.mapIndex(pos);
+                double bestToOrig = Double.MAX_VALUE;
+                for (int p = 0; p < paletteSize; p++) {
+                    bestToOrig = Math.min(bestToOrig, dist2(MapColorPalette.lab(p), orig));
+                }
+                double limit = Math.sqrt(bestToOrig) * CANDIDATE_RATIO + CANDIDATE_SLACK;
+                limit *= limit;
 
-                int chosen = MapColorPalette.mapRgb(pos);
-                float er = (r - ((chosen >> 16) & 0xFF)) * DITHER_STRENGTH;
-                float eg = (g - ((chosen >> 8) & 0xFF)) * DITHER_STRENGTH;
-                float eb = (b - (chosen & 0xFF)) * DITHER_STRENGTH;
+                int chosen = 0;
+                double chosenDist = Double.MAX_VALUE;
+                for (int p = 0; p < paletteSize; p++) {
+                    double[] lab = MapColorPalette.lab(p);
+                    if (dist2(lab, orig) > limit) continue;
+                    double d = dist2(lab, want);
+                    if (d < chosenDist) {
+                        chosenDist = d;
+                        chosen = p;
+                    }
+                }
+                out[y * SIZE + x] = MapColorPalette.mapIndex(chosen);
 
-                spread(buf, y, x + 1, w, h, er, eg, eb, 7f / 16f);
-                spread(buf, y + 1, x - 1, w, h, er, eg, eb, 3f / 16f);
-                spread(buf, y + 1, x, w, h, er, eg, eb, 5f / 16f);
-                spread(buf, y + 1, x + 1, w, h, er, eg, eb, 1f / 16f);
+                double[] c = MapColorPalette.lab(chosen);
+                double el = (want[0] - c[0]) * DITHER_STRENGTH;
+                double ea = (want[1] - c[1]) * DITHER_STRENGTH;
+                double eb = (want[2] - c[2]) * DITHER_STRENGTH;
+                spread(buf, w, h, x + 1, y, el, ea, eb, 7.0 / 16.0);
+                spread(buf, w, h, x - 1, y + 1, el, ea, eb, 3.0 / 16.0);
+                spread(buf, w, h, x, y + 1, el, ea, eb, 5.0 / 16.0);
+                spread(buf, w, h, x + 1, y + 1, el, ea, eb, 1.0 / 16.0);
             }
         }
         return out;
     }
 
-    private static void spread(float[][] buf, int y, int x, int w, int h, float er, float eg, float eb, float f) {
-        if (x < 0 || x >= w || y < 0 || y >= h) return;
-        int i = x * 3;
-        buf[y][i] += er * f;
-        buf[y][i + 1] += eg * f;
-        buf[y][i + 2] += eb * f;
+    private static double dist2(double[] a, double[] b) {
+        double dl = a[0] - b[0], da = a[1] - b[1], db = a[2] - b[2];
+        return dl * dl + da * da + db * db;
     }
 
-    private static int clamp(float v) {
-        return v < 0 ? 0 : (v > 255 ? 255 : Math.round(v));
+    private static void spread(double[][] buf, int w, int h, int x, int y, double el, double ea, double eb, double f) {
+        if (x < 0 || x >= w || y >= h) return;
+        double[] v = buf[y * w + x];
+        v[0] += el * f;
+        v[1] += ea * f;
+        v[2] += eb * f;
     }
 }