fix [1610840] Positional light doesn't work correctly on emulator

This bug was introduced when lighting computations was changed from eye-space to object-space.
The light position need to be transformed back to object-space each time the modelview matrix changes which requires us to compute the inverse of the modelview matrix. This computation was done with the assumption that normals where transformed (which was the case when the computation was made in eye-space), however, normals only require the inverse of the upper 3x3 matrix while transforming positions requires the inverse of the whole matrix.
This caused the interesting behavior that lights were more-or-less transformed properly, but not translated at all, which caused improper lighting with directional lights in particular.

There was also another smaller bug affecting directional lights: when vertices are read, only the active component are read, the other ones are ignored, later, the transformation operations are set up to ignore the unset values, howver, in the case of lighting, we use the vertex in object space (that is, before it is transformed), and therefore were using uninitalized values; in particular w.

1 files changed, 14 insertions, 16 deletions

diff --git a/opengl/libagl/light.cpp b/opengl/libagl/light.cpp
index bc9449c..8ae32cc 100644
--- a/opengl/libagl/light.cpp
+++ b/opengl/libagl/light.cpp
@@ -38,13 +38,14 @@ static void lightVertex(ogles_context_t* c, vertex_t* v);
 static void lightVertexMaterial(ogles_context_t* c, vertex_t* v);
 
 static inline void vscale3(GLfixed* d, const GLfixed* m, GLfixed s);
-static inline void vsub3w(GLfixed* d, const GLfixed* a, const GLfixed* b);
 
 static __attribute__((noinline))
 void vnorm3(GLfixed* d, const GLfixed* a);
 
 static inline void vsa3(GLfixed* d,
     const GLfixed* m, GLfixed s, const GLfixed* a);
+static inline void vss3(GLfixed* d,
+    const GLfixed* m, GLfixed s, const GLfixed* a);
 static inline void vmla3(GLfixed* d,
     const GLfixed* m0, const GLfixed* m1, const GLfixed* a);
 static inline void vmul3(GLfixed* d,
@@ -151,18 +152,10 @@ void vsa3(GLfixed* d, const GLfixed* m, GLfixed s, const GLfixed* a) {
 }
 
 static inline
-void vsub3w(GLfixed* d, const GLfixed* a, const GLfixed* b) {
-    const GLfixed wa = a[3];
-    const GLfixed wb = b[3];
-    if (ggl_likely(wa == wb)) {
-        d[0] = a[0] - b[0];
-        d[1] = a[1] - b[1];
-        d[2] = a[2] - b[2];
-    } else {
-        d[0] = gglMulSubx(a[0], wb, gglMulx(b[0], wa));
-        d[1] = gglMulSubx(a[1], wb, gglMulx(b[1], wa));
-        d[2] = gglMulSubx(a[2], wb, gglMulx(b[2], wa));
-    }
+void vss3(GLfixed* d, const GLfixed* m, GLfixed s, const GLfixed* a) {
+    d[0] = gglMulSubx(m[0], s, a[0]);
+    d[1] = gglMulSubx(m[1], s, a[1]);
+    d[2] = gglMulSubx(m[2], s, a[2]);
 }
 
 static inline
@@ -227,7 +220,7 @@ static inline void validate_light_mvi(ogles_context_t* c)
         const int i = 31 - gglClz(en);
         en &= ~(1<<i);
         light_t& l = c->lighting.lights[i];
-        c->transforms.mvui.point3(&c->transforms.mvui,
+        c->transforms.mvui.point4(&c->transforms.mvui,
                 &l.objPosition, &l.position);
         vnorm3(l.normalizedObjPosition.v, l.objPosition.v);
     }
@@ -348,7 +341,11 @@ void lightVertex(ogles_context_t* c, vertex_t* v)
         vec4_t n;
         c->arrays.normal.fetch(c, n.v,
             c->arrays.normal.element(v->index & vertex_cache_t::INDEX_MASK));
-        if (c->transforms.rescaleNormals == GL_NORMALIZE)
+
+        // TODO: right now we handle GL_RESCALE_NORMALS as if ti were
+        // GL_NORMALIZE. We could optimize this by  scaling mvui 
+        // appropriately instead.
+        if (c->transforms.rescaleNormals)
             vnorm3(n.v, n.v);
 
         const material_t& material = c->lighting.front;
@@ -365,7 +362,8 @@ void lightVertex(ogles_context_t* c, vertex_t* v)
 
             // compute vertex-to-light vector
             if (ggl_unlikely(l.position.w)) {
-                vsub3w(d.v, l.objPosition.v, v->obj.v);
+                // lightPos/1.0 - vertex/vertex.w == lightPos*vertex.w - vertex
+                vss3(d.v, l.objPosition.v, v->obj.w, v->obj.v);
                 sqDist = dot3(d.v, d.v);
                 vscale3(d.v, d.v, gglSqrtRecipx(sqDist));
             } else {