glsl/linker: check against varying limit after unused varyings are eliminated

We counted even the varyings which were later eliminated, which was
suboptimal.

Reviewed-by: Ian Romanick <ian.d.romanick@intel.com>

3 files changed, 33 insertions, 11 deletions

diff --git a/src/glsl/link_varyings.cpp b/src/glsl/link_varyings.cpp
index 4fdbdc1..5abc612 100644
--- a/src/glsl/link_varyings.cpp
+++ b/src/glsl/link_varyings.cpp
@@ -1113,16 +1113,12 @@ assign_varying_locations(struct gl_context *ctx,
       }
    }
 
-   unsigned varying_vectors = 0;
-
    if (consumer) {
       foreach_list(node, consumer->ir) {
          ir_variable *const var = ((ir_instruction *) node)->as_variable();
 
-         if ((var == NULL) || (var->mode != ir_var_shader_in))
-            continue;
-
-         if (var->is_unmatched_generic_inout) {
+         if (var && var->mode == ir_var_shader_in &&
+             var->is_unmatched_generic_inout) {
             if (prog->Version <= 120) {
                /* On page 25 (page 31 of the PDF) of the GLSL 1.20 spec:
                 *
@@ -1148,15 +1144,32 @@ assign_varying_locations(struct gl_context *ctx,
              * value is written by the previous stage.
              */
             var->mode = ir_var_auto;
-         } else if (is_varying_var(consumer->Type, var)) {
-            /* The packing rules are used for vertex shader inputs are also
-             * used for fragment shader inputs.
-             */
-            varying_vectors += count_attribute_slots(var->type);
          }
       }
    }
 
+   return true;
+}
+
+bool
+check_against_varying_limit(struct gl_context *ctx,
+                            struct gl_shader_program *prog,
+                            gl_shader *consumer)
+{
+   unsigned varying_vectors = 0;
+
+   foreach_list(node, consumer->ir) {
+      ir_variable *const var = ((ir_instruction *) node)->as_variable();
+
+      if (var && var->mode == ir_var_shader_in &&
+          is_varying_var(consumer->Type, var)) {
+         /* The packing rules used for vertex shader inputs are also
+          * used for fragment shader inputs.
+          */
+         varying_vectors += count_attribute_slots(var->type);
+      }
+   }
+
    if (ctx->API == API_OPENGLES2 || prog->IsES) {
       if (varying_vectors > ctx->Const.MaxVarying) {
          if (ctx->Const.GLSLSkipStrictMaxVaryingLimitCheck) {
diff --git a/src/glsl/link_varyings.h b/src/glsl/link_varyings.h
index ee1010a..daa9d79 100644
--- a/src/glsl/link_varyings.h
+++ b/src/glsl/link_varyings.h
@@ -232,4 +232,9 @@ assign_varying_locations(struct gl_context *ctx,
                          unsigned num_tfeedback_decls,
                          tfeedback_decl *tfeedback_decls);
 
+bool
+check_against_varying_limit(struct gl_context *ctx,
+                            struct gl_shader_program *prog,
+                            gl_shader *consumer);
+
 #endif /* GLSL_LINK_VARYINGS_H */
diff --git a/src/glsl/linker.cpp b/src/glsl/linker.cpp
index 3adcddd..6d73578 100644
--- a/src/glsl/linker.cpp
+++ b/src/glsl/linker.cpp
@@ -1929,6 +1929,10 @@ link_shaders(struct gl_context *ctx, struct gl_shader_program *prog)
       while (do_dead_code(sh_next->ir, false))
          ;
 
+      /* This must be done after all dead varyings are eliminated. */
+      if (!check_against_varying_limit(ctx, prog, sh_next))
+         goto done;
+
       next = i;
    }