postprocess: handle partial intialization failures.

This patch fixes segfaults observed when enabling the post processing
features. When the format is not supported, or a texture cannot be
created, the code must gracefully handle failure and report the error to
the calling code for proper failure handling.

To accomplish this the following changes were made to the filters.h
prototypes:

- bool return for pp_init_func
- Added pp_free_func for filter specific resource destruction

Fixes segfaults from backtraces:

* util_destroy_blit
  pp_free

* u_transfer_inline_write_vtbl
  pp_jimenezmlaa_init_run
  pp_init

This patch also uses tgsi_alloc_tokens to allocate temporary tokens in
pp_tgsi_to_state, instead of allocating the array on the stack. This
fixes the following stack corruption segfault in pp_run.c:

* _int_free
  aaline_delete_fs_state
  pp_free

Bug Number: 1021843
Reviewed-by: Brian Paul <brianp@vmware.com>

1 files changed, 109 insertions, 36 deletions

diff --git a/src/gallium/auxiliary/postprocess/pp_init.c b/src/gallium/auxiliary/postprocess/pp_init.c
index 137c82b..1130248 100644
--- a/src/gallium/auxiliary/postprocess/pp_init.c
+++ b/src/gallium/auxiliary/postprocess/pp_init.c
@@ -42,57 +42,81 @@ struct pp_queue_t *
 pp_init(struct pipe_context *pipe, const unsigned int *enabled,
         struct cso_context *cso)
 {
-
+   unsigned int num_filters = 0;
    unsigned int curpos = 0, i, tmp_req = 0;
    struct pp_queue_t *ppq;
-   pp_func *tmp_q;
 
    pp_debug("Initializing the post-processing queue.\n");
 
    /* How many filters were requested? */
    for (i = 0; i < PP_FILTERS; i++) {
       if (enabled[i])
-         curpos++;
+         num_filters++;
    }
-   if (!curpos)
+   if (num_filters == 0)
       return NULL;
 
    ppq = CALLOC(1, sizeof(struct pp_queue_t));
-   tmp_q = CALLOC(curpos, sizeof(pp_func));
-   ppq->shaders = CALLOC(curpos, sizeof(void *));
-   ppq->verts = CALLOC(curpos, sizeof(unsigned int));
 
-   if (!tmp_q || !ppq || !ppq->shaders || !ppq->verts)
+   if (ppq == NULL) {
+      pp_debug("Unable to allocate memory for ppq.\n");
       goto error;
+   }
+
+   ppq->pp_queue = CALLOC(num_filters, sizeof(pp_func));
+   if (ppq->pp_queue == NULL) {
+      pp_debug("Unable to allocate memory for pp_queue.\n");
+      goto error;
+   }
+
+   ppq->shaders = CALLOC(num_filters, sizeof(void *));
+   ppq->filters = CALLOC(num_filters, sizeof(unsigned int));
+
+   if ((ppq->shaders == NULL) ||
+       (ppq->filters == NULL)) {
+      pp_debug("Unable to allocate memory for shaders and filter arrays.\n");
+      goto error;
+   }
 
    ppq->p = pp_init_prog(ppq, pipe, cso);
-   if (!ppq->p)
+   if (ppq->p == NULL) {
+      pp_debug("pp_init_prog returned NULL.\n");
       goto error;
+   }
 
    /* Add the enabled filters to the queue, in order */
    curpos = 0;
-   ppq->pp_queue = tmp_q;
    for (i = 0; i < PP_FILTERS; i++) {
       if (enabled[i]) {
          ppq->pp_queue[curpos] = pp_filters[i].main;
          tmp_req = MAX2(tmp_req, pp_filters[i].inner_tmps);
+         ppq->filters[curpos] = i;
 
          if (pp_filters[i].shaders) {
             ppq->shaders[curpos] =
                CALLOC(pp_filters[i].shaders + 1, sizeof(void *));
-            ppq->verts[curpos] = pp_filters[i].verts;
-            if (!ppq->shaders[curpos])
+            if (!ppq->shaders[curpos]) {
+               pp_debug("Unable to allocate memory for shader list.\n");
                goto error;
+            }
          }
-         pp_filters[i].init(ppq, curpos, enabled[i]);
+
+         /* Call the initialization function for the filter. */
+         if (!pp_filters[i].init(ppq, curpos, enabled[i])) {
+            pp_debug("Initialization for filter %u failed.\n", i);
+            goto error;
+         }           
 
          curpos++;
       }
    }
 
    ppq->p->blitctx = util_create_blit(ppq->p->pipe, cso);
-   if (!ppq->p->blitctx)
+
+   if (ppq->p->blitctx == NULL) {
+      pp_debug("Unable to create a blit context.\n");
       goto error;
+   }
 
    ppq->n_filters = curpos;
    ppq->n_tmp = (curpos > 2 ? 2 : 1);
@@ -104,16 +128,18 @@ pp_init(struct pipe_context *pipe, const unsigned int *enabled,
       ppq->shaders[i][0] = ppq->p->passvs;
 
    pp_debug("Queue successfully allocated. %u filter(s).\n", curpos);
-
+   
    return ppq;
 
  error:
-   pp_debug("Error setting up pp\n");
 
-   if (ppq)
-      FREE(ppq->p);
-   FREE(ppq);
-   FREE(tmp_q);
+   if (ppq) {
+      /* Assign curpos, since we only need to destroy initialized filters. */
+      ppq->n_filters = curpos;
+
+      /* Call the common free function which must handle partial initialization. */
+      pp_free(ppq);
+   }
 
    return NULL;
 }
@@ -142,33 +168,81 @@ pp_free_fbos(struct pp_queue_t *ppq)
    ppq->fbos_init = false;
 }
 
-/** Free the pp queue. Called on context termination. */
+/** 
+ * Free the pp queue. Called on context termination and failure in
+ * pp_init.
+ */
 void
 pp_free(struct pp_queue_t *ppq)
 {
-
    unsigned int i, j;
 
    pp_free_fbos(ppq);
 
-   util_destroy_blit(ppq->p->blitctx);
+   if (ppq && ppq->p) {
+      /* Only destroy created contexts. */
+      if (ppq->p->blitctx) {
+         util_destroy_blit(ppq->p->blitctx);
+      }
 
-   for (i = 0; i < ppq->n_filters; i++) {
-      for (j = 0; j < PP_MAX_PASSES && ppq->shaders[i][j]; j++) {
-         if (j >= ppq->verts[i]) {
-            ppq->p->pipe->delete_fs_state(ppq->p->pipe, ppq->shaders[i][j]);
-            ppq->shaders[i][j] = NULL;
-         }
-         else if (ppq->shaders[i][j] != ppq->p->passvs) {
-            ppq->p->pipe->delete_vs_state(ppq->p->pipe, ppq->shaders[i][j]);
-            ppq->shaders[i][j] = NULL;
+      if (ppq->p->pipe && ppq->filters && ppq->shaders) {
+         for (i = 0; i < ppq->n_filters; i++) {
+            unsigned int filter = ppq->filters[i];
+
+            if (ppq->shaders[i] == NULL) {
+               continue;
+            }
+
+            /*
+             * Common shader destruction code for all postprocessing
+             * filters.
+             */
+            for (j = 0; j < pp_filters[filter].shaders; j++) {
+               if (ppq->shaders[i][j] == NULL) {
+                  /* We reached the end of initialized shaders. */
+                  break;
+               }
+
+               if (ppq->shaders[i][j] == ppq->p->passvs) {
+                  continue;
+               }
+
+               assert(ppq);
+               assert(ppq->p);
+               assert(ppq->p->pipe);
+ 
+               if (j >= pp_filters[filter].verts) {
+                  assert(ppq->p->pipe->delete_fs_state);
+                  ppq->p->pipe->delete_fs_state(ppq->p->pipe,
+                                                ppq->shaders[i][j]);
+                  ppq->shaders[i][j] = NULL;
+               } else {
+                  assert(ppq->p->pipe->delete_vs_state);
+                  ppq->p->pipe->delete_vs_state(ppq->p->pipe,
+                                                ppq->shaders[i][j]);
+                  ppq->shaders[i][j] = NULL;
+               }
+            }
+
+            /* Finally call each filter type's free functionality. */
+            pp_filters[filter].free(ppq, i);
          }
       }
+
+      FREE(ppq->p);
    }
 
-   FREE(ppq->p);
-   FREE(ppq->pp_queue);
-   FREE(ppq);
+   if (ppq) {
+      /*
+       * Handle partial initialization for common resource destruction
+       * in the create path.
+       */
+      FREE(ppq->filters);
+      FREE(ppq->shaders);
+      FREE(ppq->pp_queue);
+  
+      FREE(ppq);
+   }
 
    pp_debug("Queue taken down.\n");
 }
@@ -256,7 +330,6 @@ pp_init_fbos(struct pp_queue_t *ppq, unsigned int w,
    if (!ppq->stencil || !ppq->stencils)
       goto error;
 
-
    p->framebuffer.width = w;
    p->framebuffer.height = h;