From 0d05422c2001b5d0d50ea4f0d99a5cc1c301fccb Mon Sep 17 00:00:00 2001 From: Brian Paul Date: Mon, 7 Nov 2011 12:28:26 -0700 Subject: [PATCH] st/mesa: add some null pointer checking to better handle out of memory MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Reviewed-by: José Fonseca --- src/mesa/state_tracker/st_cb_bufferobjects.c | 12 +++++ src/mesa/state_tracker/st_cb_clear.c | 5 ++ src/mesa/state_tracker/st_draw.c | 54 ++++++++++++++++---- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/src/mesa/state_tracker/st_cb_bufferobjects.c b/src/mesa/state_tracker/st_cb_bufferobjects.c index aab7444e201..adac92f2c34 100644 --- a/src/mesa/state_tracker/st_cb_bufferobjects.c +++ b/src/mesa/state_tracker/st_cb_bufferobjects.c @@ -115,6 +115,11 @@ st_bufferobj_subdata(struct gl_context *ctx, if (!data) return; + if (!st_obj->buffer) { + /* we probably ran out of memory during buffer allocation */ + return; + } + /* Now that transfers are per-context, we don't have to figure out * flushing here. Usually drivers won't need to flush in this case * even if the buffer is currently referenced by hardware - they @@ -146,6 +151,11 @@ st_bufferobj_get_subdata(struct gl_context *ctx, if (!size) return; + if (!st_obj->buffer) { + /* we probably ran out of memory during buffer allocation */ + return; + } + pipe_buffer_read(st_context(ctx)->pipe, st_obj->buffer, offset, size, data); } @@ -216,6 +226,8 @@ st_bufferobj_data(struct gl_context *ctx, pipe_usage, size); if (!st_obj->buffer) { + /* out of memory */ + st_obj->Base.Size = 0; return GL_FALSE; } diff --git a/src/mesa/state_tracker/st_cb_clear.c b/src/mesa/state_tracker/st_cb_clear.c index 19a87aae29c..89273e28e89 100644 --- a/src/mesa/state_tracker/st_cb_clear.c +++ b/src/mesa/state_tracker/st_cb_clear.c @@ -165,6 +165,11 @@ draw_quad(struct st_context *st, max_slots * sizeof(st->clear.vertices)); } + if (!st->clear.vbuf) { + /* ran out of memory */ + return; + } + /* positions */ st->clear.vertices[0][0][0] = x0; st->clear.vertices[0][0][1] = y0; diff --git a/src/mesa/state_tracker/st_draw.c b/src/mesa/state_tracker/st_draw.c index ff3008a5fc0..6c62c5ec70e 100644 --- a/src/mesa/state_tracker/st_draw.c +++ b/src/mesa/state_tracker/st_draw.c @@ -338,8 +338,9 @@ is_interleaved_arrays(const struct st_vertex_program *vp, * or all live in user space. * \param vbuffer returns vertex buffer info * \param velements returns vertex element info + * \return GL_TRUE for success, GL_FALSE otherwise (probably out of memory) */ -static void +static GLboolean setup_interleaved_attribs(struct gl_context *ctx, const struct st_vertex_program *vp, const struct st_vp_variant *vpv, @@ -434,6 +435,11 @@ setup_interleaved_attribs(struct gl_context *ctx, /* all interleaved arrays in a VBO */ struct st_buffer_object *stobj = st_buffer_object(bufobj); + if (!stobj) { + /* probably out of memory */ + return GL_FALSE; + } + vbuffer->buffer = NULL; pipe_resource_reference(&vbuffer->buffer, stobj->buffer); vbuffer->buffer_offset = pointer_to_offset(low_addr); @@ -455,6 +461,8 @@ setup_interleaved_attribs(struct gl_context *ctx, st->user_attrib[0].stride = stride; st->num_user_attribs = 1; } + + return GL_TRUE; } @@ -463,8 +471,9 @@ setup_interleaved_attribs(struct gl_context *ctx, * vertex attribute. * \param vbuffer returns vertex buffer info * \param velements returns vertex element info + * \return GL_TRUE for success, GL_FALSE otherwise (probably out of memory) */ -static void +static GLboolean setup_non_interleaved_attribs(struct gl_context *ctx, const struct st_vertex_program *vp, const struct st_vp_variant *vpv, @@ -493,7 +502,11 @@ setup_non_interleaved_attribs(struct gl_context *ctx, * really an offset from the start of the VBO, not a pointer. */ struct st_buffer_object *stobj = st_buffer_object(bufobj); - assert(stobj->buffer); + + if (!stobj || !stobj->buffer) { + /* probably ran out of memory */ + return GL_FALSE; + } vbuffer[attr].buffer = NULL; pipe_resource_reference(&vbuffer[attr].buffer, stobj->buffer); @@ -533,6 +546,11 @@ setup_non_interleaved_attribs(struct gl_context *ctx, st->user_attrib[attr].element_size = element_size; st->user_attrib[attr].stride = stride; st->num_user_attribs = MAX2(st->num_user_attribs, attr + 1); + + if (!vbuffer[attr].buffer) { + /* probably ran out of memory */ + return GL_FALSE; + } } /* common-case setup */ @@ -547,6 +565,8 @@ setup_non_interleaved_attribs(struct gl_context *ctx, array->Normalized); assert(velements[attr].src_format); } + + return GL_TRUE; } @@ -775,7 +795,11 @@ translate_prim(const struct gl_context *ctx, unsigned prim) } -static void +/** + * Setup vertex arrays and buffers prior to drawing. + * \return GL_TRUE for success, GL_FALSE otherwise (probably out of memory) + */ +static GLboolean st_validate_varrays(struct gl_context *ctx, const struct gl_client_array **arrays, unsigned max_index, @@ -806,8 +830,10 @@ st_validate_varrays(struct gl_context *ctx, * Setup the vbuffer[] and velements[] arrays. */ if (is_interleaved_arrays(vp, vpv, arrays)) { - setup_interleaved_attribs(ctx, vp, vpv, arrays, vbuffer, velements, - max_index, num_instances); + if (!setup_interleaved_attribs(ctx, vp, vpv, arrays, vbuffer, velements, + max_index, num_instances)) { + return GL_FALSE; + } num_vbuffers = 1; num_velements = vpv->num_inputs; @@ -815,9 +841,12 @@ st_validate_varrays(struct gl_context *ctx, num_vbuffers = 0; } else { - setup_non_interleaved_attribs(ctx, vp, vpv, arrays, - vbuffer, velements, max_index, - num_instances); + if (!setup_non_interleaved_attribs(ctx, vp, vpv, arrays, + vbuffer, velements, max_index, + num_instances)) { + return GL_FALSE; + } + num_vbuffers = vpv->num_inputs; num_velements = vpv->num_inputs; } @@ -832,6 +861,8 @@ st_validate_varrays(struct gl_context *ctx, pipe_resource_reference(&vbuffer[attr].buffer, NULL); assert(!vbuffer[attr].buffer); } + + return GL_TRUE; } @@ -901,7 +932,10 @@ st_draw_vbo(struct gl_context *ctx, st_validate_state(st); if (new_array) { - st_validate_varrays(ctx, arrays, max_index, num_instances); + if (!st_validate_varrays(ctx, arrays, max_index, num_instances)) { + /* probably out of memory, no-op the draw call */ + return; + } } #if 0 -- 2.30.2