+ return false;
+ memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
+
+ return true;
+}
+
+/* Invalidate underlying buffer storage, reset fences, reallocate to non-busy
+ * buffer.
+ */
+void
+nouveau_buffer_invalidate(struct pipe_context *pipe,
+ struct pipe_resource *resource)
+{
+ struct nouveau_context *nv = nouveau_context(pipe);
+ struct nv04_resource *buf = nv04_resource(resource);
+ int ref = buf->base.reference.count - 1;
+
+ /* Shared buffers shouldn't get reallocated */
+ if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
+ return;
+
+ /* We can't touch persistent/coherent buffers */
+ if (buf->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
+ PIPE_RESOURCE_FLAG_MAP_COHERENT))
+ return;
+
+ /* If the buffer is sub-allocated and not currently being written, just
+ * wipe the valid buffer range. Otherwise we have to create fresh
+ * storage. (We don't keep track of fences for non-sub-allocated BO's.)
+ */
+ if (buf->mm && !nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE)) {
+ util_range_set_empty(&buf->valid_buffer_range);
+ } else {
+ nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
+ if (ref > 0) /* any references inside context possible ? */
+ nv->invalidate_resource_storage(nv, &buf->base, ref);
+ }
+}
+
+
+/* Scratch data allocation. */
+
+static inline int
+nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
+ unsigned size)
+{
+ return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
+ 4096, size, NULL, pbo);
+}
+
+static void
+nouveau_scratch_unref_bos(void *d)
+{
+ struct runout *b = d;
+ int i;
+
+ for (i = 0; i < b->nr; ++i)
+ nouveau_bo_ref(NULL, &b->bo[i]);
+
+ FREE(b);
+}
+
+void
+nouveau_scratch_runout_release(struct nouveau_context *nv)
+{
+ if (!nv->scratch.runout)
+ return;
+
+ if (!nouveau_fence_work(nv->screen->fence.current, nouveau_scratch_unref_bos,
+ nv->scratch.runout))
+ return;
+
+ nv->scratch.end = 0;
+ nv->scratch.runout = NULL;
+}
+
+/* Allocate an extra bo if we can't fit everything we need simultaneously.
+ * (Could happen for very large user arrays.)
+ */
+static inline bool
+nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
+{
+ int ret;
+ unsigned n;
+
+ if (nv->scratch.runout)
+ n = nv->scratch.runout->nr;
+ else
+ n = 0;
+ nv->scratch.runout = REALLOC(nv->scratch.runout, n == 0 ? 0 :
+ (sizeof(*nv->scratch.runout) + (n + 0) * sizeof(void *)),
+ sizeof(*nv->scratch.runout) + (n + 1) * sizeof(void *));
+ nv->scratch.runout->nr = n + 1;
+ nv->scratch.runout->bo[n] = NULL;
+
+ ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout->bo[n], size);
+ if (!ret) {
+ ret = nouveau_bo_map(nv->scratch.runout->bo[n], 0, NULL);
+ if (ret)
+ nouveau_bo_ref(NULL, &nv->scratch.runout->bo[--nv->scratch.runout->nr]);
+ }
+ if (!ret) {
+ nv->scratch.current = nv->scratch.runout->bo[n];
+ nv->scratch.offset = 0;
+ nv->scratch.end = size;
+ nv->scratch.map = nv->scratch.current->map;
+ }
+ return !ret;
+}
+
+/* Continue to next scratch buffer, if available (no wrapping, large enough).
+ * Allocate it if it has not yet been created.
+ */
+static inline bool
+nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
+{
+ struct nouveau_bo *bo;
+ int ret;
+ const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
+
+ if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
+ return false;
+ nv->scratch.id = i;
+
+ bo = nv->scratch.bo[i];
+ if (!bo) {
+ ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
+ if (ret)
+ return false;
+ nv->scratch.bo[i] = bo;
+ }
+ nv->scratch.current = bo;
+ nv->scratch.offset = 0;
+ nv->scratch.end = nv->scratch.bo_size;
+
+ ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
+ if (!ret)
+ nv->scratch.map = bo->map;
+ return !ret;
+}
+
+static bool
+nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
+{
+ bool ret;
+
+ ret = nouveau_scratch_next(nv, min_size);
+ if (!ret)
+ ret = nouveau_scratch_runout(nv, min_size);
+ return ret;
+}
+
+
+/* Copy data to a scratch buffer and return address & bo the data resides in. */
+uint64_t
+nouveau_scratch_data(struct nouveau_context *nv,
+ const void *data, unsigned base, unsigned size,
+ struct nouveau_bo **bo)
+{
+ unsigned bgn = MAX2(base, nv->scratch.offset);
+ unsigned end = bgn + size;
+
+ if (end >= nv->scratch.end) {
+ end = base + size;
+ if (!nouveau_scratch_more(nv, end))
+ return 0;
+ bgn = base;
+ }
+ nv->scratch.offset = align(end, 4);
+
+ memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
+
+ *bo = nv->scratch.current;
+ return (*bo)->offset + (bgn - base);
+}
+
+void *
+nouveau_scratch_get(struct nouveau_context *nv,
+ unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
+{
+ unsigned bgn = nv->scratch.offset;
+ unsigned end = nv->scratch.offset + size;
+
+ if (end >= nv->scratch.end) {
+ end = size;
+ if (!nouveau_scratch_more(nv, end))
+ return NULL;
+ bgn = 0;
+ }
+ nv->scratch.offset = align(end, 4);