reactos/dll/opengl/mesa/src/gallium/auxiliary/pipebuffer/pb_buffer_fenced.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2007-2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * \file
  30  * Implementation of fenced buffers.
  31  *
  32  * \author Jose Fonseca <jfonseca-at-vmware-dot-com>
  33  * \author Thomas Hellström <thellstrom-at-vmware-dot-com>
  34  */
  35
  36
  37 #include "pipe/p_config.h"
  38
  39 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
  40 #include <unistd.h>
  41 #include <sched.h>
  42 #endif
  43
  44 #include "pipe/p_compiler.h"
  45 #include "pipe/p_defines.h"
  46 #include "util/u_debug.h"
  47 #include "os/os_thread.h"
  48 #include "util/u_memory.h"
  49 #include "util/u_double_list.h"
  50
  51 #include "pb_buffer.h"
  52 #include "pb_buffer_fenced.h"
  53 #include "pb_bufmgr.h"
  54
  55
  56
  57 /**
  58  * Convenience macro (type safe).
  59  */
  60 #define SUPER(__derived) (&(__derived)->base)
  61
  62
  63 struct fenced_manager
  64 {
  65    struct pb_manager base;
  66    struct pb_manager *provider;
  67    struct pb_fence_ops *ops;
  68
  69    /**
  70     * Maximum buffer size that can be safely allocated.
  71     */
  72    pb_size max_buffer_size;
  73
  74    /**
  75     * Maximum cpu memory we can allocate before we start waiting for the
  76     * GPU to idle.
  77     */
  78    pb_size max_cpu_total_size;
  79
  80    /**
  81     * Following members are mutable and protected by this mutex.
  82     */
  83    pipe_mutex mutex;
  84
  85    /**
  86     * Fenced buffer list.
  87     *
  88     * All fenced buffers are placed in this listed, ordered from the oldest
  89     * fence to the newest fence.
  90     */
  91    struct list_head fenced;
  92    pb_size num_fenced;
  93
  94    struct list_head unfenced;
  95    pb_size num_unfenced;
  96
  97    /**
  98     * How much temporary CPU memory is being used to hold unvalidated buffers.
  99     */
 100    pb_size cpu_total_size;
 101 };
 102
 103
 104 /**
 105  * Fenced buffer.
 106  *
 107  * Wrapper around a pipe buffer which adds fencing and reference counting.
 108  */
 109 struct fenced_buffer
 110 {
 111    /*
 112     * Immutable members.
 113     */
 114
 115    struct pb_buffer base;
 116    struct fenced_manager *mgr;
 117
 118    /*
 119     * Following members are mutable and protected by fenced_manager::mutex.
 120     */
 121
 122    struct list_head head;
 123
 124    /**
 125     * Buffer with storage.
 126     */
 127    struct pb_buffer *buffer;
 128    pb_size size;
 129    struct pb_desc desc;
 130
 131    /**
 132     * Temporary CPU storage data. Used when there isn't enough GPU memory to
 133     * store the buffer.
 134     */
 135    void *data;
 136
 137    /**
 138     * A bitmask of PB_USAGE_CPU/GPU_READ/WRITE describing the current
 139     * buffer usage.
 140     */
 141    unsigned flags;
 142
 143    unsigned mapcount;
 144
 145    struct pb_validate *vl;
 146    unsigned validation_flags;
 147
 148    struct pipe_fence_handle *fence;
 149 };
 150
 151
 152 static INLINE struct fenced_manager *
 153 fenced_manager(struct pb_manager *mgr)
 154 {
 155    assert(mgr);
 156    return (struct fenced_manager *)mgr;
 157 }
 158
 159
 160 static INLINE struct fenced_buffer *
 161 fenced_buffer(struct pb_buffer *buf)
 162 {
 163    assert(buf);
 164    return (struct fenced_buffer *)buf;
 165 }
 166
 167
 168 static void
 169 fenced_buffer_destroy_cpu_storage_locked(struct fenced_buffer *fenced_buf);
 170
 171 static enum pipe_error
 172 fenced_buffer_create_cpu_storage_locked(struct fenced_manager *fenced_mgr,
 173                                         struct fenced_buffer *fenced_buf);
 174
 175 static void
 176 fenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer *fenced_buf);
 177
 178 static enum pipe_error
 179 fenced_buffer_create_gpu_storage_locked(struct fenced_manager *fenced_mgr,
 180                                         struct fenced_buffer *fenced_buf,
 181                                         boolean wait);
 182
 183 static enum pipe_error
 184 fenced_buffer_copy_storage_to_gpu_locked(struct fenced_buffer *fenced_buf);
 185
 186 static enum pipe_error
 187 fenced_buffer_copy_storage_to_cpu_locked(struct fenced_buffer *fenced_buf);
 188
 189
 190 /**
 191  * Dump the fenced buffer list.
 192  *
 193  * Useful to understand failures to allocate buffers.
 194  */
 195 static void
 196 fenced_manager_dump_locked(struct fenced_manager *fenced_mgr)
 197 {
 198 #ifdef DEBUG
 199    struct pb_fence_ops *ops = fenced_mgr->ops;
 200    struct list_head *curr, *next;
 201    struct fenced_buffer *fenced_buf;
 202
 203    debug_printf("%10s %7s %8s %7s %10s %s\n",
 204                 "buffer", "size", "refcount", "storage", "fence", "signalled");
 205
 206    curr = fenced_mgr->unfenced.next;
 207    next = curr->next;
 208    while(curr != &fenced_mgr->unfenced) {
 209       fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 210       assert(!fenced_buf->fence);
 211       debug_printf("%10p %7u %8u %7s\n",
 212                    (void *) fenced_buf,
 213                    fenced_buf->base.size,
 214                    p_atomic_read(&fenced_buf->base.reference.count),
 215                    fenced_buf->buffer ? "gpu" : (fenced_buf->data ? "cpu" : "none"));
 216       curr = next;
 217       next = curr->next;
 218    }
 219
 220    curr = fenced_mgr->fenced.next;
 221    next = curr->next;
 222    while(curr != &fenced_mgr->fenced) {
 223       int signaled;
 224       fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 225       assert(fenced_buf->buffer);
 226       signaled = ops->fence_signalled(ops, fenced_buf->fence, 0);
 227       debug_printf("%10p %7u %8u %7s %10p %s\n",
 228                    (void *) fenced_buf,
 229                    fenced_buf->base.size,
 230                    p_atomic_read(&fenced_buf->base.reference.count),
 231                    "gpu",
 232                    (void *) fenced_buf->fence,
 233                    signaled == 0 ? "y" : "n");
 234       curr = next;
 235       next = curr->next;
 236    }
 237 #else
 238    (void)fenced_mgr;
 239 #endif
 240 }
 241
 242
 243 static INLINE void
 244 fenced_buffer_destroy_locked(struct fenced_manager *fenced_mgr,
 245                              struct fenced_buffer *fenced_buf)
 246 {
 247    assert(!pipe_is_referenced(&fenced_buf->base.reference));
 248
 249    assert(!fenced_buf->fence);
 250    assert(fenced_buf->head.prev);
 251    assert(fenced_buf->head.next);
 252    LIST_DEL(&fenced_buf->head);
 253    assert(fenced_mgr->num_unfenced);
 254    --fenced_mgr->num_unfenced;
 255
 256    fenced_buffer_destroy_gpu_storage_locked(fenced_buf);
 257    fenced_buffer_destroy_cpu_storage_locked(fenced_buf);
 258
 259    FREE(fenced_buf);
 260 }
 261
 262
 263 /**
 264  * Add the buffer to the fenced list.
 265  *
 266  * Reference count should be incremented before calling this function.
 267  */
 268 static INLINE void
 269 fenced_buffer_add_locked(struct fenced_manager *fenced_mgr,
 270                          struct fenced_buffer *fenced_buf)
 271 {
 272    assert(pipe_is_referenced(&fenced_buf->base.reference));
 273    assert(fenced_buf->flags & PB_USAGE_GPU_READ_WRITE);
 274    assert(fenced_buf->fence);
 275
 276    p_atomic_inc(&fenced_buf->base.reference.count);
 277
 278    LIST_DEL(&fenced_buf->head);
 279    assert(fenced_mgr->num_unfenced);
 280    --fenced_mgr->num_unfenced;
 281    LIST_ADDTAIL(&fenced_buf->head, &fenced_mgr->fenced);
 282    ++fenced_mgr->num_fenced;
 283 }
 284
 285
 286 /**
 287  * Remove the buffer from the fenced list, and potentially destroy the buffer
 288  * if the reference count reaches zero.
 289  *
 290  * Returns TRUE if the buffer was detroyed.
 291  */
 292 static INLINE boolean
 293 fenced_buffer_remove_locked(struct fenced_manager *fenced_mgr,
 294                             struct fenced_buffer *fenced_buf)
 295 {
 296    struct pb_fence_ops *ops = fenced_mgr->ops;
 297
 298    assert(fenced_buf->fence);
 299    assert(fenced_buf->mgr == fenced_mgr);
 300
 301    ops->fence_reference(ops, &fenced_buf->fence, NULL);
 302    fenced_buf->flags &= ~PB_USAGE_GPU_READ_WRITE;
 303
 304    assert(fenced_buf->head.prev);
 305    assert(fenced_buf->head.next);
 306
 307    LIST_DEL(&fenced_buf->head);
 308    assert(fenced_mgr->num_fenced);
 309    --fenced_mgr->num_fenced;
 310
 311    LIST_ADDTAIL(&fenced_buf->head, &fenced_mgr->unfenced);
 312    ++fenced_mgr->num_unfenced;
 313
 314    if (p_atomic_dec_zero(&fenced_buf->base.reference.count)) {
 315       fenced_buffer_destroy_locked(fenced_mgr, fenced_buf);
 316       return TRUE;
 317    }
 318
 319    return FALSE;
 320 }
 321
 322
 323 /**
 324  * Wait for the fence to expire, and remove it from the fenced list.
 325  *
 326  * This function will release and re-aquire the mutex, so any copy of mutable
 327  * state must be discarded after calling it.
 328  */
 329 static INLINE enum pipe_error
 330 fenced_buffer_finish_locked(struct fenced_manager *fenced_mgr,
 331                             struct fenced_buffer *fenced_buf)
 332 {
 333    struct pb_fence_ops *ops = fenced_mgr->ops;
 334    enum pipe_error ret = PIPE_ERROR;
 335
 336 #if 0
 337    debug_warning("waiting for GPU");
 338 #endif
 339
 340    assert(pipe_is_referenced(&fenced_buf->base.reference));
 341    assert(fenced_buf->fence);
 342
 343    if(fenced_buf->fence) {
 344       struct pipe_fence_handle *fence = NULL;
 345       int finished;
 346       boolean proceed;
 347
 348       ops->fence_reference(ops, &fence, fenced_buf->fence);
 349
 350       pipe_mutex_unlock(fenced_mgr->mutex);
 351
 352       finished = ops->fence_finish(ops, fenced_buf->fence, 0);
 353
 354       pipe_mutex_lock(fenced_mgr->mutex);
 355
 356       assert(pipe_is_referenced(&fenced_buf->base.reference));
 357
 358       /*
 359        * Only proceed if the fence object didn't change in the meanwhile.
 360        * Otherwise assume the work has been already carried out by another
 361        * thread that re-aquired the lock before us.
 362        */
 363       proceed = fence == fenced_buf->fence ? TRUE : FALSE;
 364
 365       ops->fence_reference(ops, &fence, NULL);
 366
 367       if(proceed && finished == 0) {
 368          /*
 369           * Remove from the fenced list
 370           */
 371
 372          boolean destroyed;
 373
 374          destroyed = fenced_buffer_remove_locked(fenced_mgr, fenced_buf);
 375
 376          /* TODO: remove consequents buffers with the same fence? */
 377
 378          assert(!destroyed);
 379
 380          fenced_buf->flags &= ~PB_USAGE_GPU_READ_WRITE;
 381
 382          ret = PIPE_OK;
 383       }
 384    }
 385
 386    return ret;
 387 }
 388
 389
 390 /**
 391  * Remove as many fenced buffers from the fenced list as possible.
 392  *
 393  * Returns TRUE if at least one buffer was removed.
 394  */
 395 static boolean
 396 fenced_manager_check_signalled_locked(struct fenced_manager *fenced_mgr,
 397                                       boolean wait)
 398 {
 399    struct pb_fence_ops *ops = fenced_mgr->ops;
 400    struct list_head *curr, *next;
 401    struct fenced_buffer *fenced_buf;
 402    struct pipe_fence_handle *prev_fence = NULL;
 403    boolean ret = FALSE;
 404
 405    curr = fenced_mgr->fenced.next;
 406    next = curr->next;
 407    while(curr != &fenced_mgr->fenced) {
 408       fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 409
 410       if(fenced_buf->fence != prev_fence) {
 411          int signaled;
 412
 413          if (wait) {
 414             signaled = ops->fence_finish(ops, fenced_buf->fence, 0);
 415
 416             /*
 417              * Don't return just now. Instead preemptively check if the
 418              * following buffers' fences already expired, without further waits.
 419              */
 420             wait = FALSE;
 421          }
 422          else {
 423             signaled = ops->fence_signalled(ops, fenced_buf->fence, 0);
 424          }
 425
 426          if (signaled != 0) {
 427             return ret;
 428          }
 429
 430          prev_fence = fenced_buf->fence;
 431       }
 432       else {
 433          /* This buffer's fence object is identical to the previous buffer's
 434           * fence object, so no need to check the fence again.
 435           */
 436          assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0);
 437       }
 438
 439       fenced_buffer_remove_locked(fenced_mgr, fenced_buf);
 440
 441       ret = TRUE;
 442
 443       curr = next;
 444       next = curr->next;
 445    }
 446
 447    return ret;
 448 }
 449
 450
 451 /**
 452  * Try to free some GPU memory by backing it up into CPU memory.
 453  *
 454  * Returns TRUE if at least one buffer was freed.
 455  */
 456 static boolean
 457 fenced_manager_free_gpu_storage_locked(struct fenced_manager *fenced_mgr)
 458 {
 459    struct list_head *curr, *next;
 460    struct fenced_buffer *fenced_buf;
 461
 462    curr = fenced_mgr->unfenced.next;
 463    next = curr->next;
 464    while(curr != &fenced_mgr->unfenced) {
 465       fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 466
 467       /*
 468        * We can only move storage if the buffer is not mapped and not
 469        * validated.
 470        */
 471       if(fenced_buf->buffer &&
 472          !fenced_buf->mapcount &&
 473          !fenced_buf->vl) {
 474          enum pipe_error ret;
 475
 476          ret = fenced_buffer_create_cpu_storage_locked(fenced_mgr, fenced_buf);
 477          if(ret == PIPE_OK) {
 478             ret = fenced_buffer_copy_storage_to_cpu_locked(fenced_buf);
 479             if(ret == PIPE_OK) {
 480                fenced_buffer_destroy_gpu_storage_locked(fenced_buf);
 481                return TRUE;
 482             }
 483             fenced_buffer_destroy_cpu_storage_locked(fenced_buf);
 484          }
 485       }
 486
 487       curr = next;
 488       next = curr->next;
 489    }
 490
 491    return FALSE;
 492 }
 493
 494
 495 /**
 496  * Destroy CPU storage for this buffer.
 497  */
 498 static void
 499 fenced_buffer_destroy_cpu_storage_locked(struct fenced_buffer *fenced_buf)
 500 {
 501    if(fenced_buf->data) {
 502       align_free(fenced_buf->data);
 503       fenced_buf->data = NULL;
 504       assert(fenced_buf->mgr->cpu_total_size >= fenced_buf->size);
 505       fenced_buf->mgr->cpu_total_size -= fenced_buf->size;
 506    }
 507 }
 508
 509
 510 /**
 511  * Create CPU storage for this buffer.
 512  */
 513 static enum pipe_error
 514 fenced_buffer_create_cpu_storage_locked(struct fenced_manager *fenced_mgr,
 515                                         struct fenced_buffer *fenced_buf)
 516 {
 517    assert(!fenced_buf->data);
 518    if(fenced_buf->data)
 519       return PIPE_OK;
 520
 521    if (fenced_mgr->cpu_total_size + fenced_buf->size > fenced_mgr->max_cpu_total_size)
 522       return PIPE_ERROR_OUT_OF_MEMORY;
 523
 524    fenced_buf->data = align_malloc(fenced_buf->size, fenced_buf->desc.alignment);
 525    if(!fenced_buf->data)
 526       return PIPE_ERROR_OUT_OF_MEMORY;
 527
 528    fenced_mgr->cpu_total_size += fenced_buf->size;
 529
 530    return PIPE_OK;
 531 }
 532
 533
 534 /**
 535  * Destroy the GPU storage.
 536  */
 537 static void
 538 fenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer *fenced_buf)
 539 {
 540    if(fenced_buf->buffer) {
 541       pb_reference(&fenced_buf->buffer, NULL);
 542    }
 543 }
 544
 545
 546 /**
 547  * Try to create GPU storage for this buffer.
 548  *
 549  * This function is a shorthand around pb_manager::create_buffer for
 550  * fenced_buffer_create_gpu_storage_locked()'s benefit.
 551  */
 552 static INLINE boolean
 553 fenced_buffer_try_create_gpu_storage_locked(struct fenced_manager *fenced_mgr,
 554                                             struct fenced_buffer *fenced_buf)
 555 {
 556    struct pb_manager *provider = fenced_mgr->provider;
 557
 558    assert(!fenced_buf->buffer);
 559
 560    fenced_buf->buffer = provider->create_buffer(fenced_mgr->provider,
 561                                                 fenced_buf->size,
 562                                                 &fenced_buf->desc);
 563    return fenced_buf->buffer ? TRUE : FALSE;
 564 }
 565
 566
 567 /**
 568  * Create GPU storage for this buffer.
 569  */
 570 static enum pipe_error
 571 fenced_buffer_create_gpu_storage_locked(struct fenced_manager *fenced_mgr,
 572                                         struct fenced_buffer *fenced_buf,
 573                                         boolean wait)
 574 {
 575    assert(!fenced_buf->buffer);
 576
 577    /*
 578     * Check for signaled buffers before trying to allocate.
 579     */
 580    fenced_manager_check_signalled_locked(fenced_mgr, FALSE);
 581
 582    fenced_buffer_try_create_gpu_storage_locked(fenced_mgr, fenced_buf);
 583
 584    /*
 585     * Keep trying while there is some sort of progress:
 586     * - fences are expiring,
 587     * - or buffers are being being swapped out from GPU memory into CPU memory.
 588     */
 589    while(!fenced_buf->buffer &&
 590          (fenced_manager_check_signalled_locked(fenced_mgr, FALSE) ||
 591           fenced_manager_free_gpu_storage_locked(fenced_mgr))) {
 592       fenced_buffer_try_create_gpu_storage_locked(fenced_mgr, fenced_buf);
 593    }
 594
 595    if(!fenced_buf->buffer && wait) {
 596       /*
 597        * Same as before, but this time around, wait to free buffers if
 598        * necessary.
 599        */
 600       while(!fenced_buf->buffer &&
 601             (fenced_manager_check_signalled_locked(fenced_mgr, TRUE) ||
 602              fenced_manager_free_gpu_storage_locked(fenced_mgr))) {
 603          fenced_buffer_try_create_gpu_storage_locked(fenced_mgr, fenced_buf);
 604       }
 605    }
 606
 607    if(!fenced_buf->buffer) {
 608       if(0)
 609          fenced_manager_dump_locked(fenced_mgr);
 610
 611       /* give up */
 612       return PIPE_ERROR_OUT_OF_MEMORY;
 613    }
 614
 615    return PIPE_OK;
 616 }
 617
 618
 619 static enum pipe_error
 620 fenced_buffer_copy_storage_to_gpu_locked(struct fenced_buffer *fenced_buf)
 621 {
 622    uint8_t *map;
 623
 624    assert(fenced_buf->data);
 625    assert(fenced_buf->buffer);
 626
 627    map = pb_map(fenced_buf->buffer, PB_USAGE_CPU_WRITE, NULL);
 628    if(!map)
 629       return PIPE_ERROR;
 630
 631    memcpy(map, fenced_buf->data, fenced_buf->size);
 632
 633    pb_unmap(fenced_buf->buffer);
 634
 635    return PIPE_OK;
 636 }
 637
 638
 639 static enum pipe_error
 640 fenced_buffer_copy_storage_to_cpu_locked(struct fenced_buffer *fenced_buf)
 641 {
 642    const uint8_t *map;
 643
 644    assert(fenced_buf->data);
 645    assert(fenced_buf->buffer);
 646
 647    map = pb_map(fenced_buf->buffer, PB_USAGE_CPU_READ, NULL);
 648    if(!map)
 649       return PIPE_ERROR;
 650
 651    memcpy(fenced_buf->data, map, fenced_buf->size);
 652
 653    pb_unmap(fenced_buf->buffer);
 654
 655    return PIPE_OK;
 656 }
 657
 658
 659 static void
 660 fenced_buffer_destroy(struct pb_buffer *buf)
 661 {
 662    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 663    struct fenced_manager *fenced_mgr = fenced_buf->mgr;
 664
 665    assert(!pipe_is_referenced(&fenced_buf->base.reference));
 666
 667    pipe_mutex_lock(fenced_mgr->mutex);
 668
 669    fenced_buffer_destroy_locked(fenced_mgr, fenced_buf);
 670
 671    pipe_mutex_unlock(fenced_mgr->mutex);
 672 }
 673
 674
 675 static void *
 676 fenced_buffer_map(struct pb_buffer *buf,
 677                   unsigned flags, void *flush_ctx)
 678 {
 679    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 680    struct fenced_manager *fenced_mgr = fenced_buf->mgr;
 681    struct pb_fence_ops *ops = fenced_mgr->ops;
 682    void *map = NULL;
 683
 684    pipe_mutex_lock(fenced_mgr->mutex);
 685
 686    assert(!(flags & PB_USAGE_GPU_READ_WRITE));
 687
 688    /*
 689     * Serialize writes.
 690     */
 691    while((fenced_buf->flags & PB_USAGE_GPU_WRITE) ||
 692          ((fenced_buf->flags & PB_USAGE_GPU_READ) &&
 693           (flags & PB_USAGE_CPU_WRITE))) {
 694
 695       /*
 696        * Don't wait for the GPU to finish accessing it, if blocking is forbidden.
 697        */
 698       if((flags & PB_USAGE_DONTBLOCK) &&
 699           ops->fence_signalled(ops, fenced_buf->fence, 0) != 0) {
 700          goto done;
 701       }
 702
 703       if (flags & PB_USAGE_UNSYNCHRONIZED) {
 704          break;
 705       }
 706
 707       /*
 708        * Wait for the GPU to finish accessing. This will release and re-acquire
 709        * the mutex, so all copies of mutable state must be discarded.
 710        */
 711       fenced_buffer_finish_locked(fenced_mgr, fenced_buf);
 712    }
 713
 714    if(fenced_buf->buffer) {
 715       map = pb_map(fenced_buf->buffer, flags, flush_ctx);
 716    }
 717    else {
 718       assert(fenced_buf->data);
 719       map = fenced_buf->data;
 720    }
 721
 722    if(map) {
 723       ++fenced_buf->mapcount;
 724       fenced_buf->flags |= flags & PB_USAGE_CPU_READ_WRITE;
 725    }
 726
 727 done:
 728    pipe_mutex_unlock(fenced_mgr->mutex);
 729
 730    return map;
 731 }
 732
 733
 734 static void
 735 fenced_buffer_unmap(struct pb_buffer *buf)
 736 {
 737    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 738    struct fenced_manager *fenced_mgr = fenced_buf->mgr;
 739
 740    pipe_mutex_lock(fenced_mgr->mutex);
 741
 742    assert(fenced_buf->mapcount);
 743    if(fenced_buf->mapcount) {
 744       if (fenced_buf->buffer)
 745          pb_unmap(fenced_buf->buffer);
 746       --fenced_buf->mapcount;
 747       if(!fenced_buf->mapcount)
 748          fenced_buf->flags &= ~PB_USAGE_CPU_READ_WRITE;
 749    }
 750
 751    pipe_mutex_unlock(fenced_mgr->mutex);
 752 }
 753
 754
 755 static enum pipe_error
 756 fenced_buffer_validate(struct pb_buffer *buf,
 757                        struct pb_validate *vl,
 758                        unsigned flags)
 759 {
 760    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 761    struct fenced_manager *fenced_mgr = fenced_buf->mgr;
 762    enum pipe_error ret;
 763
 764    pipe_mutex_lock(fenced_mgr->mutex);
 765
 766    if(!vl) {
 767       /* invalidate */
 768       fenced_buf->vl = NULL;
 769       fenced_buf->validation_flags = 0;
 770       ret = PIPE_OK;
 771       goto done;
 772    }
 773
 774    assert(flags & PB_USAGE_GPU_READ_WRITE);
 775    assert(!(flags & ~PB_USAGE_GPU_READ_WRITE));
 776    flags &= PB_USAGE_GPU_READ_WRITE;
 777
 778    /* Buffer cannot be validated in two different lists */
 779    if(fenced_buf->vl && fenced_buf->vl != vl) {
 780       ret = PIPE_ERROR_RETRY;
 781       goto done;
 782    }
 783
 784    if(fenced_buf->vl == vl &&
 785       (fenced_buf->validation_flags & flags) == flags) {
 786       /* Nothing to do -- buffer already validated */
 787       ret = PIPE_OK;
 788       goto done;
 789    }
 790
 791    /*
 792     * Create and update GPU storage.
 793     */
 794    if(!fenced_buf->buffer) {
 795       assert(!fenced_buf->mapcount);
 796
 797       ret = fenced_buffer_create_gpu_storage_locked(fenced_mgr, fenced_buf, TRUE);
 798       if(ret != PIPE_OK) {
 799          goto done;
 800       }
 801
 802       ret = fenced_buffer_copy_storage_to_gpu_locked(fenced_buf);
 803       if(ret != PIPE_OK) {
 804          fenced_buffer_destroy_gpu_storage_locked(fenced_buf);
 805          goto done;
 806       }
 807
 808       if(fenced_buf->mapcount) {
 809          debug_printf("warning: validating a buffer while it is still mapped\n");
 810       }
 811       else {
 812          fenced_buffer_destroy_cpu_storage_locked(fenced_buf);
 813       }
 814    }
 815
 816    ret = pb_validate(fenced_buf->buffer, vl, flags);
 817    if (ret != PIPE_OK)
 818       goto done;
 819
 820    fenced_buf->vl = vl;
 821    fenced_buf->validation_flags |= flags;
 822
 823 done:
 824    pipe_mutex_unlock(fenced_mgr->mutex);
 825
 826    return ret;
 827 }
 828
 829
 830 static void
 831 fenced_buffer_fence(struct pb_buffer *buf,
 832                     struct pipe_fence_handle *fence)
 833 {
 834    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 835    struct fenced_manager *fenced_mgr = fenced_buf->mgr;
 836    struct pb_fence_ops *ops = fenced_mgr->ops;
 837
 838    pipe_mutex_lock(fenced_mgr->mutex);
 839
 840    assert(pipe_is_referenced(&fenced_buf->base.reference));
 841    assert(fenced_buf->buffer);
 842
 843    if(fence != fenced_buf->fence) {
 844       assert(fenced_buf->vl);
 845       assert(fenced_buf->validation_flags);
 846
 847       if (fenced_buf->fence) {
 848          boolean destroyed;
 849          destroyed = fenced_buffer_remove_locked(fenced_mgr, fenced_buf);
 850          assert(!destroyed);
 851       }
 852       if (fence) {
 853          ops->fence_reference(ops, &fenced_buf->fence, fence);
 854          fenced_buf->flags |= fenced_buf->validation_flags;
 855          fenced_buffer_add_locked(fenced_mgr, fenced_buf);
 856       }
 857
 858       pb_fence(fenced_buf->buffer, fence);
 859
 860       fenced_buf->vl = NULL;
 861       fenced_buf->validation_flags = 0;
 862    }
 863
 864    pipe_mutex_unlock(fenced_mgr->mutex);
 865 }
 866
 867
 868 static void
 869 fenced_buffer_get_base_buffer(struct pb_buffer *buf,
 870                               struct pb_buffer **base_buf,
 871                               pb_size *offset)
 872 {
 873    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 874    struct fenced_manager *fenced_mgr = fenced_buf->mgr;
 875
 876    pipe_mutex_lock(fenced_mgr->mutex);
 877
 878    /*
 879     * This should only be called when the buffer is validated. Typically
 880     * when processing relocations.
 881     */
 882    assert(fenced_buf->vl);
 883    assert(fenced_buf->buffer);
 884
 885    if(fenced_buf->buffer)
 886       pb_get_base_buffer(fenced_buf->buffer, base_buf, offset);
 887    else {
 888       *base_buf = buf;
 889       *offset = 0;
 890    }
 891
 892    pipe_mutex_unlock(fenced_mgr->mutex);
 893 }
 894
 895
 896 static const struct pb_vtbl
 897 fenced_buffer_vtbl = {
 898       fenced_buffer_destroy,
 899       fenced_buffer_map,
 900       fenced_buffer_unmap,
 901       fenced_buffer_validate,
 902       fenced_buffer_fence,
 903       fenced_buffer_get_base_buffer
 904 };
 905
 906
 907 /**
 908  * Wrap a buffer in a fenced buffer.
 909  */
 910 static struct pb_buffer *
 911 fenced_bufmgr_create_buffer(struct pb_manager *mgr,
 912                             pb_size size,
 913                             const struct pb_desc *desc)
 914 {
 915    struct fenced_manager *fenced_mgr = fenced_manager(mgr);
 916    struct fenced_buffer *fenced_buf;
 917    enum pipe_error ret;
 918
 919    /*
 920     * Don't stall the GPU, waste time evicting buffers, or waste memory
 921     * trying to create a buffer that will most likely never fit into the
 922     * graphics aperture.
 923     */
 924    if(size > fenced_mgr->max_buffer_size) {
 925       goto no_buffer;
 926    }
 927
 928    fenced_buf = CALLOC_STRUCT(fenced_buffer);
 929    if(!fenced_buf)
 930       goto no_buffer;
 931
 932    pipe_reference_init(&fenced_buf->base.reference, 1);
 933    fenced_buf->base.alignment = desc->alignment;
 934    fenced_buf->base.usage = desc->usage;
 935    fenced_buf->base.size = size;
 936    fenced_buf->size = size;
 937    fenced_buf->desc = *desc;
 938
 939    fenced_buf->base.vtbl = &fenced_buffer_vtbl;
 940    fenced_buf->mgr = fenced_mgr;
 941
 942    pipe_mutex_lock(fenced_mgr->mutex);
 943
 944    /*
 945     * Try to create GPU storage without stalling,
 946     */
 947    ret = fenced_buffer_create_gpu_storage_locked(fenced_mgr, fenced_buf, FALSE);
 948
 949    /*
 950     * Attempt to use CPU memory to avoid stalling the GPU.
 951     */
 952    if(ret != PIPE_OK) {
 953       ret = fenced_buffer_create_cpu_storage_locked(fenced_mgr, fenced_buf);
 954    }
 955
 956    /*
 957     * Create GPU storage, waiting for some to be available.
 958     */
 959    if(ret != PIPE_OK) {
 960       ret = fenced_buffer_create_gpu_storage_locked(fenced_mgr, fenced_buf, TRUE);
 961    }
 962
 963    /*
 964     * Give up.
 965     */
 966    if(ret != PIPE_OK) {
 967       goto no_storage;
 968    }
 969
 970    assert(fenced_buf->buffer || fenced_buf->data);
 971
 972    LIST_ADDTAIL(&fenced_buf->head, &fenced_mgr->unfenced);
 973    ++fenced_mgr->num_unfenced;
 974    pipe_mutex_unlock(fenced_mgr->mutex);
 975
 976    return &fenced_buf->base;
 977
 978 no_storage:
 979    pipe_mutex_unlock(fenced_mgr->mutex);
 980    FREE(fenced_buf);
 981 no_buffer:
 982    return NULL;
 983 }
 984
 985
 986 static void
 987 fenced_bufmgr_flush(struct pb_manager *mgr)
 988 {
 989    struct fenced_manager *fenced_mgr = fenced_manager(mgr);
 990
 991    pipe_mutex_lock(fenced_mgr->mutex);
 992    while(fenced_manager_check_signalled_locked(fenced_mgr, TRUE))
 993       ;
 994    pipe_mutex_unlock(fenced_mgr->mutex);
 995
 996    assert(fenced_mgr->provider->flush);
 997    if(fenced_mgr->provider->flush)
 998       fenced_mgr->provider->flush(fenced_mgr->provider);
 999 }
1000
1001
1002 static void
1003 fenced_bufmgr_destroy(struct pb_manager *mgr)
1004 {
1005    struct fenced_manager *fenced_mgr = fenced_manager(mgr);
1006
1007    pipe_mutex_lock(fenced_mgr->mutex);
1008
1009    /* Wait on outstanding fences */
1010    while (fenced_mgr->num_fenced) {
1011       pipe_mutex_unlock(fenced_mgr->mutex);
1012 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
1013       sched_yield();
1014 #endif
1015       pipe_mutex_lock(fenced_mgr->mutex);
1016       while(fenced_manager_check_signalled_locked(fenced_mgr, TRUE))
1017          ;
1018    }
1019
1020 #ifdef DEBUG
1021    /*assert(!fenced_mgr->num_unfenced);*/
1022 #endif
1023
1024    pipe_mutex_unlock(fenced_mgr->mutex);
1025    pipe_mutex_destroy(fenced_mgr->mutex);
1026
1027    if(fenced_mgr->provider)
1028       fenced_mgr->provider->destroy(fenced_mgr->provider);
1029
1030    fenced_mgr->ops->destroy(fenced_mgr->ops);
1031
1032    FREE(fenced_mgr);
1033 }
1034
1035
1036 struct pb_manager *
1037 fenced_bufmgr_create(struct pb_manager *provider,
1038                      struct pb_fence_ops *ops,
1039                      pb_size max_buffer_size,
1040                      pb_size max_cpu_total_size)
1041 {
1042    struct fenced_manager *fenced_mgr;
1043
1044    if(!provider)
1045       return NULL;
1046
1047    fenced_mgr = CALLOC_STRUCT(fenced_manager);
1048    if (!fenced_mgr)
1049       return NULL;
1050
1051    fenced_mgr->base.destroy = fenced_bufmgr_destroy;
1052    fenced_mgr->base.create_buffer = fenced_bufmgr_create_buffer;
1053    fenced_mgr->base.flush = fenced_bufmgr_flush;
1054
1055    fenced_mgr->provider = provider;
1056    fenced_mgr->ops = ops;
1057    fenced_mgr->max_buffer_size = max_buffer_size;
1058    fenced_mgr->max_cpu_total_size = max_cpu_total_size;
1059
1060    LIST_INITHEAD(&fenced_mgr->fenced);
1061    fenced_mgr->num_fenced = 0;
1062
1063    LIST_INITHEAD(&fenced_mgr->unfenced);
1064    fenced_mgr->num_unfenced = 0;
1065
1066    pipe_mutex_init(fenced_mgr->mutex);
1067
1068    return &fenced_mgr->base;
1069 }