backups/4jcraft
2
0
Fork 0
mirror of https://github.com/4jcraft/4jcraft.git synced 2026-08-20 09:57:10 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity
4jcraft/targets/app/windows/Iggy/gdraw/gdraw_shared.inl
MatthewBeshay 11f9bd6046 refactor: move Minecraft.Client into app
2026-04-02 16:09:32 +11:00

2694 lines
96 KiB
C++
Raw Permalink Blame History

// gdraw_shared.inl - author: Sean Barrett - copyright 2010 RAD Game Tools
//
// This file implements some common code that can be shared across
// all the sample implementations of GDraw.
#if defined(IGGY_DISABLE_GDRAW_ASSERT)
#define assert(x)
#else
#include <assert.h>
#endif
#include <stdbool.h>
#if !defined(GDRAW_MAYBE_UNUSED)
#define GDRAW_MAYBE_UNUSED
#endif
///////////////////////////////////////////////////////////////
//
// GDrawHandleCache manages resource "handles" used by Iggy
// (i.e. these handles wrap the platform resource handles,
// and this file provides those wrappers and facilities for
// LRU tracking them). Moreover, for console platforms, we
// actually implement our own managed resource pools.
//
// This is the main state machine when GDRAW_MANAGE_MEM is defined:
// (which covers all console platforms)
//
// +------+ +--------+ |
// | Live |<------->| Locked | |
// +------+ +--------+ |
// / \ ^ |
// / \ \ |
// v v \ |
// +------+ +------+ +------+ | |
// | Dead |--->| Free |<---| User | | |
// +------+ +------+ +------+ | |
// ^ ^ ^ ^ | |
// \ / \ | | |
// \ / v | | |
// +--------+ +-------+ / |
// | Pinned |<--------| Alloc |/ |
// +--------+ +-------+ |
//
// "Free" handles are not in use and available for allocation.
// "Alloc" handles have been assigned by GDraw, but do not yet
// have a system resource backing them. Resources stay in
// this state until we know that for sure that we're going
// to be able to successfully complete creation, at which
// point the resource transitions to one of the regular states.
// "Live" handles correspond to resources that may be used
// for rendering. They are kept in LRU order. Old resources
// may be evicted to make space.
// "Locked" handles cover resources that are going to be used
// in the next draw command. Once a resource is marked locked,
// it may not be evicted until it's back to "Live".
// "Dead" handles describe resources that have been freed on the
// CPU side, but are still in use by the GPU. Their memory may
// only be reclaimed once the GPU is done with them, at which
// point they are moved to the "Free" list. Items on the "Dead"
// list appear ordered by the last time they were used by the
// GPU - "most stale" first.
// "Pinned" resources can be used in any draw call without getting
// locked first. They can never be LRU-freed, but their memory
// is still managed by GDraw. Currently this is only used for
// the Iggy font cache.
// "User" (user-owned) resources are exactly that. They act much like
// pinned resources, but their memory isn't managed by GDraw.
// When a user-owned resource is freed, we really need to free
// it immediately (instead of marking it as "dead"), which might
// necessitate stalling the CPU until the GPU is finished using
// that resource. Since we don't own the memory, delayed frees
// are not an option.
//
// Without GDRAW_MANAGE_MEM, there's no "Dead" resources, and all
// frees are performed immediately.
typedef struct GDrawHandleCache GDrawHandleCache;
typedef struct GDrawHandle GDrawHandle;
typedef struct {
U64 value;
} GDrawFence;
typedef enum {
GDRAW_HANDLE_STATE_free = 0,
GDRAW_HANDLE_STATE_live,
GDRAW_HANDLE_STATE_locked,
GDRAW_HANDLE_STATE_dead,
GDRAW_HANDLE_STATE_pinned,
GDRAW_HANDLE_STATE_user_owned,
GDRAW_HANDLE_STATE_alloc,
GDRAW_HANDLE_STATE__count,
// not an actual state!
GDRAW_HANDLE_STATE_sentinel = GDRAW_HANDLE_STATE__count,
} GDrawHandleState;
struct GDrawHandle {
GDrawNativeHandle handle; // platform handle to a resource (variable size)
void* owner; // 4/8 // opaque handle used to allow freeing resources
// without calling back to owner
GDrawHandleCache* cache; // 4/8 // which cache this handle came from
GDrawHandle *next, *prev; // 8/16 // doubly-linked list
#if defined(GDRAW_MANAGE_MEM)
void* raw_ptr; // 4/8 // pointer to allocation - when you're managing
// memory manually
#if defined(GDRAW_CORRUPTION_CHECK)
U32 cached_raw_value[4];
rrbool has_check_value;
#endif
#endif
GDrawFence fence; // 8 // (optional) platform fence for resource
// 4
U32 bytes : 28; // estimated storage cost to allow setting a loose limit
U32 state : 4; // state the handle is in
};
// validate alignment to make sure structure will pack correctly
#if defined(__RAD64__)
RR_COMPILER_ASSERT((sizeof(GDrawHandle) & 7) == 0);
#else
RR_COMPILER_ASSERT((sizeof(GDrawHandle) & 3) == 0);
#endif
struct GDrawHandleCache {
S32 bytes_free;
S32 total_bytes;
S32 max_handles;
U32 is_vertex : 1; // vertex buffers have different warning codes and
// generate discard callbacks
U32 is_thrashing : 1;
U32 did_defragment : 1;
// 30 unused bits
GDrawHandle state[GDRAW_HANDLE_STATE__count]; // sentinel nodes for all of
// the state lists
#if defined(GDRAW_MANAGE_MEM)
struct gfx_allocator* alloc;
#endif
#if defined(GDRAW_MANAGE_MEM_TWOPOOL)
struct gfx_allocator* alloc_other;
#endif
GDrawFence prev_frame_start,
prev_frame_end; // fence value at start/end of previous frame, for
// thrashing detection
GDrawHandle handle[1]; // the rest of the handles must be stored right
// after this in the containing structure
};
#if defined(GDRAW_CORRUPTION_CHECK)
// values for corruption checking
#define GDRAW_CORRUPTIONCHECK_renderbegin 0x10
#define GDRAW_CORRUPTIONCHECK_renderend 0x20
#define GDRAW_CORRUPTIONCHECK_nomoregdraw 0x30
#define GDRAW_CORRUPTIONCHECK_maketexbegin 0x40
#define GDRAW_CORRUPTIONCHECK_maketexend 0x50
#define GDRAW_CORRUPTIONCHECK_wrappedcreateend 0x60
#define GDRAW_CORRUPTIONCHECK_wrappedcreatebegin 0x61
#define GDRAW_CORRUPTIONCHECK_wrappeddestroyend 0x70
#define GDRAW_CORRUPTIONCHECK_wrappeddestroybegin 0x71
#define GDRAW_CORRUPTIONCHECK_allochandle 0x80
#define GDRAW_CORRUPTIONCHECK_allochandle_begin 0x81
#define GDRAW_CORRUPTIONCHECK_allochandle_postreap 0x82
#define GDRAW_CORRUPTIONCHECK_allochandle_postfree1 0x83
#define GDRAW_CORRUPTIONCHECK_allochandle_postfree2 0x84
#define GDRAW_CORRUPTIONCHECK_allochandle_postfree3 0x85
#define GDRAW_CORRUPTIONCHECK_allochandle_postalloc1 0x86
#define GDRAW_CORRUPTIONCHECK_allochandle_postalloc2 0x87
#define GDRAW_CORRUPTIONCHECK_allochandle_postalloc3 0x88
#define GDRAW_CORRUPTIONCHECK_allochandle_defrag 0x89
#define GDRAW_CORRUPTIONCHECK_freetex 0x90
static U32* debug_raw_address(GDrawHandle* t, int choice) {
static int offset_table[4] = {0x555555, 0xaaaaaa, 0x333333, 0x6e6e6e};
U8* base = (U8*)t->raw_ptr;
int offset = offset_table[choice] & (t->bytes - 1) & ~3;
return (U32*)(base + offset);
}
static void debug_check_overlap_one(GDrawHandle* t, U8* ptr, S32 len) {
assert(len >= 0);
if (t->raw_ptr && t->raw_ptr != ptr) {
assert(t->raw_ptr < ptr || t->raw_ptr >= ptr + len);
}
}
static void debug_check_overlap(GDrawHandleCache* c, U8* ptr, S32 len) {
GDrawHandle* t = c->head;
while (t) {
debug_check_overlap_one(t, ptr, len);
t = t->next;
}
t = c->active;
while (t) {
debug_check_overlap_one(t, ptr, len);
t = t->next;
}
}
static void debug_check_raw_values(GDrawHandleCache* c) {
GDrawHandle* t = c->head;
while (t) {
if (t->raw_ptr && t->has_check_value) {
int i;
for (i = 0; i < 4; ++i) {
if (*debug_raw_address(t, i) != t->cached_raw_value[i]) {
// zlog("!Iggy texture corruption found\n");
// zlog("t=%p, t->raw_ptr=%p\n", t, t->raw_ptr);
// zlog("Cached values: %08x %08x %08x %08x\n",
// t->cached_raw_value[0], t->cached_raw_value[1],
// t->cached_raw_value[2], t->cached_raw_value[3]);
// zlog("Current values: %08x %08x %08x %08x\n",
// *debug_raw_address(t,0), *debug_raw_address(t,1),
// *debug_raw_address(t,2), *debug_raw_address(t,3));
assert(0);
}
}
}
t = t->next;
}
t = c->active;
while (t) {
if (t->raw_ptr && t->has_check_value) {
int i;
for (i = 0; i < 4; ++i) {
if (*debug_raw_address(t, i) != t->cached_raw_value[i]) {
// zlog("!Iggy texture corruption found\n");
// zlog("t=%p, t->raw_ptr=%p\n", t, t->raw_ptr);
// zlog("Cached values: %08x %08x %08x %08x\n",
// t->cached_raw_value[0], t->cached_raw_value[1],
// t->cached_raw_value[2], t->cached_raw_value[3]);
// zlog("Current values: %08x %08x %08x %08x\n",
// *debug_raw_address(t,0), *debug_raw_address(t,1),
// *debug_raw_address(t,2), *debug_raw_address(t,3));
assert(0);
}
}
}
t = t->next;
}
}
#if !defined(GDRAW_CORRUPTION_MASK)
#define GDRAW_CORRUPTION_MASK 0
#endif
#define debug_check_raw_values_if(c, v) \
if ((GDRAW_CORRUPTION_CHECK & ~GDRAW_CORRUPTION_MASK) == \
((v) & ~GDRAW_CORRUPTION_MASK)) \
debug_check_raw_values(c); \
else
static void debug_set_raw_value(GDrawHandle* t) {
if (t->raw_ptr) {
int i;
for (i = 0; i < 4; ++i)
t->cached_raw_value[i] = *debug_raw_address(t, i);
t->has_check_value = true;
}
}
static void debug_unset_raw_value(GDrawHandle* t) {
t->has_check_value = false;
}
static void debug_check_value_is_unreferenced(GDrawHandleCache* c, void* ptr) {
GDrawHandle* t = c->head;
while (t) {
assert(t->raw_ptr != ptr);
t = t->next;
}
t = c->active;
while (t) {
assert(t->raw_ptr != ptr);
t = t->next;
}
}
#else
#define debug_check_overlap(c, p, len)
#define debug_set_raw_value(t)
#define debug_check_value_is_unreferenced(c, p)
#define debug_unset_raw_value(t)
#define debug_check_raw_values(c)
#define debug_check_raw_values_if(c, v)
#endif
#if defined(SUPERDEBUG)
static void check_lists(GDrawHandleCache* c) {
GDrawHandle *sentinel, *t;
U32 state;
// for all lists, verify that they are consistent and
// properly linked
for (state = 0; state < GDRAW_HANDLE_STATE__count; state++) {
S32 count = 0;
sentinel = &c->state[state];
assert(!sentinel->cache);
assert(sentinel->state == GDRAW_HANDLE_STATE_sentinel);
for (t = sentinel->next; t != sentinel; t = t->next) {
count++;
assert(t->cache == c);
assert(t->state == state);
assert(t->prev->next == t);
assert(t->next->prev == t);
assert(count < 50000);
}
}
// for dead list, additionally verify that it's in the right
// order (namely, sorted by ascending fence index)
sentinel = &c->state[GDRAW_HANDLE_STATE_dead];
for (t = sentinel->next; t != sentinel; t = t->next) {
assert(t->prev == sentinel || t->fence.value >= t->prev->fence.value);
}
}
#include <stdio.h>
static const char* gdraw_StateName(U32 state) {
switch (state) {
case GDRAW_HANDLE_STATE_free:
return "free";
case GDRAW_HANDLE_STATE_live:
return "live";
case GDRAW_HANDLE_STATE_locked:
return "locked";
case GDRAW_HANDLE_STATE_dead:
return "dead";
case GDRAW_HANDLE_STATE_pinned:
return "pinned";
case GDRAW_HANDLE_STATE_user_owned:
return "user-owned";
case GDRAW_HANDLE_STATE_alloc:
return "alloc";
case GDRAW_HANDLE_STATE_sentinel:
return "<sentinel>";
default:
return "???";
}
}
#else
static RADINLINE void check_lists(GDrawHandleCache* c) {
RR_UNUSED_VARIABLE(c);
}
#endif
static void gdraw_HandleTransitionInsertBefore(GDrawHandle* t,
GDrawHandleState new_state,
GDrawHandle* succ) {
check_lists(t->cache);
assert(t->state !=
GDRAW_HANDLE_STATE_sentinel); // sentinels should never get here!
assert(t->state != (U32)new_state); // code should never call "transition"
// if it's not transitioning!
// unlink from prev state
t->prev->next = t->next;
t->next->prev = t->prev;
// add to list for new state
t->next = succ;
t->prev = succ->prev;
t->prev->next = t;
t->next->prev = t;
#if defined(SUPERDEBUG)
printf("GD %chandle %p %s->%s\n", t->cache->is_vertex ? 'v' : 't', t,
gdraw_StateName(t->state), gdraw_StateName(new_state));
#endif
t->state = new_state;
check_lists(t->cache);
}
static RADINLINE void gdraw_HandleTransitionTo(GDrawHandle* t,
GDrawHandleState new_state) {
gdraw_HandleTransitionInsertBefore(t, new_state,
&t->cache->state[new_state]);
}
#if defined(GDRAW_MANAGE_MEM_TWOPOOL)
static rrbool gdraw_MigrateResource(GDrawHandle* t, GDrawStats* stats);
static void gdraw_res_free(GDrawHandle* t, GDrawStats* stats);
#endif
static rrbool gdraw_HandleCacheLockStats(GDrawHandle* t, void* owner,
GDrawStats* stats) {
RR_UNUSED_VARIABLE(stats);
// if the GPU memory is owned by the user, then we never spontaneously
// free it, and we can always report true. moreover, Iggy doesn't bother
// keeping 'owner' consistent in this case, so we must check this before
// verifying t->owner.
if (t->state == GDRAW_HANDLE_STATE_user_owned) return true;
// if t->owner has changed, then Iggy is trying to lock an old version
// of this handle from before (the handle has already been recycled to
// point to a new resource)
if (t->owner != owner) return false;
// otherwise, it's a valid resource and we should lock it until the next
// unlock call
assert(t->state == GDRAW_HANDLE_STATE_live ||
t->state == GDRAW_HANDLE_STATE_locked ||
t->state == GDRAW_HANDLE_STATE_pinned);
if (t->state == GDRAW_HANDLE_STATE_live) {
#if defined(GDRAW_MANAGE_MEM_TWOPOOL)
// if we defragmented this frame, we can't just make resources live;
// we need to migrate them to their new location. (which might fail
// if we don't have enough memory left in the new pool)
if (t->cache->did_defragment) {
if (!gdraw_MigrateResource(t, stats)) {
gdraw_res_free(t, stats);
return false;
}
}
#endif
gdraw_HandleTransitionTo(t, GDRAW_HANDLE_STATE_locked);
}
return true;
}
static rrbool gdraw_HandleCacheLock(GDrawHandle* t, void* owner) {
return gdraw_HandleCacheLockStats(t, owner, NULL);
}
static void gdraw_HandleCacheUnlock(GDrawHandle* t) {
assert(t->state == GDRAW_HANDLE_STATE_locked ||
t->state == GDRAW_HANDLE_STATE_pinned ||
t->state == GDRAW_HANDLE_STATE_user_owned);
if (t->state == GDRAW_HANDLE_STATE_locked)
gdraw_HandleTransitionTo(t, GDRAW_HANDLE_STATE_live);
}
static void gdraw_HandleCacheUnlockAll(GDrawHandleCache* c) {
GDrawHandle* sentinel = &c->state[GDRAW_HANDLE_STATE_locked];
while (sentinel->next != sentinel)
gdraw_HandleTransitionTo(sentinel->next, GDRAW_HANDLE_STATE_live);
}
static void gdraw_HandleCacheInit(GDrawHandleCache* c, S32 num_handles,
S32 bytes) {
S32 i;
assert(num_handles > 0);
c->max_handles = num_handles;
c->total_bytes = bytes;
c->bytes_free = c->total_bytes;
c->is_vertex = false;
c->is_thrashing = false;
c->did_defragment = false;
for (i = 0; i < GDRAW_HANDLE_STATE__count; i++) {
c->state[i].owner = NULL;
c->state[i].cache =
NULL; // should never follow cache link from sentinels!
c->state[i].next = c->state[i].prev = &c->state[i];
#if defined(GDRAW_MANAGE_MEM)
c->state[i].raw_ptr = NULL;
#endif
c->state[i].fence.value = 0;
c->state[i].bytes = 0;
c->state[i].state = GDRAW_HANDLE_STATE_sentinel;
}
for (i = 0; i < num_handles; ++i) {
c->handle[i].cache = c;
c->handle[i].prev =
(i == 0) ? &c->state[GDRAW_HANDLE_STATE_free] : &c->handle[i - 1];
c->handle[i].next = (i == num_handles - 1)
? &c->state[GDRAW_HANDLE_STATE_free]
: &c->handle[i + 1];
c->handle[i].bytes = 0;
c->handle[i].state = GDRAW_HANDLE_STATE_free;
#if defined(GDRAW_MANAGE_MEM)
c->handle[i].raw_ptr = NULL;
#endif
}
c->state[GDRAW_HANDLE_STATE_free].next = &c->handle[0];
c->state[GDRAW_HANDLE_STATE_free].prev = &c->handle[num_handles - 1];
c->prev_frame_start.value = 0;
c->prev_frame_end.value = 0;
#if defined(GDRAW_MANAGE_MEM)
c->alloc = NULL;
#endif
#if defined(GDRAW_MANAGE_MEM_TWOPOOL)
c->alloc_other = NULL;
#endif
check_lists(c);
}
static GDrawHandle* gdraw_HandleCacheAllocateBegin(GDrawHandleCache* c) {
GDrawHandle* free_list = &c->state[GDRAW_HANDLE_STATE_free];
GDrawHandle* t = NULL;
if (free_list->next != free_list) {
t = free_list->next;
gdraw_HandleTransitionTo(t, GDRAW_HANDLE_STATE_alloc);
t->bytes = 0;
t->owner = 0;
#if defined(GDRAW_MANAGE_MEM)
t->raw_ptr = NULL;
#endif
#if defined(GDRAW_CORRUPTION_CHECK)
t->has_check_value = false;
#endif
}
return t;
}
static void gdraw_HandleCacheAllocateEnd(GDrawHandle* t, S32 bytes, void* owner,
GDrawHandleState new_state) {
assert(t->cache);
assert(t->bytes == 0);
assert(t->owner == 0);
assert(t->state == GDRAW_HANDLE_STATE_alloc);
if (bytes == 0)
assert(new_state == GDRAW_HANDLE_STATE_user_owned);
else
assert(new_state == GDRAW_HANDLE_STATE_locked ||
new_state == GDRAW_HANDLE_STATE_pinned);
t->bytes = bytes;
t->owner = owner;
t->cache->bytes_free -= bytes;
gdraw_HandleTransitionTo(t, new_state);
}
static void gdraw_HandleCacheFree(GDrawHandle* t) {
GDrawHandleCache* c = t->cache;
assert(t->state != GDRAW_HANDLE_STATE_alloc &&
t->state != GDRAW_HANDLE_STATE_sentinel);
c->bytes_free += t->bytes;
t->bytes = 0;
t->owner = 0;
#if defined(GDRAW_MANAGE_MEM)
t->raw_ptr = 0;
#endif
#if defined(GDRAW_CORRUPTION_CHECK)
t->has_check_value = false;
#endif
gdraw_HandleTransitionTo(t, GDRAW_HANDLE_STATE_free);
}
static void gdraw_HandleCacheAllocateFail(GDrawHandle* t) {
assert(t->state == GDRAW_HANDLE_STATE_alloc);
gdraw_HandleTransitionTo(t, GDRAW_HANDLE_STATE_free);
}
static GDrawHandle* gdraw_HandleCacheGetLRU(GDrawHandleCache* c) {
// TransitionTo always inserts at the end, which means that the resources
// at the front of the LRU list are the oldest ones, since in-use resources
// will get appended on every transition from "locked" to "live".
GDrawHandle* sentinel = &c->state[GDRAW_HANDLE_STATE_live];
return (sentinel->next != sentinel) ? sentinel->next : NULL;
}
static void gdraw_HandleCacheTick(GDrawHandleCache* c, GDrawFence now) {
c->prev_frame_start = c->prev_frame_end;
c->prev_frame_end = now;
// reset these flags every frame
c->is_thrashing = false;
c->did_defragment = false;
}
#if defined(GDRAW_MANAGE_MEM)
static void gdraw_HandleCacheInsertDead(GDrawHandle* t) {
GDrawHandle *s, *sentinel;
assert(t->state == GDRAW_HANDLE_STATE_live ||
t->state == GDRAW_HANDLE_STATE_locked ||
t->state == GDRAW_HANDLE_STATE_pinned);
// figure out where t belongs in the dead list in "chronological order"
// do this by finding its (chronological) successor s
sentinel = &t->cache->state[GDRAW_HANDLE_STATE_dead];
s = sentinel->next;
while (s != sentinel && s->fence.value <= t->fence.value) s = s->next;
// and then insert it there
gdraw_HandleTransitionInsertBefore(t, GDRAW_HANDLE_STATE_dead, s);
}
#endif
////////////////////////////////////////////////////////////////////////
//
// Set transformation matrices
//
// Our vertex shaders use this convention:
// world: our world matrices always look like this
// m00 m01 0 t0
// m10 m11 0 t1
// 0 0 0 d
// 0 0 0 1
//
// we just store the first two rows and insert d
// in the first row, third column. our input position vectors are
// always (x,y,0,1) or (x,y,0,0), so we can still just use dp4 to
// compute final x/y. after that it's a single move to set the
// correct depth value.
//
// viewproj: our view-projection matrix is always just a 2D scale+translate,
// i.e. the matrix looks like this:
//
// p[0] 0 0 p[2]
// 0 p[1] 0 p[3]
// 0 0 1 0
// 0 0 0 1
//
// just store (p[0],p[1],p[2],p[3]) in a 4-component vector and the
// projection transform is a single multiply-add.
//
// The output is volatile since it's often in Write-Combined memory where we
// really don't want compiler reordering.
static RADINLINE void gdraw_PixelSpace(volatile F32* RADRESTRICT vvec) {
// 1:1 pixel mapping - just identity since our "view space" is pixels
vvec[0] = 1.0f;
vvec[1] = 0.0f;
vvec[2] = 0.0f;
vvec[3] = 0.0f;
vvec[4] = 0.0f;
vvec[5] = 1.0f;
vvec[6] = 0.0f;
vvec[7] = 0.0f;
}
static RADINLINE void gdraw_WorldSpace(volatile F32* RADRESTRICT vvec,
F32* RADRESTRICT world_to_pixel,
F32 depth, F32 misc) {
// World->pixel space transform is just a scale
vvec[0] = world_to_pixel[0];
vvec[1] = 0.0f;
vvec[2] = depth;
vvec[3] = 0.0f;
vvec[4] = 0.0f;
vvec[5] = world_to_pixel[1];
vvec[6] = misc;
vvec[7] = 0.0f;
}
static RADINLINE void gdraw_ObjectSpace(volatile F32* RADRESTRICT vvec,
gswf_matrix* RADRESTRICT xform,
F32 depth, F32 misc) {
// Object->pixel transform is a 2D homogeneous matrix transform
F32 m00 = xform->m00;
F32 m01 = xform->m01;
F32 m10 = xform->m10;
F32 m11 = xform->m11;
F32 trans0 = xform->trans[0];
F32 trans1 = xform->trans[1];
vvec[0] = m00;
vvec[1] = m01;
vvec[2] = depth;
vvec[3] = trans0;
vvec[4] = m10;
vvec[5] = m11;
vvec[6] = misc;
vvec[7] = trans1;
}
static void gdraw_GetObjectSpaceMatrix(F32* RADRESTRICT mat,
gswf_matrix* RADRESTRICT xform,
F32* RADRESTRICT proj, F32 depth,
int out_col_major) {
int row = out_col_major ? 1 : 4;
int col = out_col_major ? 4 : 1;
F32 xs = proj[0];
F32 ys = proj[1];
mat[0 * row + 0 * col] = xform->m00 * xs;
mat[0 * row + 1 * col] = xform->m01 * xs;
mat[0 * row + 2 * col] = 0.0f;
mat[0 * row + 3 * col] = xform->trans[0] * xs + proj[2];
mat[1 * row + 0 * col] = xform->m10 * ys;
mat[1 * row + 1 * col] = xform->m11 * ys;
mat[1 * row + 2 * col] = 0.0f;
mat[1 * row + 3 * col] = xform->trans[1] * ys + proj[3];
mat[2 * row + 0 * col] = 0.0f;
mat[2 * row + 1 * col] = 0.0f;
mat[2 * row + 2 * col] = 0.0f;
mat[2 * row + 3 * col] = depth;
mat[3 * row + 0 * col] = 0.0f;
mat[3 * row + 1 * col] = 0.0f;
mat[3 * row + 2 * col] = 0.0f;
mat[3 * row + 3 * col] = 1.0f;
}
////////////////////////////////////////////////////////////////////////
//
// Blurs
//
// symmetrically expand a rectangle by ex/ey pixels on both sides, then clamp to
// tile bounds
static void gdraw_ExpandRect(gswf_recti* out, gswf_recti const* in, S32 ex,
S32 ey, S32 w, S32 h) {
out->x0 = RR_MAX(in->x0 - ex, 0);
out->y0 = RR_MAX(in->y0 - ey, 0);
out->x1 = RR_MIN(in->x1 + ex, w);
out->y1 = RR_MIN(in->y1 + ey, h);
}
static void gdraw_ShiftRect(gswf_recti* out, gswf_recti const* in, S32 dx,
S32 dy) {
out->x0 = in->x0 + dx;
out->y0 = in->y0 + dy;
out->x1 = in->x1 + dx;
out->y1 = in->y1 + dy;
}
#define MAX_TAPS 9 // max # of bilinear samples in one 'convolution' step
enum {
// basic shader family
VAR_tex0 = 0,
VAR_tex1,
VAR_cmul,
VAR_cadd,
VAR_focal,
// filter family
VAR_filter_tex0 = 0,
VAR_filter_tex1,
VAR_filter_color,
VAR_filter_tc_off,
VAR_filter_tex2,
VAR_filter_clamp0,
VAR_filter_clamp1,
VAR_filter_color2,
MAX_VARS,
// blur family
VAR_blur_tex0 = 0,
VAR_blur_tap,
VAR_blur_clampv,
// color matrix family
VAR_colormatrix_tex0 = 0,
VAR_colormatrix_data,
// ihud family
VAR_ihudv_worldview = 0,
VAR_ihudv_material,
VAR_ihudv_textmode,
};
typedef struct {
S32 w, h, frametex_width, frametex_height;
void (*BlurPass)(GDrawRenderState* r, int taps, float* data, gswf_recti* s,
float* tc, float height_max, float* clampv,
GDrawStats* gstats);
} GDrawBlurInfo;
static GDrawTexture* gdraw_BlurPass(GDrawFunctions* g, GDrawBlurInfo* c,
GDrawRenderState* r, int taps, float* data,
gswf_recti* draw_bounds,
gswf_recti* sample_bounds,
GDrawStats* gstats) {
F32 tc[4];
F32 clamp[4];
F32 t = 0;
F32 texel_scale_s = 1.0f / c->frametex_width;
F32 texel_scale_t = 1.0f / c->frametex_height;
S32 i;
for (i = 0; i < taps; ++i) t += data[4 * i + 2];
assert(t >= 0.99f && t <= 1.01f);
tc[0] = texel_scale_s * draw_bounds->x0;
tc[1] = texel_scale_t * draw_bounds->y0;
tc[2] = texel_scale_s * draw_bounds->x1;
tc[3] = texel_scale_t * draw_bounds->y1;
// sample_bounds is (x0,y0) inclusive, (x1,y1) exclusive
// texel centers are offset by 0.5 from integer coordinates and we don't
// want to sample outside sample_bounds
clamp[0] = texel_scale_s * (sample_bounds->x0 + 0.5f);
clamp[1] = texel_scale_t * (sample_bounds->y0 + 0.5f);
clamp[2] = texel_scale_s * (sample_bounds->x1 - 0.5f);
clamp[3] = texel_scale_t * (sample_bounds->y1 - 0.5f);
if (!g->TextureDrawBufferBegin(
draw_bounds, GDRAW_TEXTURE_FORMAT_rgba32,
GDRAW_TEXTUREDRAWBUFFER_FLAGS_needs_color |
GDRAW_TEXTUREDRAWBUFFER_FLAGS_needs_alpha,
0, gstats))
return r->tex[0];
c->BlurPass(r, taps, data, draw_bounds, tc, (F32)c->h / c->frametex_height,
clamp, gstats);
return g->TextureDrawBufferEnd(gstats);
}
static GDrawTexture* gdraw_BlurPassDownsample(
GDrawFunctions* g, GDrawBlurInfo* c, GDrawRenderState* r, int taps,
float* data, gswf_recti* draw_bounds, int axis, int divisor, int tex_w,
int tex_h, gswf_recti* sample_bounds, GDrawStats* gstats) {
S32 i;
F32 t = 0;
F32 tc[4];
F32 clamp[4];
F32 texel_scale_s = 1.0f / tex_w;
F32 texel_scale_t = 1.0f / tex_h;
gswf_recti z;
for (i = 0; i < taps; ++i) t += data[4 * i + 2];
assert(t >= 0.99f && t <= 1.01f);
// following must be integer divides!
if (axis == 0) {
z.x0 = draw_bounds->x0 / divisor;
z.x1 = (draw_bounds->x1 - 1) / divisor + 1;
z.y0 = draw_bounds->y0;
z.y1 = draw_bounds->y1;
tc[0] = ((z.x0 - 0.5f) * divisor + 0.5f) * texel_scale_s;
tc[2] = ((z.x1 - 0.5f) * divisor + 0.5f) * texel_scale_s;
tc[1] = z.y0 * texel_scale_t;
tc[3] = z.y1 * texel_scale_t;
} else {
z.x0 = draw_bounds->x0;
z.x1 = draw_bounds->x1;
z.y0 = draw_bounds->y0 / divisor;
z.y1 = (draw_bounds->y1 - 1) / divisor + 1;
tc[0] = z.x0 * texel_scale_s;
tc[2] = z.x1 * texel_scale_s;
tc[1] = ((z.y0 - 0.5f) * divisor + 0.5f) * texel_scale_t;
tc[3] = ((z.y1 - 0.5f) * divisor + 0.5f) * texel_scale_t;
}
if (!g->TextureDrawBufferBegin(
&z, GDRAW_TEXTURE_FORMAT_rgba32,
GDRAW_TEXTUREDRAWBUFFER_FLAGS_needs_color |
GDRAW_TEXTUREDRAWBUFFER_FLAGS_needs_alpha,
0, gstats))
return r->tex[0];
clamp[0] = texel_scale_s * (sample_bounds->x0 + 0.5f);
clamp[1] = texel_scale_t * (sample_bounds->y0 + 0.5f);
clamp[2] = texel_scale_s * (sample_bounds->x1 - 0.5f);
clamp[3] = texel_scale_t * (sample_bounds->y1 - 0.5f);
assert(clamp[0] <= clamp[2]);
assert(clamp[1] <= clamp[3]);
c->BlurPass(r, taps, data, &z, tc, (F32)c->h / c->frametex_height, clamp,
gstats);
return g->TextureDrawBufferEnd(gstats);
}
#define unmap(t, a, b) (((t) - (a)) / (F32)((b) - (a)))
#define linear_remap(t, a, b, c, d) ((c) + unmap(t, a, b) * ((d) - (c)))
static void gdraw_BlurAxis(S32 axis, GDrawFunctions* g, GDrawBlurInfo* c,
GDrawRenderState* r, F32 blur_width, F32 texel,
gswf_recti* draw_bounds, gswf_recti* sample_bounds,
GDrawTexture* protect, GDrawStats* gstats) {
GDrawTexture* t;
F32 data[MAX_TAPS][4];
S32 off_axis = 1 - axis;
S32 w = ((S32)ceil((blur_width - 1) / 2)) * 2 +
1; // 1.2 => 3, 2.8 => 3, 3.2 => 5
F32 edge_weight =
1 - (w - blur_width) / 2; // 3 => 0 => 1; 1.2 => 1.8 => 0.9 => 0.1
F32 inverse_weight = 1.0f / blur_width;
w = ((w - 1) >> 1) +
1; // 3 => 2, 5 => 3, 7 => 4 (number of texture samples)
if (!r->tex[0]) return;
// horizontal filter
if (w > 1) {
if (w <= MAX_TAPS) {
// we have enough taps to just do it
// use 'w' taps
S32 i, expand;
// just go through and place all the taps in the right place
// if w is 2 (sample from -1,0,1)
// 0 => -0.5
// 1 => 1
// if w is 3:
// 0 => -1.5 samples from -2,-1
// 1 => 0.5 samples from 0,1
// 2 => 2 samples from 2
// if w is 4:
// 0 => -2.5 samples from -3,-2
// 1 => -0.5 samples from -1,0
// 2 => 1.5 samples from 1,2
// 3 => 3 samples from 3
for (i = 0; i < w; ++i) {
// first texsample samples from -w+1 and -w+2, e.g. w=2 =>
// -1,0,1
data[i][axis] = (-w + 1.5f + i * 2) * texel;
data[i][off_axis] = 0;
data[i][2] = 2 * inverse_weight; // 2 full-weight samples
data[i][3] = 0;
}
// now reweight the last one
data[i - 1][axis] = (w - 1) * texel;
data[i - 1][2] = edge_weight * inverse_weight;
// now reweight the first one
// (ew*0 + 1*1)/(1+ew) = 1/(1+ew)
data[0][axis] = (-w + 1.0f + 1 / (edge_weight + 1)) * texel;
data[0][2] = (edge_weight + 1) * inverse_weight;
expand = w - 1;
gdraw_ExpandRect(draw_bounds, draw_bounds, axis ? 0 : expand,
axis ? expand : 0, c->w, c->h);
t = gdraw_BlurPass(g, c, r, w, data[0], draw_bounds, sample_bounds,
gstats);
if (r->tex[0] != protect && r->tex[0] != t)
g->FreeTexture(r->tex[0], 0, gstats);
r->tex[0] = t;
gdraw_ExpandRect(sample_bounds, draw_bounds, 1, 1, c->w,
c->h); // for next pass
} else {
// @OPTIMIZE: for symmetrical blurs we can get a 2-wide blur in the
// *off* axis at the same time we get N-wide in the on axis, which
// could double our max width
S32 i, expand;
// @HACK: this is really a dumb way to do it, i kind of had a brain
// fart, you could get the exact same result by just doing the
// downsample the naive way and then the final sample uses texture
// samples spaced by a texel rather than spaced by two texels -- the
// current method is just as inefficient, it just puts the
// inefficiency in the way the downsampled texture is
// self-overlapping, so the downsampled texture is twice as larger
// as it should be.
// we COULD be exact by generating a mipmap, then sampling some
// number of samples from the mipmap and some from the original, but
// that would require being polyphase. instead we just are
// approximate. the mipmap weights the edge pixels by one half and
// overlaps them by one sample, so then in phase two we sample N
// slightly-overlapping mipmap samples
//
// instead we do the following.
// divide the source data up into clusters that are K samples
// long.
// ...K0... ...K1... ...K2... ...K3...
//
// Suppose K[i] is the average of all the items in cluster i.
//
// We compute a downsampled texture where T[i] = K[i] + K[i+1].
//
// Now, we sample N taps from adjacent elements of T, allowing the
// texture unit to bilerp. Suppose a given sample falls at
// coordinate i with sub-position p. Then tap #j will compute:
// T[i+j]*(1-p) + T[i+j+1]*p
// But tap #j+1 will compute:
// T[i+j+1]*(1-p) + T[i+j+2]*p
// so we end up computing:
// sum(T[i+j]) except for the end samples.
//
// So, how do we create these initial clusters? That's easy, we use
// K taps to sample 2K texels.
//
// What value of k do we use? Well, we're constrained to using
// MAX_TAPS on each pass. So at the high end, we're bounded by:
// K = MAX_TAPS
// S = MAX_TAPS (S is number of samples in second pass)
// S addresses S*2-1 texels of T, and each texel adds K more
// samples, so (ignoring the edges) we basically have w = K*S
// if w == MAX_TAPS*MAX_TAPS, then k = MAX_TAPS
// if w == MAX_TAPS+1, then k = 2
//
// suppose we have 3 taps, then we can sample 5 samples in one pass,
// so then our max coverage is 25 samples, or a filter width of 13.
// with 7 taps, we sample 13 samples in one pass, max coverage is
// 13*13 samples or (13*13-1)/2 width, which is ((2T-1)*(2T-1)-1)/2
// or (4T^2 - 4T + 1 -1)/2 or 2T^2 - 2T or 2T*(T-1)
S32 w_mip = (S32)ceil(linear_remap(
w, MAX_TAPS + 1, MAX_TAPS * MAX_TAPS, 2, MAX_TAPS));
S32 downsample = w_mip;
F32 sample_spacing = texel;
if (downsample < 2) downsample = 2;
if (w_mip > MAX_TAPS) {
// if w_mip > MAX_TAPS, then we ought to use more than one
// mipmap pass, but since that's a huge filter ( > 80 pixels)
// let's just try subsampling and see if it's good enough.
sample_spacing *= w_mip / MAX_TAPS;
w_mip = MAX_TAPS;
} else {
assert(w / downsample <= MAX_TAPS);
}
inverse_weight = 1.0f / (2 * w_mip);
for (i = 0; i < w_mip; ++i) {
data[i][axis] = (-w_mip + 1 + i * 2 + 0.5f) * sample_spacing;
data[i][off_axis] = 0;
data[i][2] = 2 * inverse_weight;
data[i][3] = 0;
}
w = w * 2 / w_mip;
// @TODO: compute the correct bboxes for this size
// the downsampled texture samples from -w_mip+1 to w_mip
// the sample from within that samples w spots within that,
// or w/2 of those, but they're overlapping by 50%.
// so if a sample is a point i, it samples from the original
// from -w_mip+1 to w_mip + i*w_mip.
// So then the minimum is: -w_mip+1 + (w/2)*w_mip, and
// the maximum is w_mip + (w/2)*w_mip
expand = (((w + 1) >> 1) + 1) * w_mip + 1;
gdraw_ExpandRect(draw_bounds, draw_bounds, axis ? 0 : expand,
axis ? expand : 0, c->w, c->h);
t = gdraw_BlurPassDownsample(
g, c, r, w_mip, data[0], draw_bounds, axis, downsample,
c->frametex_width, c->frametex_height, sample_bounds, gstats);
if (r->tex[0] != protect && r->tex[0] != t)
g->FreeTexture(r->tex[0], 0, gstats);
r->tex[0] = t;
gdraw_ExpandRect(sample_bounds, draw_bounds, 1, 1, c->w, c->h);
if (!r->tex[0]) return;
// now do a regular blur pass sampling from that
// the raw texture now contains 'downsample' samples per texel
if (w > 2 * MAX_TAPS) {
sample_spacing = texel * (w - 1) / (2 * MAX_TAPS - 1);
w = 2 * MAX_TAPS;
} else {
sample_spacing = texel;
}
// sample_spacing *= 1.0f/2;
assert(w >= 2 && w <= 2 * MAX_TAPS);
if (w & 1) {
// we just want to evenly weight even-spaced samples
inverse_weight = 1.0f / w;
// just go through and place all the taps in the right place
w = (w + 1) >> 1;
for (i = 0; i < w; ++i) {
data[i][axis] = (-w + 1.0f + 0.5f + i * 2) * sample_spacing;
data[i][off_axis] = 0;
data[i][2] = 2 * inverse_weight; // 2 full-weight samples
data[i][3] = 0;
}
// fix up the last tap
// the following test is always true, but we're testing it here
// explicitly so as to make VS2012's static analyzer not
// complain
if (i > 0) {
data[i - 1][axis] =
(-w + 1.0f + (i - 1) * 2) * sample_spacing;
data[i - 1][2] = inverse_weight;
}
} else {
// we just want to evenly weight even-spaced samples
inverse_weight = 1.0f / w;
// just go through and place all the taps in the right place
w >>= 1;
for (i = 0; i < w; ++i) {
data[i][axis] = (-w + 1.0f + i * 2) * sample_spacing;
data[i][off_axis] = 0;
data[i][2] = 2 * inverse_weight; // 2 full-weight samples
data[i][3] = 0;
}
}
t = gdraw_BlurPassDownsample(
g, c, r, w, data[0], draw_bounds, axis, 1,
axis == 0 ? c->frametex_width * downsample : c->frametex_width,
axis == 1 ? c->frametex_height * downsample
: c->frametex_height,
sample_bounds, gstats);
if (r->tex[0] != protect && r->tex[0] != t)
g->FreeTexture(r->tex[0], 0, gstats);
r->tex[0] = t;
gdraw_ExpandRect(sample_bounds, draw_bounds, 1, 1, c->w, c->h);
}
}
}
static void gdraw_Blur(GDrawFunctions* g, GDrawBlurInfo* c, GDrawRenderState* r,
gswf_recti* draw_bounds, gswf_recti* sample_bounds,
GDrawStats* gstats) {
S32 p;
GDrawTexture* protect = r->tex[0];
gswf_recti sbounds;
// compute texel offset size
F32 dx = 1.0f / c->frametex_width;
F32 dy = 1.0f / c->frametex_height;
// blur = 1 => 1 tap
// blur = 1.2 => 3 taps (0.1, 1, 0.1)
// blur = 2.2 => 3 taps (0.6, 1, 0.6)
// blur = 2.8 => 3 taps (0.9, 1, 0.9)
// blur = 3 => 3 taps (1 , 1, 1 )
// blur = 3.2 => 5 taps (0.1, 1, 1, 1, 0.1)
// S32 w = ((S32) ceil((r->blur_x-1)/2))*2+1; // 1.2 => (1.2-1)/2 => 0.1
// => 1.0 => 1 => 2 => 3 S32 h = ((S32) ceil((r->blur_y-1)/2))*2+1; // 3
// => (3-1)/2 => 1.0 => 1 => 2 => 3
// gdraw puts 1 border pixel around everything when producing rendertargets
// and we use this so expand the input sample bounds accordingly
gdraw_ExpandRect(&sbounds, sample_bounds, 1, 1, c->w, c->h);
for (p = 0; p < r->blur_passes; ++p) {
{
// do the filter separably
gdraw_BlurAxis(0, g, c, r, r->blur_x, dx, draw_bounds, &sbounds,
protect, gstats);
gdraw_BlurAxis(1, g, c, r, r->blur_y, dy, draw_bounds, &sbounds,
protect, gstats);
}
}
}
#if defined(GDRAW_MANAGE_MEM)
static void make_pool_aligned(void** start, S32* num_bytes, U32 alignment) {
UINTa addr_orig = (UINTa)*start;
UINTa addr_aligned = (addr_orig + alignment - 1) & ~((UINTa)alignment - 1);
if (addr_aligned != addr_orig) {
S32 diff = (S32)(addr_aligned - addr_orig);
if (*num_bytes < diff) {
*start = NULL;
*num_bytes = 0;
return;
} else {
*start = (void*)addr_aligned;
*num_bytes -= diff;
}
}
}
// Very simple arena allocator
typedef struct {
U8* begin;
U8* current;
U8* end;
} GDrawArena;
static void gdraw_arena_init(GDrawArena* arena, void* start, U32 size) {
arena->begin = (U8*)start;
arena->current = (U8*)start;
arena->end = (U8*)start + size;
}
static GDRAW_MAYBE_UNUSED void gdraw_arena_reset(GDrawArena* arena) {
arena->current = arena->begin;
}
static void* gdraw_arena_alloc(GDrawArena* arena, U32 size, U32 align) {
UINTa start_addr =
((UINTa)arena->current + align - 1) & ~((UINTa)align - 1);
U8* ptr = (U8*)start_addr;
UINTa remaining = arena->end - arena->current;
UINTa total_size = (ptr - arena->current) + size;
if (remaining < total_size) // doesn't fit
return NULL;
arena->current = ptr + size;
return ptr;
}
// Allocator for graphics memory.
// Graphics memory is assumed to be write-combined and slow to read for the
// CPU, so we keep all heap management information separately in main memory.
//
// There's a constant management of about 1k (2k for 64bit) to create a heap,
// plus a per-block overhead. The maximum number of blocks the allocator can
// ever use is bounded by 2*max_allocs+1; since GDraw manages a limited
// amount of handles, max_allocs is a known value at heap creation time.
//
// The allocator uses a best-fit heuristic to minimize fragmentation.
// Currently, there are no size classes or other auxiliary data structures to
// speed up this process, since the number of free blocks at any point in time
// is assumed to be fairly low.
//
// The allocator maintains a number of invariants:
// - The free list and physical block list are proper double-linked lists.
// (i.e. block->next->prev == block->prev->next == block)
// - All allocated blocks are also kept in a hash table, indexed by their
// pointer (to allow free to locate the corresponding block_info quickly).
// There's a single-linked, NULL-terminated list of elements in each hash
// bucket.
// - The physical block list is ordered. It always contains all currently
// active blocks and spans the whole managed memory range. There are no
// gaps between blocks, and all blocks have nonzero size.
// - There are no two adjacent free blocks; if two such blocks would be created,
// they are coalesced immediately.
// - The maximum number of blocks that could ever be necessary is allocated
// on initialization. All block_infos not currently in use are kept in a
// single-linked, NULL-terminated list of unused blocks. Every block is either
// in the physical block list or the unused list, and the total number of
// blocks is constant.
// These invariants always hold before and after an allocation/free.
#if !defined(GFXALLOC_ASSERT)
#define GFXALLOC_ASSERT(x)
#endif
typedef struct gfx_block_info {
U8* ptr;
gfx_block_info *prev,
*next; // for free blocks this is the free list, for allocated blocks
// it's a (single-linked!) list of elements in the corresponding
// hash bucket
gfx_block_info *prev_phys, *next_phys;
U32 is_free : 1;
U32 is_unused : 1;
U32 size : 30;
} gfx_block_info;
// 24 bytes/block on 32bit, 48 bytes/block on 64bit.
#define GFXALLOC_HASH_SIZE 256
typedef struct gfx_allocator {
U8* mem_base;
U8* mem_end;
U32 max_allocs;
U32 block_align;
U32 block_shift;
S32 actual_bytes_free;
#if defined(GFXALLOC_CHECK)
int num_blocks;
int num_unused;
int num_alloc;
int num_free;
#endif
GDrawHandleCache* cache;
gfx_block_info* unused_list; // next unused block_info (single-linked list)
gfx_block_info* hash[GFXALLOC_HASH_SIZE]; // allocated blocks
gfx_block_info blocks[1]; // first block is head of free list AND head of
// physical block list (sentinel)
} gfx_allocator;
// about 1k (32bit), 2k (64bit) with 256 hash buckets (the default). dominated
// by hash table.
#if defined(GFXALLOC_CHECK)
#define GFXALLOC_IF_CHECK(x) x
#else
#define GFXALLOC_IF_CHECK(x)
#endif
static U32 gfxalloc_get_hash_code(gfx_allocator* alloc, void* ptr) {
U32 a = (U32)(((U8*)ptr - alloc->mem_base) >> alloc->block_shift);
// integer hash function by Bob Jenkins
// (http://burtleburtle.net/bob/hash/integer.html) I use this function
// because integer mults are slow on PPC and large literal constants take
// multiple instrs to set up on all RISC CPUs.
a -= (a << 6);
a ^= (a >> 17);
a -= (a << 9);
a ^= (a << 4);
a -= (a << 3);
a ^= (a << 10);
a ^= (a >> 15);
return a & (GFXALLOC_HASH_SIZE - 1);
}
#if defined(SUPERDEBUG) || defined(COMPLETE_DEBUG)
#include <stdlib.h>
#define MAX_REGIONS 8192
typedef struct {
U32 begin, end;
} gfx_region;
static gfx_region region[MAX_REGIONS];
static int region_sort(const void* p, const void* q) {
U32 a = *(U32*)p;
U32 b = *(U32*)q;
if (a < b) return -1;
if (a > b) return 1;
return 0;
}
static void gfxalloc_check1(gfx_allocator* alloc) {
assert(alloc->max_allocs * 2 + 1 < MAX_REGIONS);
int i, n = 0;
for (i = 0; i < GFXALLOC_HASH_SIZE; ++i) {
gfx_block_info* b = alloc->hash[i];
while (b) {
region[n].begin = (UINTa)b->ptr;
region[n].end = region[n].begin + b->size;
++n;
b = b->next;
}
}
gfx_block_info* b = alloc->blocks[0].next;
while (b != &alloc->blocks[0]) {
region[n].begin = (UINTa)b->ptr;
region[n].end = region[n].begin + b->size;
++n;
b = b->next;
}
qsort(region, n, sizeof(region[0]), region_sort);
for (i = 0; i + 1 < n; ++i) {
assert(region[i].end == region[i + 1].begin);
}
}
#else
#define gfxalloc_check1(a)
#endif
#if defined(COMPLETE_DEBUG)
static void verify_against_blocks(int num_regions, void* vptr, S32 len) {
U32* ptr = (U32*)vptr;
// binary search for ptr amongst regions
S32 s = 0, e = num_regions - 1;
assert(len != 0);
while (s < e) {
S32 i = (s + e + 1) >> 1;
// invariant: b[s] <= ptr <= b[e]
if (region[i].begin <= (UINTa)ptr)
s = i;
else
e = i - 1;
// consider cases:
// s=0,e=1: i = 0, how do we get i to be 1?
}
// at this point, s >= e
assert(s < num_regions && region[s].begin == (UINTa)ptr &&
(UINTa)ptr + len <= region[s].end);
}
static void debug_complete_check(gfx_allocator* alloc, void* ptr, S32 len,
void* skip) {
GDrawHandleCache* c = alloc->cache;
assert(alloc->max_allocs * 2 + 1 < MAX_REGIONS);
int i, n = 0;
for (i = 0; i < GFXALLOC_HASH_SIZE; ++i) {
gfx_block_info* b = alloc->hash[i];
while (b) {
region[n].begin = (UINTa)b->ptr;
region[n].end = region[n].begin + b->size;
++n;
b = b->next;
}
}
gfx_block_info* b = alloc->blocks[0].next;
while (b != &alloc->blocks[0]) {
region[n].begin = (UINTa)b->ptr;
region[n].end = region[n].begin + b->size;
++n;
b = b->next;
}
for (i = 0; i < n; ++i) assert(region[i].end > region[i].begin);
qsort(region, n, sizeof(region[0]), region_sort);
for (i = 0; i + 1 < n; ++i) {
assert(region[i].end == region[i + 1].begin);
}
if (ptr) verify_against_blocks(n, ptr, len);
if (c) {
GDrawHandle* t = c->head;
while (t) {
if (t->raw_ptr && t->raw_ptr != skip)
verify_against_blocks(n, t->raw_ptr, t->bytes);
t = t->next;
}
t = c->active;
while (t) {
if (t->raw_ptr && t->raw_ptr != skip)
verify_against_blocks(n, t->raw_ptr, t->bytes);
t = t->next;
}
}
}
#else
#define debug_complete_check(a, p, len, s)
#endif
#if defined(GFXALLOC_CHECK)
static void gfxalloc_check2(gfx_allocator* alloc) {
int n = 0;
gfx_block_info* b = alloc->unused_list;
while (b) {
++n;
b = b->next;
}
GFXALLOC_ASSERT(n == alloc->num_unused);
b = alloc->blocks->next;
n = 0;
while (b != alloc->blocks) {
++n;
b = b->next;
}
GFXALLOC_ASSERT(n == alloc->num_free);
GFXALLOC_ASSERT(alloc->num_blocks ==
alloc->num_unused + alloc->num_free + alloc->num_alloc);
}
#define gfxalloc_check(a) \
do { \
gfxalloc_check1(a); \
gfxalloc_check2(a); \
} while (0)
#else
#define gfxalloc_check2(a)
#define gfxalloc_check(a)
#endif
static gfx_block_info* gfxalloc_pop_unused(gfx_allocator* alloc) {
GFXALLOC_ASSERT(alloc->unused_list != NULL);
GFXALLOC_ASSERT(alloc->unused_list->is_unused);
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(alloc->num_unused);)
gfx_block_info* b = alloc->unused_list;
alloc->unused_list = b->next;
GFXALLOC_ASSERT(alloc->unused_list);
b->is_unused = 0;
GFXALLOC_IF_CHECK(--alloc->num_unused;)
return b;
}
static void gfxalloc_push_unused(gfx_allocator* alloc, gfx_block_info* b) {
GFXALLOC_ASSERT(!b->is_unused);
b->is_unused = 1;
b->next = alloc->unused_list;
alloc->unused_list = b;
GFXALLOC_IF_CHECK(++alloc->num_unused);
}
static void gfxalloc_add_free(gfx_allocator* alloc, gfx_block_info* b) {
gfx_block_info* head = alloc->blocks;
b->is_free = 1;
b->next = head->next;
b->prev = head;
head->next->prev = b;
head->next = b;
GFXALLOC_IF_CHECK(++alloc->num_free;)
}
static void gfxalloc_rem_free(gfx_allocator* alloc, gfx_block_info* b) {
RR_UNUSED_VARIABLE(alloc);
b->is_free = 0;
b->prev->next = b->next;
b->next->prev = b->prev;
GFXALLOC_IF_CHECK(--alloc->num_free;)
}
static void gfxalloc_split_free(gfx_allocator* alloc, gfx_block_info* b,
U32 pos) {
gfx_block_info* n = gfxalloc_pop_unused(alloc);
GFXALLOC_ASSERT(b->is_free);
GFXALLOC_ASSERT(pos > 0 && pos < b->size);
// set up new free block
n->ptr = b->ptr + pos;
n->prev_phys = b;
n->next_phys = b->next_phys;
n->next_phys->prev_phys = n;
n->size = b->size - pos;
assert(n->size != 0);
gfxalloc_add_free(alloc, n);
// fix original block
b->next_phys = n;
b->size = pos;
assert(b->size != 0);
debug_complete_check(alloc, n->ptr, n->size, 0);
debug_complete_check(alloc, b->ptr, b->size, 0);
}
static gfx_allocator* gfxalloc_create(void* mem, U32 mem_size, U32 align,
U32 max_allocs) {
gfx_allocator* a;
U32 i, max_blocks, size;
if (!align ||
(align & (align - 1)) != 0) // align must be >0 and a power of 2
return NULL;
// for <= max_allocs live allocs, there's <= 2*max_allocs+1 blocks. worst
// case: [free][used][free] .... [free][used][free]
max_blocks = max_allocs * 2 + 1;
size = sizeof(gfx_allocator) + max_blocks * sizeof(gfx_block_info);
a = (gfx_allocator*)IggyGDrawMalloc(size);
if (!a) return NULL;
memset(a, 0, size);
GFXALLOC_IF_CHECK(a->num_blocks = max_blocks;)
GFXALLOC_IF_CHECK(a->num_alloc = 0;)
GFXALLOC_IF_CHECK(a->num_free = 1;)
GFXALLOC_IF_CHECK(a->num_unused = max_blocks - 1;)
GFXALLOC_IF_CHECK(
GFXALLOC_ASSERT(a->num_blocks ==
a->num_alloc + a->num_free + a->num_unused);)
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(a->num_free <= a->num_blocks + 1);)
a->actual_bytes_free = mem_size;
a->mem_base = (U8*)mem;
a->mem_end = a->mem_base + mem_size;
a->max_allocs = max_allocs;
a->block_align = align;
a->block_shift = 0;
while ((1u << a->block_shift) < a->block_align) a->block_shift++;
// init sentinel block
a->blocks[0].prev = a->blocks[0].next =
&a->blocks[1]; // point to free block
a->blocks[0].prev_phys = a->blocks[0].next_phys = &a->blocks[1]; // same
// init first free block
a->blocks[1].ptr = a->mem_base;
a->blocks[1].prev = a->blocks[1].next = &a->blocks[0];
a->blocks[1].prev_phys = a->blocks[1].next_phys = &a->blocks[0];
a->blocks[1].is_free = 1;
a->blocks[1].size = mem_size;
// init "unused" list
a->unused_list = a->blocks + 2;
for (i = 2; i < max_blocks; i++) {
a->blocks[i].is_unused = 1;
a->blocks[i].next = a->blocks + (i + 1);
}
a->blocks[i].is_unused = 1;
gfxalloc_check(a);
debug_complete_check(a, NULL, 0, 0);
return a;
}
static void* gfxalloc_alloc(gfx_allocator* alloc, U32 size_in_bytes) {
gfx_block_info *cur, *best = NULL;
U32 i, best_wasted = ~0u;
U32 size = size_in_bytes;
debug_complete_check(alloc, NULL, 0, 0);
gfxalloc_check(alloc);
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(alloc->num_blocks ==
alloc->num_alloc + alloc->num_free +
alloc->num_unused);)
GFXALLOC_IF_CHECK(
GFXALLOC_ASSERT(alloc->num_free <= alloc->num_blocks + 1);)
// round up to multiple of our block alignment
size = (size + alloc->block_align - 1) & ~(alloc->block_align - 1);
assert(size >= size_in_bytes);
assert(size != 0);
// find best fit among all free blocks. this is O(N)!
for (cur = alloc->blocks[0].next; cur != alloc->blocks; cur = cur->next) {
if (cur->size >= size) {
U32 wasted = cur->size - size;
if (wasted < best_wasted) {
best_wasted = wasted;
best = cur;
if (!wasted) break; // can't get better than perfect
}
}
}
// return the best fit, if we found any suitable block
if (best) {
debug_check_overlap(alloc->cache, best->ptr, best->size);
// split off allocated part
if (size != best->size) gfxalloc_split_free(alloc, best, size);
debug_complete_check(alloc, best->ptr, best->size, 0);
// remove from free list and add to allocated hash table
GFXALLOC_ASSERT(best->size == size);
gfxalloc_rem_free(alloc, best);
i = gfxalloc_get_hash_code(alloc, best->ptr);
best->next = alloc->hash[i];
alloc->hash[i] = best;
alloc->actual_bytes_free -= size;
GFXALLOC_ASSERT(alloc->actual_bytes_free >= 0);
GFXALLOC_IF_CHECK(++alloc->num_alloc;)
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(alloc->num_blocks ==
alloc->num_alloc + alloc->num_free +
alloc->num_unused);)
GFXALLOC_IF_CHECK(
GFXALLOC_ASSERT(alloc->num_free <= alloc->num_blocks + 1);)
debug_complete_check(alloc, best->ptr, best->size, 0);
gfxalloc_check(alloc);
debug_check_overlap(alloc->cache, best->ptr, best->size);
return best->ptr;
} else
return NULL; // not enough space!
}
static void gfxalloc_free(gfx_allocator* alloc, void* ptr) {
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(alloc->num_blocks ==
alloc->num_alloc + alloc->num_free +
alloc->num_unused);)
GFXALLOC_IF_CHECK(
GFXALLOC_ASSERT(alloc->num_free <= alloc->num_blocks + 1);)
// find the block in the hash table
gfx_block_info *b, *t, **prevnext;
U32 i = gfxalloc_get_hash_code(alloc, ptr);
prevnext = &alloc->hash[i];
b = alloc->hash[i];
while (b) {
if (b->ptr == ptr) break;
prevnext = &b->next;
b = b->next;
}
if (!b) {
GFXALLOC_ASSERT(0); // trying to free a non-allocated block
return;
}
debug_complete_check(alloc, b->ptr, b->size, 0);
GFXALLOC_IF_CHECK(--alloc->num_alloc;)
// remove it from the hash table
*prevnext = b->next;
alloc->actual_bytes_free += b->size;
// merge with previous block if it's free, else add it to free list
t = b->prev_phys;
if (t->is_free) {
t->size += b->size;
t->next_phys = b->next_phys;
t->next_phys->prev_phys = t;
gfxalloc_push_unused(alloc, b);
b = t;
} else
gfxalloc_add_free(alloc, b);
// try to merge with next block
t = b->next_phys;
if (t->is_free) {
b->size += t->size;
b->next_phys = t->next_phys;
t->next_phys->prev_phys = b;
gfxalloc_rem_free(alloc, t);
gfxalloc_push_unused(alloc, t);
}
debug_complete_check(alloc, 0, 0, ptr);
gfxalloc_check(alloc);
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(alloc->num_blocks ==
alloc->num_alloc + alloc->num_free +
alloc->num_unused);)
GFXALLOC_IF_CHECK(
GFXALLOC_ASSERT(alloc->num_free <= alloc->num_blocks + 1);)
}
#if defined(GDRAW_MANAGE_MEM_TWOPOOL)
static rrbool gfxalloc_is_empty(gfx_allocator* alloc) {
gfx_block_info* first_free = alloc->blocks[0].next;
// we want to check whether there's exactly one free block that
// covers the entire pool.
if (first_free == alloc->blocks) // 0 free blocks
return false;
if (first_free->next != alloc->blocks) // >1 free block
return false;
return first_free->ptr == alloc->mem_base &&
first_free->ptr + first_free->size == alloc->mem_end;
}
static rrbool gfxalloc_mem_contains(gfx_allocator* alloc, void* ptr) {
return alloc->mem_base <= (U8*)ptr && (U8*)ptr < alloc->mem_end;
}
#endif
#if defined(GDRAW_DEBUG)
static void gfxalloc_dump(gfx_allocator* alloc) {
static const char* type[] = {
"allocated",
"free",
};
for (gfx_block_info* b = alloc->blocks[0].next_phys; b != alloc->blocks;
b = b->next_phys) {
U8* start = b->ptr;
U8* end = b->ptr + b->size;
printf("%p-%p: %s (%d bytes)\n", start, end, type[b->is_free], b->size);
}
}
#endif
#endif
#if defined(GDRAW_DEFRAGMENT)
#define GDRAW_DEFRAGMENT_may_overlap \
1 // self-overlap for individual copies is OK
// Defragmentation code for graphics memory.
// The platform implementation must provide a GPU memcpy function and handle all
// necessary synchronization. It must also adjust its resource descriptors to
// match the new addresses after defragmentation.
static void gdraw_gpu_memcpy(GDrawHandleCache* c, void* dst, void* src,
U32 num_bytes);
static void gdraw_Defragment_memmove(GDrawHandleCache* c, U8* dst, U8* src,
U32 num_bytes, U32 flags,
GDrawStats* stats) {
if (dst == src) return;
assert(num_bytes != 0);
stats->nonzero_flags |= GDRAW_STATS_defrag;
stats->defrag_objects += 1;
stats->defrag_bytes += num_bytes;
if ((flags & GDRAW_DEFRAGMENT_may_overlap) || dst + num_bytes <= src ||
src + num_bytes <= dst) // no problematic overlap
gdraw_gpu_memcpy(c, dst, src, num_bytes);
else {
// need to copy in multiple chunks
U32 chunk_size, pos = 0;
if (dst < src)
chunk_size = (U32)(src - dst);
else
chunk_size = (U32)(dst - src);
while (pos < num_bytes) {
U32 amount = num_bytes - pos;
if (amount > chunk_size) amount = chunk_size;
gdraw_gpu_memcpy(c, dst + pos, src + pos, amount);
pos += amount;
}
}
}
static rrbool gdraw_CanDefragment(GDrawHandleCache* c) {
// we can defragment (and extract some gain from it) if and only if there's
// more than one free block. since gfxalloc coalesces free blocks
// immediately and keeps them in a circular linked list, this is very easy
// to detect: just check if the "next" pointer of the first free block
// points to the sentinel. (this is only the case if there are 0 or 1 free
// blocks)
gfx_allocator* alloc = c->alloc;
return alloc->blocks[0].next->next != alloc->blocks;
}
static void gdraw_DefragmentMain(GDrawHandleCache* c, U32 flags,
GDrawStats* stats) {
gfx_allocator* alloc = c->alloc;
gfx_block_info *b, *n;
U8* p;
S32 i;
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(alloc->num_blocks ==
alloc->num_alloc + alloc->num_free +
alloc->num_unused);)
GFXALLOC_IF_CHECK(
GFXALLOC_ASSERT(alloc->num_free <= alloc->num_blocks + 1);)
// go over all allocated memory blocks and clear the "prev" pointer
// (unused for allocated blocks, we'll use it to store a back-pointer to the
// corresponding handle)
for (b = alloc->blocks[0].next_phys; b != alloc->blocks; b = b->next_phys)
if (!b->is_free) b->prev = NULL;
// go through all handles and store a pointer to the handle in the
// corresponding memory block
for (i = 0; i < c->max_handles; i++)
if (c->handle[i].raw_ptr) {
assert(c->handle[i].bytes != 0);
for (b = alloc->hash[gfxalloc_get_hash_code(alloc,
c->handle[i].raw_ptr)];
b; b = b->next)
if (b->ptr == c->handle[i].raw_ptr) {
void* block = &c->handle[i];
b->prev = (gfx_block_info*)block;
break;
}
GFXALLOC_ASSERT(b != NULL); // didn't find this block anywhere!
}
// clear alloc hash table (we rebuild it during defrag)
memset(alloc->hash, 0, sizeof(alloc->hash));
// defragmentation proper: go over all blocks again, remove all free blocks
// from the physical block list and compact the remaining blocks together.
p = alloc->mem_base;
for (b = alloc->blocks[0].next_phys; b != alloc->blocks; b = n) {
n = b->next_phys;
if (!b->is_free) {
U32 h;
// move block if necessary
if (p != b->ptr) {
assert(b->size != 0);
gdraw_Defragment_memmove(c, p, b->ptr, b->size, flags, stats);
b->ptr = p;
assert(b->prev);
if (b->prev) ((GDrawHandle*)b->prev)->raw_ptr = p;
}
// re-insert into hash table
h = gfxalloc_get_hash_code(alloc, p);
b->next = alloc->hash[h];
alloc->hash[h] = b;
p += b->size;
} else {
// free block: remove it from the physical block list
b->prev_phys->next_phys = b->next_phys;
b->next_phys->prev_phys = b->prev_phys;
gfxalloc_rem_free(alloc, b);
gfxalloc_push_unused(alloc, b);
}
}
// the free list should be empty now
assert(alloc->blocks[0].next == &alloc->blocks[0]);
// unless all memory is allocated, we now need to add a new block for the
// free space at the end
if (p != alloc->mem_end) {
b = gfxalloc_pop_unused(alloc);
b->ptr = p;
b->prev_phys = alloc->blocks[0].prev_phys;
b->next_phys = &alloc->blocks[0];
b->prev_phys->next_phys = b;
b->next_phys->prev_phys = b;
b->size = alloc->mem_end - p;
gfxalloc_add_free(alloc, b);
}
GFXALLOC_IF_CHECK(GFXALLOC_ASSERT(alloc->num_blocks ==
alloc->num_alloc + alloc->num_free +
alloc->num_unused);)
GFXALLOC_IF_CHECK(
GFXALLOC_ASSERT(alloc->num_free <= alloc->num_blocks + 1);)
}
#endif
#if defined(GDRAW_MANAGE_MEM_TWOPOOL)
// Defragmentation code for graphics memory, using two-pool strategy.
//
// The platform implementation must provide a GPU memcpy function and handle
// all necessary synchronization. It must also adjust its resource descriptors
// to match the new addresses after defragmentation.
//
// The high concept for two-pool is that we can't update the resource pools
// mid-frame; instead, while preparing for a frame, we need to produce a memory
// configuration that is suitable for rendering a whole frame at once (in
// contrast to our normal incremental strategy, where we can decide to
// defragment mid-frame if things are getting desperate). This is for tiled
// renderers.
//
// Two-pool works like this:
// - As the name suggests, each handle cache has two memory pools and
// corresponding backing
// allocators. The currently used allocator, "alloc", and a second allocator,
// "alloc_other".
// - Any resource used in a command buffer gets locked and *stays locked* until
// we're done
// preparing that command buffer (i.e. no unlocking after every draw as in the
// normal incremental memory management).
// - All allocations happen from "alloc", always. We mostly do our normal LRU
// cache freeing
// to make space when required.
// - We can still run out of space (no surprise) and get into a configuration
// where we have
// to defragment. This is the only tricky part, and where the second pool
// comes in. To defragment, we switch the roles of "alloc" and "alloc_other",
// and allocate new backing storage for all currently "locked" and "pinned"
// resources (i.e. everything we've used in the currently pending frame).
// - In general, we have the invariant that all resources we're using for
// batches we're
// working on must be in the "alloc" (fresh) pool, not in the "other" (stale)
// pool. Therefore, after a defragment/pool switch, any "live" resource (which
// means it's present in the stale pool) has to be copied to the "fresh" pool
// as it's getting locked to maintain this invariant.
//
// What this does is give us a guarantee that any given frame either only
// references resources in one pool (the common case), or does a defragment, in
// which case it looks like this:
//
// +------------------------------+
// | |
// | | pool A is fresh (=alloc), pool B is stale
// (=alloc_other) | | all resources referenced
// in here are in pool A | | | | | |
// +------------------------------+ <-- defragment! pools flip roles here
// | |
// | |
// | | pool B is fresh (=alloc), pool A is stale
// (=alloc_other) | | all resources referenced
// in here are in pool B | |
// +------------------------------+
//
// Now, at the end of the frame, we need to decide what to do with the
// resources that remain "live" (i.e. they're in the old pool but weren't
// referenced in the current frame so they didn't get copied). As of this
// writing, we simply free them, to maximize the amount of free memory in the
// new pool (and hopefully minimize the chance that we'll have to defragment
// again soon). It would also be possible to copy some of them though, assuming
// there's enough space.
//
// Freeing resources is an interesting case. When the CPU side of GDraw does a
// "free", we can't immediately reclaim the resource memory, since the GPU will
// generally still have outstanding commands that reference that resource. So
// our freed resources first enter the "Dead" state and only actually get freed
// once the GPU is done with them. What this means is that the list of
// resources in the "dead" state can end up holding references to both the
// fresh and the stale pool; the free implementation needs to be aware of this
// and return the memory to the right allocator.
//
// When we defragment, it's important to make sure that the pool we're flipping
// to is actually empty. What this means is that right before a defragment, we
// need to wait for all stale "dead" resources to actually become free. If the
// last defragment was several frames ago, this is fast - we haven't generated
// any new commands referencing the stale resources in several frames, so most
// likely they're all immediately free-able. By contrast, if we just
// defragmented last frame, this will be a slow operation since we need to wait
// for the GPU pipeline to drain - but if you're triggering defragments in
// several consecutive frames, you're thrashing the resource pools badly and
// are getting really bad performance anyway.
static void gdraw_gpu_memcpy(GDrawHandleCache* c, void* dst, void* src,
U32 num_bytes);
static void gdraw_gpu_wait_for_transfer_completion();
static void gdraw_resource_moved(GDrawHandle* t);
static rrbool gdraw_CanDefragment(GDrawHandleCache* c) {
// we can defragment (and extract some gain from it) if and only if there's
// more than one free block. since gfxalloc coalesces free blocks
// immediately and keeps them in a circular linked list, this is very easy
// to detect: just check if the "next" pointer of the first free block
// points to the sentinel. (this is only the case if there are 0 or 1 free
// blocks)
gfx_allocator* alloc = c->alloc;
if (!c->alloc_other) // if we don't have a second pool, we can't defrag at
// all.
return false;
return alloc->blocks[0].next->next != alloc->blocks;
}
static rrbool gdraw_MigrateResource(GDrawHandle* t, GDrawStats* stats) {
GDrawHandleCache* c = t->cache;
void* ptr = NULL;
assert(t->state == GDRAW_HANDLE_STATE_live ||
t->state == GDRAW_HANDLE_STATE_locked ||
t->state == GDRAW_HANDLE_STATE_pinned);
// anything we migrate should be in the "other" (old) pool
assert(gfxalloc_mem_contains(c->alloc_other, t->raw_ptr));
ptr = gfxalloc_alloc(c->alloc, t->bytes);
if (ptr) {
// update stats
stats->nonzero_flags |= GDRAW_STATS_defrag;
stats->defrag_objects += 1;
stats->defrag_bytes += t->bytes;
// copy contents to new storage
gdraw_gpu_memcpy(c, ptr, t->raw_ptr, t->bytes);
// free old storage
gfxalloc_free(c->alloc_other, t->raw_ptr);
// adjust pointers to point to new location
t->raw_ptr = ptr;
gdraw_resource_moved(t);
return true;
} else
return false;
}
static rrbool gdraw_MigrateAllResources(GDrawHandle* sentinel,
GDrawStats* stats) {
GDrawHandle* h;
for (h = sentinel->next; h != sentinel; h = h->next) {
if (!gdraw_MigrateResource(h, stats)) return false;
}
return true;
}
static rrbool gdraw_TwoPoolDefragmentMain(GDrawHandleCache* c,
GDrawStats* stats) {
gfx_allocator* t;
// swap allocators
t = c->alloc;
c->alloc = c->alloc_other;
c->alloc_other = t;
// immediately migrate all currently pinned and locked resources
rrbool ok = true;
ok = ok &&
gdraw_MigrateAllResources(&c->state[GDRAW_HANDLE_STATE_pinned], stats);
ok = ok &&
gdraw_MigrateAllResources(&c->state[GDRAW_HANDLE_STATE_locked], stats);
return ok;
}
static rrbool gdraw_StateListIsEmpty(GDrawHandle* head) {
// a list is empty when the head sentinel is the only node
return head->next == head;
}
static void gdraw_CheckAllPointersUpdated(GDrawHandle* head) {
#if defined(GDRAW_DEBUG)
GDrawHandle* h;
for (h = head->next; h != head; h = h->next) {
assert(gfxalloc_mem_contains(h->cache->alloc, h->raw_ptr));
}
#endif
}
static void gdraw_PostDefragmentCleanup(GDrawHandleCache* c,
GDrawStats* stats) {
// if we defragmented during this scene, this is the spot where
// we need to nuke all references to resources that weren't
// carried over into the new pool.
if (c->did_defragment) {
GDrawHandle* h;
// alloc list should be empty at this point
assert(gdraw_StateListIsEmpty(&c->state[GDRAW_HANDLE_STATE_alloc]));
// free all remaining live resources (these are the resources we didn't
// touch this frame, hence stale)
h = &c->state[GDRAW_HANDLE_STATE_live];
while (!gdraw_StateListIsEmpty(h)) gdraw_res_free(h->next, stats);
// "live" is now empty, and we already checked that "alloc" was empty
// earlier. "dead" may hold objects on the old heap still (that were
// freed before we swapped allocators). "user owned" is not managed by
// us. that leaves "locked" and "pinned" resources, both of which better
// be only pointing into the new heap now!
gdraw_CheckAllPointersUpdated(&c->state[GDRAW_HANDLE_STATE_locked]);
gdraw_CheckAllPointersUpdated(&c->state[GDRAW_HANDLE_STATE_pinned]);
gdraw_gpu_wait_for_transfer_completion();
}
}
#endif
// Image processing code
// Compute average of 4 RGBA8888 pixels passed as U32.
// Variables are named assuming the values are stored as big-endian, but all
// bytes are treated equally, so this code will work just fine on little-endian
// data.
static U32 gdraw_Avg4_rgba8888(U32 p0, U32 p1, U32 p2, U32 p3) {
U32 mask = 0x00ff00ff;
U32 bias = 0x00020002;
U32 gasum = ((p0 >> 0) & mask) + ((p1 >> 0) & mask) + ((p2 >> 0) & mask) +
((p3 >> 0) & mask) + bias;
U32 rbsum = ((p0 >> 8) & mask) + ((p1 >> 8) & mask) + ((p2 >> 8) & mask) +
((p3 >> 8) & mask) + bias;
return ((gasum >> 2) & mask) | ((rbsum << 6) & ~mask);
}
// Compute average of 2 RGBA8888 pixels passed as U32
static U32 gdraw_Avg2_rgba8888(U32 p0, U32 p1) {
return (p0 | p1) - (((p0 ^ p1) >> 1) & 0x7f7f7f7f);
}
// 2:1 downsample in both horizontal and vertical direction, for one line.
// width is width of destination line.
static void gdraw_Downsample_2x2_line(U8* dst, U8* line0, U8* line1, U32 width,
U32 bpp) {
U32 x;
if (bpp == 4) {
U32* in0 = (U32*)line0;
U32* in1 = (U32*)line1;
U32* out = (U32*)dst;
for (x = 0; x < width; x++, in0 += 2, in1 += 2)
*out++ = gdraw_Avg4_rgba8888(in0[0], in0[1], in1[0], in1[1]);
} else if (bpp == 1) {
for (x = 0; x < width; x++, line0 += 2, line1 += 2)
*dst++ = (line0[0] + line0[1] + line1[0] + line1[1] + 2) / 4;
} else
RR_BREAK();
}
// 2:1 downsample in horizontal but not vertical direction.
static void gdraw_Downsample_2x1_line(U8* dst, U8* src, U32 width, U32 bpp) {
U32 x;
if (bpp == 4) {
U32* in = (U32*)src;
U32* out = (U32*)dst;
for (x = 0; x < width; x++, in += 2)
*out++ = gdraw_Avg2_rgba8888(in[0], in[1]);
} else if (bpp == 1) {
for (x = 0; x < width; x++, src += 2)
*dst++ = (src[0] + src[1] + 1) / 2;
} else
RR_BREAK();
}
// 2:1 downsample in vertical but not horizontal direction.
static void gdraw_Downsample_1x2(U8* dst, S32 dstpitch, U8* src, S32 srcpitch,
U32 height, U32 bpp) {
U32 y;
if (bpp == 4) {
for (y = 0; y < height; y++, dst += dstpitch, src += 2 * srcpitch)
*((U32*)dst) =
gdraw_Avg2_rgba8888(*((U32*)src), *((U32*)(src + srcpitch)));
} else if (bpp == 1) {
for (y = 0; y < height; y++, dst += dstpitch, src += 2 * srcpitch)
*dst = (src[0] + src[srcpitch] + 1) / 2;
} else
RR_BREAK();
}
// 2:1 downsample (for mipmaps)
// dst: Pointer to destination buffer
// dstpitch: Pitch for destination buffer
// width: Width of *destination* image (i.e. downsampled version)
// height: Height of *destination* image (i.e. downsampled version)
// src: Pointer to source buffer
// srcpitch: Pitch of source buffer
// bpp: Bytes per pixel for image data
//
// can be used for in-place resizing if src==dst and dstpitch <= srcpitch!
static GDRAW_MAYBE_UNUSED void gdraw_Downsample(U8* dst, S32 dstpitch,
U32 width, U32 height, U8* src,
S32 srcpitch, U32 bpp) {
U32 y;
assert(bpp == 1 || bpp == 4);
// @TODO gamma?
if (!height) // non-square texture, height was reduced to 1 in a previous
// step
gdraw_Downsample_2x1_line(dst, src, width, bpp);
else if (!width) // non-square texture, width was reduced to 1 in a
// previous step
gdraw_Downsample_1x2(dst, dstpitch, src, srcpitch, height, bpp);
else {
for (y = 0; y < height; y++) {
gdraw_Downsample_2x2_line(dst, src, src + srcpitch, width, bpp);
dst += dstpitch;
src += 2 * srcpitch;
}
}
}
#if !defined(GDRAW_NO_STREAMING_MIPGEN)
#define GDRAW_MAXMIPS 16 // maximum number of mipmaps supported.
typedef struct GDrawMipmapContext {
U32 width; // width of the texture being mipmapped
U32 height; // height of the texture being mipmapped
U32 mipmaps; // number of mipmaps
U32 bpp; // bytes per pixel
U32 partial_row; // bit N: is mipmap N currently storing a partial row?
U32 bheight; // height of the buffer at miplevel 0
U8* pixels[GDRAW_MAXMIPS];
U32 pitch[GDRAW_MAXMIPS];
} GDrawMipmapContext;
static rrbool gdraw_MipmapBegin(GDrawMipmapContext* c, U32 width, U32 height,
U32 mipmaps, U32 bpp, U8* buffer,
U32 buffer_size) {
U32 i;
U8* p;
if (mipmaps > GDRAW_MAXMIPS) return false;
c->width = width;
c->height = height;
c->mipmaps = mipmaps;
c->bpp = bpp;
c->partial_row = 0;
// determine how many lines to buffer
// we try to use roughly 2/3rds of the buffer for the first miplevel (less
// than 3/4 since with our partial line buffers, we have extra buffer space
// for lower mip levels).
c->bheight = (2 * buffer_size) / (3 * width * bpp);
// round down to next-smaller power of 2 (in case we need to swizzle;
// swizzling works on pow2-sized blocks)
while (c->bheight & (c->bheight - 1)) // while not a power of 2...
c->bheight &= c->bheight - 1; // clear least significant bit set
// then keep lowering the number of buffered lines until they fit (or we
// reach zero, i.e. it doesn't fit)
while (c->bheight) {
p = buffer;
for (i = 0; i < c->mipmaps; i++) {
U32 mw = c->width >> i;
U32 bh = c->bheight >> i;
if (!mw) mw++;
if (!bh) mw *= 2, bh++; // need space for line of previous miplevel
c->pixels[i] = p;
c->pitch[i] = mw * bpp;
p += c->pitch[i] * bh;
}
// if it fits, we're done
if (p <= buffer + buffer_size) {
if (c->bheight >
height) // buffer doesn't need to be larger than the image!
c->bheight = height;
return true;
}
// need to try a smaller line buffer...
c->bheight >>= 1;
}
// can't fit even one line into our buffer. ouch!
return false;
}
// returns true if there was data generated for this miplevel, false otherwise.
static rrbool gdraw_MipmapAddLines(GDrawMipmapContext* c, U32 level) {
U32 bw, bh;
assert(level > 0); // doesn't make sense to call this on level 0
if (level == 0 || level >= c->mipmaps)
return false; // this level doesn't exist
bw = c->width >> level; // buffer width at this level
bh = c->bheight >> level; // buffer height at this level
if (bh) { // we can still do regular downsampling
gdraw_Downsample(c->pixels[level], c->pitch[level], bw, bh,
c->pixels[level - 1], c->pitch[level - 1], c->bpp);
return true;
} else if (c->height >> level) { // need to buffer partial lines, but still
// doing vertical 2:1 downsampling
if ((c->partial_row ^= (1 << level)) &
(1 << level)) { // no buffered partial row for this miplevel yet,
// make one
memcpy(c->pixels[level], c->pixels[level - 1], bw * 2 * c->bpp);
return false;
} else { // have one buffered row, can generate output pixels
gdraw_Downsample_2x2_line(c->pixels[level], c->pixels[level],
c->pixels[level - 1], bw, c->bpp);
return true;
}
} else { // finish off with a chain of Nx1 miplevels
gdraw_Downsample_2x1_line(c->pixels[level], c->pixels[level - 1], bw,
c->bpp);
return true;
}
}
#endif
#if defined(GDRAW_CHECK_BLOCK)
static void check_block_alloc(gfx_allocator* alloc, void* ptr,
rrbool allocated) {
int i, n = 0, m = 0;
for (i = 0; i < GFXALLOC_HASH_SIZE; ++i) {
gfx_block_info* b = alloc->hash[i];
while (b) {
if (b->ptr == ptr) ++n;
b = b->next;
}
}
gfx_block_info* b = alloc->blocks[0].next;
while (b != &alloc->blocks[0]) {
if (b->ptr == ptr) ++m;
b = b->next;
}
if (allocated)
assert(n == 1 && m == 0);
else
assert(n == 0 && m == 1);
}
#else
#define check_block_alloc(a, p, f)
#endif
#if defined(GDRAW_BUFFER_RING)
////////////////////////////////////////////////////////////////////////
//
// Buffer ring
//
// Implements a dynamic buffer backed by multiple physical buffers, with
// the usual append-only, DISCARD/NOOVERWRITE semantics.
//
// This can be used for dynamic vertex buffers, constant buffers, etc.
#define GDRAW_BUFRING_MAXSEGS 4 // max number of backing segments
typedef struct gdraw_bufring_seg {
struct gdraw_bufring_seg* next; // next segment in ring
U8* data; // pointer to the allocation
GDrawFence fence; // fence for this segment
U32 used; // number of bytes used
} gdraw_bufring_seg;
typedef struct gdraw_bufring {
gdraw_bufring_seg* cur; // active ring segment
U32 seg_size; // size of one segment
U32 align; // alignment of segment allocations
gdraw_bufring_seg all_segs[GDRAW_BUFRING_MAXSEGS];
} gdraw_bufring;
// forwards
static GDrawFence put_fence();
static void wait_on_fence(GDrawFence fence);
static void gdraw_bufring_init(gdraw_bufring* RADRESTRICT ring, void* ptr,
U32 size, U32 nsegs, U32 align) {
U32 i, seg_size;
ring->seg_size = 0;
if (!ptr || nsegs < 1 ||
size < nsegs * align) // bail if no ring buffer memory or too small
return;
if (nsegs > GDRAW_BUFRING_MAXSEGS) nsegs = GDRAW_BUFRING_MAXSEGS;
// align needs to be a positive power of two
assert(align >= 1 && (align & (align - 1)) == 0);
// buffer really needs to be properly aligned
assert(((UINTa)ptr & (align - 1)) == 0);
seg_size = (size / nsegs) & ~(align - 1);
for (i = 0; i < nsegs; ++i) {
ring->all_segs[i].next = &ring->all_segs[(i + 1) % nsegs];
ring->all_segs[i].data = (U8*)ptr + i * seg_size;
ring->all_segs[i].fence.value = 0;
ring->all_segs[i].used = 0;
}
ring->cur = ring->all_segs;
ring->seg_size = seg_size;
ring->align = align;
}
static void gdraw_bufring_shutdown(gdraw_bufring* RADRESTRICT ring) {
ring->cur = NULL;
ring->seg_size = 0;
}
static void* gdraw_bufring_alloc(gdraw_bufring* RADRESTRICT ring, U32 size,
U32 align) {
U32 align_up;
gdraw_bufring_seg* seg;
if (size > ring->seg_size) return NULL; // nope, won't fit
assert(align <= ring->align);
// check if it fits in the active segment first
seg = ring->cur;
align_up = (seg->used + align - 1) & -align;
if ((align_up + size) <= ring->seg_size) {
void* ptr = seg->data + align_up;
seg->used = align_up + size;
return ptr;
}
// doesn't fit, we have to start a new ring segment.
seg->fence = put_fence();
// switch to the next segment, wait till GPU is done with it
seg = ring->cur = seg->next;
wait_on_fence(seg->fence);
// allocate from the new segment. we assume that segment offsets
// satisfy the highest alignment requirements we ever ask for!
seg->used = size;
return seg->data;
}
#endif
////////////////////////////////////////////////////////////////////////
//
// General resource manager
//
#if !defined(GDRAW_FENCE_FLUSH)
#define GDRAW_FENCE_FLUSH()
#endif
#if defined(GDRAW_MANAGE_MEM)
// functions the platform must implement
#if !defined(GDRAW_BUFFER_RING // avoid "redundant redeclaration" warning)
static void wait_on_fence(GDrawFence fence);
#endif
static rrbool is_fence_pending(GDrawFence fence);
static void gdraw_defragment_cache(GDrawHandleCache* c, GDrawStats* stats);
// functions we implement
static void gdraw_res_reap(GDrawHandleCache* c, GDrawStats* stats);
#endif
// If GDRAW_MANAGE_MEM is not #defined, this needs to perform the
// actual free using whatever API we're targeting.
//
// If GDRAW_MANAGE_MEM is #defined, the shared code handles the
// memory management part, but you might still need to update
// your state caching.
static void api_free_resource(GDrawHandle* r);
// Actually frees a resource and releases all allocated resources
static void gdraw_res_free(GDrawHandle* r, GDrawStats* stats) {
assert(r->state == GDRAW_HANDLE_STATE_live ||
r->state == GDRAW_HANDLE_STATE_locked ||
r->state == GDRAW_HANDLE_STATE_dead ||
r->state == GDRAW_HANDLE_STATE_pinned ||
r->state == GDRAW_HANDLE_STATE_user_owned);
#if defined(GDRAW_MANAGE_MEM)
GDRAW_FENCE_FLUSH();
// make sure resource isn't in use before we actually free the memory
wait_on_fence(r->fence);
if (r->raw_ptr) {
#if !defined(GDRAW_MANAGE_MEM_TWOPOOL)
gfxalloc_free(r->cache->alloc, r->raw_ptr);
#else
GDrawHandleCache* c = r->cache;
if (gfxalloc_mem_contains(c->alloc, r->raw_ptr))
gfxalloc_free(c->alloc, r->raw_ptr);
else {
assert(gfxalloc_mem_contains(c->alloc_other, r->raw_ptr));
gfxalloc_free(c->alloc_other, r->raw_ptr);
}
#endif
}
#endif
api_free_resource(r);
stats->nonzero_flags |= GDRAW_STATS_frees;
stats->freed_objects += 1;
stats->freed_bytes += r->bytes;
gdraw_HandleCacheFree(r);
}
// Frees the LRU resource in the given cache.
static rrbool gdraw_res_free_lru(GDrawHandleCache* c, GDrawStats* stats) {
GDrawHandle* r = gdraw_HandleCacheGetLRU(c);
if (!r) return false;
if (c->is_vertex && r->owner) // check for r->owner since it may already be
// killed (if player destroyed first)
IggyDiscardVertexBufferCallback(r->owner, r);
// was it referenced since end of previous frame (=in this frame)?
// if some, we're thrashing; report it to the user, but only once per frame.
if (c->prev_frame_end.value < r->fence.value && !c->is_thrashing) {
IggyGDrawSendWarning(NULL, c->is_vertex
? "GDraw Thrashing vertex memory"
: "GDraw Thrashing texture memory");
c->is_thrashing = true;
}
gdraw_res_free(r, stats);
return true;
}
static void gdraw_res_flush(GDrawHandleCache* c, GDrawStats* stats) {
c->is_thrashing = true; // prevents warnings being generated from free_lru
gdraw_HandleCacheUnlockAll(c);
while (gdraw_res_free_lru(c, stats));
}
static GDrawHandle* gdraw_res_alloc_outofmem(GDrawHandleCache* c,
GDrawHandle* t,
char const* failed_type) {
if (t) gdraw_HandleCacheAllocateFail(t);
IggyGDrawSendWarning(NULL,
c->is_vertex ? "GDraw Out of static vertex buffer %s"
: "GDraw Out of texture %s",
failed_type);
return NULL;
}
#if !defined(GDRAW_MANAGE_MEM)
static GDrawHandle* gdraw_res_alloc_begin(GDrawHandleCache* c, S32 size,
GDrawStats* stats) {
GDrawHandle* t;
if (size > c->total_bytes)
gdraw_res_alloc_outofmem(
c, NULL, "memory (single resource larger than entire pool)");
else {
// given how much data we're going to allocate, throw out
// data until there's "room" (this basically lets us use
// managed memory and just bound our usage, without actually
// packing it and being exact)
while (c->bytes_free < size) {
if (!gdraw_res_free_lru(c, stats)) {
gdraw_res_alloc_outofmem(c, NULL, "memory");
break;
}
}
}
// now try to allocate a handle
t = gdraw_HandleCacheAllocateBegin(c);
if (!t) {
// it's possible we have no free handles, because all handles
// are in use without exceeding the max storage above--in that
// case, just free one texture to give us a free handle (ideally
// we'd trade off cost of regenerating)
if (gdraw_res_free_lru(c, stats)) {
t = gdraw_HandleCacheAllocateBegin(c);
if (t == NULL) {
gdraw_res_alloc_outofmem(c, NULL, "handles");
}
}
}
return t;
}
#else
// Returns whether this resource holds pointers to one of the GDraw-managed
// pools.
static rrbool gdraw_res_is_managed(GDrawHandle* r) {
return r->state == GDRAW_HANDLE_STATE_live ||
r->state == GDRAW_HANDLE_STATE_locked ||
r->state == GDRAW_HANDLE_STATE_dead ||
r->state == GDRAW_HANDLE_STATE_pinned;
}
// "Reaps" dead resources. Even if the user requests that a
// resource be freed, it might still be in use in a pending
// command buffer. So we can't free the associated memory
// immediately; instead, we flag the resource as "dead" and
// periodically check whether we can actually free the
// pending memory of dead resources ("reap" them).
static void gdraw_res_reap(GDrawHandleCache* c, GDrawStats* stats) {
GDrawHandle* sentinel = &c->state[GDRAW_HANDLE_STATE_dead];
GDrawHandle* t;
GDRAW_FENCE_FLUSH();
// reap all dead resources that aren't in use anymore
while ((t = sentinel->next) != sentinel && !is_fence_pending(t->fence))
gdraw_res_free(t, stats);
}
// "Kills" a resource. This means GDraw won't use it anymore
// (it's dead), but there might still be outstanding references
// to it in a pending command buffer, so we can't physically
// free the associated memory until that's all processed.
static void gdraw_res_kill(GDrawHandle* r, GDrawStats* stats) {
GDRAW_FENCE_FLUSH(); // dead list is sorted by fence index - make sure all
// fence values are current.
r->owner = NULL;
gdraw_HandleCacheInsertDead(r);
gdraw_res_reap(r->cache, stats);
}
static GDrawHandle* gdraw_res_alloc_begin(GDrawHandleCache* c, S32 size,
GDrawStats* stats) {
GDrawHandle* t;
void* ptr = NULL;
gdraw_res_reap(c, stats); // NB this also does GDRAW_FENCE_FLUSH();
if (size > c->total_bytes)
return gdraw_res_alloc_outofmem(
c, NULL, "memory (single resource larger than entire pool)");
// now try to allocate a handle
t = gdraw_HandleCacheAllocateBegin(c);
if (!t) {
// it's possible we have no free handles, because all handles
// are in use without exceeding the max storage above--in that
// case, just free one texture to give us a free handle (ideally
// we'd trade off cost of regenerating)
gdraw_res_free_lru(c, stats);
t = gdraw_HandleCacheAllocateBegin(c);
if (!t) return gdraw_res_alloc_outofmem(c, NULL, "handles");
}
// try to allocate first
if (size) {
ptr = gfxalloc_alloc(c->alloc, size);
if (!ptr) {
// doesn't currently fit. try to free some allocations to get space
// to breathe.
S32 want_free = RR_MAX(size + (size / 2), GDRAW_MIN_FREE_AMOUNT);
if (want_free > c->total_bytes)
want_free = size; // okay, *really* big resource, just try to
// allocate its real size
// always keep freeing textures until want_free bytes are free.
while (c->alloc->actual_bytes_free < want_free) {
if (!gdraw_res_free_lru(c, stats))
return gdraw_res_alloc_outofmem(c, t, "memory");
}
// now, keep trying to allocate and free some more memory when it
// still doesn't fit
while (!(ptr = gfxalloc_alloc(c->alloc, size))) {
if (c->alloc->actual_bytes_free >=
3 * size || // if we should have enough free bytes to
// satisfy the request by now
(c->alloc->actual_bytes_free >= size &&
size * 2 >=
c->total_bytes)) // or the resource is very big and
// the alloc doesn't fit
{
// before we actually consider defragmenting, we want to
// free all stale resources (not referenced in the previous
// 2 frames). and if that frees up enough memory so we don't
// have to defragment, all the better! also, never
// defragment twice in a frame, just assume we're thrashing
// when we get in that situation and free up as much as
// possible.
if (!c->did_defragment &&
c->prev_frame_start.value <= c->handle->fence.value) {
// defragment.
defrag:
if (gdraw_CanDefragment(
c)) { // only try defrag if it has a chance of
// helping.
gdraw_defragment_cache(c, stats);
c->did_defragment = true;
}
ptr = gfxalloc_alloc(c->alloc, size);
if (!ptr)
return gdraw_res_alloc_outofmem(
c, t, "memory (fragmentation)");
break;
}
}
// keep trying to free some more
if (!gdraw_res_free_lru(c, stats)) {
if (c->alloc->actual_bytes_free >=
size) // nothing left to free but we should be good -
// defrag again, even if it's the second time in
// a frame
goto defrag;
return gdraw_res_alloc_outofmem(c, t, "memory");
}
}
}
}
t->fence.value = 0; // hasn't been used yet
t->raw_ptr = ptr;
return t;
}
#endif
Reference in a new issue View git blame Copy permalink