#include "sys_tilestore.h"
#include "system.h"
#include "idxmap.h"
#include "stack32.h"
#include "srswq.h"
#include "sys_log.h"
#include "sys_vr.h"
#include "scenefile.h"
#include "platform.h"
#include "imgload.h"
#include <stdlib.h>
#include "sys_render.h"

#ifdef _DEBUG
#define GLERRCHECK {GLuint glerr = glGetError(); if(glerr) yrLog(0, "OpenGL error: %x", glerr);}
#else
#define GLERRCHECK
#endif

#define TICK_MARGIN 1414

#define N_IMPORTANT (128)
#define N_BLANK (128)
#define N_EXBLANK (64)
#define N_READERS (4)
#define N_WRITERS (4)
#define JOBS_RECOVER_BREAK (16)
#define MIPS_PER_FULL (8)
#define EXPAND_STEP (4)

#define RQ_EMPTY (0)
#define RQ_MIPS (1)
#define RQ_FULL (2)

#define JobMask		(0x000000FF00000000ull)
#define JobExpand	(0x0000000100000000ull)
#define JobShift	(0x0000000200000000ull)
#define JobUpload	(0x0000000300000000ull)
#define JobFlush	(0x0000000400000000ull)
#define JobIOMask	(0x00000000FFFFFFFFull)

static const size_t mipoff[9] = {
	0,
	256 * 256,
	256 * 256 + 128 * 128,
	256 * 256 + 128 * 128 + 64 * 64,
	256 * 256 + 128 * 128 + 64 * 64 + 32 * 32,
	256 * 256 + 128 * 128 + 64 * 64 + 32 * 32 + 16 * 16,
	256 * 256 + 128 * 128 + 64 * 64 + 32 * 32 + 16 * 16 + 8 * 8,
	256 * 256 + 128 * 128 + 64 * 64 + 32 * 32 + 16 * 16 + 8 * 8 + 4 * 4,
	256 * 256 + 128 * 128 + 64 * 64 + 32 * 32 + 16 * 16 + 8 * 8 + 4 * 4 + 2 * 2,
};

static int yrTileStore_init(void* loadtoken);
static int yrTileStore_tick(void);
static int yrTileStore_save(void* savetoken, void* saveevent, int* savefail)  { yrEvent_set(saveevent, 1); return 0; }
static int yrTileStore_canfreeze(void) { return 1; }
static int yrTileStore_freeze(void) { return 0; }
static int yrTileStore_unfreeze(void) { return 0; }
static int yrTileStore_shutdown(void);

void yrTileStore_reg(void)
{
	yrSystem_register(sysTileStore,
					  yrTileStore_init,
					  yrTileStore_tick,
					  yrTileStore_save,
					  yrTileStore_canfreeze,
					  yrTileStore_freeze,
					  yrTileStore_unfreeze,
					  yrTileStore_shutdown,
					  (1<<sysLog)|(1<<sysRender));
}

/********
* Structs
*********/
struct tilerecord
{
	TileID id;
	uint8_t rq_extra;
	uint8_t rq_ideal;
	uint8_t dirty;
	uint8_t _padding;
	GLuint full_glid;
	GLuint mips_glid;
	uint64_t full_handle;
	uint64_t mips_handle;
};

struct rqlist
{
	size_t count;
	TileID* tile;
	float* metric;
};

struct ideal
{
	size_t count;
	TileID* tile;
	TileIndex* idx;
};

/************
* Static vars
*************/
static yrIdxmap*	idxmap = NULL;
static int			shutting_down = 0;
static int			shutting_down_jobs = 0;
static yrSceneFile* scenefile = NULL;
static GLuint		empty_glid = 0;
static uint32_t*	empty_data = NULL;
static struct tilerecord*	store = NULL;
static int					store_maxed = 0;
static yrEvent				store_expand_done = NULL;
static size_t	capacity_store = 0;
static size_t	capacity_mips = 0;
static size_t	capacity_full = 0;
static size_t	allocated_mips = 0;
static size_t	allocated_full = 0;

static struct rqlist rqlists[2] = {0};
static struct rqlist* rqlist_main = NULL;
static struct rqlist* rqlist_ideal = NULL;
static struct ideal ideals[2] = {0};
static struct ideal* ideal_ideal = NULL;
static struct ideal* ideal_jobs = NULL;

static TileID list_important[N_IMPORTANT];
static TileID list_blank[N_BLANK + N_EXBLANK];
static size_t list_important_i = 0;
static size_t list_blank_i = 0;
static size_t list_blank_free = N_BLANK + N_EXBLANK;

static yrStack32* texfull_reserve = NULL;
static yrStack32* texmips_reserve = NULL;
static yrSRSWQ* important_q = NULL;
static yrSRSWQ* delete_q = NULL;
static yrSRSWQ* blank_q = NULL;
static yrSRSWQ* jobs_q = NULL;

static int64_t perf_longest_job = 0;
static int64_t perf_last_tick = 0;

static struct iojob {
	yrThread thread;
	yrEvent done;
	yrEvent go;
	TileID tile;
	TileIndex idx;
	void* data;
}
io_read[N_READERS] = {0},
io_write[N_WRITERS] = {0};
static yrStack32* ioread_reserve = NULL;
static yrStack32* iowrite_reserve = NULL;

static yrEvent ideal_go = NULL;
static yrEvent ideal_done = NULL;
static yrThread ideal_thread = NULL;

static size_t jobs_stop_loop = 0;
static size_t jobs_i_ideal = 0;
static size_t jobs_i_store = 0;
static void* jobs_pending_delete = NULL;
static TileID jobs_next_id = {0};
static yrThread jobs_thread = NULL;

/*********************
* Forward declarations
**********************/
static int process_job(void);
static int handle_job_expand(void);
static void handle_job_flush(size_t writer);
static void reader_threadfunc(void* param);
static void writer_threadfunc(void* param);
static void ideal_threadfunc(void* param);
static void jobs_threadfunc(void* param);

/*****************
* Public Functions
******************/
TileID yrTileStore_new(GLuint* tex_id, GLuint* mip_id)
{
	GLERRCHECK;
	//get a new blank tile
	void* blankfoo = NULL;
	TileIndex blank;
	while(!(blankfoo = yrSRSWQ_read(blank_q)))
		process_job();
	blank.i = (uint32_t)(size_t) blankfoo;
	//wipe tile
	for(GLint lvl = 0; lvl < 9; lvl += 1)
		glCopyImageSubData(empty_glid, GL_TEXTURE_2D, lvl, 0,0,0, store[blank.i].full_glid, GL_TEXTURE_2D, lvl, 0,0,0, 256 >> lvl, 256 >> lvl, 1);
	for(GLint lvl = 2; lvl < 9; lvl += 1)
		glCopyImageSubData(empty_glid, GL_TEXTURE_2D, lvl, 0,0,0, store[blank.i].mips_glid, GL_TEXTURE_2D, lvl - 2, 0,0,0, 256 >> lvl, 256 >> lvl, 1);	
	//get the bindless handles
	store[blank.i].full_handle = glGetTextureHandleARB(store[blank.i].full_glid);
	store[blank.i].mips_handle = glGetTextureHandleARB(store[blank.i].mips_glid);
	GLERRCHECK;
	//return id + free textures
	*tex_id = store[blank.i].full_glid;
	*mip_id = store[blank.i].mips_glid;
	return store[blank.i].id;
}

TileID yrTileStore_copy(TileID tile, GLuint* tex_id, GLuint* mip_id)
{
	GLERRCHECK;
	//wait for source tile
	TileIndex srcidx;
	while(invalid_item == (srcidx.i = yrIdxmap_lookup(idxmap, tile.t)))
		process_job();
	while(!store[srcidx.i].full_handle) //TODO: put a timeout on this
		process_job();
	//get a new blank tile
	void* blankfoo = NULL;
	TileIndex blank;
	while(!(blankfoo = yrSRSWQ_read(blank_q)))
		process_job();
	blank.i = (uint32_t)(size_t) blankfoo;
	//copy tile
	for(GLint lvl = 0; lvl < 9; lvl += 1)
		glCopyImageSubData(store[srcidx.i].full_glid, GL_TEXTURE_2D, lvl, 0,0,0, store[blank.i].full_glid, GL_TEXTURE_2D, lvl, 0,0,0, 256 >> lvl, 256 >> lvl, 1);
	for(GLint lvl = 2; lvl < 9; lvl += 1)
		glCopyImageSubData(store[srcidx.i].full_glid, GL_TEXTURE_2D, lvl, 0,0,0, store[blank.i].mips_glid, GL_TEXTURE_2D, lvl - 2, 0,0,0, 256 >> lvl, 256 >> lvl, 1);	
	//get the bindless handles
	store[blank.i].full_handle = glGetTextureHandleARB(store[blank.i].full_glid);
	store[blank.i].mips_handle = glGetTextureHandleARB(store[blank.i].mips_glid);
	GLERRCHECK;
	//return id + free textures
	*tex_id = store[blank.i].full_glid;
	*mip_id = store[blank.i].mips_glid;
	return store[blank.i].id;
}

void yrTileStore_finishtile(TileID tile)
{
	TileIndex idx;
	idx.i = yrIdxmap_lookup(idxmap, tile.t);
	YR_ASSERT(idx.i != invalid_item);
	store[idx.i].dirty = 1;
}

void yrTileStore_remove(TileID tile)
{
	//prefix value so tile 0 doesn't look like the NULL pointer
	while(!yrSRSWQ_write(delete_q, (void*)(0x3900000000 | (size_t) tile.t)))
		process_job();
}

void yrTileStore_important(size_t count, TileID* tiles)
{
	for(size_t i = 0; i < count; ++i) {
		//prefix value so tile 0 doesn't look like the NULL pointer
		while(!yrSRSWQ_write(important_q, (void*)(0x3900000000 | (size_t) tiles[i].t)))
			process_job();
	}
}

uint64_t yrTileStore_request(TileID tile, float priority, TileIndex* lookup_hint, int do_lookup)
{
	//save request
	YR_ASSERT(rqlist_main->count < 64 * yrSystem_visible_quads()); 
	rqlist_main->tile[rqlist_main->count] = tile;
	rqlist_main->metric[rqlist_main->count] = priority;
	rqlist_main->count += 1;
	//check hint
	if(do_lookup) {
		lookup_hint->i = yrIdxmap_lookup(idxmap, tile.t);
	} else {
		if(lookup_hint->i != invalid_item &&
		   store[lookup_hint->i].id.t != tile.t)
		{
			lookup_hint->i = invalid_item;
		}
	}
	//return whatever is currently available
	if(lookup_hint->i == invalid_item) return 0;
	if(store[lookup_hint->i].full_handle) return store[lookup_hint->i].full_handle;
	if(store[lookup_hint->i].mips_handle) return store[lookup_hint->i].mips_handle;
	return 0;
}

/****************************************
* System functions (init, tick, shutdown)
*****************************************/
static int yrTileStore_init(void* loadtoken)
{
	shutting_down = 0;
	shutting_down_jobs = 0;
	scenefile = yrSystem_scenefile();
	perf_longest_job = 0;
	
	//empty tile
	empty_data = malloc((mipoff[8] + 1) * sizeof(uint32_t));
	if(!empty_data) { yrLog(0, "Out of memory"); return -1; }
	memset(empty_data, 0, (mipoff[8] + 1) * sizeof(uint32_t));
	glGenTextures(1, &empty_glid);
	glBindTexture(GL_TEXTURE_2D, empty_glid);
	glTexStorage2D(GL_TEXTURE_2D, 9, GL_SRGB8_ALPHA8, 256, 256);
	glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, 256, 256, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV, empty_data);
	glGenerateMipmap(GL_TEXTURE_2D);
	glBindTexture(GL_TEXTURE_2D, 0);
	GLenum glerr = glGetError();
	if(glerr != GL_NO_ERROR) { yrLog(0, "OpenGL error during empty tile creation: %x", glerr); goto onerror; }

	//calculate capacities
	size_t max_quads = yrSystem_visible_quads();
	size_t max_bytes = yrSystem_get_tilestore_bytes();
	size_t size_mips = 4*(1*1 + 2*2 + 4*4 + 8*8 + 16*16 + 32*32 + 64*64);
	size_t size_full = 4*(128*128 + 256*256) + size_mips;
	size_t min_bytes = (N_IMPORTANT + N_BLANK + N_EXBLANK) * (size_full + MIPS_PER_FULL * size_mips);
	if(max_bytes < min_bytes) max_bytes = min_bytes;
	capacity_full = max_bytes / (size_full + MIPS_PER_FULL * size_mips);
	capacity_mips = MIPS_PER_FULL * capacity_full;
	capacity_store = capacity_mips + N_IMPORTANT + N_BLANK + N_EXBLANK + 1;

	//allocate store and idxmap
	allocated_mips = 0;
	allocated_full = 0;
	store_maxed = 0;
	store_expand_done = yrEvent_create();
	if(!store_expand_done) { yrLog(0, "OS failed to create synchronization event"); goto onerror; }

	store = malloc(capacity_store * sizeof(struct tilerecord));
	if(!store) { yrLog(0, "Out of memory"); goto onerror; }
	memset(store, 0, capacity_store * sizeof(struct tilerecord));
	for(size_t i = 0; i < capacity_store; ++i)
		store[i].id.t = invalid_item;

	idxmap = yrIdxmap_create(capacity_store);
	if(!idxmap) { yrLog(0, "Out of memory"); goto onerror; }

	//request lists
	for(size_t i = 0; i < 2; ++i) {
		rqlists[i].count = 0;
		rqlists[i].tile = malloc(max_quads * 64 * sizeof(TileID));
		if(!rqlists[i].tile) { yrLog(0, "Out of memory"); goto onerror; }
		rqlists[i].metric = malloc(max_quads * 64 * sizeof(float));
		if(!rqlists[i].metric) { yrLog(0, "Out of memory"); goto onerror; }
	}
	rqlist_main = rqlists + 0;
	rqlist_ideal = rqlists + 1;

	//ideals
	for(size_t i = 0; i < 2; ++i) {
		ideals[i].count = 0;
		ideals[i].tile = malloc(capacity_mips * sizeof(TileID));
		if(!ideals[i].tile) { yrLog(0, "Out of memory"); goto onerror; }
		ideals[i].idx = malloc(capacity_mips * sizeof(TileIndex));
		if(!ideals[i].idx) { yrLog(0, "Out of memory"); goto onerror; }
		memset(ideals[i].idx, -1, capacity_mips * sizeof(TileIndex));
	}
	ideal_ideal = ideals + 0;
	ideal_jobs = ideals + 1;

	//important and blank lists
	memset(list_important, -1, sizeof(list_important));
	memset(list_blank, -1, sizeof(list_blank));
	list_important_i = 0;
	list_blank_i = 0;
	list_blank_free = N_BLANK + N_EXBLANK;

	//texture reserves
	texfull_reserve = yrStack32_create(capacity_full);
	if(!texfull_reserve) goto onerror;
	texmips_reserve = yrStack32_create(capacity_mips);
	if(!texmips_reserve) goto onerror;
	
	//communication queues
	important_q = yrSRSWQ_create(N_IMPORTANT);
	if(!important_q) { yrLog(0, "Out of memory"); goto onerror; }
	delete_q = yrSRSWQ_create(128);
	if(!delete_q) { yrLog(0, "Out of memory"); goto onerror; }
	blank_q = yrSRSWQ_create(N_BLANK);
	if(!blank_q) { yrLog(0, "Out of memory"); goto onerror; }
	jobs_q = yrSRSWQ_create(128);
	if(!jobs_q) { yrLog(0, "Out of memory"); goto onerror; }

	//io threads
	ioread_reserve = yrStack32_create(N_READERS);
	if(!ioread_reserve) goto onerror;
	iowrite_reserve = yrStack32_create(N_WRITERS);
	if(!iowrite_reserve) goto onerror;
	for(size_t i = 0; i < N_READERS; ++i)
	{
		io_read[i].data = NULL;
		io_read[i].tile.t = invalid_item;
		io_read[i].idx.i = invalid_item;
		io_read[i].go = yrEvent_create();			if(!io_read[i].go) { yrLog(0, "OS failed to create synchronization event"); goto onerror; }
		io_read[i].done = yrEvent_create();			if(!io_read[i].done) { yrLog(0, "OS failed to create synchronization event"); goto onerror; }
		io_read[i].thread = yrThread_create(reader_threadfunc, (void*)i);
													if(!io_read[i].thread) { yrLog(0, "OS failed to create tile reader thread"); goto onerror; }
		yrStack32_push(ioread_reserve, (uint32_t)i);
	}
	for(size_t i = 0; i < N_WRITERS; ++i)
	{
		io_write[i].data = NULL;
		io_write[i].tile.t = invalid_item;
		io_write[i].idx.i = invalid_item;
		io_write[i].go = yrEvent_create();			if(!io_write[i].go) { yrLog(0, "OS failed to create synchronization event"); goto onerror; }
		io_write[i].done = yrEvent_create();		if(!io_write[i].done) { yrLog(0, "OS failed to create synchronization event"); goto onerror; }
		io_write[i].thread = yrThread_create(writer_threadfunc, (void*)i);
													if(!io_write[i].thread) { yrLog(0, "OS failed to create tile writer thread"); goto onerror; }
		yrStack32_push(iowrite_reserve, (uint32_t)i);
	}
	
	//ideal thread
	ideal_go = yrEvent_create();		if(!ideal_go) { yrLog(0, "OS failed to create synchronization event"); goto onerror; }
	ideal_done = yrEvent_create();		if(!ideal_done) { yrLog(0, "OS failed to create synchronization event"); goto onerror; }
	yrEvent_set(ideal_done, 1);
	ideal_thread = yrThread_create(ideal_threadfunc, NULL);
										if(!ideal_thread) { yrLog(0, "OS failed to create tile ideal thread"); goto onerror; }

	//initialize the minimum amount of full textures before creating jobs
	size_t required_full = N_IMPORTANT + N_BLANK + N_EXBLANK;
	for(size_t i = 0; i < (required_full/EXPAND_STEP); ++i) {//no extra iteration needed to compensate for rounding down because no rounding happens
		int exp_err = handle_job_expand();
		if(exp_err) break;
	}
	if(allocated_full < required_full) {
		yrLog(0, "Could not pre-allocate %llu tiles", required_full);
		goto onerror;
	}
	yrEvent_set(store_expand_done, 0);

	//jobs thread
	jobs_stop_loop = capacity_store;
	jobs_i_ideal = 0;
	jobs_i_store = 0;
	jobs_pending_delete = NULL;
	jobs_thread = yrThread_create(jobs_threadfunc, NULL);
	if(!jobs_thread) { yrLog(0, "OS failed to create tile jobs thread"); goto onerror; }

	return 0;
onerror:
	yrTileStore_shutdown();
	return -1;
}

static int yrTileStore_shutdown(void)
{
#ifdef _DEBUG
	yrLog(0, "Info: longest tilestore job duration: %lld", perf_longest_job);
#endif
	int lost_data = 0;
	glBindTexture(GL_TEXTURE_2D, 0);
	//shut down jobs thread
	shutting_down_jobs = 1;
	if(jobs_thread) {
		yrEvent_set(ideal_done, 1);
		int err = yrThread_join(jobs_thread, 111000);// give it 111ms to quit before abandoning it
		if(err > 0) yrLog(1, "Thread join timeout (tile jobs).");
	
		//finish off jobs
		while(process_job());
		//handle finished readers
		while(yrStack32_get_top(ioread_reserve) != N_READERS)
		{
			for(unsigned r = 0; r < N_READERS; ++r) {
				if(yrEvent_wait(io_read[r].done, 0) == 0)
				{
					yrEvent_set(io_read[r].done, 0);
					yrStack32_push(ioread_reserve, r);
					free(io_read[r].data); //discard read data
					io_read[r].data = NULL;
				}
			}
		}
		//handle finished writers
		while(yrStack32_get_top(iowrite_reserve) != N_WRITERS)
		{
			for(unsigned w = 0; w < N_WRITERS; ++w) {
				if(yrEvent_wait(io_write[w].done, 0) == 0)
				{
					yrEvent_set(io_write[w].done, 0);
					yrStack32_push(iowrite_reserve, w);
					//check write error
					if(io_write[w].data) { //data was left as a tombstone
						free(io_write[w].data);
						io_write[w].data = NULL;
						io_write[w].tile.t = invalid_item;
						continue;
					}
					store[io_write[w].idx.i].dirty = 0;
					io_write[w].tile.t = invalid_item;
				}
			}
		}
		//finish off deletion queue
		void* next_delete = jobs_pending_delete;
		if(!next_delete) next_delete = yrSRSWQ_read(delete_q);
		while(next_delete)
		{
			TileID nextdel = {(uint32_t)(size_t) next_delete};
			TileIndex delidx = {yrIdxmap_lookup(idxmap, nextdel.t)};
			if(delidx.i != invalid_item)
				store[delidx.i].dirty = 0;
			yrSFTile_delete(scenefile, nextdel.t);
			next_delete = yrSRSWQ_read(delete_q);
		}
		//delete unused blank tiles
		void* blankfoo = NULL;
		TileIndex blank;
		while(blankfoo = yrSRSWQ_read(blank_q)) {
			blank.i = (uint32_t)(size_t) blankfoo;
			yrSFTile_delete(scenefile, store[blank.i].id.t);
			store[blank.i].dirty = 0;
		}
		//write dirty tiles
		for(size_t i = 0; i < capacity_store; ++i)
		{
			if(store[i].id.t != invalid_item && store[i].dirty == 1)
			{
				io_write[0].idx.i = (uint32_t)i;
				io_write[0].tile = store[i].id;
				handle_job_flush(0);
				yrEvent_wait(io_read[0].done, 0xFFFFFFFFFFFFFFFFull);
				yrEvent_set(io_read[0].done, 0);
				if(io_write[0].data) {
					yrLog(0, "Could not flush tile %u", store[i].id.t);
					lost_data = 1;
					free(io_write[0].data);
					io_write[0].data = NULL;
				}
			}
		}
	}
	jobs_thread = NULL;

	//shut down ideals thread
	shutting_down = 1;
		if(ideal_thread) {
		yrEvent_set(ideal_go, 1);
		int err = yrThread_join(ideal_thread, 1200000); //give 1200ms to shut down, after that leak the thread
		if(err > 0) yrLog(1, "Thread join timeout (tile ideal).");
	}
	yrEvent_destroy(ideal_done);
	yrEvent_destroy(ideal_go);
	ideal_thread = NULL;
	ideal_done = NULL;
	ideal_go = NULL;

	//shut down io threads
	for(size_t i = 0; i < N_WRITERS; ++i)
	{
		if(io_write[i].thread) {
			yrEvent_set(io_write[i].go, 1);
			int err = yrThread_join(io_write[i].thread, 200000);
			if(err > 0) yrLog(1, "Thread join timeout (tile writer).");
		}
		yrEvent_destroy(io_write[i].done);
		yrEvent_destroy(io_write[i].go);
	}
	for(size_t i = 0; i < N_READERS; ++i)
	{
		if(io_read[i].thread) {
			yrEvent_set(io_read[i].go, 1);
			int err = yrThread_join(io_read[i].thread, 200000);
			if(err > 0) yrLog(1, "Thread join timeout (tile reader).");
		}
		yrEvent_destroy(io_read[i].done);
		yrEvent_destroy(io_read[i].go);
	}
	memset(io_write, 0, sizeof(io_write));
	memset(io_read, 0, sizeof(io_read));
	yrStack32_destroy(iowrite_reserve);
	yrStack32_destroy(ioread_reserve);
	iowrite_reserve = NULL;
	ioread_reserve = NULL;

	//communication queues
	yrSRSWQ_destroy(important_q);
	yrSRSWQ_destroy(delete_q);
	yrSRSWQ_destroy(blank_q);
	yrSRSWQ_destroy(jobs_q);
	important_q = NULL;
	delete_q = NULL;
	blank_q = NULL;
	jobs_q = NULL;

	//texture reserves
	if(store) {
		for(size_t i = 0; i < capacity_store; ++i)
		{
			if(store[i].id.t == invalid_item) continue;
			if(store[i].full_glid) glDeleteTextures(1, &store[i].full_glid);
			if(store[i].mips_glid) glDeleteTextures(1, &store[i].mips_glid);
		}
	}
	if(texfull_reserve) {
		GLsizei count = (GLsizei) yrStack32_get_top(texfull_reserve);
		GLuint* ptr = (GLuint*) yrStack32_get_ptr(texfull_reserve);
		glDeleteTextures(count, ptr);
		yrStack32_destroy(texfull_reserve);
	}
	if(texmips_reserve) {
		GLsizei count = (GLsizei) yrStack32_get_top(texmips_reserve);
		GLuint* ptr = (GLuint*) yrStack32_get_ptr(texmips_reserve);
		glDeleteTextures(count, ptr);
		yrStack32_destroy(texmips_reserve);
	}
	texfull_reserve = NULL;
	texmips_reserve = NULL;
	
	//important and blank lists
	;

	//ideals
	for(size_t i = 0; i < 2; ++i) {
		free(ideals[i].tile);
		free(ideals[i].idx);
		ideals[i].count = 0;
		ideals[i].tile = NULL;
		ideals[i].idx = NULL;
	}
	ideal_ideal = NULL;
	ideal_jobs = NULL;

	//request lists
	for(size_t i = 0; i < 2; ++i) {
		free(rqlists[i].tile);
		free(rqlists[i].metric);
		rqlists[i].count = 0;
		rqlists[i].tile = NULL;
		rqlists[i].metric = NULL;
	}
	rqlist_main = NULL;
	rqlist_ideal = NULL;

	//store and idxmap
	yrIdxmap_destroy(idxmap);
	free(store);
	yrEvent_destroy(store_expand_done);
	idxmap = NULL;
	store = NULL;
	store_expand_done = NULL;

	//capacities
	capacity_full = 0;
	capacity_mips = 0;
	capacity_store = 0;
		
	//empty tile
	if(empty_glid) glDeleteTextures(1, &empty_glid);
	empty_glid = 0;
	free(empty_data);
	empty_data = NULL;

	scenefile = NULL;
	if(lost_data) yrLogAlert(0, 0xFF8080FF, 3000000, "Tile cleanup error. Data may have been lost.");
	return 0;
}

static int yrTileStore_tick(void)
{
	//process jobs until it is time to stop
	perf_last_tick += 11111 - TICK_MARGIN; //don't start a new job if there is estimated to only be a little time left in the tick
	uint64_t timeout = 0;
	yrEvent sync = yrVR_sync_waitgetposes();
	while(yrEvent_wait(sync, timeout) > 0)
	{
		int64_t now = yr_get_microtime();
	
		timeout = process_job() ? 0 : 1000;
		if(now > perf_last_tick) timeout = 9999;
		
		int64_t dur = yr_get_microtime() - now;
		if(dur > perf_longest_job) {
			perf_longest_job = dur;
		}
	}
	yrEvent_set(sync, 0);
	//reset requests
	rqlist_main->count = 0;
	perf_last_tick = yr_get_microtime();
	return 0;
}

/***************
* Job Processing
****************/
static int handle_job_expand(void);
static void handle_job_shift(void);
static int handle_job_upload(struct iojob* jobdata);
static void handle_job_flush(size_t writer);

static int process_job(void)
{
	void* job = yrSRSWQ_read(jobs_q);
	int done = 0;
	switch((size_t)job & JobMask)
	{
		case JobExpand:	handle_job_expand(); done = 1; break;
		case JobShift:	handle_job_shift(); done = 1; break;
		case JobFlush:	handle_job_flush((size_t)job & JobIOMask);  done = 1; break;
		default:		done = handle_job_upload(job); break; //upload is the only job that is actually a pointer to a datastructure because its reader thread has already been recycled
	}
	return done;
}

//swap the requestlist so ideal_thread can create a new ideal for jobs_thread
static void handle_job_shift(void)
{
	if(rqlist_main->count != 0) {
		void* swap = rqlist_ideal;
		rqlist_ideal = rqlist_main;
		rqlist_main = swap;
	}
	rqlist_main->count = 0;
	yrEvent_set(ideal_go, 1);
}

#ifdef _DEBUG
#pragma push_macro("GLERRCHECK")
#undef GLERRCHECK
#define GLERRCHECK {GLuint glerr = glGetError(); if(glerr) {yrLog(0, "OpenGL error: %x", glerr); goto finish;}}
#endif

//create new tiles for the store
static int handle_job_expand(void)
{
	int err = 1;
	size_t expand_full = capacity_full - allocated_full;
	size_t expand_mips = capacity_mips - allocated_mips;
	if(expand_full > EXPAND_STEP) expand_full = EXPAND_STEP;
	if(expand_mips > MIPS_PER_FULL * EXPAND_STEP) expand_mips = MIPS_PER_FULL * EXPAND_STEP;
	size_t top_full = yrStack32_get_top(texfull_reserve);
	size_t top_mips = yrStack32_get_top(texmips_reserve);
	GLuint* ptr_full = yrStack32_get_ptr(texfull_reserve) + top_full;
	GLuint* ptr_mips = yrStack32_get_ptr(texmips_reserve) + top_mips;
	GLenum glerr = GL_NO_ERROR;

	for(int full = 1; full >= 0; full -= 1)
	{
		size_t cap = full ? expand_full : expand_mips;
		GLuint** dst = full ? &ptr_full : &ptr_mips;
		size_t* top = full ? &top_full : &top_mips;
		size_t* alloc = full ? &allocated_full : &allocated_mips;
		unsigned F = full ? 0 : 2;

		for(size_t i = 0; i < cap; ++i) {
			glGenTextures(1, *dst);
			GLERRCHECK;
			glBindTexture(GL_TEXTURE_2D, **dst);

			glTexStorage2D(GL_TEXTURE_2D, 9 - F, GL_SRGB8_ALPHA8, 256 >> F, 256 >> F);
			GLERRCHECK;
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR_MIPMAP_LINEAR);
			glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
			glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAX_ANISOTROPY_EXT, 16.0f);
			glBindTexture(GL_TEXTURE_2D, 0);

			uint64_t handle = glGetTextureHandleARB(**dst);
			GLERRCHECK;
			glMakeTextureHandleResidentARB(handle);
			if((glerr = glGetError()) != GL_NO_ERROR) {
				yrLog(0, "OpenGL error during tile store expansion: %x", glerr);
				goto finish;
			}
			*dst += 1;
			*top += 1;
			*alloc += 1;
		}
	}
	err = 0;
finish:
	yrStack32_set_top(texfull_reserve, top_full);
	yrStack32_set_top(texmips_reserve, top_mips);
	yrEvent_set(store_expand_done, 1);
	return err;
}

#ifdef _DEBUG
#pragma pop_macro("GLERRCHECK")
#endif

static int handle_job_upload(struct iojob* jobdata)
{
	if(!jobdata) return 0;
	GLERRCHECK;
	TileIndex idx = jobdata->idx;
	uint32_t* levels = jobdata->data;
	free(jobdata);
	//upload full to gpu
	if(store[idx.i].full_glid)
	{
		glBindTexture(GL_TEXTURE_2D, store[idx.i].full_glid);
		for(int l = 0; l < 9; ++l) {
			glTexSubImage2D(GL_TEXTURE_2D, l, 0, 0, 256 >> l, 256 >> l, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV, (const GLvoid*) (levels + mipoff[l]));
		}
		store[idx.i].full_handle = glGetTextureHandleARB(store[idx.i].full_glid);
		GLERRCHECK;
	}
	//upload mips to gpu
	if(store[idx.i].mips_glid)
	{
		glBindTexture(GL_TEXTURE_2D, store[idx.i].mips_glid);
		for(int l = 2; l < 9; ++l) {
			glTexSubImage2D(GL_TEXTURE_2D, l - 2, 0, 0, 256 >> l, 256 >> l, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV, (const GLvoid*) (levels + mipoff[l]));
		}
		store[idx.i].mips_handle = glGetTextureHandleARB(store[idx.i].mips_glid);
		GLERRCHECK;
	}
	//done
	glBindTexture(GL_TEXTURE_2D, 0);
	if(levels != empty_data) free(levels);
	return 1;
}

static void handle_job_flush(size_t writer)
{
	io_write[writer].data = malloc(256 * 256 * 4);
	if(io_write[writer].data) {
		glGetError(); //clear last error
		glBindTexture(GL_TEXTURE_2D, store[io_write[writer].idx.i].full_glid);
		glGetTexImage(GL_TEXTURE_2D, 0, GL_RGBA, GL_UNSIGNED_INT_8_8_8_8_REV, io_write[writer].data);
		GLenum glerr = glGetError();
		if(glerr != GL_NO_ERROR) {
			yrLog(0, "OpenGL error during tile flush: %x", glerr);
			free(io_write[writer].data);
			io_write[writer].data = NULL;
		}
	}
	yrEvent_set(io_write[writer].go, 1);
}

/*************
* Ideal Thread
**************/
static void sort_requests(size_t count, TileID* tile, float* metric);
static void build_ideal(void);

static void ideal_threadfunc(void* param)
{
	for(;;)
	{
		//wait for new requests list or end
		yrEvent_wait(ideal_go, 0xFFFFFFFFFFFFFFFFull);
		yrEvent_set(ideal_go, 0);
		if(shutting_down) break;
		//do work
		sort_requests(rqlist_ideal->count, rqlist_ideal->tile, rqlist_ideal->metric);
		build_ideal();
		//signal new ideal
		yrEvent_set(ideal_done, 1);
		//wait 1000ms to reduce cpu load
		yr_sleep(1000000);
	}
}

static void build_ideal(void)
{
	size_t maxsize = capacity_mips - N_IMPORTANT - N_BLANK - N_EXBLANK;
	size_t count = (rqlist_ideal->count < maxsize) ? rqlist_ideal->count : maxsize;
	
	ideal_ideal->count = count;
	memcpy(ideal_ideal->tile, rqlist_ideal->tile, count * sizeof(TileID));
}

static void sort_requests(size_t count, TileID* tile, float* metric)
{
	if(count == 0) return;
	if(count == 1) return;

	size_t pivi = count/2;
	float pivv = metric[pivi];
	size_t idxtop = 0;
	size_t idxbot = count - 1;
	while(idxtop < idxbot && idxbot != 0xFFFFFFFFFFFFFFFFull) {
		while(idxtop != idxbot && metric[idxtop] >= pivv) ++idxtop;
		while(idxtop != idxbot && metric[idxbot] < pivv) --idxbot;
		if(idxtop == idxbot) break;

		TileID swapt = tile[idxtop];
		float swapm = metric[idxtop];
		tile[idxtop] = tile[idxbot];
		metric[idxtop] = metric[idxbot];
		tile[idxbot] = swapt;
		metric[idxbot] = swapm;
		idxtop += 1;
		idxbot -= 1;
	}
	if(idxtop == idxbot && metric[idxtop] >= pivv) ++idxtop;
	if(metric[0] == metric[count -1]) return;

	if(idxtop) {
		if(idxtop == count) {
			TileID swapt = tile[idxtop-1];
			float swapm = metric[idxtop-1];
			tile[idxtop-1] = tile[pivi];
			metric[idxtop-1] = metric[pivi];
			tile[pivi] = swapt;
			metric[pivi] = swapm;
			idxtop -= 1;
		}
		sort_requests(idxtop, tile, metric);
	}
	sort_requests(count - idxtop, tile + idxtop, metric + idxtop);
}

/************
* Jobs Thread
*************/
static int jobsub_finish_reads(void);
static int jobsub_finish_writes(void);
static int jobsub_delete_tiles(void);
static int jobsub_expand_storage(void);
static int jobsub_refill_blanks(void);
static int jobsub_do_importants(void);
static int jobsub_make_ideal(void);

static void jobs_shift_ideal(void);
static void jobs_tile_update(TileIndex idx, int allow_upgrade);
static void jobs_start_flush(TileIndex idx);
static void jobs_start_load(TileIndex idx);

static void jobs_threadfunc(void* param)
{
	for(;;) {
		int act = 1;
		while(yrEvent_wait(ideal_done, act ? 0 : 3000) > 0)
		{
			//report queue sizes and texture/idx availability
			yrRender_barstat(bsBlankReserve, yrSRSWQ_getload(blank_q));
			yrRender_barstat(bsImporantQueue, yrSRSWQ_getload(important_q));
			yrRender_barstat(bsFullAvailable, yrStack32_is_empty(texfull_reserve) ? 0.0f : 1.0f);
			yrRender_barstat(bsMipsAvailable, yrStack32_is_empty(texmips_reserve) ? 0.0f : 1.0f);
			yrRender_barstat(bsIdxAvailable, yrIdxmap_isfull(idxmap) ? 0.0f : 1.0f);
			//update
			act = 0;
			act = jobsub_finish_reads();	if(act) continue;
			act = jobsub_finish_writes();	if(act) continue;
			act = jobsub_delete_tiles();	if(act) continue;
			act = jobsub_expand_storage();	if(act) continue;
			act = jobsub_refill_blanks();	if(act) continue;
			act = jobsub_do_importants();	if(act) continue;
			act = jobsub_make_ideal();		if(act) continue;
		}
		if(shutting_down_jobs) break;
		yrEvent_set(ideal_done, 0);
		jobs_shift_ideal();
		yrSRSWQ_write_block(jobs_q, (void*) JobShift);
	}
}

static int jobsub_finish_reads(void)
{
	int act = 0;
	for(unsigned r = 0; r < N_READERS; ++r) {
		if(yrEvent_wait(io_read[r].done, 0) == 0)
		{
			act = 1;
			//recover thread for future use
			yrEvent_set(io_read[r].done, 0);
			yrStack32_push(ioread_reserve, r);
			//check whether the tile was deleted
			if(io_read[r].tile.t != store[io_read[r].idx.i].id.t) {
				yrLog(1, "TileID mismatch while reading, tile %x was probably deleted", io_read[r].tile.t);
				io_read[r].tile.t = invalid_item;
				continue;
			}
			//check for read error, replace with empty tile if needed
			if(!io_read[r].data) {
				yrLogAlert(0, 0xFFFFFFFFul, 1000000, "Tile load failed.");
				yrLog(0, "Could not read tile data for tile %x", io_read[r].tile.t);
				io_read[r].data = empty_data;
			}
			//post job to the main thread
			struct iojob* job = malloc(sizeof(struct iojob));
			if(job) {
				job->idx = io_read[r].idx;
				job->data = io_read[r].data;
				yrSRSWQ_write_block(jobs_q, job);
			}
			else {
				if(io_read[r].data != empty_data)
					free(io_read[r].data);
				yrLog(0, "Out of memory");
			}
			io_read[r].tile.t = invalid_item;
		}
	}
	return act;
}

static int jobsub_finish_writes(void)
{
	int act = 0;
	for(unsigned w = 0; w < N_WRITERS; ++w)
	{
		if(yrEvent_wait(io_write[w].done, 0) == 0)
		{
			act = 1;
			//recover thread for future use
			yrEvent_set(io_write[w].done, 0);
			yrStack32_push(iowrite_reserve, w);
			//check write error
			if(io_write[w].data) { //data was left as a tombstone
				yrLogAlert(0, 0xFF8080FF, 5000000, \
							"An error occured while saving whiteboard data.\n"\
							"The data drive is most likely full.\n"\
							"This will impact performance, and data will be lost if this is not resolved before the application is closed.\n");
				free(io_write[w].data);
				io_write[w].data = NULL;
				io_write[w].tile.t = invalid_item;
				continue;
			}
			//unmark dirty and recover resources
			store[io_write[w].idx.i].dirty = 0;
			jobs_tile_update(io_write[w].idx, 0);
			io_write[w].tile.t = invalid_item;
		}
	}
	return act;
}

static int jobsub_delete_tiles(void)
{
	void* next_delete = jobs_pending_delete;
	if(!next_delete) next_delete = yrSRSWQ_read(delete_q);
	jobs_pending_delete = NULL;

	if(next_delete)
	{
		TileID nextdel = {(uint32_t)(size_t) next_delete};
		TileIndex delidx = {yrIdxmap_lookup(idxmap, nextdel.t)};
		if(delidx.i != invalid_item)
		{
			//check whether any writer thread is currently handling this tile
			unsigned w = 0;
			for(; w < N_WRITERS; ++w)
				if(io_write[w].tile.t == nextdel.t)
					break;
			if(w != N_WRITERS) {
				//save tile for retry
				jobs_pending_delete = (void*)(0x3900000000ull | (size_t) nextdel.t);
				return 0;
			}
			//remove from important list
			for(size_t i = 0; i < N_IMPORTANT; ++i)
				if(list_important[i].t == nextdel.t)
					list_important[i].t = invalid_item;
			//remove from exblank list
			for(size_t i = 0; i < N_EXBLANK; ++i) {
				size_t idx = (i + list_blank_i) % (N_BLANK + N_EXBLANK);
				if(list_blank[idx].t == nextdel.t)
					list_blank[idx].t = invalid_item;
			}
			//remove from ideal
			for(size_t i = 0; i < ideal_jobs->count; ++i)
				if(ideal_jobs->tile[i].t == nextdel.t)
					ideal_jobs->tile[i].t = invalid_item;
			//unmark dirty, set requested to zero
			store[delidx.i].dirty = 0;
			store[delidx.i].rq_extra = 0;
			store[delidx.i].rq_ideal = RQ_EMPTY;
			//recover resources
			jobs_tile_update(delidx, 0);
		}
		//delete from file
		yrSFTile_delete(scenefile, nextdel.t);
		return 1;
	}
	return 0;
}

static int jobsub_expand_storage(void)
{
	if(!store_maxed && (yrStack32_is_empty(texfull_reserve) || yrStack32_is_empty(texmips_reserve)))
	{
		yrEvent_set(store_expand_done, 0);
		yrSRSWQ_write_block(jobs_q, (void*) JobExpand);
		//blocking wait for success or fail
		yrEvent_wait(store_expand_done, 0xFFFFFFFFFFFFFFFFull);
		yrEvent_set(store_expand_done, 0);
		//failed or max capacity reached
		if(	yrStack32_is_empty(texfull_reserve) ||
			yrStack32_is_empty(texmips_reserve) ||
			allocated_mips == capacity_mips ||
			allocated_full == capacity_full)
		{
			store_maxed = 1;
			capacity_full = allocated_full;
			capacity_mips = allocated_mips;
		}
		return 1;
	}
	return 0;
}

static int jobsub_refill_blanks(void)
{
	if(!yrStack32_is_empty(iowrite_reserve) &&
	   !yrSRSWQ_isfull(blank_q))
	{
		//free a slot in the blank list
		if(!list_blank_free) { //detect a run of deleted tiles
			for(size_t i = 0; i < (N_BLANK + N_EXBLANK); i += 1) {
				size_t idx = (i + list_blank_i) % (N_BLANK + N_EXBLANK);
				if(list_blank[idx].t != invalid_item) break;
				list_blank_free += 1;
			}
		}
		if(!list_blank_free) {
			TileIndex idx = {yrIdxmap_lookup(idxmap, list_blank[list_blank_i].t)};
			YR_ASSERT(idx.i != invalid_item);
			store[idx.i].rq_extra -= 1;
			jobs_tile_update(idx, 0);
			list_blank_free += 1;
		}
		//check texture & idx availability
		if(yrStack32_is_empty(texfull_reserve)) return 0;
		if(yrStack32_is_empty(texmips_reserve)) return 0;
		if(yrIdxmap_isfull(idxmap)) return 0;
		
		//alloc id
		for(;;) {
			if(jobs_next_id.t == invalid_item) jobs_next_id.t += 1;
			int exists = yrSFTile_exists(scenefile, jobs_next_id.t);
			if(!exists) break;
			jobs_next_id.t += 1;
		}
		TileID tile = jobs_next_id;
		jobs_next_id.t += 1;
		//create "file"
		size_t empty = 0;
		int tile_err = yrSFTile_write(scenefile, tile.t, 8, &empty); //TODO: might want to put actual png data here	
		if(tile_err) return 0;
		//get idx
		TileIndex idx = {yrIdxmap_insert(idxmap, tile.t)};
		if(idx.i == invalid_item) { yrLog(0, "The indexmap is full when it shouldn't be."); return 0; }
		//get textures
		GLuint fulltex = (GLuint) yrStack32_pop(texfull_reserve);
		GLuint mipstex = (GLuint) yrStack32_pop(texmips_reserve);

		//put it all together
		store[idx.i].id = tile;
		store[idx.i].full_glid = fulltex;
		store[idx.i].mips_glid = mipstex;
		store[idx.i].dirty = 2;
		store[idx.i].rq_extra = 1;
		store[idx.i].rq_ideal = 0;
		
		list_blank_free -= 1;
		list_blank[list_blank_i] = tile;
		list_blank_i = (list_blank_i + 1) % (N_BLANK + N_EXBLANK);
		
		//add to blank queue
		yrSRSWQ_write_block(blank_q, (void*)(0x3900000000ull | (size_t) idx.i));

		return 1;
	}
	return 0;
}

static int jobsub_do_importants(void)
{
	void* next_important = NULL;
	while(!yrStack32_is_empty(iowrite_reserve) &&
		  !yrStack32_is_empty(ioread_reserve) &&
		  !yrStack32_is_empty(texfull_reserve) &&
		  !yrStack32_is_empty(texmips_reserve) &&
		  !yrIdxmap_isfull(idxmap) &&
		  (next_important = yrSRSWQ_read(important_q)))
	{
		TileID nextimp = {(uint32_t)(size_t) next_important};
		if(nextimp.t == invalid_item) continue;
		//free a slot in the important list
		if(list_important[list_important_i].t != invalid_item)
		{
			TileIndex idx = {yrIdxmap_lookup(idxmap, list_important[list_important_i].t)};
			YR_ASSERT(idx.i != invalid_item);
			store[idx.i].rq_extra -= 1;
			jobs_tile_update(idx, 0);
		}
		//check texture & idx availability
		if(yrStack32_is_empty(texfull_reserve)) return 0;
		if(yrStack32_is_empty(texmips_reserve)) return 0;
		if(yrIdxmap_isfull(idxmap)) return 0;
		
		//get index and prep record
		TileIndex idx = {yrIdxmap_lookup(idxmap, nextimp.t)};
		if(idx.i == invalid_item) {
			idx.i = yrIdxmap_insert(idxmap, nextimp.t);
			YR_ASSERT(idx.i != invalid_item);
			store[idx.i].id = nextimp;
		}

		//add to important list
		list_important[list_important_i] = nextimp;
		list_important_i = (list_important_i + 1) % N_IMPORTANT;
		store[idx.i].rq_extra += 1;

		//load if necessary
		jobs_tile_update(idx, 1);
		return 1;
	}
	return 0;
}

static int jobsub_make_ideal(void)
{
	int act = 0;

	//loop ideal for loading
	size_t real_cap_full = (capacity_full - N_IMPORTANT - N_BLANK - N_EXBLANK - 1);
	size_t real_cap_mips = (capacity_mips - N_IMPORTANT - N_BLANK - N_EXBLANK - 1);
	size_t limit = (ideal_jobs->count < real_cap_mips) ? ideal_jobs->count : real_cap_mips;
	for(; jobs_i_ideal < limit; ++jobs_i_ideal)
	{
		if(yrStack32_is_empty(ioread_reserve) ||
		   yrStack32_is_empty(texfull_reserve) ||
		   yrStack32_is_empty(texmips_reserve) ||
		   yrIdxmap_isfull(idxmap))
		{
			break;
		}
		//get index and prep record
		if(ideal_jobs->tile[jobs_i_ideal].t == invalid_item) break;
		TileIndex idx = ideal_jobs->idx[jobs_i_ideal];
		if(idx.i == invalid_item) idx.i = yrIdxmap_lookup(idxmap, ideal_jobs->tile[jobs_i_ideal].t);
		if(idx.i == invalid_item) {
			idx.i = yrIdxmap_insert(idxmap, ideal_jobs->tile[jobs_i_ideal].t);
			store[idx.i].id = ideal_jobs->tile[jobs_i_ideal];
			store[idx.i].rq_ideal = (jobs_i_ideal < real_cap_full) ? RQ_FULL : RQ_MIPS;
		}
		if(idx.i == invalid_item) { yrLog(0, "The indexmap is full when it shouldn't be."); break; }
		ideal_jobs->idx[jobs_i_ideal] = idx;
		
		//load if necessary
		jobs_tile_update(idx, 1);
		act = 1;
	}

	//loop store for unloading
	size_t recover_break = 0;
	for(; jobs_i_store != jobs_stop_loop; jobs_i_store = (jobs_i_store + 1) % capacity_store)
	{
		//set stop condition on first iteration after ideal shift
		if(jobs_stop_loop == capacity_store)
			jobs_stop_loop = jobs_i_store;
		//unload
		if(yrStack32_is_empty(iowrite_reserve)) break;
		TileIndex idx = {(uint32_t) jobs_i_store};
		if(store[jobs_i_store].id.t != invalid_item &&
		   store[jobs_i_store].dirty != 2)
		{
			jobs_tile_update(idx, 0);
		}
		act = 1;
		//give other jobs a chance to be generated
		recover_break += 1;
		if(recover_break >= JOBS_RECOVER_BREAK) break;
	}
	return act;
}

static void jobs_shift_ideal(void)
{
	size_t real_cap_full = (capacity_full - N_IMPORTANT - N_BLANK - N_EXBLANK - 1); //reserve one extra as swap space
	size_t real_cap_mips = (capacity_mips - N_IMPORTANT - N_BLANK - N_EXBLANK - 1);

	//swap
	void* swap = ideal_jobs;
	ideal_jobs = ideal_ideal;
	ideal_ideal = swap;

	//reset store loop
	jobs_i_ideal = 0;
	jobs_stop_loop = capacity_store;

	//set loadlevels
	for(size_t i = 0; i < capacity_store; ++i)
	{
		store[i].rq_ideal = RQ_EMPTY;
	}
	for(size_t i = 0; i < ideal_jobs->count; ++i)
	{
		//check cached idx value
		TileIndex idx = ideal_jobs->idx[i];
		if(idx.i == invalid_item ||
		   store[idx.i].id.t != ideal_jobs->tile[i].t)
		{
			idx.i = yrIdxmap_lookup(idxmap, ideal_jobs->tile[i].t);
			if(idx.i == invalid_item) {
				idx.i = yrIdxmap_insert(idxmap, ideal_jobs->tile[i].t);//allocate idx as soon as possible
				if(idx.i != invalid_item)
					store[idx.i].id = ideal_jobs->tile[i];
			}
			ideal_jobs->idx[i] = idx;
		}
		//set requested loadlevel
		if(idx.i != invalid_item)
		{
			int rq = RQ_EMPTY;
			if(i < real_cap_full) rq = RQ_FULL;
			else if(i < real_cap_mips) rq = RQ_MIPS;

			store[idx.i].rq_ideal = rq;
		}
	}
}

static void jobs_tile_update(TileIndex idx, int allow_upgrade)
{
	YR_ASSERT(idx.i != invalid_item);
	YR_ASSERT(store[idx.i].id.t != invalid_item);
	YR_ASSERT(store[idx.i].dirty != 2);
	struct tilerecord* rec = store + idx.i;
	int do_load = 0;
	//flush
	if(!allow_upgrade && rec->dirty == 1) {
		jobs_start_flush(idx);
		return;
	}
	int needfull = rec->rq_extra || (rec->rq_ideal >= RQ_FULL);
	int needmips = needfull || (rec->rq_ideal >= RQ_MIPS);
	int needidx  = needmips || (rec->rq_ideal > RQ_EMPTY);

	//-full
	if(!needfull && rec->full_glid) {
		while(!rec->full_handle) jobsub_finish_reads(); //image is still loading so can't recover the glid just yet
		yrStack32_push(texfull_reserve, rec->full_glid);
		rec->full_glid = 0;
		rec->full_handle = 0;
	}
	//-mips
	if(!needmips && rec->mips_glid) {
		while(!rec->mips_handle) jobsub_finish_reads(); //image is still loading so can't recover the glid just yet
		yrStack32_push(texmips_reserve, rec->mips_glid);
		rec->mips_glid = 0;
		rec->mips_handle = 0;
	}
	//-idx
	if(!needidx) {
		yrIdxmap_remove(idxmap, rec->id.t);
		rec->id.t = invalid_item;
	}
	//+mips
	if(allow_upgrade && needmips && !rec->mips_glid) {
		rec->mips_glid = yrStack32_pop(texmips_reserve);
		do_load = 1;
	}
	//+full
	if(allow_upgrade && needfull && !rec->full_glid) {
		rec->full_glid = yrStack32_pop(texfull_reserve);
		do_load = 1;
	}
	//load texture
	if(do_load) {
		jobs_start_load(idx);
	}
}

static void jobs_start_load(TileIndex idx)
{
	unsigned r = yrStack32_pop(ioread_reserve);
	io_read[r].idx = idx;
	io_read[r].tile = store[idx.i].id;
	yrEvent_set(io_read[r].go, 1);
}

static void jobs_start_flush(TileIndex idx)
{
	unsigned w = yrStack32_pop(iowrite_reserve);
	io_write[w].idx = idx;
	io_write[w].tile = store[idx.i].id;
	yrSRSWQ_write_block(jobs_q, (void*)(JobFlush | w));
}

/**********************
* Reader/Writer threads
***********************/
static unsigned char* reader_read(size_t myid);
static void reader_unmangle(size_t myid, unsigned char* pngdata);
static unsigned char* writer_mangle(size_t myid);
static int	writer_write(size_t myid, unsigned char* pngdata);

static void reader_threadfunc(void* param)
{
	size_t myid = (size_t) param;
	for(;;) {
		yrEvent_wait(io_read[myid].go, 0xFFFFFFFFFFFFFFFF);
		yrEvent_set(io_read[myid].go, 0);
		if(shutting_down) break;

		io_read[myid].data = NULL;
		unsigned char* pngdata = reader_read(myid);
		if(pngdata) reader_unmangle(myid, pngdata);
		free(pngdata);

		yrEvent_set(io_read[myid].done, 1);
	}
}

static void writer_threadfunc(void* param)
{
	size_t myid = (size_t) param;
	for(;;) {
		yrEvent_wait(io_write[myid].go, 0xFFFFFFFFFFFFFFFF);
		yrEvent_set(io_write[myid].go, 0);
		if(shutting_down) break;

		if(io_write[myid].data)
		{
			unsigned char* pngdata = writer_mangle(myid);
			if(pngdata && writer_write(myid, pngdata)) {
				free(io_write[myid].data);
				io_write[myid].data = NULL; //otherwise leave the data as an indicator things went wrong
			}
			free(pngdata);
		}
		
		yrEvent_set(io_write[myid].done, 1);
	}
}

static unsigned char* reader_read(size_t myid)
{
	unsigned char* img = NULL;
	unsigned int width = 0;
	unsigned int height = 0;
	
	size_t len;
	void* buffer;
	int err = yrSFTile_read(scenefile, io_read[myid].tile.t, &len, &buffer);
	if(err) return NULL;

	const char* errtxt = yrImgLoad_decode(len, buffer, &width, &height, &img);
	free(buffer);
	if(errtxt) {
		yrLog(0,"Error while decoding tile: %s", errtxt);
		free(img);
		return NULL;
	}
	if(width != 256 || height != 256) {
		yrLog(0,"Error while decoding tile: the tile has an invalid size");
		free(img);
		return NULL;
	}
	else return img;
}

static int writer_write(size_t myid, unsigned char* pngdata)
{
	size_t len;
	void* buffer;
	const char* errtxt = yrImgLoad_encode(&len, &buffer, 256, 256, pngdata); //TODO: inconsistent use of the variable name 'pngdata', it points to raw pixels here
	if(errtxt) {
		yrLog(0,"Error while reading tile: %s", errtxt);
		return 0;
	}

	int err = yrSFTile_write(scenefile, io_write[myid].tile.t, len, buffer);
	free(buffer);
	if(err) return 0;

	return 1;
}

static void	reader_unmangle(size_t myid, unsigned char* pngdata)
{
	io_read[myid].data = malloc((mipoff[8] + 1) * sizeof(uint32_t));
	if(!io_read[myid].data) { yrLog(0, "Out of memory"); return;}
	uint32_t* data = io_read[myid].data;

	//mangle byte order and build mips
	uint32_t line0[4 * 256] = {0};
	uint32_t line1[4 * 128] = {0};
	uint32_t line2[4 * 64] = {0};
	uint32_t line3[4 * 32] = {0};
	uint32_t line4[4 * 16] = {0};
	uint32_t line5[4 * 8] = {0};
	uint32_t line6[4 * 4] = {0};
	uint32_t line7[4 * 2] = {0};
	uint32_t line8[4 * 1] = {0};
	uint32_t* lines[9] = {line0,line1,line2,line3,line4,line5,line6,line7,line8};
	uint32_t* levels[9] = {
		data + mipoff[0],
		data + mipoff[1],
		data + mipoff[2],
		data + mipoff[3],
		data + mipoff[4],
		data + mipoff[5],
		data + mipoff[6],
		data + mipoff[7],
		data + mipoff[8],
	};
	for(unsigned line = 0; line < 256; ++line) {
		//init
		for(unsigned pixel = 0; pixel < 256; ++pixel) {
			line0[pixel * 4 + 0] = pngdata[4 * 256 * line + pixel * 4 + 3];
			line0[pixel * 4 + 1] = pngdata[4 * 256 * line + pixel * 4 + 2];
			line0[pixel * 4 + 2] = pngdata[4 * 256 * line + pixel * 4 + 1];
			line0[pixel * 4 + 3] = pngdata[4 * 256 * line + pixel * 4 + 0];
		}
		//accumulate
		for(unsigned level = 1; level < 9; ++level) {
			if((line + 1) % (1 << (level-1)) == 0) { //if the line is complete for the previous level
				unsigned dim = 256 >> level;
				for(unsigned pixel = 0; pixel < dim; ++pixel) {
					lines[level][pixel * 4 + 0] += lines[level - 1][2 * pixel * 4 + 0] + lines[level - 1][2 * pixel * 4 + 4 + 0];
					lines[level][pixel * 4 + 1] += lines[level - 1][2 * pixel * 4 + 1] + lines[level - 1][2 * pixel * 4 + 4 + 1];
					lines[level][pixel * 4 + 2] += lines[level - 1][2 * pixel * 4 + 2] + lines[level - 1][2 * pixel * 4 + 4 + 2];
					lines[level][pixel * 4 + 3] += lines[level - 1][2 * pixel * 4 + 3] + lines[level - 1][2 * pixel * 4 + 4 + 3];
				}
			}
		}
		//process target line
		for(unsigned level = 0; level < 9; ++level) {
			unsigned dim = 256 >> level;
			if((line + 1) % (1 << level) == 0) { //if the line is complete for this level
				//process
				unsigned target_line = dim - (line >> level) - 1;
				for(unsigned pixel = 0; pixel < dim; ++pixel) {
					levels[level][dim * target_line + pixel] = lines[level][4 * pixel + 0] >> (level * 2); //R
					levels[level][dim * target_line + pixel] <<= 8;
					levels[level][dim * target_line + pixel] |= lines[level][4 * pixel + 1] >> (level * 2); //G
					levels[level][dim * target_line + pixel] <<= 8;
					levels[level][dim * target_line + pixel] |= lines[level][4 * pixel + 2] >> (level * 2); //B
					levels[level][dim * target_line + pixel] <<= 8;
					levels[level][dim * target_line + pixel] |= lines[level][4 * pixel + 3] >> (level * 2); //A
				}
				//reset
				memset(lines[level], 0, 4 * dim * sizeof(uint32_t));
			}
		}
	}
	//done
}

static unsigned char* writer_mangle(size_t myid)
{
	//alloc space
	unsigned char* pngdata = malloc(256*256*4);
	if(!pngdata) { yrLog(0, "Out of memory"); return 0;}
	//re-order bytes
	unsigned char* src = io_write[myid].data;
	for(unsigned r = 0; r < 256; ++r)
	for(unsigned c = 0; c < 256; ++c) {
		pngdata[(r*256+c)*4 + 0] = src[((255-r)*256+c)*4 + 0];//R
		pngdata[(r*256+c)*4 + 1] = src[((255-r)*256+c)*4 + 1];//G
		pngdata[(r*256+c)*4 + 2] = src[((255-r)*256+c)*4 + 2];//B
		pngdata[(r*256+c)*4 + 3] = src[((255-r)*256+c)*4 + 3];//A
	}
	//done
	return pngdata;
}