From d2661874c61d518324a6b05fa5155dd793dc00e3 Mon Sep 17 00:00:00 2001 From: Jon Atkins Date: Mon, 7 Oct 2013 20:29:05 +0100 Subject: [PATCH] some work-in-progress. from a read of the far-less-obfsucated code on the stock site it looks like map data can be retrieved as an update to an earlier query i.e. pass the timestamp of the last data request, and the server should only send the changed data rather than everything --- code/map_data_cache.js | 10 ++++-- code/map_data_debug.js | 3 +- code/map_data_request.js | 74 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 81 insertions(+), 6 deletions(-) diff --git a/code/map_data_cache.js b/code/map_data_cache.js index c143e9c9..6dcda1a8 100644 --- a/code/map_data_cache.js +++ b/code/map_data_cache.js @@ -2,8 +2,14 @@ // cache for map data tiles. window.DataCache = function() { - this.REQUEST_CACHE_FRESH_AGE = 60; // if younger than this, use data in the cache rather than fetching from the server - this.REQUEST_CACHE_MAX_AGE = 180; // maximum cache age. entries are deleted from the cache after this time + // stock site nemesis.dashboard.DataManager.CACHE_EXPIRY_MS_ = 18E4 - so should be 2 mins cache time + this.REQUEST_CACHE_FRESH_AGE = 120; // if younger than this, use data in the cache rather than fetching from the server + + // stale cache entries can be updated (that's what the optional 'timestampMs' field in getThinnedEntnties is + // for, retrieving deltas) so use a long max age to take advantage of this + // however, ther must be an overall limit on the maximum age of data from the servers, otherwise the deletedEntity + // entries would grow indefinitely. an hour seems reasonable from experience with the data, so 55 mins max cache time + this.REQUEST_CACHE_MAX_AGE = 55*60; // maximum cache age. entries are deleted from the cache after this time if (L.Browser.mobile) { // on mobile devices, smaller cache size diff --git a/code/map_data_debug.js b/code/map_data_debug.js index 4ecc27a8..dcede257 100644 --- a/code/map_data_debug.js +++ b/code/map_data_debug.js @@ -37,7 +37,8 @@ window.RenderDebugTiles.prototype.setState = function(id,state) { var col = '#f0f'; var fill = '#f0f'; switch(state) { - case 'ok': col='#0f0'; fill='#0f0'; break; + case 'ok': col='#080'; fill='#080'; break; + case 'ok-delta': col='#0f0'; fill='#0f0'; break; case 'error': col='#f00'; fill='#f00'; break; case 'cache-fresh': col='#0f0'; fill='#ff0'; break; case 'cache-stale': col='#f00'; fill='#ff0'; break; diff --git a/code/map_data_request.js b/code/map_data_request.js index e6fcb799..ff839663 100644 --- a/code/map_data_request.js +++ b/code/map_data_request.js @@ -11,6 +11,7 @@ window.MapDataRequest = function() { this.activeRequestCount = 0; this.requestedTiles = {}; + this.staleTileData = {}; this.idle = false; @@ -45,7 +46,7 @@ window.MapDataRequest = function() { // additionally, a delay before processing the queue after requeueing tiles // (this way, if multiple requeue delays finish within a short time of each other, they're all processed in one queue run) - this.RERUN_QUEUE_DELAY = 2; + this.RERUN_QUEUE_DELAY = 1; this.REFRESH_CLOSE = 120; // refresh time to use for close views z>12 when not idle and not moving this.REFRESH_FAR = 600; // refresh time for far views z <= 12 @@ -172,6 +173,9 @@ window.MapDataRequest.prototype.refresh = function() { // fill tileBounds with the data needed to request each tile this.tileBounds = {}; + // clear the stale tile data + this.staleTileData = {}; + var bounds = clampLatLngBounds(map.getBounds()); var zoom = getPortalDataZoom(); @@ -238,6 +242,7 @@ window.MapDataRequest.prototype.refresh = function() { this.render.processTileData (this.cache.get(tile_id)); this.cachedTileCount += 1; } else { + // no fresh data - queue a request var boundsParams = generateBoundsParams( tile_id, @@ -247,6 +252,30 @@ window.MapDataRequest.prototype.refresh = function() { lngEast ); + // however, the server does support delta requests - only returning the entities changed since a particular timestamp + // retrieve the stale cache entry and use it, if possible + var stale = (this.cache && this.cache.get(tile_id)); + var lastTimestamp = undefined; + if (stale) { + // find the timestamp of the latest entry in the stale records. the stock site appears to use the browser + // clock, but this isn't reliable. ideally the data set should include it's retrieval timestamp, set by the + // server, for use here. a good approximation is the highest timestamp of all entities + + for (var i in stale.gameEntities) { + var ent = stale.gameEntities[i]; + if (lastTimestamp===undefined || ent[1] > lastTimestamp) { + lastTimestamp = ent[1]; + } + } + +console.log('stale tile '+tile_id+': newest mtime '+lastTimestamp+(lastTimestamp?' '+new Date(lastTimestamp).toString():'')); + if (lastTimestamp) { + // we can issue a useful delta request - store the previous data, as we can't rely on the cache still having it later + this.staleTileData[tile_id] = stale; + boundsParams.timestampMs = lastTimestamp; + } + } + this.tileBounds[tile_id] = boundsParams; this.requestedTileCount += 1; } @@ -358,7 +387,7 @@ window.MapDataRequest.prototype.sendTileRequest = function(tiles) { this.debugTiles.setState (id, 'requested'); - this.requestedTiles[id] = true; + this.requestedTiles[id] = { staleData: this.staleTileData[id] }; var boundsParams = this.tileBounds[id]; if (boundsParams) { @@ -473,13 +502,52 @@ window.MapDataRequest.prototype.handleResponse = function (data, tiles, success) // no error for this data tile - process it successTiles.push (id); + var stale = this.requestedTiles[id].staleData; + if (stale) { + // we have stale data. therefore, a delta request was made for this tile. we need to merge the results with + // the existing stale data before proceeding + + var dataObj = {}; + // copy all entities from the stale data... + for (var i in stale.gameEntities||[]) { + var ent = stale.gameEntities[i]; + dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] }; + } +var oldEntityCount = Object.keys(dataObj).length; + // and remove any entities in the deletedEntnties list + for (var i in val.deletedEntities||[]) { + var guid = val.deletedEntities[i]; + delete dataObj[guid]; + } +var oldEntityCount2 = Object.keys(dataObj).length; + // then add all entities from the new data + for (var i in val.gameEntities||[]) { + var ent = val.gameEntities[i]; + dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] }; + } +var newEntityCount = Object.keys(dataObj).length; +console.log('processed delta mapData request:'+id+': '+oldEntityCount+' original entities, '+oldEntityCount2+' after deletion, '+val.gameEntities.length+' entities in the response'); + + // now reconstruct a new gameEntities array in val, with the updated data + val.gameEntities = []; + for (var guid in dataObj) { + var ent = [guid, dataObj[guid].timestamp, dataObj[guid].data]; + val.gameEntities.push(ent); + } + + // we can leave the returned 'deletedEntities' data unmodified - it's not critial to how IITC works anyway + + // also delete any staleTileData entries for this tile - no longer required + delete this.staleTileData[id]; + } + // store the result in the cache this.cache && this.cache.store (id, val); // if this tile was in the render list, render it // (requests aren't aborted when new requests are started, so it's entirely possible we don't want to render it!) if (id in this.tileBounds) { - this.debugTiles.setState (id, 'ok'); + this.debugTiles.setState (id, stale?'ok-delta':'ok'); this.render.processTileData (val);