diff --git a/code/map_data_cache.js b/code/map_data_cache.js index c143e9c9..6dcda1a8 100644 --- a/code/map_data_cache.js +++ b/code/map_data_cache.js @@ -2,8 +2,14 @@ // cache for map data tiles. window.DataCache = function() { - this.REQUEST_CACHE_FRESH_AGE = 60; // if younger than this, use data in the cache rather than fetching from the server - this.REQUEST_CACHE_MAX_AGE = 180; // maximum cache age. entries are deleted from the cache after this time + // stock site nemesis.dashboard.DataManager.CACHE_EXPIRY_MS_ = 18E4 - so should be 2 mins cache time + this.REQUEST_CACHE_FRESH_AGE = 120; // if younger than this, use data in the cache rather than fetching from the server + + // stale cache entries can be updated (that's what the optional 'timestampMs' field in getThinnedEntnties is + // for, retrieving deltas) so use a long max age to take advantage of this + // however, ther must be an overall limit on the maximum age of data from the servers, otherwise the deletedEntity + // entries would grow indefinitely. an hour seems reasonable from experience with the data, so 55 mins max cache time + this.REQUEST_CACHE_MAX_AGE = 55*60; // maximum cache age. entries are deleted from the cache after this time if (L.Browser.mobile) { // on mobile devices, smaller cache size diff --git a/code/map_data_debug.js b/code/map_data_debug.js index 4ecc27a8..dcede257 100644 --- a/code/map_data_debug.js +++ b/code/map_data_debug.js @@ -37,7 +37,8 @@ window.RenderDebugTiles.prototype.setState = function(id,state) { var col = '#f0f'; var fill = '#f0f'; switch(state) { - case 'ok': col='#0f0'; fill='#0f0'; break; + case 'ok': col='#080'; fill='#080'; break; + case 'ok-delta': col='#0f0'; fill='#0f0'; break; case 'error': col='#f00'; fill='#f00'; break; case 'cache-fresh': col='#0f0'; fill='#ff0'; break; case 'cache-stale': col='#f00'; fill='#ff0'; break; diff --git a/code/map_data_request.js b/code/map_data_request.js index e6fcb799..ff839663 100644 --- a/code/map_data_request.js +++ b/code/map_data_request.js @@ -11,6 +11,7 @@ window.MapDataRequest = function() { this.activeRequestCount = 0; this.requestedTiles = {}; + this.staleTileData = {}; this.idle = false; @@ -45,7 +46,7 @@ window.MapDataRequest = function() { // additionally, a delay before processing the queue after requeueing tiles // (this way, if multiple requeue delays finish within a short time of each other, they're all processed in one queue run) - this.RERUN_QUEUE_DELAY = 2; + this.RERUN_QUEUE_DELAY = 1; this.REFRESH_CLOSE = 120; // refresh time to use for close views z>12 when not idle and not moving this.REFRESH_FAR = 600; // refresh time for far views z <= 12 @@ -172,6 +173,9 @@ window.MapDataRequest.prototype.refresh = function() { // fill tileBounds with the data needed to request each tile this.tileBounds = {}; + // clear the stale tile data + this.staleTileData = {}; + var bounds = clampLatLngBounds(map.getBounds()); var zoom = getPortalDataZoom(); @@ -238,6 +242,7 @@ window.MapDataRequest.prototype.refresh = function() { this.render.processTileData (this.cache.get(tile_id)); this.cachedTileCount += 1; } else { + // no fresh data - queue a request var boundsParams = generateBoundsParams( tile_id, @@ -247,6 +252,30 @@ window.MapDataRequest.prototype.refresh = function() { lngEast ); + // however, the server does support delta requests - only returning the entities changed since a particular timestamp + // retrieve the stale cache entry and use it, if possible + var stale = (this.cache && this.cache.get(tile_id)); + var lastTimestamp = undefined; + if (stale) { + // find the timestamp of the latest entry in the stale records. the stock site appears to use the browser + // clock, but this isn't reliable. ideally the data set should include it's retrieval timestamp, set by the + // server, for use here. a good approximation is the highest timestamp of all entities + + for (var i in stale.gameEntities) { + var ent = stale.gameEntities[i]; + if (lastTimestamp===undefined || ent[1] > lastTimestamp) { + lastTimestamp = ent[1]; + } + } + +console.log('stale tile '+tile_id+': newest mtime '+lastTimestamp+(lastTimestamp?' '+new Date(lastTimestamp).toString():'')); + if (lastTimestamp) { + // we can issue a useful delta request - store the previous data, as we can't rely on the cache still having it later + this.staleTileData[tile_id] = stale; + boundsParams.timestampMs = lastTimestamp; + } + } + this.tileBounds[tile_id] = boundsParams; this.requestedTileCount += 1; } @@ -358,7 +387,7 @@ window.MapDataRequest.prototype.sendTileRequest = function(tiles) { this.debugTiles.setState (id, 'requested'); - this.requestedTiles[id] = true; + this.requestedTiles[id] = { staleData: this.staleTileData[id] }; var boundsParams = this.tileBounds[id]; if (boundsParams) { @@ -473,13 +502,52 @@ window.MapDataRequest.prototype.handleResponse = function (data, tiles, success) // no error for this data tile - process it successTiles.push (id); + var stale = this.requestedTiles[id].staleData; + if (stale) { + // we have stale data. therefore, a delta request was made for this tile. we need to merge the results with + // the existing stale data before proceeding + + var dataObj = {}; + // copy all entities from the stale data... + for (var i in stale.gameEntities||[]) { + var ent = stale.gameEntities[i]; + dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] }; + } +var oldEntityCount = Object.keys(dataObj).length; + // and remove any entities in the deletedEntnties list + for (var i in val.deletedEntities||[]) { + var guid = val.deletedEntities[i]; + delete dataObj[guid]; + } +var oldEntityCount2 = Object.keys(dataObj).length; + // then add all entities from the new data + for (var i in val.gameEntities||[]) { + var ent = val.gameEntities[i]; + dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] }; + } +var newEntityCount = Object.keys(dataObj).length; +console.log('processed delta mapData request:'+id+': '+oldEntityCount+' original entities, '+oldEntityCount2+' after deletion, '+val.gameEntities.length+' entities in the response'); + + // now reconstruct a new gameEntities array in val, with the updated data + val.gameEntities = []; + for (var guid in dataObj) { + var ent = [guid, dataObj[guid].timestamp, dataObj[guid].data]; + val.gameEntities.push(ent); + } + + // we can leave the returned 'deletedEntities' data unmodified - it's not critial to how IITC works anyway + + // also delete any staleTileData entries for this tile - no longer required + delete this.staleTileData[id]; + } + // store the result in the cache this.cache && this.cache.store (id, val); // if this tile was in the render list, render it // (requests aren't aborted when new requests are started, so it's entirely possible we don't want to render it!) if (id in this.tileBounds) { - this.debugTiles.setState (id, 'ok'); + this.debugTiles.setState (id, stale?'ok-delta':'ok'); this.render.processTileData (val);