some work-in-progress. from a read of the far-less-obfsucated code on the stock site it looks like map data can be retrieved as an update to an earlier query

i.e. pass the timestamp of the last data request, and the server should only send the changed data rather than everything

code/map_data_cache.js

@@ -2,8 +2,14 @@
 // cache for map data tiles. 
 
 window.DataCache = function() {
-  this.REQUEST_CACHE_FRESH_AGE = 60;  // if younger than this, use data in the cache rather than fetching from the server
+  // stock site nemesis.dashboard.DataManager.CACHE_EXPIRY_MS_ = 18E4 - so should be 2 mins cache time
-  this.REQUEST_CACHE_MAX_AGE = 180;  // maximum cache age. entries are deleted from the cache after this time
+  this.REQUEST_CACHE_FRESH_AGE = 120;  // if younger than this, use data in the cache rather than fetching from the server
+
+  // stale cache entries can be updated (that's what the optional 'timestampMs' field in getThinnedEntnties is
+  // for, retrieving deltas) so use a long max age to take advantage of this
+  // however, ther must be an overall limit on the maximum age of data from the servers, otherwise the deletedEntity
+  // entries would grow indefinitely. an hour seems reasonable from experience with the data, so 55 mins max cache time
+  this.REQUEST_CACHE_MAX_AGE = 55*60;  // maximum cache age. entries are deleted from the cache after this time
 
   if (L.Browser.mobile) {
     // on mobile devices, smaller cache size

code/map_data_debug.js

@@ -37,7 +37,8 @@ window.RenderDebugTiles.prototype.setState = function(id,state) {
   var col = '#f0f';
   var fill = '#f0f';
   switch(state) {
-    case 'ok': col='#0f0'; fill='#0f0'; break;
+    case 'ok': col='#080'; fill='#080'; break;
+    case 'ok-delta': col='#0f0'; fill='#0f0'; break;
     case 'error': col='#f00'; fill='#f00'; break;
     case 'cache-fresh': col='#0f0'; fill='#ff0'; break;
     case 'cache-stale': col='#f00'; fill='#ff0'; break;

code/map_data_request.js

@@ -11,6 +11,7 @@ window.MapDataRequest = function() {
 
   this.activeRequestCount = 0;
   this.requestedTiles = {};
+  this.staleTileData = {};
 
   this.idle = false;
 
@@ -45,7 +46,7 @@ window.MapDataRequest = function() {
 
   // additionally, a delay before processing the queue after requeueing tiles
   // (this way, if multiple requeue delays finish within a short time of each other, they're all processed in one queue run)
-  this.RERUN_QUEUE_DELAY = 2;
+  this.RERUN_QUEUE_DELAY = 1;
 
   this.REFRESH_CLOSE = 120;  // refresh time to use for close views z>12 when not idle and not moving
   this.REFRESH_FAR = 600;  // refresh time for far views z <= 12
@@ -172,6 +173,9 @@ window.MapDataRequest.prototype.refresh = function() {
   // fill tileBounds with the data needed to request each tile
   this.tileBounds = {};
 
+  // clear the stale tile data
+  this.staleTileData = {};
+
 
   var bounds = clampLatLngBounds(map.getBounds());
   var zoom = getPortalDataZoom();
@@ -238,6 +242,7 @@ window.MapDataRequest.prototype.refresh = function() {
         this.render.processTileData (this.cache.get(tile_id));
         this.cachedTileCount += 1;
       } else {
+
         // no fresh data - queue a request
         var boundsParams = generateBoundsParams(
           tile_id,
@@ -247,6 +252,30 @@ window.MapDataRequest.prototype.refresh = function() {
           lngEast
         );
 
+        // however, the server does support delta requests - only returning the entities changed since a particular timestamp
+        // retrieve the stale cache entry and use it, if possible
+        var stale = (this.cache && this.cache.get(tile_id));
+        var lastTimestamp = undefined;
+        if (stale) {
+          // find the timestamp of the latest entry in the stale records. the stock site appears to use the browser
+          // clock, but this isn't reliable. ideally the data set should include it's retrieval timestamp, set by the
+          // server, for use here. a good approximation is the highest timestamp of all entities
+
+          for (var i in stale.gameEntities) {
+            var ent = stale.gameEntities[i];
+            if (lastTimestamp===undefined || ent[1] > lastTimestamp) {
+              lastTimestamp = ent[1];
+            }
+          }
+
+console.log('stale tile '+tile_id+': newest mtime '+lastTimestamp+(lastTimestamp?' '+new Date(lastTimestamp).toString():''));
+          if (lastTimestamp) {
+            // we can issue a useful delta request - store the previous data, as we can't rely on the cache still having it later
+            this.staleTileData[tile_id] = stale;
+            boundsParams.timestampMs = lastTimestamp;
+          }
+        }
+
         this.tileBounds[tile_id] = boundsParams;
         this.requestedTileCount += 1;
       }
@@ -358,7 +387,7 @@ window.MapDataRequest.prototype.sendTileRequest = function(tiles) {
 
     this.debugTiles.setState (id, 'requested');
 
-    this.requestedTiles[id] = true;
+    this.requestedTiles[id] = { staleData: this.staleTileData[id] };
 
     var boundsParams = this.tileBounds[id];
     if (boundsParams) {
@@ -473,13 +502,52 @@ window.MapDataRequest.prototype.handleResponse = function (data, tiles, success)
         // no error for this data tile - process it
         successTiles.push (id);
 
+        var stale = this.requestedTiles[id].staleData;
+        if (stale) {
+          // we have stale data. therefore, a delta request was made for this tile. we need to merge the results with
+          // the existing stale data before proceeding
+
+          var dataObj = {};
+          // copy all entities from the stale data...
+          for (var i in stale.gameEntities||[]) {
+            var ent = stale.gameEntities[i];
+            dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] };
+          }
+var oldEntityCount = Object.keys(dataObj).length;
+          // and remove any entities in the deletedEntnties list
+          for (var i in val.deletedEntities||[]) {
+            var guid = val.deletedEntities[i];
+            delete dataObj[guid];
+          }
+var oldEntityCount2 = Object.keys(dataObj).length;
+          // then add all entities from the new data
+          for (var i in val.gameEntities||[]) {
+            var ent = val.gameEntities[i];
+            dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] };
+          }
+var newEntityCount = Object.keys(dataObj).length;
+console.log('processed delta mapData request:'+id+': '+oldEntityCount+' original entities, '+oldEntityCount2+' after deletion, '+val.gameEntities.length+' entities in the response');
+
+          // now reconstruct a new gameEntities array in val, with the updated data
+          val.gameEntities = [];
+          for (var guid in dataObj) {
+            var ent = [guid, dataObj[guid].timestamp, dataObj[guid].data];
+            val.gameEntities.push(ent);
+          }
+
+          // we can leave the returned 'deletedEntities' data unmodified - it's not critial to how IITC works anyway
+
+          // also delete any staleTileData entries for this tile - no longer required
+          delete this.staleTileData[id];
+        }
+
         // store the result in the cache
         this.cache && this.cache.store (id, val);
 
         // if this tile was in the render list, render it
         // (requests aren't aborted when new requests are started, so it's entirely possible we don't want to render it!)
         if (id in this.tileBounds) {
-          this.debugTiles.setState (id, 'ok');
+          this.debugTiles.setState (id, stale?'ok-delta':'ok');
 
           this.render.processTileData (val);