some work-in-progress. from a read of the far-less-obfsucated code on the stock site it looks like map data can be retrieved as an update to an earlier query

i.e. pass the timestamp of the last data request, and the server should only send the changed data rather than everything

code/map_data_request.js

@@ -11,6 +11,7 @@ window.MapDataRequest = function() {
 
   this.activeRequestCount = 0;
   this.requestedTiles = {};
+  this.staleTileData = {};
 
   this.idle = false;
 
@@ -45,7 +46,7 @@ window.MapDataRequest = function() {
 
   // additionally, a delay before processing the queue after requeueing tiles
   // (this way, if multiple requeue delays finish within a short time of each other, they're all processed in one queue run)
-  this.RERUN_QUEUE_DELAY = 2;
+  this.RERUN_QUEUE_DELAY = 1;
 
   this.REFRESH_CLOSE = 120;  // refresh time to use for close views z>12 when not idle and not moving
   this.REFRESH_FAR = 600;  // refresh time for far views z <= 12
@@ -172,6 +173,9 @@ window.MapDataRequest.prototype.refresh = function() {
   // fill tileBounds with the data needed to request each tile
   this.tileBounds = {};
 
+  // clear the stale tile data
+  this.staleTileData = {};
+
 
   var bounds = clampLatLngBounds(map.getBounds());
   var zoom = getPortalDataZoom();
@@ -238,6 +242,7 @@ window.MapDataRequest.prototype.refresh = function() {
         this.render.processTileData (this.cache.get(tile_id));
         this.cachedTileCount += 1;
       } else {
+
         // no fresh data - queue a request
         var boundsParams = generateBoundsParams(
           tile_id,
@@ -247,6 +252,30 @@ window.MapDataRequest.prototype.refresh = function() {
           lngEast
         );
 
+        // however, the server does support delta requests - only returning the entities changed since a particular timestamp
+        // retrieve the stale cache entry and use it, if possible
+        var stale = (this.cache && this.cache.get(tile_id));
+        var lastTimestamp = undefined;
+        if (stale) {
+          // find the timestamp of the latest entry in the stale records. the stock site appears to use the browser
+          // clock, but this isn't reliable. ideally the data set should include it's retrieval timestamp, set by the
+          // server, for use here. a good approximation is the highest timestamp of all entities
+
+          for (var i in stale.gameEntities) {
+            var ent = stale.gameEntities[i];
+            if (lastTimestamp===undefined || ent[1] > lastTimestamp) {
+              lastTimestamp = ent[1];
+            }
+          }
+
+console.log('stale tile '+tile_id+': newest mtime '+lastTimestamp+(lastTimestamp?' '+new Date(lastTimestamp).toString():''));
+          if (lastTimestamp) {
+            // we can issue a useful delta request - store the previous data, as we can't rely on the cache still having it later
+            this.staleTileData[tile_id] = stale;
+            boundsParams.timestampMs = lastTimestamp;
+          }
+        }
+
         this.tileBounds[tile_id] = boundsParams;
         this.requestedTileCount += 1;
       }
@@ -358,7 +387,7 @@ window.MapDataRequest.prototype.sendTileRequest = function(tiles) {
 
     this.debugTiles.setState (id, 'requested');
 
-    this.requestedTiles[id] = true;
+    this.requestedTiles[id] = { staleData: this.staleTileData[id] };
 
     var boundsParams = this.tileBounds[id];
     if (boundsParams) {
@@ -473,13 +502,52 @@ window.MapDataRequest.prototype.handleResponse = function (data, tiles, success)
         // no error for this data tile - process it
         successTiles.push (id);
 
+        var stale = this.requestedTiles[id].staleData;
+        if (stale) {
+          // we have stale data. therefore, a delta request was made for this tile. we need to merge the results with
+          // the existing stale data before proceeding
+
+          var dataObj = {};
+          // copy all entities from the stale data...
+          for (var i in stale.gameEntities||[]) {
+            var ent = stale.gameEntities[i];
+            dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] };
+          }
+var oldEntityCount = Object.keys(dataObj).length;
+          // and remove any entities in the deletedEntnties list
+          for (var i in val.deletedEntities||[]) {
+            var guid = val.deletedEntities[i];
+            delete dataObj[guid];
+          }
+var oldEntityCount2 = Object.keys(dataObj).length;
+          // then add all entities from the new data
+          for (var i in val.gameEntities||[]) {
+            var ent = val.gameEntities[i];
+            dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] };
+          }
+var newEntityCount = Object.keys(dataObj).length;
+console.log('processed delta mapData request:'+id+': '+oldEntityCount+' original entities, '+oldEntityCount2+' after deletion, '+val.gameEntities.length+' entities in the response');
+
+          // now reconstruct a new gameEntities array in val, with the updated data
+          val.gameEntities = [];
+          for (var guid in dataObj) {
+            var ent = [guid, dataObj[guid].timestamp, dataObj[guid].data];
+            val.gameEntities.push(ent);
+          }
+
+          // we can leave the returned 'deletedEntities' data unmodified - it's not critial to how IITC works anyway
+
+          // also delete any staleTileData entries for this tile - no longer required
+          delete this.staleTileData[id];
+        }
+
         // store the result in the cache
         this.cache && this.cache.store (id, val);
 
         // if this tile was in the render list, render it
         // (requests aren't aborted when new requests are started, so it's entirely possible we don't want to render it!)
         if (id in this.tileBounds) {
-          this.debugTiles.setState (id, 'ok');
+          this.debugTiles.setState (id, stale?'ok-delta':'ok');
 
           this.render.processTileData (val);