diff --git a/code/map_data.js b/code/map_data.js index 471eb2fb..5dfdbec3 100644 --- a/code/map_data.js +++ b/code/map_data.js @@ -3,242 +3,17 @@ // map. They also keep them up to date, unless interrupted by user // action. -// cache for data tiles. indexed by the query key (qk) -window.cache = undefined; -window.render = undefined; -window.debugTiles = undefined; +window.request = undefined; -// due to the cache (and race conditions in the server) - and now also oddities in the returned data -// we need to remember the deleted entities list across multiple requests -window._deletedEntityGuid = {} // requests map data for current viewport. For details on how this // works, refer to the description in “MAP DATA REQUEST CALCULATORS” window.requestData = function() { - if (window.cache === undefined) window.cache = new DataCache(); - if (window.render === undefined) window.render = new Render(); + if (window.request === undefined) window.request = new Request(); - console.log('refreshing data'); - requests.abort(); - window.statusTotalMapTiles = 0; - window.statusCachedMapTiles = 0; - window.statusSuccessMapTiles = 0; - window.statusStaleMapTiles = 0; - window.statusErrorMapTiles = 0; + request.start(); - // clear the list of returned deleted entities - window._deletedEntityGuid = {} - - if (!debugTiles) debugTiles = new RenderDebugTiles(); - debugTiles.reset(); - - cache.expire(); - - - render.startRenderPass(); - - - //a limit on the number of map tiles to be pulled in a single request - var MAX_TILES_PER_BUCKET = 18; - // the number of separate buckets. more can be created if the size exceeds MAX_TILES_PER_BUCKET - var BUCKET_COUNT = 4; - - var bounds = clampLatLngBounds(map.getBounds()); - - //we query the server as if the zoom level was this. it may not match the actual map zoom level - var z = getPortalDataZoom(); - console.log('requesting data tiles at zoom '+z+' (L'+getMinPortalLevelForZoom(z)+'+ portals), map zoom is '+map.getZoom()); - - var x1 = lngToTile(bounds.getWest(), z); - var x2 = lngToTile(bounds.getEast(), z); - var y1 = latToTile(bounds.getNorth(), z); - var y2 = latToTile(bounds.getSouth(), z); - - // will group requests by second-last quad-key quadrant - tiles = {}; - fullBucketCount = 0; - - var requestTileCount = 0; - - // y goes from left to right - for (var y = y1; y <= y2; y++) { - // x goes from bottom to top(?) - for (var x = x1; x <= x2; x++) { - var tile_id = pointToTileId(z, x, y); - var latNorth = tileToLat(y,z); - var latSouth = tileToLat(y+1,z); - var lngWest = tileToLng(x,z); - var lngEast = tileToLng(x+1,z); - - debugTiles.create(tile_id,[[latSouth,lngWest],[latNorth,lngEast]]); - window.statusTotalMapTiles++; - - // TODO?: if the selected portal is in this tile, always fetch the data - if (cache.isFresh(tile_id)) { - var tiledata = cache.get(tile_id); - - render.processTileData (tiledata); - - debugTiles.setColour(tile_id,'#0f0','#ff0'); - window.statusCachedMapTiles++; - } else { - // group requests into buckets based on the tile count retrieved via the network. - var bucket = requestTileCount % BUCKET_COUNT; - - if (!tiles[bucket]) { - //create empty bucket - tiles[bucket] = []; - } - else if(tiles[bucket].length >= MAX_TILES_PER_BUCKET) { - //too many items in bucket. rename it, and create a new empty one - tiles[bucket+'_'+fullBucketCount] = tiles[bucket]; - fullBucketCount++; - tiles[bucket] = []; - } - - requestTileCount++; - - var boundsParam = generateBoundsParams( - tile_id, - latSouth, - lngWest, - latNorth, - lngEast - ); - - tiles[bucket].push(boundsParam); - - debugTiles.setColour(tile_id,'#00f','#000'); - } - - } - } - - - // send ajax requests - console.log('requesting '+requestTileCount+' tiles in '+Object.keys(tiles).length+' requests'); - $.each(tiles, function(ind, tls) { - // sort the tiles by the cache age - oldest/missing first. the server often times out requests and the first listed - // are more likely to succeed. this will ensure we're more likely to have fresh data - tls.sort(function(a,b) { - var timea = cache.getTime(a.qk); - var timeb = cache.getTime(b.qk); - if (timea < timeb) return -1; - if (timea > timeb) return 1; - return 0; - }); - - data = { }; - data.boundsParamsList = tls; - // keep a list of tile_ids with each request. in the case of a server error, we can try and use cached tiles if available - var tile_ids = [] - $.each(tls,function(i,req) { tile_ids.push(req.qk); }); - window.requests.add(window.postAjax('getThinnedEntitiesV4', data, function(data, textStatus, jqXHR) { window.handleDataResponse(data,false,tile_ids); }, function() { window.handleFailedRequest(tile_ids); })); - }); - - if(tiles.length == 0) { - // if everything was cached, we immediately end the render pass - // otherwise, the render pass will be ended in the callbacks - render.endRenderPass(); - } -} - -// Handle failed map data request -window.handleFailedRequest = function(tile_ids) { - console.log('request failed: tiles '+tile_ids.join(',')); - - var cachedData = { result: { map: {} } }; - $.each(tile_ids, function(ind,tile_id) { - var cached = cache.get(tile_id); - if (cached) { - // we have stale cached data - use it - cachedData.result.map[tile_id] = cached; - debugTiles.setColour(tile_id,'#800','#ff0'); - console.log('(using stale cache entry for map tile '+tile_id+')'); - window.statusStaleMapTiles++; - } else { - // no cached data - debugTiles.setColour(tile_id,'#800','#f00'); - window.statusErrorMapTiles++; - } - }); - if(Object.keys(cachedData.result.map).length > 0) { - handleDataResponse(cachedData, true); - } - - if(requests.isLastRequest('getThinnedEntitiesV4')) { - render.endRenderPass(); -// var leftOverPortals = portalRenderLimit.mergeLowLevelPortals(null); -// handlePortalsRender(leftOverPortals); - } - runHooks('requestFinished', {success: false}); -} - -// works on map data response and ensures entities are drawn/updated. -window.handleDataResponse = function(data, fromCache, tile_ids) { - // remove from active ajax queries list - if(!data || !data.result) { - window.failedRequestCount++; - console.warn(data); - handleFailedRequest(tile_ids); - return; - } - - var m = data.result.map; - // defer rendering of portals because there is no z-index in SVG. - // this means that what’s rendered last ends up on top. While the - // portals can be brought to front, this costs extra time. They need - // to be in the foreground, or they cannot be clicked. See - // https://github.com/Leaflet/Leaflet/issues/185 - var ppp = {}; - var p2f = {}; - $.each(m, function(qk, val) { - - // if this request wasn't from the cache, check it's status. store in the cache if good - // for debugging, we set the debug tile colours. cached tiles have colours set elsewhere so are not set here - if (!fromCache) { - - if('error' in val) { - console.log('map data tile '+qk+' response error: '+val.error); - - // try to use data in the cache, even if it's stale - var cacheVal = cache.get(qk); - - if (!cacheVal) { - debugTiles.setColour(qk, '#f00','#f00'); - // no data in cache for this tile. continue processing - it's possible it also has some valid data - window.statusErrorMapTiles++; - } else { - // stale cache entry exists - use it - val = cacheVal; - debugTiles.setColour(qk, '#f00','#ff0'); - console.log('(using stale cache entry for map tile '+qk+')'); - window.statusStaleMapTiles++; - } - } else { - // not an error - store this tile into the cache - cache.store(qk,val); - debugTiles.setColour(qk, '#0f0','#0f0'); - window.statusSuccessMapTiles++; - } - - } - - render.processTileData(val); - - }); - - - resolvePlayerNames(); - renderUpdateStatus(); - - if(requests.isLastRequest('getThinnedEntitiesV4')) { - render.endRenderPass(); -// var leftOverPortals = portalRenderLimit.mergeLowLevelPortals(null); -// handlePortalsRender(leftOverPortals); - } - runHooks('requestFinished', {success: true}); } diff --git a/code/map_data_debug.js b/code/map_data_debug.js index 3b12b376..340531fb 100644 --- a/code/map_data_debug.js +++ b/code/map_data_debug.js @@ -15,7 +15,7 @@ window.RenderDebugTiles.prototype.reset = function() { } window.RenderDebugTiles.prototype.create = function(id,bounds) { - var s = {color: '#000', weight: 3, opacity: 0.7, fillColor: '#000', fillOpacity: 0.5, clickable: false}; + var s = {color: '#666', weight: 3, opacity: 0.4, fillColor: '#666', fillOpacity: 0.2, clickable: false}; var bounds = new L.LatLngBounds(bounds); bounds = bounds.pad(-0.02); @@ -28,7 +28,21 @@ window.RenderDebugTiles.prototype.create = function(id,bounds) { window.RenderDebugTiles.prototype.setColour = function(id,bordercol,fillcol) { var l = this.debugTileToRectangle[id]; if (l) { - var s = {color: bordercol, weight: 3, opacity: 0.3, fillColor: fillcol, fillOpacity: 0.1, clickable: false}; + var s = {color: bordercol, weight: 3, opacity: 0.4, fillColor: fillcol, fillOpacity: 0.2, clickable: false}; l.setStyle(s); } } + +window.RenderDebugTiles.prototype.setState = function(id,state) { + var col = '#666'; + var fill = '#666'; + switch(state) { + case 'ok': col='#0f0'; fill='#0f0'; break; + case 'error': col='#f00'; fill='#f00'; break; + case 'cache-fresh': col='#0f0'; fill='#ff0'; break; + case 'cache-stale': col='#f00'; fill='#ff0'; break; + case 'requested': col='#00f'; fill='#00f'; break; + case 'retrying': col='#666'; fill='#666'; break; + } + this.setColour (id, col, fill); +} diff --git a/code/map_data_request.js b/code/map_data_request.js index 9beba535..79211cc7 100644 --- a/code/map_data_request.js +++ b/code/map_data_request.js @@ -10,11 +10,11 @@ window.Request = function() { this.debugTiles = new RenderDebugTiles(); this.activeRequestCount = 0; + this.requestedTiles = {}; this.MAX_REQUESTS = 4; this.MAX_TILES_PER_REQUEST = 12; - } @@ -27,7 +27,7 @@ window.Request.prototype.start = function() { var bounds = clampLatLngBounds(map.getBounds()); var zoom = getPortalDataZoom(); - var minPortalLevel = getPortalLevelForZoom(zoom); + var minPortalLevel = getMinPortalLevelForZoom(zoom); this.render.startRenderPass(bounds); @@ -35,27 +35,27 @@ window.Request.prototype.start = function() { console.log('requesting data tiles at zoom '+zoom+' (L'+minPortalLevel+'+ portals), map zoom is '+map.getZoom()); - var x1 = lngToTile(bounds.getWest(), z); - var x2 = lngToTile(bounds.getEast(), z); - var y1 = latToTile(bounds.getNorth(), z); - var y2 = latToTile(bounds.getSouth(), z); - + // fill tileBounds with the data needed to request each tile this.tileBounds = {}; + var x1 = lngToTile(bounds.getWest(), zoom); + var x2 = lngToTile(bounds.getEast(), zoom); + var y1 = latToTile(bounds.getNorth(), zoom); + var y2 = latToTile(bounds.getSouth(), zoom); // y goes from left to right for (var y = y1; y <= y2; y++) { // x goes from bottom to top(?) for (var x = x1; x <= x2; x++) { - var tile_id = pointToTileId(z, x, y); - var latNorth = tileToLat(y,z); - var latSouth = tileToLat(y+1,z); - var lngWest = tileToLng(x,z); - var lngEast = tileToLng(x+1,z); + var tile_id = pointToTileId(zoom, x, y); + var latNorth = tileToLat(y,zoom); + var latSouth = tileToLat(y+1,zoom); + var lngWest = tileToLng(x,zoom); + var lngEast = tileToLng(x+1,zoom); this.debugTiles.create(tile_id,[[latSouth,lngWest],[latNorth,lngEast]]); - var boundsParam = generateBoundsParams( + var boundsParams = generateBoundsParams( tile_id, latSouth, lngWest, @@ -63,7 +63,7 @@ window.Request.prototype.start = function() { lngEast ); - this.tileBounds[tile_id] = boundsParam; + this.tileBounds[tile_id] = boundsParams; } } @@ -71,22 +71,19 @@ window.Request.prototype.start = function() { this.renderCachedTiles(); - // if nothing left in the queue, end the render. otherwise, send network requests - if (Object.keys(this.tileBounds).length == 0) { - this.render.endRenderPass(); - // TODO: start timer for next refresh cycle here - } else { - this.processRequestQueue(); - } + this.processRequestQueue(); } window.Request.prototype.renderCachedTiles = function() { for (var tile_id in this.tileBounds) { - if (this.cache.isFresh(tile_id) { + if (this.cache.isFresh(tile_id)) { // fresh cache data + + this.debugTiles.setState(tile_id, 'cache-fresh'); + var data = this.cache.get(tile_id); // render it... // TODO?? make rendering sort-of asynchronous? rendering a large number of dense tiles is slow - so perhaps use a timer and render one/a few at a time?? @@ -101,51 +98,108 @@ window.Request.prototype.renderCachedTiles = function() { window.Request.prototype.processRequestQueue = function() { + console.log("Request.processRequestQueue..."); + + // if nothing left in the queue, end the render. otherwise, send network requests + if (Object.keys(this.tileBounds).length == 0) { + this.render.endRenderPass(); + // TODO: start timer for next refresh cycle here + console.log("Request.processRequestQueue: ended"); + + return; + } + + + // create a list of tiles that aren't requested over the network + var pendingTiles = {}; + for (var id in this.tileBounds) { + if (!(id in this.requestedTiles) ) { + pendingTiles[id] = true; + } + } + + console.log("Request.processRequestQueue: "+Object.keys(pendingTiles).length+" tiles waiting"); + + + while (this.activeRequestCount < this.MAX_REQUESTS && Object.keys(pendingTiles).length > 0) { + // let's distribute the requests evenly throughout the pending list. + + var pendingTilesArray = Object.keys(pendingTiles); + + var mod = Math.ceil(Object.keys(pendingTiles).length / this.MAX_TILES_PER_REQUEST); + + var tiles = []; + for (var i in pendingTilesArray) { + if ((i % mod) == 0) { + id = pendingTilesArray[i]; + tiles.push(id); + delete pendingTiles[id]; +// if (tiles.length >= this.MAX_TILES_PER_REQUEST) { +// break; +// } + } + } + + console.log("Request.processRequestQueue: asking for "+tiles.length+" tiles in one request"); + this.sendTileRequest(tiles); + } } window.Request.prototype.sendTileRequest = function(tiles) { - var boundsParamList = []; + var boundsParamsList = []; - for (var id in tiles) { - var boundsParam = this.tileBounds[id]; - if (boundsParam) { - push (boundsParamList, boundsParam); + for (var i in tiles) { + var id = tiles[i]; + + this.debugTiles.setState (id, 'requested'); + + this.requestedTiles[id] = true; + + var boundsParams = this.tileBounds[id]; + if (boundsParams) { + boundsParamsList.push (boundsParams); } else { console.warn('failed to find bounds for tile id '+id); } } - var data = { boundsParamList: boundsParamList }; + var data = { boundsParamsList: boundsParamsList }; this.activeRequestCount += 1; + var savedThis = this; + window.requests.add (window.postAjax('getThinnedEntitiesV4', data, - function(data, textStatus, jqXHR) { this.handleResponse (data, tiles, true); }, // request successful callback - function() { this.handleResponse (undefined, tiles, false); } // request failed callback + function(data, textStatus, jqXHR) { savedThis.handleResponse (data, tiles, true); }, // request successful callback + function() { savedThis.handleResponse (undefined, tiles, false); } // request failed callback )); } window.Request.prototype.requeueTile = function(id, error) { if (id in this.tileBounds) { - // tile is currently wanted, requeue + // tile is currently wanted... if (error) { // if error is true, it was a 'bad' error - in this case we limit the number of retries (or prefer stale cached data?) var data = this.cache.get(id); if (data) { // we have cached data - use it, even though it's stale + this.debugTiles.setState (id, 'cache-stale'); this.render.processTileData (data); delete this.tileBounds[id]; } else { // no cached data + this.debugTiles.setState (id, 'error'); } + // and delete from the pending requests... + delete this.tileBounds[id]; } else { // if false, was a 'timeout', so unlimited retries (as the stock site does) - + this.debugTiles.setState (id, 'retrying'); } } } @@ -155,13 +209,24 @@ window.Request.prototype.handleResponse = function (data, tiles, success) { this.activeRequestCount -= 1; + for (var i in tiles) { + var id = tiles[i]; + delete this.requestedTiles[id]; + } + if (!success || !data || !data.result) { + console.warn("Request.handleResponse: request failed - requeing..."); + //request failed - requeue all the tiles(?) - for (var id in tiles) { + for (var i in tiles) { + var id = tiles[i]; this.requeueTile(id, true); } } else { + + // TODO: use result.minLevelOfDetail ??? stock site doesn't use it yet... + var m = data.result.map; for (var id in m) { @@ -171,6 +236,7 @@ window.Request.prototype.handleResponse = function (data, tiles, success) { // server returned an error for this individual data tile if (val.error == "TIMEOUT") { + console.log('map data tile '+id+' returned TIMEOUT'); // TIMEOUT errors for individual tiles are 'expected'(!) - and result in a silent unlimited retries this.requeueTile(id, false); } else { @@ -179,6 +245,7 @@ window.Request.prototype.handleResponse = function (data, tiles, success) { } } else { // no error for this data tile - process it + console.log('map data tile '+id+' is good...'); // store the result in the cache this.cache.store (id, val); @@ -186,6 +253,8 @@ window.Request.prototype.handleResponse = function (data, tiles, success) { // if this tile was in the render list, render it // (requests aren't aborted when new requests are started, so it's entirely possible we don't want to render it!) if (id in this.tileBounds) { + this.debugTiles.setState (id, 'ok'); + this.render.processTileData (val); delete this.tileBounds[id]; @@ -195,4 +264,5 @@ window.Request.prototype.handleResponse = function (data, tiles, success) { } } + this.processRequestQueue(); }