From e7fe4bb1dddc7fefc4b197642afcda31193787e9 Mon Sep 17 00:00:00 2001 From: Jon Atkins Date: Tue, 29 Oct 2013 20:29:08 +0000 Subject: [PATCH] first pass at a fix for protocol changes - #618 --- code/map_data_debug.js | 3 +- code/map_data_request.js | 120 +++++------------------------- code/utils_misc.js | 154 +++++++++++++-------------------------- 3 files changed, 72 insertions(+), 205 deletions(-) diff --git a/code/map_data_debug.js b/code/map_data_debug.js index dcede257..4ecc27a8 100644 --- a/code/map_data_debug.js +++ b/code/map_data_debug.js @@ -37,8 +37,7 @@ window.RenderDebugTiles.prototype.setState = function(id,state) { var col = '#f0f'; var fill = '#f0f'; switch(state) { - case 'ok': col='#080'; fill='#080'; break; - case 'ok-delta': col='#0f0'; fill='#0f0'; break; + case 'ok': col='#0f0'; fill='#0f0'; break; case 'error': col='#f00'; fill='#f00'; break; case 'cache-fresh': col='#0f0'; fill='#ff0'; break; case 'cache-stale': col='#f00'; fill='#ff0'; break; diff --git a/code/map_data_request.js b/code/map_data_request.js index 07ee7e32..1c112a88 100644 --- a/code/map_data_request.js +++ b/code/map_data_request.js @@ -11,7 +11,6 @@ window.MapDataRequest = function() { this.activeRequestCount = 0; this.requestedTiles = {}; - this.staleTileData = {}; this.idle = false; @@ -173,11 +172,8 @@ window.MapDataRequest.prototype.refresh = function() { // a 'set' to keep track of hard failures for tiles this.tileErrorCount = {}; - // fill tileBounds with the data needed to request each tile - this.tileBounds = {}; - - // clear the stale tile data - this.staleTileData = {}; + // the 'set' of requested tile QKs + this.queuedTiles = {}; var bounds = clampLatLngBounds(map.getBounds()); @@ -255,43 +251,7 @@ window.MapDataRequest.prototype.refresh = function() { } // queue a request - var boundsParams = generateBoundsParams( - tile_id, - latSouth, - lngWest, - latNorth, - lngEast - ); - -/* After some testing, this doesn't seem to actually work. The server returns the same data with and without the parameter - * Also, closer study of the stock site code shows the parameter isn't actually set anywhere. - * so, disabling for now... - // however, the server does support delta requests - only returning the entities changed since a particular timestamp - // retrieve the stale cache entry and use it, if possible - var stale = (this.cache && this.cache.get(tile_id)); - var lastTimestamp = undefined; - if (stale) { - // find the timestamp of the latest entry in the stale records. the stock site appears to use the browser - // clock, but this isn't reliable. ideally the data set should include it's retrieval timestamp, set by the - // server, for use here. a good approximation is the highest timestamp of all entities - - for (var i in stale.gameEntities) { - var ent = stale.gameEntities[i]; - if (lastTimestamp===undefined || ent[1] > lastTimestamp) { - lastTimestamp = ent[1]; - } - } - -console.log('stale tile '+tile_id+': newest mtime '+lastTimestamp+(lastTimestamp?' '+new Date(lastTimestamp).toString():'')); - if (lastTimestamp) { - // we can issue a useful delta request - store the previous data, as we can't rely on the cache still having it later - this.staleTileData[tile_id] = stale; - boundsParams.timestampMs = lastTimestamp; - } - } -*/ - - this.tileBounds[tile_id] = boundsParams; + this.queuedTiles[tile_id] = tile_id; this.requestedTileCount += 1; } } @@ -305,7 +265,7 @@ console.log('stale tile '+tile_id+': newest mtime '+lastTimestamp+(lastTimestamp console.log ('done request preperation (cleared out-of-bounds and invalid for zoom, and rendered cached data)'); - if (Object.keys(this.tileBounds).length > 0) { + if (Object.keys(this.queuedTiles).length > 0) { // queued requests - don't start processing the download queue immediately - start it after a short delay this.delayProcessRequestQueue (this.DOWNLOAD_DELAY,true); } else { @@ -326,7 +286,7 @@ window.MapDataRequest.prototype.delayProcessRequestQueue = function(seconds,isFi window.MapDataRequest.prototype.processRequestQueue = function(isFirstPass) { // if nothing left in the queue, end the render. otherwise, send network requests - if (Object.keys(this.tileBounds).length == 0) { + if (Object.keys(this.queuedTiles).length == 0) { this.render.endRenderPass(); @@ -354,7 +314,7 @@ window.MapDataRequest.prototype.processRequestQueue = function(isFirstPass) { // create a list of tiles that aren't requested over the network var pendingTiles = []; - for (var id in this.tileBounds) { + for (var id in this.queuedTiles) { if (!(id in this.requestedTiles) ) { pendingTiles.push(id); } @@ -404,24 +364,23 @@ window.MapDataRequest.prototype.processRequestQueue = function(isFirstPass) { window.MapDataRequest.prototype.sendTileRequest = function(tiles) { - var boundsParamsList = []; + var tilesList = []; for (var i in tiles) { var id = tiles[i]; this.debugTiles.setState (id, 'requested'); - this.requestedTiles[id] = { staleData: this.staleTileData[id] }; + this.requestedTiles[id] = true; - var boundsParams = this.tileBounds[id]; - if (boundsParams) { - boundsParamsList.push (boundsParams); + if (id in this.queuedTiles) { + tilesList.push (id); } else { - console.warn('failed to find bounds for tile id '+id); + console.warn('no queue entry for tile id '+id); } } - var data = { boundsParamsList: boundsParamsList }; + var data = { quadKeys: tilesList }; this.activeRequestCount += 1; @@ -435,7 +394,7 @@ window.MapDataRequest.prototype.sendTileRequest = function(tiles) { } window.MapDataRequest.prototype.requeueTile = function(id, error) { - if (id in this.tileBounds) { + if (id in this.queuedTiles) { // tile is currently wanted... // first, see if the error can be ignored due to retry counts @@ -461,7 +420,7 @@ window.MapDataRequest.prototype.requeueTile = function(id, error) { this.failedTileCount += 1; } // and delete from the pending requests... - delete this.tileBounds[id]; + delete this.queuedTiles[id]; } else { // if false, was a 'timeout' or we're retrying, so unlimited retries (as the stock site does) @@ -471,9 +430,8 @@ window.MapDataRequest.prototype.requeueTile = function(id, error) { // proper queue, just an object with guid as properties. Javascript standards don't guarantee the order of properties // within an object. however, all current browsers do keep property order, and new properties are added at the end. // therefore, delete and re-add the requeued tile and it will be added to the end of the queue - var boundsData = this.tileBounds[id]; - delete this.tileBounds[id]; - this.tileBounds[id] = boundsData; + delete this.queuedTiles[id]; + this.queuedTiles[id] = id; } } // else the tile wasn't currently wanted (an old non-cancelled request) - ignore @@ -525,57 +483,17 @@ window.MapDataRequest.prototype.handleResponse = function (data, tiles, success) // no error for this data tile - process it successTiles.push (id); - var stale = this.requestedTiles[id].staleData; - if (stale) { -//NOTE: currently this code path won't be used, as we never set the staleData to anything other than 'undefined' - // we have stale data. therefore, a delta request was made for this tile. we need to merge the results with - // the existing stale data before proceeding - - var dataObj = {}; - // copy all entities from the stale data... - for (var i in stale.gameEntities||[]) { - var ent = stale.gameEntities[i]; - dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] }; - } -var oldEntityCount = Object.keys(dataObj).length; - // and remove any entities in the deletedEntnties list - for (var i in val.deletedEntities||[]) { - var guid = val.deletedEntities[i]; - delete dataObj[guid]; - } -var oldEntityCount2 = Object.keys(dataObj).length; - // then add all entities from the new data - for (var i in val.gameEntities||[]) { - var ent = val.gameEntities[i]; - dataObj[ent[0]] = { timestamp: ent[1], data: ent[2] }; - } -var newEntityCount = Object.keys(dataObj).length; -console.log('processed delta mapData request:'+id+': '+oldEntityCount+' original entities, '+oldEntityCount2+' after deletion, '+val.gameEntities.length+' entities in the response'); - - // now reconstruct a new gameEntities array in val, with the updated data - val.gameEntities = []; - for (var guid in dataObj) { - var ent = [guid, dataObj[guid].timestamp, dataObj[guid].data]; - val.gameEntities.push(ent); - } - - // we can leave the returned 'deletedEntities' data unmodified - it's not critial to how IITC works anyway - - // also delete any staleTileData entries for this tile - no longer required - delete this.staleTileData[id]; - } - // store the result in the cache this.cache && this.cache.store (id, val); // if this tile was in the render list, render it // (requests aren't aborted when new requests are started, so it's entirely possible we don't want to render it!) - if (id in this.tileBounds) { - this.debugTiles.setState (id, stale?'ok-delta':'ok'); + if (id in this.queuedTiles) { + this.debugTiles.setState (id, 'ok'); this.render.processTileData (val); - delete this.tileBounds[id]; + delete this.queuedTiles[id]; this.successTileCount += 1; } // else we don't want this tile (from an old non-cancelled request) - ignore diff --git a/code/utils_misc.js b/code/utils_misc.js index f41755e5..5ca68549 100644 --- a/code/utils_misc.js +++ b/code/utils_misc.js @@ -129,104 +129,46 @@ window.digits = function(d) { window.requestParameterMunges = [ // now obsolete (they don't have some of the new parameters) munge sets deleted - - // set 3 - in the update of 2013-09-30 (addition of 'alerts' chat tab) + // set 6 - 2013-10-29 { - 'dashboard.getGameScore': 'fhlzntzkl5v7hcfh', // GET_GAME_SCORE - 'dashboard.getPaginatedPlextsV2': 'wzuitnswoda7w028', // GET_PAGINATED_PLEXTS - 'dashboard.getThinnedEntitiesV4': 'scgrm4lf2371esgw', // GET_THINNED_ENTITIES - 'dashboard.getPlayersByGuids': '81l6usczczoi3lfi', // LOOKUP_PLAYERS - 'dashboard.redeemReward': '8kop2koeld9b4c26', // REDEEM_REWARD - 'dashboard.sendInviteEmail': 't0ccodsm1nuo5uso', // SEND_INVITE_EMAIL - 'dashboard.sendPlext': 'k04cfjwwsg3h3827', // SEND_PLEXT + 'dashboard.getGameScore': 'vzjhib746rvkre04', // GET_GAME_SCORE + 'dashboard.getPaginatedPlextsV2': 'gqa96zhqpddtfmkl', // GET_PAGINATED_PLEXTS + 'dashboard.getThinnedEntitiesV4': '18lmw7lytgxji0dk', // GET_THINNED_ENTITIES + 'dashboard.getPlayersByGuids': 'emb5xrj8rav1i0be', // LOOKUP_PLAYERS + 'dashboard.redeemReward': '4xqof5pldqab63rb', // REDEEM_REWARD + 'dashboard.sendInviteEmail': 'yq5wxjlnud0tj6hu', // SEND_INVITE_EMAIL + 'dashboard.sendPlext': 'e1ipqdxjlwd3l7zb', // SEND_PLEXT - method: '22ux2z96jwq5zn78', - version: 'kf6hgl9yau03ws0o', //guessed parameter name - only seen munged - version_parameter: '4608f4356a6f55690f127fb542f557f98de66169', // passed as the value to the above parameter - boundsParamsList: '29t16cmsn6l3r2xg', - id: '7rogqhp5pzcqobcw', - minLatE6: 'yzbnp7z9bd28p0yr', - minLngE6: '2pdhntvo85cd90bw', - maxLatE6: 'c4ivr013h4dr68pd', - maxLngE6: '4p8oorcrwalc1mzf', - timestampMs: 'vd2rsa9v6f8q606s', - qk: 'cblh9xe0bgwjy5ij', - desiredNumItems: '3ymaq7slb165porj', - minTimestampMs: 's9jf2seni33y3gyu', - maxTimestampMs: '2kh3vti98rhp3g29', - chatTab: '7n7ocqfq1p18352b', //guessed parameter name - only seen munged - ascendingTimestampOrder: 'p88a2ztchtjhiazl', - message: 'e8qm0kptw2trrcrw', - latE6: 'fja1phtsqxm71dqm', - lngE6: 'iut1tb7c0x726hwn', - guids: '5hyiwhwc0jyljvro', - inviteeEmailAddress: 's9z6zt03eymzxhkj', - }, + // common parameters + method: 'wg7gyxoanqc1si5r', + version: 'adlo9o4kjvho5q94', //guessed parameter name - only seen munged + version_parameter: '56036a6497ea344a9fffa38b171a77c092c1f220', // passed as the value to the above parameter - // set 4 - second update of 2013-09-30 - { - 'dashboard.getGameScore': 'ija9jgrf5hj7wm9r', // GET_GAME_SCORE - 'dashboard.getPaginatedPlextsV2': '0elftx739mkbzi1b', // GET_PAGINATED_PLEXTS - 'dashboard.getThinnedEntitiesV4': 'prv0ez8cbsykh63g', // GET_THINNED_ENTITIES - 'dashboard.getPlayersByGuids': 'i0lxy6nc695z9ka3', // LOOKUP_PLAYERS - 'dashboard.redeemReward': '376oivna8rf8qbfj', // REDEEM_REWARD - 'dashboard.sendInviteEmail': '96y930v5q96nrcrw', // SEND_INVITE_EMAIL - 'dashboard.sendPlext': 'c04kceytofuqvyqg', // SEND_PLEXT + // GET_THINNED_ENTITIES + quadKeys: '6vcl0ivqz4aj5sfu', //guessed parameter name - only seen munged - method: '9we4b31i48ui4sdm', - version: 'q402kn5zqisuo1ym', //guessed parameter name - only seen munged - version_parameter: 'dbad4485024d446ae946e3d287b5d640029ef3e3', // passed as the value to the above parameter - boundsParamsList: '3r5ctyvc2f653zjd', - id: 'izey8ciqg2dz2oqc', - minLatE6: 'cein0n4jrifa7ui2', - minLngE6: 'lbd1juids3johtdo', - maxLatE6: 'h4kyot9kmvd3g284', - maxLngE6: 'sbci6jjc2d5g9uy4', - timestampMs: '2wurn9giagbvv6bt', - qk: 'hq73mwpjqyvcp6ul', - desiredNumItems: 'kyo6vh5n58hmrnua', - minTimestampMs: 'hu4swdftcp7mvkdi', - maxTimestampMs: 'ly6ylae5lv1z9072', - chatTab: 'q5kxut5rmbtlqbf9', //guessed parameter name - only seen munged - ascendingTimestampOrder: 'hvfd0io35rahwjgr', - message: 'z4hf7tzl27o14455', - latE6: 'zyzh3bdxyd47vk1x', - lngE6: 'n5d1f8pql51t641x', - guids: 'gl16ehqoc3i3oi07', - inviteeEmailAddress: 'orc9ufg7rp7g1y9j', - }, + // GET_PAGINATED_PLEXTS + desiredNumItems: '6jd5b49wn748diye', + minLatE6: '891ebsryg45b8cxb', + minLngE6: 'mvepdcx1k6noya15', + maxLatE6: 's3rh3fhji5mcjlof', + maxLngE6: 'yqdgfuukrxj8byzj', + minTimestampMs: 'btf0kpztxrkt6sl6', + maxTimestampMs: 'hg8vhtehxf53n5cu', + chatTab: '6bk9rmebtk1ux6da', //guessed parameter name - only seen munged + ascendingTimestampOrder: '4zw3v6xwp117r47w', - // set 5 - second update of 2013-10-16 - { - 'dashboard.getGameScore': '3b48kl956b33brrl', // GET_GAME_SCORE - 'dashboard.getPaginatedPlextsV2': 'h785pmet6wrx6xoa', // GET_PAGINATED_PLEXTS - 'dashboard.getThinnedEntitiesV4': '4gux7b0n3euu7e8y', // GET_THINNED_ENTITIES - 'dashboard.getPlayersByGuids': 'nqm1kocgzspecpzv', // LOOKUP_PLAYERS - 'dashboard.redeemReward': 'g618n6peb74u2ae9', // REDEEM_REWARD - 'dashboard.sendInviteEmail': 'bsl4280bm39bkl3a', // SEND_INVITE_EMAIL - 'dashboard.sendPlext': 'jym2hbw15i6uru7g', // SEND_PLEXT + // SEND_PLEXT + message: '55vpsci0hji0ai5x', + latE6: 'lyhrt4miuwc7w29d', + lngE6: 'c1yl2qmzfu5j23ao', +// chatTab: '6bk9rmebtk1ux6da', //guessed parameter name - only seen munged - method: 'g9cmy5g6vpxpmcxz', - version: 'blq7574e6kkg0fig', //guessed parameter name - only seen munged - version_parameter: '465c62b22b3bc9ecae01e08b30703752186a1dc9', // passed as the value to the above parameter - boundsParamsList: '45k478vh10jt1ik7', - id: '3eh1ynwxjy8c8rd5', - minLatE6: 'krpywcgq1voq71z3', - minLngE6: 'yo6lte88zvoneqi6', - maxLatE6: 'dncli54tfafmtk6y', - maxLngE6: '76pq437r7vm3osx9', - timestampMs: '2zlgpsg1x6i9720s', - qk: 'pzejivoj28p6kkry', - desiredNumItems: 'u3uxpkqd4pn37ydn', - minTimestampMs: 'msw5gcxhuuk46rb2', - maxTimestampMs: 'bps0ekgdzakdfvr0', - chatTab: 'pm4fm8bjvotjm30h', //guessed parameter name - only seen munged - ascendingTimestampOrder: '7qp8gv50ogelh7cs', - message: 'y599irwyfs45adp4', - latE6: '19ko11fmx32sjfqk', - lngE6: 'i8yjq6v2mjhze29d', - guids: 'szebfshb9f3uo2h9', - inviteeEmailAddress: 'qq4t7lhqphq7wqvh', + // LOOKUP_PLAYERS + guids: 'k76phw8ey9z21z7c', + + // SEND_INVITE_EMAIL + inviteeEmailAddress: 'x16pe9u4i8bidbi2', }, ]; @@ -234,22 +176,30 @@ window.activeRequestMungeSet = undefined; // attempt to guess the munge set in use, by looking therough the functions of the stock intel page for one of the munged params window.detectActiveMungeSet = function() { - if (window.requestParameterMunges.length == 1) { - // no point in searching through the code when there's only one set in use - window.activeRequestMungeSet = 0; - return; - } // try and find the stock page functions // FIXME? revert to searching through all the code? is that practical? - var stockFunc = nemesis.dashboard.network.DataFetcher.prototype.sendRequest_.toString() - for (var i in window.requestParameterMunges) { - if (stockFunc.indexOf (window.requestParameterMunges[i]['method']) >= 0) { - console.log('IITC: found request munge set index '+i+' in stock intel function nemesis.dashboard.network.DataFetcher.prototype.sendRequest_'); - window.activeRequestMungeSet = i; + var stockFunc; + try { + stockFunc = nemesis.dashboard.network.XhrController.prototype.sendRequest.toString(); + } catch(e) { + try { + stockFunc = nemesis.dashboard.network.DataFetcher.prototype.sendRequest_.toString(); + } catch(e) { } } + if(stockFunc) { + for (var i in window.requestParameterMunges) { + if (stockFunc.indexOf (window.requestParameterMunges[i]['method']) >= 0) { + console.log('IITC: found request munge set index '+i+' in stock intel site'); + window.activeRequestMungeSet = i; + } + } + } else { + console.error('IITC: failed to find the stock site function for detecting munge set'); + } + if (window.activeRequestMungeSet===undefined) { console.error('IITC: failed to find request munge set - IITC will likely fail'); window.activeRequestMungeSet = 0;