back(function toTheFuture(failure, backoff) {
options.backoff = backoff;
mana.debug(
'Starting request again to %s after back off attempt %s/%s',
options.uri,
backoff.attempt,
backoff.retries
);
if (!failure) return request(options, parse);
//
// Okay, we can assume that shit is seriously wrong here.
//
mana.debug('We failed to fetch %s, all servers are down.', options.uri);
failure = new Error('Failed to process request: '+ err.message);
failure.url = options.url; // URL we connected with.
failure.statusCode = 500; // Returned status code.
failure.errors = errors; // Reference to errors.
failure.body = ''; // Returned body>
failure.data = {}; // Parsed response.
failure.remaining = mana.remaining; // Rate remaining.
failure.ratereset = mana.ratereset; // Rate reset.
failure.ratelimit = mana.ratelimit; // Rate limit.
assign.destroy(failure);
}, options.backoff);
}, function received(err, res, body) {
if (err) return assignee.destroy(err);
if (res.statusCode !== 200) {
err = new Error('Received an invalid statusCode, expected statusCode 200');
err.statusCode = res.statusCode;
return assignee.destroy(err);
}
var data;
try { data = JSON.parse(body); }
catch (e) {
assignee.destroy(new Error('Failed to parse the JSON response: '+ e.message));
return;
}
assignee.write(data, true);
});
_setImmediate(function immediate() {
if (err) return assign.destroy(err);
assign.write(data, { end: true });
});
/**
* Handle the requests.
*
* @param {Error} err Optional error argument.
* @param {Object} res HTTP response object.
* @param {String} body The registry response.
* @api private
*/
function parse(err, res, body) {
if (err) {
mana.debug('Received an error (%s) for URL %s', err.message, options.uri);
err.url = options.uri; // The URL we accessed.
err.statusCode = 500; // Force status code.
err.errors = errors; // Previous errors.
err.body = body; // The response body.
err.data = {}; // The parsed data.
err.remaining = mana.remaining; // Rate remaining.
err.ratereset = mana.ratereset; // Rate reset.
err.ratelimit = mana.ratelimit; // Rate limit.
return assign.destroy(err);
}
//
// Even if we get an error response, it could still contain an updated
// rate limit, so make sure we parse that out before we start handling
// potential errors.
//
var ratereset = +res.headers['x-ratelimit-reset']
, ratelimit = +res.headers['x-ratelimit-limit']
, remaining = +res.headers['x-ratelimit-remaining'];
mana.ratereset = isNaN(ratereset) ? mana.ratereset : ratereset;
mana.ratelimit = isNaN(ratelimit) ? mana.ratelimit : ratelimit;
mana.remaining = isNaN(remaining) ? mana.remaining : remaining;
//
// We had a successful cache hit, use our cached result as response
// body.
//
if (304 === res.statusCode && cache) {
mana.debug('CACHE HIT, using cached data for URL', options.uri);
return assign.write(cache.data, { end: true });
}
//
// This 403 was a result of a reached API limit, but we've got multiple
// API / OAuth tokens that we can use to request this API. So we're
// going to roll the dice and hopefully get a working API key
//
if (
mana.tokens.length
&& (
(403 === res.statusCode && 0 === mana.remaining)
|| 401 === res.statusCode
)
) {
if (401 === res.statusCode) mana.debug('Received a 401 on our access token, trying another token');
else mana.debug('Weve reached our API limit, trying another token');
if (mana.roll()) {
options.headers.Authorization = mana.authorization;
mana.debug('Successfully switched from authorization tokens');
//
// We've upgrade the header with new authorization information so
// attempt again with the same URL.
//
return downgraded(undefined, root, next, errors);
}
}
if (200 !== res.statusCode && 404 !== res.statusCode) {
//
// Assume that the server is returning an unknown response and that we
// should try a different server.
//
mana.debug('Received an invalid statusCode (%s) for URL %s', res.statusCode, options.uri);
err = new Error('Received a non 200 status code: '+ res.statusCode);
err.url = options.uri; // The URL we accessed.
err.statusCode = 500; // Force status code.
err.errors = errors; // Previous errors.
err.body = body; // The response body.
err.data = {}; // Parsed response.
err.remaining = mana.remaining; // Rate remaining.
err.ratereset = mana.ratereset; // Rate reset.
err.ratelimit = mana.ratelimit; // Rate limit.
return next(err);
}
//
// In this case I prefer to manually parse the JSON response as it
// allows us to return more readable error messages.
//
var data = body;
if (
'string' === typeof data
&& !~options.headers.Accept.indexOf('text')
&& !~options.headers.Accept.indexOf('html')
&& 'HEAD' !== options.method
) {
try { data = JSON.parse(body); }
catch (e) {
//
// Failed to parse response, it was something completely different
// then we originally expected so it could be that the server
// is down and returned an error page, so we need to continue to
// a different server.
//
mana.debug('Failed to parse JSON: %s for URL %s', e.message, options.uri);
e.url = options.uri; // The URL we accessed.
e.statusCode = 500; // Force status code.
e.errors = errors; // Previous errors.
e.body = body; // The response body.
e.data = {}; // Parsed response.
e.remaining = mana.remaining; // Rate remaining.
e.ratereset = mana.ratereset; // Rate reset.
e.ratelimit = mana.ratelimit; // Rate limit.
return next(e);
}
}
//
// Special case for 404 requests, it's technically not an error, but
// it's also not a valid response from the server so if we're going to
// write it to our Assign instance it could cause issues as the data
// format might differ. So instead we're going to call this as an error.
//
if (res.statusCode === 404 && 'HEAD' !== options.method) {
err = new Error('Invalid status code: 404');
err.url = options.uri; // URL of the request.
err.statusCode = 404; // Status code.
err.errors = errors; // Previous errors.
err.body = body; // The response body.
err.data = data; // Parsed response.
err.remaining = mana.remaining; // Rate remaining.
err.ratereset = mana.ratereset; // Rate reset.
err.ratelimit = mana.ratelimit; // Rate limit.
return assign.destroy(err);
}
//
// Check if we need to cache the data internally. Caching is done using
// a fire and forget pattern so we don't slow down in returning this
// result. Most cache implementation would be done in a fraction of
// a second anyways. We should also only cache GET requests as POST
// responses body usually referrer back to the content that got posted.
//
if (res.headers.etag && 'GET' === options.method) {
if (cache) mana.debug('CACHE MISS, updating cache for URL %s', options.uri);
mana.fireforget('set', {
etag: res.headers.etag,
key: args.str,
data: data
});
}
if ('HEAD' !== options.method) assign.write(data, { end: true });
else assign.write({ res: res, data: data }, { end: true });
}
)(function immediate() {
if (err) assign.destroy(err);
assign.write(data, { end: true });
});