list.js 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185
  1. var path = require('path');
  2. var url = require('url');
  3. var async = require('async');
  4. var request = require('request');
  5. var replay = require('request-replay');
  6. var Cache = require('./util/Cache');
  7. var createError = require('./util/createError');
  8. function list(callback) {
  9. var data = [];
  10. var that = this;
  11. var registry = this._config.registry.search;
  12. var total = registry.length;
  13. var index = 0;
  14. // If no registry entries were passed, simply
  15. // error with package not found
  16. if (!total) {
  17. return callback(null, []);
  18. }
  19. // List packages in series in each registry
  20. async.doUntil(function (next) {
  21. var remote = url.parse(registry[index]);
  22. var listCache = that._listCache[remote.host];
  23. // If offline flag is passed, only query the cache
  24. if (that._config.offline) {
  25. return listCache.get('list', function (err, results) {
  26. if (err || !results) {
  27. return next(err);
  28. }
  29. // Add each result
  30. results.forEach(function (result) {
  31. addResult.call(that, data, result);
  32. });
  33. next();
  34. });
  35. }
  36. // Otherwise make a request to always obtain fresh data
  37. doRequest.call(that, index, function (err, results) {
  38. if (err || !results) {
  39. return next(err);
  40. }
  41. // Add each result
  42. results.forEach(function (result) {
  43. addResult(data, result);
  44. });
  45. // Store in cache for future offline usage
  46. listCache.set('list', results, getMaxAge(), next);
  47. });
  48. }, function () {
  49. // Until there's still registries to test
  50. return ++index === total;
  51. }, function (err) {
  52. // Clear runtime cache, keeping the persistent data
  53. // in files for future offline usage
  54. resetCache();
  55. // If some of the registry entries failed, error out
  56. if (err) {
  57. return callback(err);
  58. }
  59. callback(null, data);
  60. });
  61. }
  62. function addResult(accumulated, result) {
  63. var exists = accumulated.some(function (current) {
  64. return current.name === result.name;
  65. });
  66. if (!exists) {
  67. accumulated.push(result);
  68. }
  69. }
  70. function doRequest(index, callback) {
  71. var req;
  72. var msg;
  73. var requestUrl = this._config.registry.search[index] + '/packages';
  74. var remote = url.parse(requestUrl);
  75. var headers = {};
  76. var that = this;
  77. if (this._config.userAgent) {
  78. headers['User-Agent'] = this._config.userAgent;
  79. }
  80. req = replay(request.get(requestUrl, {
  81. proxy: remote.protocol === 'https:' ? this._config.httpsProxy : this._config.proxy,
  82. ca: this._config.ca.search[index],
  83. headers: headers,
  84. strictSSL: this._config.strictSsl,
  85. timeout: this._config.timeout,
  86. json: true
  87. }, function (err, response, body) {
  88. // If there was an internal error (e.g. timeout)
  89. if (err) {
  90. return callback(createError('Request to ' + requestUrl + ' failed: ' + err.message, err.code));
  91. }
  92. // Abort if there was an error (range different than 2xx)
  93. if (response.statusCode < 200 || response.statusCode > 299) {
  94. return callback(createError('Request to ' + requestUrl + ' failed with ' + response.statusCode, 'EINVRES'));
  95. }
  96. // Validate response body, since we are expecting a JSON object
  97. // If the server returns an invalid JSON, it's still a string
  98. if (typeof body !== 'object') {
  99. return callback(createError('Response of request to ' + requestUrl + ' is not a valid json', 'EINVRES'));
  100. }
  101. callback(null, body);
  102. }));
  103. if (this._logger) {
  104. req.on('replay', function (replay) {
  105. msg = 'Request to ' + requestUrl + ' failed with ' + replay.error.code + ', ';
  106. msg += 'retrying in ' + (replay.delay / 1000).toFixed(1) + 's';
  107. that._logger.warn('retry', msg);
  108. });
  109. }
  110. }
  111. function getMaxAge() {
  112. // Make it 5 minutes
  113. return 5 * 60 * 60 * 1000;
  114. }
  115. function initCache() {
  116. this._listCache = this._cache.list || {};
  117. // Generate a cache instance for each registry endpoint
  118. this._config.registry.search.forEach(function (registry) {
  119. var cacheDir;
  120. var host = url.parse(registry).host;
  121. // Skip if there's a cache for the same host
  122. if (this._listCache[host]) {
  123. return;
  124. }
  125. if (this._config.cache) {
  126. cacheDir = path.join(this._config.cache, encodeURIComponent(host), 'list');
  127. }
  128. this._listCache[host] = new Cache(cacheDir, {
  129. max: 250,
  130. // If offline flag is passed, we use stale entries from the cache
  131. useStale: this._config.offline
  132. });
  133. }, this);
  134. }
  135. function clearCache(callback) {
  136. var listCache = this._listCache;
  137. var remotes = Object.keys(listCache);
  138. // There's only one key, which is 'list'..
  139. // But we clear everything anyway
  140. async.forEach(remotes, function (remote, next) {
  141. listCache[remote].clear(next);
  142. }, callback);
  143. }
  144. function resetCache() {
  145. var remote;
  146. for (remote in this._listCache) {
  147. this._listCache[remote].reset();
  148. }
  149. }
  150. list.initCache = initCache;
  151. list.clearCache = clearCache;
  152. list.resetCache = resetCache;
  153. module.exports = list;