search.js 6.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201
  1. var path = require('path');
  2. var url = require('url');
  3. var async = require('async');
  4. var request = require('request');
  5. var replay = require('request-replay');
  6. var Cache = require('./util/Cache');
  7. var createError = require('./util/createError');
  8. // TODO:
  9. // The search cache simply stores a specific search result
  10. // into a file. This is a very rudimentary algorithm but
  11. // works to support elementary offline support
  12. // Once the registry server is rewritten, a better strategy
  13. // can be implemented (with diffs and local search), similar to npm.
  14. function search(name, callback) {
  15. var data = [];
  16. var that = this;
  17. var registry = this._config.registry.search;
  18. var total = registry.length;
  19. var index = 0;
  20. // If no registry entries were passed, simply
  21. // error with package not found
  22. if (!total) {
  23. return callback(null, []);
  24. }
  25. // Search package in series in each registry,
  26. // merging results together
  27. async.doUntil(function (next) {
  28. var remote = url.parse(registry[index]);
  29. var searchCache = that._searchCache[remote.host];
  30. // If offline flag is passed, only query the cache
  31. if (that._config.offline) {
  32. return searchCache.get(name, function (err, results) {
  33. if (err || !results || !results.length) {
  34. return next(err);
  35. }
  36. // Add each result
  37. results.forEach(function (result) {
  38. addResult.call(that, data, result);
  39. });
  40. next();
  41. });
  42. }
  43. // Otherwise make a request to always obtain fresh data
  44. doRequest.call(that, name, index, function (err, results) {
  45. if (err || !results || !results.length) {
  46. return next(err);
  47. }
  48. // Add each result
  49. results.forEach(function (result) {
  50. addResult.call(that, data, result);
  51. });
  52. // Store in cache for future offline usage
  53. searchCache.set(name, results, getMaxAge(), next);
  54. });
  55. }, function () {
  56. // Until the data is unknown or there's still registries to test
  57. return ++index === total;
  58. }, function (err) {
  59. // Clear runtime cache, keeping the persistent data
  60. // in files for future offline usage
  61. resetCache();
  62. // If some of the registry entries failed, error out
  63. if (err) {
  64. return callback(err);
  65. }
  66. callback(null, data);
  67. });
  68. }
  69. function addResult(accumulated, result) {
  70. var exists = accumulated.some(function (current) {
  71. return current.name === result.name;
  72. });
  73. if (!exists) {
  74. accumulated.push(result);
  75. }
  76. }
  77. function doRequest(name, index, callback) {
  78. var req;
  79. var msg;
  80. var requestUrl = this._config.registry.search[index] + '/packages/search/' + encodeURIComponent(name);
  81. var remote = url.parse(requestUrl);
  82. var headers = {};
  83. var that = this;
  84. if (this._config.userAgent) {
  85. headers['User-Agent'] = this._config.userAgent;
  86. }
  87. req = replay(request.get(requestUrl, {
  88. proxy: remote.protocol === 'https:' ? this._config.httpsProxy : this._config.proxy,
  89. headers: headers,
  90. ca: this._config.ca.search[index],
  91. strictSSL: this._config.strictSsl,
  92. timeout: this._config.timeout,
  93. json: true
  94. }, function (err, response, body) {
  95. // If there was an internal error (e.g. timeout)
  96. if (err) {
  97. return callback(createError('Request to ' + requestUrl + ' failed: ' + err.message, err.code));
  98. }
  99. // Abort if there was an error (range different than 2xx)
  100. if (response.statusCode < 200 || response.statusCode > 299) {
  101. return callback(createError('Request to ' + requestUrl + ' failed with ' + response.statusCode, 'EINVRES'));
  102. }
  103. // Validate response body, since we are expecting a JSON object
  104. // If the server returns an invalid JSON, it's still a string
  105. if (typeof body !== 'object') {
  106. return callback(createError('Response of request to ' + requestUrl + ' is not a valid json', 'EINVRES'));
  107. }
  108. callback(null, body);
  109. }));
  110. if (this._logger) {
  111. req.on('replay', function (replay) {
  112. msg = 'Request to ' + requestUrl + ' failed with ' + replay.error.code + ', ';
  113. msg += 'retrying in ' + (replay.delay / 1000).toFixed(1) + 's';
  114. that._logger.warn('retry', msg);
  115. });
  116. }
  117. }
  118. function getMaxAge() {
  119. // Make it 5 minutes
  120. return 5 * 60 * 60 * 1000;
  121. }
  122. function initCache() {
  123. this._searchCache = this._cache.search || {};
  124. // Generate a cache instance for each registry endpoint
  125. this._config.registry.search.forEach(function (registry) {
  126. var cacheDir;
  127. var host = url.parse(registry).host;
  128. // Skip if there's a cache for the same host
  129. if (this._searchCache[host]) {
  130. return;
  131. }
  132. if (this._config.cache) {
  133. cacheDir = path.join(this._config.cache, encodeURIComponent(host), 'search');
  134. }
  135. this._searchCache[host] = new Cache(cacheDir, {
  136. max: 250,
  137. // If offline flag is passed, we use stale entries from the cache
  138. useStale: this._config.offline
  139. });
  140. }, this);
  141. }
  142. function clearCache(name, callback) {
  143. var searchCache = this._searchCache;
  144. var remotes = Object.keys(searchCache);
  145. if (typeof name === 'function') {
  146. callback = name;
  147. name = null;
  148. }
  149. // Simply erase everything since other searches could
  150. // contain the "name" package
  151. // One possible solution would be to read every entry from the cache and
  152. // delete if the package is contained in the search results
  153. // But this is too expensive
  154. async.forEach(remotes, function (remote, next) {
  155. searchCache[remote].clear(next);
  156. }, callback);
  157. }
  158. function resetCache() {
  159. var remote;
  160. for (remote in this._searchCache) {
  161. this._searchCache[remote].reset();
  162. }
  163. }
  164. search.initCache = initCache;
  165. search.clearCache = clearCache;
  166. search.resetCache = resetCache;
  167. module.exports = search;