ResolveCache.js 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. var fs = require('graceful-fs');
  2. var path = require('path');
  3. var mout = require('mout');
  4. var Q = require('q');
  5. var mkdirp = require('mkdirp');
  6. var rimraf = require('rimraf');
  7. var LRU = require('lru-cache');
  8. var lockFile = require('lockfile');
  9. var semver = require('../util/semver');
  10. var readJson = require('../util/readJson');
  11. var copy = require('../util/copy');
  12. var md5 = require('../util/md5');
  13. function ResolveCache(config) {
  14. // TODO: Make some config entries, such as:
  15. // - Max MB
  16. // - Max versions per source
  17. // - Max MB per source
  18. // - etc..
  19. this._config = config;
  20. this._dir = this._config.storage.packages;
  21. this._lockDir = this._config.storage.packages;
  22. mkdirp.sync(this._lockDir);
  23. // Cache is stored/retrieved statically to ensure singularity
  24. // among instances
  25. this._cache = this.constructor._cache.get(this._dir);
  26. if (!this._cache) {
  27. this._cache = new LRU({
  28. max: 100,
  29. maxAge: 60 * 5 * 1000 // 5 minutes
  30. });
  31. this.constructor._cache.set(this._dir, this._cache);
  32. }
  33. // Ensure dir is created
  34. mkdirp.sync(this._dir);
  35. }
  36. // -----------------
  37. ResolveCache.prototype.retrieve = function (source, target) {
  38. var sourceId = md5(source);
  39. var dir = path.join(this._dir, sourceId);
  40. var that = this;
  41. target = target || '*';
  42. return this._getVersions(sourceId)
  43. .spread(function (versions) {
  44. var suitable;
  45. // If target is a semver, find a suitable version
  46. if (semver.validRange(target)) {
  47. suitable = semver.maxSatisfying(versions, target, true);
  48. if (suitable) {
  49. return suitable;
  50. }
  51. }
  52. // If target is '*' check if there's a cached '_wildcard'
  53. if (target === '*') {
  54. return mout.array.find(versions, function (version) {
  55. return version === '_wildcard';
  56. });
  57. }
  58. // Otherwise check if there's an exact match
  59. return mout.array.find(versions, function (version) {
  60. return version === target;
  61. });
  62. })
  63. .then(function (version) {
  64. var canonicalDir;
  65. if (!version) {
  66. return [];
  67. }
  68. // Resolve with canonical dir and package meta
  69. canonicalDir = path.join(dir, encodeURIComponent(version));
  70. return that._readPkgMeta(canonicalDir)
  71. .then(function (pkgMeta) {
  72. return [canonicalDir, pkgMeta];
  73. }, function () {
  74. // If there was an error, invalidate the in-memory cache,
  75. // delete the cached package and try again
  76. that._cache.del(sourceId);
  77. return Q.nfcall(rimraf, canonicalDir)
  78. .then(function () {
  79. return that.retrieve(source, target);
  80. });
  81. });
  82. });
  83. };
  84. ResolveCache.prototype.store = function (canonicalDir, pkgMeta) {
  85. var sourceId;
  86. var release;
  87. var dir;
  88. var pkgLock;
  89. var promise;
  90. var that = this;
  91. promise = pkgMeta ? Q.resolve(pkgMeta) : this._readPkgMeta(canonicalDir);
  92. return promise
  93. .then(function (pkgMeta) {
  94. sourceId = md5(pkgMeta._source);
  95. release = that._getPkgRelease(pkgMeta);
  96. dir = path.join(that._dir, sourceId, release);
  97. pkgLock = path.join(that._lockDir, sourceId + '-' + release + '.lock');
  98. // Check if destination directory exists to prevent issuing lock at all times
  99. return Q.nfcall(fs.stat, dir)
  100. .fail(function (err) {
  101. var lockParams = { wait: 250, retries: 25, stale: 60000 };
  102. return Q.nfcall(lockFile.lock, pkgLock, lockParams).then(function () {
  103. // Ensure other process didn't start copying files before lock was created
  104. return Q.nfcall(fs.stat, dir)
  105. .fail(function (err) {
  106. // If stat fails, it is expected to return ENOENT
  107. if (err.code !== 'ENOENT') {
  108. throw err;
  109. }
  110. // Create missing directory and copy files there
  111. return Q.nfcall(mkdirp, path.dirname(dir)).then(function () {
  112. return Q.nfcall(fs.rename, canonicalDir, dir)
  113. .fail(function (err) {
  114. // If error is EXDEV it means that we are trying to rename
  115. // across different drives, so we copy and remove it instead
  116. if (err.code !== 'EXDEV') {
  117. throw err;
  118. }
  119. return copy.copyDir(canonicalDir, dir);
  120. });
  121. });
  122. });
  123. }).finally(function () {
  124. lockFile.unlockSync(pkgLock);
  125. });
  126. }).finally(function () {
  127. // Ensure no tmp dir is left on disk.
  128. return Q.nfcall(rimraf, canonicalDir);
  129. });
  130. })
  131. .then(function () {
  132. var versions = that._cache.get(sourceId);
  133. // Add it to the in memory cache
  134. // and sort the versions afterwards
  135. if (versions && versions.indexOf(release) === -1) {
  136. versions.push(release);
  137. that._sortVersions(versions);
  138. }
  139. // Resolve with the final location
  140. return dir;
  141. });
  142. };
  143. ResolveCache.prototype.eliminate = function (pkgMeta) {
  144. var sourceId = md5(pkgMeta._source);
  145. var release = this._getPkgRelease(pkgMeta);
  146. var dir = path.join(this._dir, sourceId, release);
  147. var that = this;
  148. return Q.nfcall(rimraf, dir)
  149. .then(function () {
  150. var versions = that._cache.get(sourceId) || [];
  151. mout.array.remove(versions, release);
  152. // If this was the last package in the cache,
  153. // delete the parent folder (source)
  154. // For extra security, check against the file system
  155. // if this was really the last package
  156. if (!versions.length) {
  157. that._cache.del(sourceId);
  158. return that._getVersions(sourceId)
  159. .spread(function (versions) {
  160. if (!versions.length) {
  161. // Do not keep in-memory cache if it's completely
  162. // empty
  163. that._cache.del(sourceId);
  164. return Q.nfcall(rimraf, path.dirname(dir));
  165. }
  166. });
  167. }
  168. });
  169. };
  170. ResolveCache.prototype.clear = function () {
  171. return Q.nfcall(rimraf, this._dir)
  172. .then(function () {
  173. return Q.nfcall(fs.mkdir, this._dir);
  174. }.bind(this))
  175. .then(function () {
  176. this._cache.reset();
  177. }.bind(this));
  178. };
  179. ResolveCache.prototype.reset = function () {
  180. this._cache.reset();
  181. return this;
  182. };
  183. ResolveCache.prototype.versions = function (source) {
  184. var sourceId = md5(source);
  185. return this._getVersions(sourceId)
  186. .spread(function (versions) {
  187. return versions.filter(function (version) {
  188. return semver.valid(version);
  189. });
  190. });
  191. };
  192. ResolveCache.prototype.list = function () {
  193. var promises;
  194. var dirs = [];
  195. var that = this;
  196. // Get the list of directories
  197. return Q.nfcall(fs.readdir, this._dir)
  198. .then(function (sourceIds) {
  199. promises = sourceIds.map(function (sourceId) {
  200. return Q.nfcall(fs.readdir, path.join(that._dir, sourceId))
  201. .then(function (versions) {
  202. versions.forEach(function (version) {
  203. var dir = path.join(that._dir, sourceId, version);
  204. dirs.push(dir);
  205. });
  206. }, function (err) {
  207. // Ignore lurking files, e.g.: .DS_Store if the user
  208. // has navigated throughout the cache
  209. if (err.code === 'ENOTDIR' && err.path) {
  210. return Q.nfcall(rimraf, err.path);
  211. }
  212. throw err;
  213. });
  214. });
  215. return Q.all(promises);
  216. })
  217. // Read every package meta
  218. .then(function () {
  219. promises = dirs.map(function (dir) {
  220. return that._readPkgMeta(dir)
  221. .then(function (pkgMeta) {
  222. return {
  223. canonicalDir: dir,
  224. pkgMeta: pkgMeta
  225. };
  226. }, function () {
  227. // If it fails to read, invalidate the in memory
  228. // cache for the source and delete the entry directory
  229. var sourceId = path.basename(path.dirname(dir));
  230. that._cache.del(sourceId);
  231. return Q.nfcall(rimraf, dir);
  232. });
  233. });
  234. return Q.all(promises);
  235. })
  236. // Sort by name ASC & release ASC
  237. .then(function (entries) {
  238. // Ignore falsy entries due to errors reading
  239. // package metas
  240. entries = entries.filter(function (entry) {
  241. return !!entry;
  242. });
  243. return entries.sort(function (entry1, entry2) {
  244. var pkgMeta1 = entry1.pkgMeta;
  245. var pkgMeta2 = entry2.pkgMeta;
  246. var comp = pkgMeta1.name.localeCompare(pkgMeta2.name);
  247. // Sort by name
  248. if (comp) {
  249. return comp;
  250. }
  251. // Sort by version
  252. if (pkgMeta1.version && pkgMeta2.version) {
  253. return semver.compare(pkgMeta1.version, pkgMeta2.version);
  254. }
  255. if (pkgMeta1.version) {
  256. return -1;
  257. }
  258. if (pkgMeta2.version) {
  259. return 1;
  260. }
  261. // Sort by target
  262. return pkgMeta1._target.localeCompare(pkgMeta2._target);
  263. });
  264. });
  265. };
  266. // ------------------------
  267. ResolveCache.clearRuntimeCache = function () {
  268. // Note that _cache refers to the static _cache variable
  269. // that holds other caches per dir!
  270. // Do not confuse it with the instance cache
  271. // Clear cache of each directory
  272. this._cache.forEach(function (cache) {
  273. cache.reset();
  274. });
  275. // Clear root cache
  276. this._cache.reset();
  277. };
  278. // ------------------------
  279. ResolveCache.prototype._getPkgRelease = function (pkgMeta) {
  280. var release = pkgMeta.version || (pkgMeta._target === '*' ? '_wildcard' : pkgMeta._target);
  281. // Encode some dangerous chars such as / and \
  282. release = encodeURIComponent(release);
  283. return release;
  284. };
  285. ResolveCache.prototype._readPkgMeta = function (dir) {
  286. var filename = path.join(dir, '.bower.json');
  287. return readJson(filename)
  288. .spread(function (json) {
  289. return json;
  290. });
  291. };
  292. ResolveCache.prototype._getVersions = function (sourceId) {
  293. var dir;
  294. var versions = this._cache.get(sourceId);
  295. var that = this;
  296. if (versions) {
  297. return Q.resolve([versions, true]);
  298. }
  299. dir = path.join(this._dir, sourceId);
  300. return Q.nfcall(fs.readdir, dir)
  301. .then(function (versions) {
  302. // Sort and cache in memory
  303. that._sortVersions(versions);
  304. versions = versions.map(decodeURIComponent);
  305. that._cache.set(sourceId, versions);
  306. return [versions, false];
  307. }, function (err) {
  308. // If the directory does not exists, resolve
  309. // as an empty array
  310. if (err.code === 'ENOENT') {
  311. versions = [];
  312. that._cache.set(sourceId, versions);
  313. return [versions, false];
  314. }
  315. throw err;
  316. });
  317. };
  318. ResolveCache.prototype._sortVersions = function (versions) {
  319. // Sort DESC
  320. versions.sort(function (version1, version2) {
  321. var validSemver1 = semver.valid(version1);
  322. var validSemver2 = semver.valid(version2);
  323. // If both are semvers, compare them
  324. if (validSemver1 && validSemver2) {
  325. return semver.rcompare(version1, version2);
  326. }
  327. // If one of them are semvers, give higher priority
  328. if (validSemver1) {
  329. return -1;
  330. }
  331. if (validSemver2) {
  332. return 1;
  333. }
  334. // Otherwise they are considered equal
  335. return 0;
  336. });
  337. };
  338. // ------------------------
  339. ResolveCache._cache = new LRU({
  340. max: 5,
  341. maxAge: 60 * 30 * 1000 // 30 minutes
  342. });
  343. module.exports = ResolveCache;