codekingpro/portable-devtools

1'use strict'
2
3const crypto = require('crypto')
4const {
5  appendFile,
6  mkdir,
7  readFile,
8  readdir,
9  rm,
10  writeFile,
11} = require('fs/promises')
12const { Minipass } = require('minipass')
13const path = require('path')
14const ssri = require('ssri')
15const { tmpName } = require('./util/tmp')
16
17const contentPath = require('./content/path')
18const hashToSegments = require('./util/hash-to-segments')
19const indexV = require('../package.json')['cache-version'].index
20const { moveFile } = require('@npmcli/fs')
21
22const lsStreamConcurrency = 5
23
24module.exports.NotFoundError = class NotFoundError extends Error {
25  constructor (cache, key) {
26    super(`No cache entry for ${key} found in ${cache}`)
27    this.code = 'ENOENT'
28    this.cache = cache
29    this.key = key
30  }
31}
32
33module.exports.compact = compact
34
35async function compact (cache, key, matchFn, opts = {}) {
36  const bucket = bucketPath(cache, key)
37  const entries = await bucketEntries(bucket)
38  const newEntries = []
39  // we loop backwards because the bottom-most result is the newest
40  // since we add new entries with appendFile
41  for (let i = entries.length - 1; i >= 0; --i) {
42    const entry = entries[i]
43    // a null integrity could mean either a delete was appended
44    // or the user has simply stored an index that does not map
45    // to any content. we determine if the user wants to keep the
46    // null integrity based on the validateEntry function passed in options.
47    // if the integrity is null and no validateEntry is provided, we break
48    // as we consider the null integrity to be a deletion of everything
49    // that came before it.
50    if (entry.integrity === null && !opts.validateEntry) {
51      break
52    }
53
54    // if this entry is valid, and it is either the first entry or
55    // the newEntries array doesn't already include an entry that
56    // matches this one based on the provided matchFn, then we add
57    // it to the beginning of our list
58    if ((!opts.validateEntry || opts.validateEntry(entry) === true) &&
59      (newEntries.length === 0 ||
60        !newEntries.find((oldEntry) => matchFn(oldEntry, entry)))) {
61      newEntries.unshift(entry)
62    }
63  }
64
65  const newIndex = '\n' + newEntries.map((entry) => {
66    const stringified = JSON.stringify(entry)
67    const hash = hashEntry(stringified)
68    return `${hash}\t${stringified}`
69  }).join('\n')
70
71  const setup = async () => {
72    const target = tmpName(cache, opts.tmpPrefix)
73    await mkdir(path.dirname(target), { recursive: true })
74    return {
75      target,
76      moved: false,
77    }
78  }
79
80  const teardown = async (tmp) => {
81    if (!tmp.moved) {
82      return rm(tmp.target, { recursive: true, force: true })
83    }
84  }
85
86  const write = async (tmp) => {
87    await writeFile(tmp.target, newIndex, { flag: 'wx' })
88    await mkdir(path.dirname(bucket), { recursive: true })
89    // we use @npmcli/move-file directly here because we
90    // want to overwrite the existing file
91    await moveFile(tmp.target, bucket)
92    tmp.moved = true
93  }
94
95  // write the file atomically
96  const tmp = await setup()
97  try {
98    await write(tmp)
99  } finally {
100    await teardown(tmp)
101  }
102
103  // we reverse the list we generated such that the newest
104  // entries come first in order to make looping through them easier
105  // the true passed to formatEntry tells it to keep null
106  // integrity values, if they made it this far it's because
107  // validateEntry returned true, and as such we should return it
108  return newEntries.reverse().map((entry) => formatEntry(cache, entry, true))
109}
110
111module.exports.insert = insert
112
113async function insert (cache, key, integrity, opts = {}) {
114  const { metadata, size, time } = opts
115  const bucket = bucketPath(cache, key)
116  const entry = {
117    key,
118    integrity: integrity && ssri.stringify(integrity),
119    time: time || Date.now(),
120    size,
121    metadata,
122  }
123  try {
124    await mkdir(path.dirname(bucket), { recursive: true })
125    const stringified = JSON.stringify(entry)
126    // NOTE - Cleverness ahoy!
127    //
128    // This works because it's tremendously unlikely for an entry to corrupt
129    // another while still preserving the string length of the JSON in
130    // question. So, we just slap the length in there and verify it on read.
131    //
132    // Thanks to @isaacs for the whiteboarding session that ended up with
133    // this.
134    await appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
135  } catch (err) {
136    if (err.code === 'ENOENT') {
137      return undefined
138    }
139
140    throw err
141  }
142  return formatEntry(cache, entry)
143}
144
145module.exports.find = find
146
147async function find (cache, key) {
148  const bucket = bucketPath(cache, key)
149  try {
150    const entries = await bucketEntries(bucket)
151    return entries.reduce((latest, next) => {
152      if (next && next.key === key) {
153        return formatEntry(cache, next)
154      } else {
155        return latest
156      }
157    }, null)
158  } catch (err) {
159    if (err.code === 'ENOENT') {
160      return null
161    } else {
162      throw err
163    }
164  }
165}
166
167module.exports.delete = del
168
169function del (cache, key, opts = {}) {
170  if (!opts.removeFully) {
171    return insert(cache, key, null, opts)
172  }
173
174  const bucket = bucketPath(cache, key)
175  return rm(bucket, { recursive: true, force: true })
176}
177
178module.exports.lsStream = lsStream
179
180function lsStream (cache) {
181  const indexDir = bucketDir(cache)
182  const stream = new Minipass({ objectMode: true })
183
184  // Set all this up to run on the stream and then just return the stream
185  Promise.resolve().then(async () => {
186    const { default: pMap } = await import('p-map')
187    const buckets = await readdirOrEmpty(indexDir)
188    await pMap(buckets, async (bucket) => {
189      const bucketPath = path.join(indexDir, bucket)
190      const subbuckets = await readdirOrEmpty(bucketPath)
191      await pMap(subbuckets, async (subbucket) => {
192        const subbucketPath = path.join(bucketPath, subbucket)
193
194        // "/cachename/<bucket 0xFF>/<bucket 0xFF>./*"
195        const subbucketEntries = await readdirOrEmpty(subbucketPath)
196        await pMap(subbucketEntries, async (entry) => {
197          const entryPath = path.join(subbucketPath, entry)
198          try {
199            const entries = await bucketEntries(entryPath)
200            // using a Map here prevents duplicate keys from showing up
201            // twice, I guess?
202            const reduced = entries.reduce((acc, entry) => {
203              acc.set(entry.key, entry)
204              return acc
205            }, new Map())
206            // reduced is a map of key => entry
207            for (const entry of reduced.values()) {
208              const formatted = formatEntry(cache, entry)
209              if (formatted) {
210                stream.write(formatted)
211              }
212            }
213          } catch (err) {
214            if (err.code === 'ENOENT') {
215              return undefined
216            }
217            throw err
218          }
219        },
220        { concurrency: lsStreamConcurrency })
221      },
222      { concurrency: lsStreamConcurrency })
223    },
224    { concurrency: lsStreamConcurrency })
225    stream.end()
226    return stream
227  }).catch(err => stream.emit('error', err))
228
229  return stream
230}
231
232module.exports.ls = ls
233
234async function ls (cache) {
235  const entries = await lsStream(cache).collect()
236  return entries.reduce((acc, xs) => {
237    acc[xs.key] = xs
238    return acc
239  }, {})
240}
241
242module.exports.bucketEntries = bucketEntries
243
244async function bucketEntries (bucket, filter) {
245  const data = await readFile(bucket, 'utf8')
246  return _bucketEntries(data, filter)
247}
248
249function _bucketEntries (data) {
250  const entries = []
251  data.split('\n').forEach((entry) => {
252    if (!entry) {
253      return
254    }
255
256    const pieces = entry.split('\t')
257    if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {
258      // Hash is no good! Corruption or malice? Doesn't matter!
259      // EJECT EJECT
260      return
261    }
262    let obj
263    try {
264      obj = JSON.parse(pieces[1])
265    } catch (_) {
266      // eslint-ignore-next-line no-empty-block
267    }
268    // coverage disabled here, no need to test with an entry that parses to something falsey
269    // istanbul ignore else
270    if (obj) {
271      entries.push(obj)
272    }
273  })
274  return entries
275}
276
277module.exports.bucketDir = bucketDir
278
279function bucketDir (cache) {
280  return path.join(cache, `index-v${indexV}`)
281}
282
283module.exports.bucketPath = bucketPath
284
285function bucketPath (cache, key) {
286  const hashed = hashKey(key)
287  return path.join.apply(
288    path,
289    [bucketDir(cache)].concat(hashToSegments(hashed))
290  )
291}
292
293module.exports.hashKey = hashKey
294
295function hashKey (key) {
296  return hash(key, 'sha256')
297}
298
299module.exports.hashEntry = hashEntry
300
301function hashEntry (str) {
302  return hash(str, 'sha1')
303}
304
305function hash (str, digest) {
306  return crypto
307    .createHash(digest)
308    .update(str)
309    .digest('hex')
310}
311
312function formatEntry (cache, entry, keepAll) {
313  // Treat null digests as deletions. They'll shadow any previous entries.
314  if (!entry.integrity && !keepAll) {
315    return null
316  }
317
318  return {
319    key: entry.key,
320    integrity: entry.integrity,
321    path: entry.integrity ? contentPath(cache, entry.integrity) : undefined,
322    size: entry.size,
323    time: entry.time,
324    metadata: entry.metadata,
325  }
326}
327
328function readdirOrEmpty (dir) {
329  return readdir(dir).catch((err) => {
330    if (err.code === 'ENOENT' || err.code === 'ENOTDIR') {
331      return []
332    }
333
334    throw err
335  })
336}
337