diff --git a/.cspell.json b/.cspell.json index 6b8c792..fad8e89 100644 --- a/.cspell.json +++ b/.cspell.json @@ -13,6 +13,7 @@ "OSSEC", "Pieter", "Termux", + "abortable", "autoconf", "automake", "backlinks", diff --git a/package-lock.json b/package-lock.json index b6adee6..ef4b493 100644 --- a/package-lock.json +++ b/package-lock.json @@ -483,7 +483,7 @@ }, "@types/debug": { "version": "4.1.5", - "resolved": false, + "resolved": "", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, "@types/events": { @@ -534,7 +534,7 @@ }, "@types/highlight.js": { "version": "9.12.3", - "resolved": false, + "resolved": "", "integrity": "sha512-pGF/zvYOACZ/gLGWdQH8zSwteQS1epp68yRcVLJMgUck/MjEn/FBYmPub9pXT8C1e4a8YZfHo1CKyV8q1vKUnQ==", "dev": true }, @@ -576,7 +576,7 @@ }, "@types/koa-mount": { "version": "4.0.0", - "resolved": false, + "resolved": "", "integrity": "sha512-56iBULArwY3uKLl28eRFchZ2v0diEoJzJbDaHH/ehgruF/s2/KMHyWsKcIhvDJ3tGdKu9oZNQvxaMg++1IKFdA==", "dev": true, "requires": { @@ -594,7 +594,7 @@ }, "@types/koa-static": { "version": "4.0.1", - "resolved": false, + "resolved": "", "integrity": "sha512-SSpct5fEcAeRkBHa3RiwCIRfDHcD1cZRhwRF///ZfvRt8KhoqRrhK6wpDlYPk/vWHVFE9hPGqh68bhzsHkir4w==", "dev": true, "requires": { @@ -604,7 +604,7 @@ }, "@types/koa__router": { "version": "8.0.2", - "resolved": false, + "resolved": "", "integrity": "sha512-3ZWfVAEcErHrZA31fWUC2YyZyAgoG4eKtQPy2XwBzdSpQealxjL7GcEEtGY925qPPs1wurW59qDl0KuRB39rrw==", "dev": true, "requires": { @@ -625,7 +625,7 @@ }, "@types/markdown-it": { "version": "0.0.9", - "resolved": false, + "resolved": "", "integrity": "sha512-IFSepyZXbF4dgSvsk8EsgaQ/8Msv1I5eTL0BZ0X3iGO9jw6tCVtPG8HchIPm3wrkmGdqZOD42kE0zplVi1gYDA==", "dev": true, "requires": { @@ -652,7 +652,7 @@ }, "@types/mkdirp": { "version": "1.0.0", - "resolved": false, + "resolved": "", "integrity": "sha512-ONFY9//bCEr3DWKON3iDv/Q8LXnhaYYaNDeFSN0AtO5o4sLf9F0pstJKKKjQhXE0kJEeHs8eR6SAsROhhc2Csw==", "dev": true, "requires": { @@ -666,7 +666,7 @@ }, "@types/nodemon": { "version": "1.19.0", - "resolved": false, + "resolved": "", "integrity": "sha512-nf0PKjNv3wo0pyhlPkHjfpU2oUYMdOIsCXceiFG6kvhYxbTwn2ne9mz2iWvJ/4QtCrJUEPJLNXuyGleyTacdaw==", "dev": true, "requires": { @@ -693,7 +693,7 @@ }, "@types/pull-stream": { "version": "3.6.0", - "resolved": false, + "resolved": "", "integrity": "sha512-d7cobAytjsYaTluqFTJopLbdZRs1HBkU+N07/zCtsXANZ5s9riDazRkvxkAUfmRzCvXqYAsGgnJHgX2lInIOQg==", "dev": true }, @@ -779,7 +779,7 @@ }, "@types/yargs": { "version": "15.0.4", - "resolved": false, + "resolved": "", "integrity": "sha512-9T1auFmbPZoxHz0enUFlUuKRy3it01R+hlggyVUMtnCTQRunsQYifnSGb8hET4Xo8yiC0o0r1paW3ud5+rbURg==", "dev": true, "requires": { @@ -994,7 +994,8 @@ "aproba": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz", - "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==" + "integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==", + "optional": true }, "archy": { "version": "1.0.0", @@ -1006,6 +1007,7 @@ "version": "1.1.5", "resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz", "integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==", + "optional": true, "requires": { "delegates": "^1.0.0", "readable-stream": "^2.0.6" @@ -1664,7 +1666,8 @@ "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "optional": true }, "ci-info": { "version": "2.0.0", @@ -2028,7 +2031,8 @@ "console-control-strings": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz", - "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=" + "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=", + "optional": true }, "cont": { "version": "1.0.3", @@ -2621,6 +2625,7 @@ "version": "4.2.1", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz", "integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==", + "optional": true, "requires": { "mimic-response": "^2.0.0" } @@ -2740,7 +2745,8 @@ "detect-libc": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz", - "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=" + "integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=", + "optional": true }, "detective": { "version": "5.2.0", @@ -3906,6 +3912,7 @@ "version": "2.7.4", "resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz", "integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=", + "optional": true, "requires": { "aproba": "^1.0.3", "console-control-strings": "^1.0.0", @@ -3920,12 +3927,14 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "optional": true }, "is-fullwidth-code-point": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz", "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=", + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -3934,6 +3943,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz", "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=", + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -3944,6 +3954,7 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -4208,7 +4219,8 @@ "has-unicode": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", - "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" + "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=", + "optional": true }, "has-yarn": { "version": "2.1.0", @@ -6287,7 +6299,8 @@ "mimic-response": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz", - "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==" + "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==", + "optional": true }, "min-indent": { "version": "1.0.0", @@ -6737,6 +6750,7 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz", "integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==", + "optional": true, "requires": { "are-we-there-yet": "~1.1.2", "console-control-strings": "~1.1.0", @@ -7551,9 +7565,9 @@ "dev": true }, "pull-abortable": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/pull-abortable/-/pull-abortable-4.0.0.tgz", - "integrity": "sha1-cBephMO4NN53usOMELd28i38GEM=" + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/pull-abortable/-/pull-abortable-4.1.1.tgz", + "integrity": "sha1-s61a77QRayWRbSbbiTk6yY0NzqE=" }, "pull-box-stream": { "version": "1.0.13", @@ -7686,6 +7700,13 @@ "requires": { "pull-abortable": "~4.0.0", "pull-stream": "^3.4.5" + }, + "dependencies": { + "pull-abortable": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/pull-abortable/-/pull-abortable-4.0.0.tgz", + "integrity": "sha1-cBephMO4NN53usOMELd28i38GEM=" + } } }, "pull-level": { @@ -8735,12 +8756,14 @@ "simple-concat": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz", - "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=" + "integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=", + "optional": true }, "simple-get": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz", "integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==", + "optional": true, "requires": { "decompress-response": "^4.2.0", "once": "^1.3.1", @@ -10182,7 +10205,7 @@ "dependencies": { "@babel/helpers": { "version": "7.8.4", - "resolved": false, + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.8.4.tgz", "integrity": "sha512-VPbe7wcQ4chu4TDQjimHv/5tj73qz88o12EPkO2ValS2QiQS/1F2SsjyIGNnAD0vF/nZS6Cf9i+vW6HIlnaR8w==", "requires": { "@babel/template": "^7.8.3", @@ -10192,7 +10215,7 @@ }, "@babel/runtime": { "version": "7.8.7", - "resolved": false, + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.8.7.tgz", "integrity": "sha512-+AATMUFppJDw6aiR5NVPHqIQBlV/Pj8wY/EZH+lmvRdUo9xBaz/rF3alAwFJQavvKfeOlPE7oaaDHVbcySbCsg==", "requires": { "regenerator-runtime": "^0.13.4" @@ -11246,6 +11269,7 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", + "optional": true, "requires": { "string-width": "^1.0.2 || 2" } diff --git a/package.json b/package.json index 33a934f..0dca51c 100644 --- a/package.json +++ b/package.json @@ -39,6 +39,7 @@ "open": "^7.0.3", "piexifjs": "^1.0.4", "pretty-ms": "^6.0.0", + "pull-abortable": "^4.1.1", "pull-paramap": "^1.2.2", "pull-sort": "^1.0.2", "pull-stream": "^3.6.12", diff --git a/src/index.js b/src/index.js index cbdfb11..7f0a6a1 100755 --- a/src/index.js +++ b/src/index.js @@ -145,6 +145,8 @@ const { about, blob, friend, meta, post, vote } = require("./models")({ isPublic: config.public, }); +const nameWarmup = about._startNameWarmup(); + // enhance the users' input text by expanding @name to [@name](@feedPub.key) // and slurps up blob uploads and appends a markdown link for it to the text (see handleBlobUpload) const preparePreview = async function (ctx) { @@ -1058,16 +1060,8 @@ const app = http({ host, port, middleware, allowHost }); // If we close the database after each test it throws lots of really fun "parent // stream closing" errors everywhere and breaks the tests. :/ app._close = () => { + nameWarmup.close(); cooler.close(); - // HACK: app._close is called when everything is supposed to have finished; - // cooler.close successfully stops the ssb-db process. we create a timeout - // to definitively kill the server and thereby circumvent the node process - // staying alive despite all signals pointing to the app exiting. more - // context when it was originally - // introduced in this PR: https://github.com/fraction/oasis/pull/462 - setTimeout(() => { - process.exit(); - }, 20000); }; module.exports = app; diff --git a/src/models.js b/src/models.js index 3e91d19..760bb37 100644 --- a/src/models.js +++ b/src/models.js @@ -4,6 +4,7 @@ const debug = require("debug")("oasis"); const { isRoot, isReply: isComment } = require("ssb-thread-schema"); const lodash = require("lodash"); const prettyMs = require("pretty-ms"); +const pullAbortable = require("pull-abortable"); const pullParallelMap = require("pull-paramap"); const pull = require("pull-stream"); const pullSort = require("pull-sort"); @@ -195,55 +196,6 @@ module.exports = ({ cooler, isPublic }) => { }); }; - cooler.open().then((ssb) => { - console.time("about-name-warmup"); // benchmark the time it takes to stream all existing about messages - pull( - ssb.query.read({ - live: true, // keep streaming new messages as they arrive - query: [ - { - $filter: { - // all messages of type:about that have a name field that is typeof string - value: { - content: { - type: "about", - name: { $is: "string" }, - }, - }, - }, - }, - ], - }), - pull.filter((msg) => { - // backlog of data is done, only new values from now on - if (msg.sync && msg.sync === true) { - console.timeEnd("about-name-warmup"); - transposeLookupTable(); // fire once now - setInterval(transposeLookupTable, 1000 * 60); // and then every 60 seconds - return false; - } - // only pick messages about self - return msg.value.author == msg.value.content.about; - }), - pull.drain((msg) => { - const name = msg.value.content.name; - const ts = msg.value.timestamp; - const feed = msg.value.author; - - const newEntry = { name, ts }; - const currentEntry = feeds_to_name[feed]; - if (typeof currentEntry == "undefined") { - dirty = true; - feeds_to_name[feed] = newEntry; - } else if (currentEntry.ts < ts) { - // overwrite entry if it's newer - dirty = true; - feeds_to_name[feed] = newEntry; - } - }) - ); - }); - models.about = { publicWebHosting: async (feedId) => { const result = await getAbout({ @@ -306,6 +258,70 @@ module.exports = ({ cooler, isPublic }) => { })) || ""; return raw; }, + // This needs to run in the background but also needs to be aborted + // in index.js when the server closes. There's also an interval that + // needs to be cleared. TODO: Ensure that there's never more than + // one interval running at a time. + _startNameWarmup() { + const abortable = pullAbortable(); + let intervals = []; + cooler.open().then((ssb) => { + console.time("about-name-warmup"); // benchmark the time it takes to stream all existing about messages + pull( + ssb.query.read({ + live: true, // keep streaming new messages as they arrive + query: [ + { + $filter: { + // all messages of type:about that have a name field that is typeof string + value: { + content: { + type: "about", + name: { $is: "string" }, + }, + }, + }, + }, + ], + }), + abortable, + pull.filter((msg) => { + // backlog of data is done, only new values from now on + if (msg.sync && msg.sync === true) { + console.timeEnd("about-name-warmup"); + transposeLookupTable(); // fire once now + intervals.push(setInterval(transposeLookupTable, 1000 * 60)); // and then every 60 seconds + return false; + } + // only pick messages about self + return msg.value.author == msg.value.content.about; + }), + pull.drain((msg) => { + const name = msg.value.content.name; + const ts = msg.value.timestamp; + const feed = msg.value.author; + + const newEntry = { name, ts }; + const currentEntry = feeds_to_name[feed]; + if (typeof currentEntry == "undefined") { + dirty = true; + feeds_to_name[feed] = newEntry; + } else if (currentEntry.ts < ts) { + // overwrite entry if it's newer + dirty = true; + feeds_to_name[feed] = newEntry; + } + }) + ); + }); + + return { + close: () => { + abortable.abort(); + intervals.forEach((i) => clearInterval(i)); + }, + }; + }, }; models.blob = { diff --git a/test/basic.js b/test/basic.js index 2f0fca5..ef50631 100644 --- a/test/basic.js +++ b/test/basic.js @@ -15,7 +15,7 @@ const paths = [ "/profile/edit", "/public/latest", "/public/latest/extended", - // "/public/latest/summaries", + "/public/latest/summaries", "/public/latest/threads", "/public/latest/topics", "/public/popular/day",