3wordchant
/
oasis
0
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Fix tests and remove workaround 

Problem: Recently there was a PR [0] merged with a quickfix to avoid
some test failures, which is something I've been trying to avoid. While
`process.exit()` works fine, I'm worried that it means we don't
understand what's happening under the hood, *plus* I have the [maybe
unjustified?] worry that it might kill the process during a database
write or something dangerous. It looks like this particular test hang
was caused by both a stream and some number of intervals that were left
open.

Solution: Provide a way to close the stream and intervals in `index.js`
and ensure that we do that before closing the server.

[0]: https://github.com/fraction/oasis/pull/462
This commit is contained in:
Christian Bundy 2020-11-11 08:25:16 -08:00
parent 4e0960d2d1
commit 0aacf3bbc3
6 changed files with 118 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.cspell.json
View File

@ -13,6 +13,7 @@
"OSSEC",
"Pieter",
"Termux",
"abortable",
"autoconf",
"automake",
"backlinks",

70
package-lock.json generated
View File

@ -483,7 +483,7 @@
},
"@types/debug": {
"version": "4.1.5",
"resolved": false,
"resolved": "",
"integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ=="
},
"@types/events": {
@ -534,7 +534,7 @@
},
"@types/highlight.js": {
"version": "9.12.3",
"resolved": false,
"resolved": "",
"integrity": "sha512-pGF/zvYOACZ/gLGWdQH8zSwteQS1epp68yRcVLJMgUck/MjEn/FBYmPub9pXT8C1e4a8YZfHo1CKyV8q1vKUnQ==",
"dev": true
},
@ -576,7 +576,7 @@
},
"@types/koa-mount": {
"version": "4.0.0",
"resolved": false,
"resolved": "",
"integrity": "sha512-56iBULArwY3uKLl28eRFchZ2v0diEoJzJbDaHH/ehgruF/s2/KMHyWsKcIhvDJ3tGdKu9oZNQvxaMg++1IKFdA==",
"dev": true,
"requires": {
@ -594,7 +594,7 @@
},
"@types/koa-static": {
"version": "4.0.1",
"resolved": false,
"resolved": "",
"integrity": "sha512-SSpct5fEcAeRkBHa3RiwCIRfDHcD1cZRhwRF///ZfvRt8KhoqRrhK6wpDlYPk/vWHVFE9hPGqh68bhzsHkir4w==",
"dev": true,
"requires": {
@ -604,7 +604,7 @@
},
"@types/koa__router": {
"version": "8.0.2",
"resolved": false,
"resolved": "",
"integrity": "sha512-3ZWfVAEcErHrZA31fWUC2YyZyAgoG4eKtQPy2XwBzdSpQealxjL7GcEEtGY925qPPs1wurW59qDl0KuRB39rrw==",
"dev": true,
"requires": {
@ -625,7 +625,7 @@
},
"@types/markdown-it": {
"version": "0.0.9",
"resolved": false,
"resolved": "",
"integrity": "sha512-IFSepyZXbF4dgSvsk8EsgaQ/8Msv1I5eTL0BZ0X3iGO9jw6tCVtPG8HchIPm3wrkmGdqZOD42kE0zplVi1gYDA==",
"dev": true,
"requires": {
@ -652,7 +652,7 @@
},
"@types/mkdirp": {
"version": "1.0.0",
"resolved": false,
"resolved": "",
"integrity": "sha512-ONFY9//bCEr3DWKON3iDv/Q8LXnhaYYaNDeFSN0AtO5o4sLf9F0pstJKKKjQhXE0kJEeHs8eR6SAsROhhc2Csw==",
"dev": true,
"requires": {
@ -666,7 +666,7 @@
},
"@types/nodemon": {
"version": "1.19.0",
"resolved": false,
"resolved": "",
"integrity": "sha512-nf0PKjNv3wo0pyhlPkHjfpU2oUYMdOIsCXceiFG6kvhYxbTwn2ne9mz2iWvJ/4QtCrJUEPJLNXuyGleyTacdaw==",
"dev": true,
"requires": {
@ -693,7 +693,7 @@
},
"@types/pull-stream": {
"version": "3.6.0",
"resolved": false,
"resolved": "",
"integrity": "sha512-d7cobAytjsYaTluqFTJopLbdZRs1HBkU+N07/zCtsXANZ5s9riDazRkvxkAUfmRzCvXqYAsGgnJHgX2lInIOQg==",
"dev": true
},
@ -779,7 +779,7 @@
},
"@types/yargs": {
"version": "15.0.4",
"resolved": false,
"resolved": "",
"integrity": "sha512-9T1auFmbPZoxHz0enUFlUuKRy3it01R+hlggyVUMtnCTQRunsQYifnSGb8hET4Xo8yiC0o0r1paW3ud5+rbURg==",
"dev": true,
"requires": {
@ -994,7 +994,8 @@
"aproba": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/aproba/-/aproba-1.2.0.tgz",
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw=="
"integrity": "sha512-Y9J6ZjXtoYh8RnXVCMOU/ttDmk1aBjunq9vO0ta5x85WDQiQfUF9sIPBITdbiiIVcBo03Hi3jMxigBtsddlXRw==",
"optional": true
},
"archy": {
"version": "1.0.0",
@ -1006,6 +1007,7 @@
"version": "1.1.5",
"resolved": "https://registry.npmjs.org/are-we-there-yet/-/are-we-there-yet-1.1.5.tgz",
"integrity": "sha512-5hYdAkZlcG8tOLujVDTgCT+uPX0VnpAH28gWsLfzpXYm7wP6mp5Q/gYyR7YQ0cKVJcXJnl3j2kpBan13PtQf6w==",
"optional": true,
"requires": {
"delegates": "^1.0.0",
"readable-stream": "^2.0.6"
@ -1664,7 +1666,8 @@
"chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
"optional": true
},
"ci-info": {
"version": "2.0.0",
@ -2028,7 +2031,8 @@
"console-control-strings": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4="
"integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
"optional": true
},
"cont": {
"version": "1.0.3",
@ -2621,6 +2625,7 @@
"version": "4.2.1",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-4.2.1.tgz",
"integrity": "sha512-jOSne2qbyE+/r8G1VU+G/82LBs2Fs4LAsTiLSHOCOMZQl2OKZ6i8i4IyHemTe+/yIXOtTcRQMzPcgyhoFlqPkw==",
"optional": true,
"requires": {
"mimic-response": "^2.0.0"
}
@ -2740,7 +2745,8 @@
"detect-libc": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
"integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups="
"integrity": "sha1-+hN8S9aY7fVc1c0CrFWfkaTEups=",
"optional": true
},
"detective": {
"version": "5.2.0",
@ -3906,6 +3912,7 @@
"version": "2.7.4",
"resolved": "https://registry.npmjs.org/gauge/-/gauge-2.7.4.tgz",
"integrity": "sha1-LANAXHU4w51+s3sxcCLjJfsBi/c=",
"optional": true,
"requires": {
"aproba": "^1.0.3",
"console-control-strings": "^1.0.0",
@ -3920,12 +3927,14 @@
"ansi-regex": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8="
"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
"optional": true
},
"is-fullwidth-code-point": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
"integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -3934,6 +3943,7 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
"integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -3944,6 +3954,7 @@
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
"optional": true,
"requires": {
"ansi-regex": "^2.0.0"
}
@ -4208,7 +4219,8 @@
"has-unicode": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz",
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk="
"integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=",
"optional": true
},
"has-yarn": {
"version": "2.1.0",
@ -6287,7 +6299,8 @@
"mimic-response": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
"integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA=="
"integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==",
"optional": true
},
"min-indent": {
"version": "1.0.0",
@ -6737,6 +6750,7 @@
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
"integrity": "sha512-2uUqazuKlTaSI/dC8AzicUck7+IrEaOnN/e0jd3Xtt1KcGpwx30v50mL7oPyr/h9bL3E4aZccVwpwP+5W9Vjkg==",
"optional": true,
"requires": {
"are-we-there-yet": "~1.1.2",
"console-control-strings": "~1.1.0",
@ -7551,9 +7565,9 @@
"dev": true
},
"pull-abortable": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/pull-abortable/-/pull-abortable-4.0.0.tgz",
"integrity": "sha1-cBephMO4NN53usOMELd28i38GEM="
"version": "4.1.1",
"resolved": "https://registry.npmjs.org/pull-abortable/-/pull-abortable-4.1.1.tgz",
"integrity": "sha1-s61a77QRayWRbSbbiTk6yY0NzqE="
},
"pull-box-stream": {
"version": "1.0.13",
@ -7686,6 +7700,13 @@
"requires": {
"pull-abortable": "~4.0.0",
"pull-stream": "^3.4.5"
},
"dependencies": {
"pull-abortable": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/pull-abortable/-/pull-abortable-4.0.0.tgz",
"integrity": "sha1-cBephMO4NN53usOMELd28i38GEM="
}
}
},
"pull-level": {
@ -8735,12 +8756,14 @@
"simple-concat": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.0.tgz",
"integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY="
"integrity": "sha1-c0TLuLbib7J9ZrL8hvn21Zl1IcY=",
"optional": true
},
"simple-get": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-3.1.0.tgz",
"integrity": "sha512-bCR6cP+aTdScaQCnQKbPKtJOKDp/hj9EDLJo3Nw4y1QksqaovlW/bnptB6/c1e+qmNIDHRK+oXFDdEqBT8WzUA==",
"optional": true,
"requires": {
"decompress-response": "^4.2.0",
"once": "^1.3.1",
@ -10182,7 +10205,7 @@
"dependencies": {
"@babel/helpers": {
"version": "7.8.4",
"resolved": false,
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.8.4.tgz",
"integrity": "sha512-VPbe7wcQ4chu4TDQjimHv/5tj73qz88o12EPkO2ValS2QiQS/1F2SsjyIGNnAD0vF/nZS6Cf9i+vW6HIlnaR8w==",
"requires": {
"@babel/template": "^7.8.3",
@ -10192,7 +10215,7 @@
},
"@babel/runtime": {
"version": "7.8.7",
"resolved": false,
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.8.7.tgz",
"integrity": "sha512-+AATMUFppJDw6aiR5NVPHqIQBlV/Pj8wY/EZH+lmvRdUo9xBaz/rF3alAwFJQavvKfeOlPE7oaaDHVbcySbCsg==",
"requires": {
"regenerator-runtime": "^0.13.4"
@ -11246,6 +11269,7 @@
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz",
"integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==",
"optional": true,
"requires": {
"string-width": "^1.0.2 || 2"
}

1
package.json
View File

@ -39,6 +39,7 @@
"open": "^7.0.3",
"piexifjs": "^1.0.4",
"pretty-ms": "^6.0.0",
"pull-abortable": "^4.1.1",
"pull-paramap": "^1.2.2",
"pull-sort": "^1.0.2",
"pull-stream": "^3.6.12",

12
src/index.js
View File

@ -145,6 +145,8 @@ const { about, blob, friend, meta, post, vote } = require("./models")({
isPublic: config.public,
});
const nameWarmup = about._startNameWarmup();
// enhance the users' input text by expanding @name to [@name](@feedPub.key)
// and slurps up blob uploads and appends a markdown link for it to the text (see handleBlobUpload)
const preparePreview = async function (ctx) {
@ -1058,16 +1060,8 @@ const app = http({ host, port, middleware, allowHost });
// If we close the database after each test it throws lots of really fun "parent
// stream closing" errors everywhere and breaks the tests. :/
app._close = () => {
nameWarmup.close();
cooler.close();
// HACK: app._close is called when everything is supposed to have finished;
// cooler.close successfully stops the ssb-db process. we create a timeout
// to definitively kill the server and thereby circumvent the node process
// staying alive despite all signals pointing to the app exiting. more
// context when it was originally
// introduced in this PR: https://github.com/fraction/oasis/pull/462
setTimeout(() => {
process.exit();
}, 20000);
};
module.exports = app;

114
src/models.js
View File

@ -4,6 +4,7 @@ const debug = require("debug")("oasis");
const { isRoot, isReply: isComment } = require("ssb-thread-schema");
const lodash = require("lodash");
const prettyMs = require("pretty-ms");
const pullAbortable = require("pull-abortable");
const pullParallelMap = require("pull-paramap");
const pull = require("pull-stream");
const pullSort = require("pull-sort");
@ -195,55 +196,6 @@ module.exports = ({ cooler, isPublic }) => {
});
};
cooler.open().then((ssb) => {
console.time("about-name-warmup"); // benchmark the time it takes to stream all existing about messages
pull(
ssb.query.read({
live: true, // keep streaming new messages as they arrive
query: [
{
$filter: {
// all messages of type:about that have a name field that is typeof string
value: {
content: {
type: "about",
name: { $is: "string" },
},
},
},
},
],
}),
pull.filter((msg) => {
// backlog of data is done, only new values from now on
if (msg.sync && msg.sync === true) {
console.timeEnd("about-name-warmup");
transposeLookupTable(); // fire once now
setInterval(transposeLookupTable, 1000 * 60); // and then every 60 seconds
return false;
}
// only pick messages about self
return msg.value.author == msg.value.content.about;
}),
pull.drain((msg) => {
const name = msg.value.content.name;
const ts = msg.value.timestamp;
const feed = msg.value.author;
const newEntry = { name, ts };
const currentEntry = feeds_to_name[feed];
if (typeof currentEntry == "undefined") {
dirty = true;
feeds_to_name[feed] = newEntry;
} else if (currentEntry.ts < ts) {
// overwrite entry if it's newer
dirty = true;
feeds_to_name[feed] = newEntry;
}
})
);
});
models.about = {
publicWebHosting: async (feedId) => {
const result = await getAbout({
@ -306,6 +258,70 @@ module.exports = ({ cooler, isPublic }) => {
})) || "";
return raw;
},
// This needs to run in the background but also needs to be aborted
// in index.js when the server closes. There's also an interval that
// needs to be cleared. TODO: Ensure that there's never more than
// one interval running at a time.
_startNameWarmup() {
const abortable = pullAbortable();
let intervals = [];
cooler.open().then((ssb) => {
console.time("about-name-warmup"); // benchmark the time it takes to stream all existing about messages
pull(
ssb.query.read({
live: true, // keep streaming new messages as they arrive
query: [
{
$filter: {
// all messages of type:about that have a name field that is typeof string
value: {
content: {
type: "about",
name: { $is: "string" },
},
},
},
},
],
}),
abortable,
pull.filter((msg) => {
// backlog of data is done, only new values from now on
if (msg.sync && msg.sync === true) {
console.timeEnd("about-name-warmup");
transposeLookupTable(); // fire once now
intervals.push(setInterval(transposeLookupTable, 1000 * 60)); // and then every 60 seconds
return false;
}
// only pick messages about self
return msg.value.author == msg.value.content.about;
}),
pull.drain((msg) => {
const name = msg.value.content.name;
const ts = msg.value.timestamp;
const feed = msg.value.author;
const newEntry = { name, ts };
const currentEntry = feeds_to_name[feed];
if (typeof currentEntry == "undefined") {
dirty = true;
feeds_to_name[feed] = newEntry;
} else if (currentEntry.ts < ts) {
// overwrite entry if it's newer
dirty = true;
feeds_to_name[feed] = newEntry;
}
})
);
});
return {
close: () => {
abortable.abort();
intervals.forEach((i) => clearInterval(i));
},
};
},
};
models.blob = {

2
test/basic.js
View File

@ -15,7 +15,7 @@ const paths = [
"/profile/edit",
"/public/latest",
"/public/latest/extended",
// "/public/latest/summaries",
"/public/latest/summaries",
"/public/latest/threads",
"/public/latest/topics",
"/public/popular/day",