change name->feed lookup structure
we had duplicates and _old_ names in the first attempt, with this new strucutre only one name per feed is kept (the most recent) also: tweak regexp some more should not correctly match end of string and NOT match inside md link https://regex101.com/r/Uu6iTj/3/
This commit is contained in:
parent
28f1821fd5
commit
8f9ef5e3c7
|
@ -159,7 +159,7 @@ const preparePreview = async function(ctx) {
|
||||||
// This matches for @string followed by a space or other punctuations like ! , or .
|
// This matches for @string followed by a space or other punctuations like ! , or .
|
||||||
// The idea here is to match a plain @name but not [@name](...)
|
// The idea here is to match a plain @name but not [@name](...)
|
||||||
// also: re.exec is stateful => regex is consumed and thus needs to be re-instantiated for each call
|
// also: re.exec is stateful => regex is consumed and thus needs to be re-instantiated for each call
|
||||||
const rex = /@([a-zA-Z0-9-]+)[\s.,!?]{0,1}/g
|
const rex = /(?!\[)@([a-zA-Z0-9-]+)([\s\.,!?]{1}|$)/g
|
||||||
|
|
||||||
// find @mentions using rex and use about.named() to get the info for them
|
// find @mentions using rex and use about.named() to get the info for them
|
||||||
let m
|
let m
|
||||||
|
|
108
src/models.js
108
src/models.js
|
@ -119,14 +119,79 @@ module.exports = ({ cooler, isPublic }) => {
|
||||||
// gotcha two is: there is no way to express (where msg.author == msg.value.content.about) so we need to do it as a pull.filter()
|
// gotcha two is: there is no way to express (where msg.author == msg.value.content.about) so we need to do it as a pull.filter()
|
||||||
// one drawback: is, it gives us all the abouts from forever, not just the latest
|
// one drawback: is, it gives us all the abouts from forever, not just the latest
|
||||||
// TODO: an alternative would be using ssb.names if available and just loading this as a fallback
|
// TODO: an alternative would be using ssb.names if available and just loading this as a fallback
|
||||||
const all_the_names = {}
|
|
||||||
|
// Two lookup tables to remove old and duplicate names
|
||||||
|
const feeds_to_name = {}
|
||||||
|
let all_the_names = {}
|
||||||
|
|
||||||
cooler.open().then((ssb) => {
|
cooler.open().then((ssb) => {
|
||||||
|
|
||||||
|
let dirty = false // just stop mindless work (nothing changed) could be smarter thou
|
||||||
|
let running = false // don't run twice
|
||||||
|
|
||||||
|
// this flips the lookup around (form feed->name to name->feed)
|
||||||
|
// and also enhances the entries with image and relationship info
|
||||||
|
const transposeLookupTable = () => {
|
||||||
|
if (!dirty) return
|
||||||
|
if (running) return
|
||||||
|
running = true
|
||||||
|
|
||||||
|
// invalidate old cache
|
||||||
|
// regenerate a new thing because we don't know which entries will be gone
|
||||||
|
all_the_names = {}
|
||||||
|
|
||||||
|
const allFeeds = Object.keys(feeds_to_name)
|
||||||
|
console.log(`updating ${allFeeds.length} feeds`)
|
||||||
|
console.time('transpose-name-index')
|
||||||
|
|
||||||
|
const lookups = []
|
||||||
|
for (const feed of allFeeds) {
|
||||||
|
const e = feeds_to_name[feed]
|
||||||
|
let pair = { feed, name: e.name }
|
||||||
|
lookups.push(enhanceFeedInfo(pair))
|
||||||
|
}
|
||||||
|
|
||||||
|
// wait for all image and follow lookups
|
||||||
|
Promise.all(lookups).then(() => {
|
||||||
|
dirty = false // all updated
|
||||||
|
running = false
|
||||||
|
console.timeEnd('transpose-name-index')
|
||||||
|
}).catch((err) => {
|
||||||
|
running = false
|
||||||
|
console.warn('lookup transposition failed:', err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// this function adds the avater image and relationship to the all_the_names lookup table
|
||||||
|
const enhanceFeedInfo = ({feed, name}) => {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
getAbout({feedId: feed, key: "image"}).then((img) => {
|
||||||
|
if (img !== null && typeof img !== "string" && typeof img === "object" && typeof img.link === "string") {
|
||||||
|
img = img.link
|
||||||
|
} else if (img === null) {
|
||||||
|
img = nullImage // default empty image if we dont have one
|
||||||
|
}
|
||||||
|
|
||||||
|
models.friend.getRelationship(feed).then((rel) => {
|
||||||
|
// append and update lookup table
|
||||||
|
let feeds_named = all_the_names[name] || []
|
||||||
|
feeds_named.push({feed, name, rel, img })
|
||||||
|
all_the_names[name.toLowerCase()] = feeds_named
|
||||||
|
resolve()
|
||||||
|
|
||||||
|
// TODO: append if these fail!?
|
||||||
|
}).catch(reject)
|
||||||
|
}).catch(reject)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
console.time('about-name-warmup') // benchmark the time it takes to stream all existing abouts
|
||||||
pull(
|
pull(
|
||||||
ssb.query.read({
|
ssb.query.read({
|
||||||
live: true,
|
live: true, // keep streaming new messages as they arrive
|
||||||
query: [
|
query: [
|
||||||
{
|
{
|
||||||
$filter: {
|
$filter: { // all messages of type:about that have a name field that is typeof string
|
||||||
value: {
|
value: {
|
||||||
content: {
|
content: {
|
||||||
type: "about",
|
type: "about",
|
||||||
|
@ -137,8 +202,15 @@ module.exports = ({ cooler, isPublic }) => {
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}),
|
}),
|
||||||
pull.filter((msg) => { // only pick messages about self
|
pull.filter((msg) => {
|
||||||
if (msg.sync && msg.sync === true) { return false } // live query blurp
|
// backlog of data is done, only new values from now on
|
||||||
|
if (msg.sync && msg.sync === true) {
|
||||||
|
console.timeEnd('about-name-warmup')
|
||||||
|
transposeLookupTable() // fire once now
|
||||||
|
setInterval(transposeLookupTable, 1000*60) // and then every 60 seconds
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// only pick messages about self
|
||||||
return msg.value.author == msg.value.content.about
|
return msg.value.author == msg.value.content.about
|
||||||
}),
|
}),
|
||||||
pull.unique((msg) => { // ignore duplicates
|
pull.unique((msg) => { // ignore duplicates
|
||||||
|
@ -146,23 +218,18 @@ module.exports = ({ cooler, isPublic }) => {
|
||||||
}),
|
}),
|
||||||
pull.drain((msg) => {
|
pull.drain((msg) => {
|
||||||
const name = msg.value.content.name
|
const name = msg.value.content.name
|
||||||
|
const ts = msg.value.timestamp
|
||||||
const feed = msg.value.author
|
const feed = msg.value.author
|
||||||
|
|
||||||
getAbout({feedId: feed, key: "image"}).then((img) => {
|
const newEntry = { name, ts }
|
||||||
if (img !== null && typeof img !== "string" && typeof img === "object" && typeof img.link === "string") {
|
const currentEntry = feeds_to_name[feed]
|
||||||
img = img.link
|
if (typeof currentEntry == 'undefined') {
|
||||||
}
|
dirty = true
|
||||||
models.friend.getRelationship(feed).then((rel) => {
|
feeds_to_name[feed] = newEntry
|
||||||
let feeds_named = all_the_names[name] || []
|
} else if (currentEntry.ts < ts) { // overwrite entry if it's newer
|
||||||
feeds_named.push({
|
dirty = true
|
||||||
feed: feed,
|
feeds_to_name[feed] = newEntry
|
||||||
name: name,
|
}
|
||||||
img: img,
|
|
||||||
rel: rel,
|
|
||||||
})
|
|
||||||
all_the_names[name.toLowerCase()] = feeds_named
|
|
||||||
}).catch(console.warn)
|
|
||||||
}).catch(console.warn)
|
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
});
|
});
|
||||||
|
@ -180,6 +247,7 @@ module.exports = ({ cooler, isPublic }) => {
|
||||||
return "Redacted";
|
return "Redacted";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: could possibly use all_the_names
|
||||||
return (
|
return (
|
||||||
(await getAbout({
|
(await getAbout({
|
||||||
key: "name",
|
key: "name",
|
||||||
|
|
Loading…
Reference in New Issue