Merge pull request 'Add the seventh installment: Latest Posts and Names' (#2) from part_7 into main

Reviewed-on: #2

Cargo.toml

@@ -6,5 +6,6 @@ members = [
     #"part_3_database_follows",
     #"part_4_posts_streams",
     #"part_5_task_loop",
-    #"part_6_update_ui"
+    #"part_6_ui_layout",
+    #"part_7_latest_posts"
 ]

README.md

@@ -13,10 +13,11 @@ Author: [@glyph](https://mycelial.technology/)
  - Part 3: Database and Follows
  - Part 4: Posts and Message Streams
  - Part 5: Task Loop
- - Part 6: Update UI and Display Peers List
- - Part 7: Fetch Latest Posts
- - Part 8: Read, Unread and Delete Posts
- - Part 9: Extension Ideas and Conclusion
+ - Part 6: UI Layout and Peers List
+ - Part 7: Latest Posts and Names
+ - Part 8: Posts List and Post Content
+ - Part 9: Read, Unread and Delete
+ - Part 10: Extension Ideas and Conclusion
 
 ## Links
 

part_6_ui_layout/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "part_6_update_ui"
+name = "part_6_ui_layout"
 version = "0.1.0"
 edition = "2021"
 

part_6_ui_layout/README.md

@@ -1,6 +1,6 @@
 # lykin tutorial
 
-## Part 6: Update UI and Display Peers List
+## Part 6: UI Layout and Peers List
 
 ### Introduction
 

part_7_latest_posts/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "part_7_latest_posts"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-std = "1.10"
+bincode = "1.3"
+chrono = "0.4"
+futures = "0.3"
+golgi = { git = "https://git.coopcloud.tech/golgi-ssb/golgi.git" }
+log = "0.4"
+rocket = "0.5.0-rc.1"
+rocket_dyn_templates = { version = "0.1.0-rc.1", features = ["tera"] }
+serde = "1"
+serde_json = "1"
+sled = "0.34"
+xdg = "2.4.1"

part_7_latest_posts/README.md

@@ -0,0 +1,229 @@
+# lykin tutorial
+
+## Part 7: Latest Posts and Names
+
+### Introduction
+
+In the last tutorial installment we updated the user interface of our application and added the ability to display a list of peer subscriptions. Today we'll turn our attention to staying up-to-date with the latest posts authored by the peers we subscribe to. In doing so, we'll add the ability to keep track of the latest sequence number for each peer we follow - as well as syncing only the latest posts for each peer.
+
+This installment will be a short one, since much of the groundwork has already been done in previous installments.
+
+### Outline
+
+ - Update the database to store the latest sequence number
+ - Update the sequence number when fetching posts
+ - Add a task to fetch the latest posts
+ - Add a route handler to invoke the `FetchLatestPosts` task
+ - Update the navigation template
+
+### Update the Database to Store the Latest Sequence Number
+
+The main objective of this tutorial installment is to be able to request only the latest messages for each peer we subscribe to from the sbot. In order to do so, we need to know the sequence number of the most recently published message already in our key-value store. With that information, we can say to the sbot: "please give me all messages for peer X with sequence number greater than Y".
+
+We're going to add a `latest_sequence` field to the `Peer` struct in our database code, as well as a method for updating that value:
+
+`src/db.rs`
+
+```rust
+/// Scuttlebutt peer data.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Peer {
+    pub public_key: String,
+    pub name: String,
+    pub latest_sequence: u64,
+}
+
+impl Peer {
+    pub fn new(public_key: &str) -> Peer {
+        Peer {
+            public_key: public_key.to_string(),
+            name: "".to_string(),
+            // Set the value of latest_sequence to 0.
+            latest_sequence: 0,
+        }
+    }
+
+    // ...
+
+    // Modify the latest_sequence field of an instance of the Peer struct,
+    // leaving the other values unchanged.
+    pub fn set_latest_sequence(self, latest_sequence: u64) -> Peer {
+        Self {
+            latest_sequence,
+            ..self
+        }
+    }
+}
+```
+
+### Update the Sequence Number When Fetching Posts
+
+Now that we have a way to store and update the latest sequence number for each peer in our database, we need to update our post-fetching function in the task loop accordingly.
+
+`src/task_loop.rs`
+
+```rust
+async fn fetch_posts_and_update_db(db: &Database, peer_id: String, after_sequence: u64) {
+    let peer_msgs = sbot::get_message_stream(&peer_id, after_sequence).await;
+    let (latest_sequence, root_posts) = sbot::get_root_posts(peer_msgs).await;
+
+    match db.add_post_batch(&peer_id, root_posts) {
+        Ok(_) => {
+            info!(
+                "Inserted batch of posts into database post tree for peer: {}",
+                &peer_id
+            )
+        }
+        Err(e) => warn!(
+            "Failed to insert batch of posts into database post tree for peer: {}: {}",
+            &peer_id, e
+        ),
+    }
+
+    // Update the value of the latest sequence number for
+    // the peer (this is stored in the database).
+    if let Ok(Some(peer)) = db.get_peer(&peer_id) {
+        db.add_peer(peer.set_latest_sequence(latest_sequence))
+            .unwrap();
+    }
+}
+```
+
+### Add a Task to Fetch the Latest Posts
+
+We already have a `FetchAllPosts` variant of the `Task` enum in our task loop. Let's add a `FetchLatestPosts` variant, along with a match statement and sbot-related function call: 
+
+`src/task_loop.rs`
+
+```rust
+pub enum Task {
+    Cancel,
+    FetchAllPosts(String),
+    FetchLatestPosts(String),
+    FetchLatestName(String),
+}
+
+// Spawn an asynchronous loop which receives tasks over an unbounded channel
+// and invokes task functions accordingly.
+pub async fn spawn(db: Database, rx: Receiver<Task>) {
+    task::spawn(async move {
+        while let Ok(task) = rx.recv().await {
+            match task {
+                // Fetch only the latest messages authored by the given peer,
+                // ie. messages with sequence numbers greater than those
+                // which are already stored in the database.
+                //
+                // Retrieve the root posts from those messages and insert them
+                // into the posts tree of the database.
+                Task::FetchLatestPosts(peer_id) => {
+                    if let Ok(Some(peer)) = db.get_peer(&peer_id) {
+                        info!("Fetching latest posts for peer: {}", peer_id);
+                        fetch_posts_and_update_db(&db, peer_id, peer.latest_sequence).await;
+                    }
+                }
+                // ...
+            }
+        }
+    }
+}
+```
+
+You'll notice that the same function (`fetch_posts_and_update_db()`) is called by both the `FetchAllPosts` and `FetchLatestPosts` tasks; the difference is the value passed in for the third parameter: `after_sequence`. When fetching all posts we pass in a value of 0, while the value of `peer.latest_sequence` is passed when fetching only the latest posts. This relatively simple addition to our code has provided a very efficient means of syncing the latest posts from our local go-sbot instance to the key-value database.
+
+### Add a Route Handler to Invoke the FetchLatestPosts Task
+
+Now we can begin exposing a means for the user to invoke the `FetchLatestPosts` task. This will be done by clicking an icon on the navigation bar of the web interface. Once clicked, a GET request will be sent to `/posts/download_latest`. Let's write the route handler to accept the request and invoke the task for each peer we're subscribed to.
+
+`src/routes.rs`
+
+```rust
+#[get("/posts/download_latest")]
+pub async fn download_latest_posts(db: &State<Database>, tx: &State<Sender<Task>>) -> Redirect {
+    // Iterate through the list of peers in the key-value database.
+    // These are all the peers we're subscribed to via lykin.
+    for peer in db.get_peers() {
+        // Fetch the latest root posts authored by each peer we're
+        // subscribed to. Posts will be added to the key-value database.
+        if let Err(e) = tx
+            .send(Task::FetchLatestPosts(peer.public_key.clone()))
+            .await
+        {
+            warn!("Task loop error: {}", e)
+        }
+
+        // Fetch the latest name for each peer we're subscribed to and update
+        // the database.
+        if let Err(e) = tx.send(Task::FetchLatestName(peer.public_key)).await {
+            warn!("Task loop error: {}", e)
+        }
+    }
+
+    Redirect::to(uri!(home))
+}
+```
+
+You'll notice in the code above that we also invoke the `FetchLatestName` task for each peer. This ensures that our application stays up-to-date with the ways our peers have chosen to name themselves.
+
+Now we need to mount the `download_latest_posts` route to our Rocket application:
+
+`src/main.rs`
+
+```rust
+#[launch]
+async fn rocket() -> _ {
+    // ...
+
+    rocket::build()
+        .manage(db)
+        .manage(tx)
+        .attach(Template::fairing())
+        .mount(
+            "/",
+            routes![
+                home,
+                subscribe_form,
+                unsubscribe_form,
+                // Here we add the route we just wrote.
+                download_latest_posts
+            ],
+        )
+        .mount("/", FileServer::from(relative!("static")))
+        .attach(AdHoc::on_shutdown("cancel task loop", |_| {
+            Box::pin(async move {
+                tx_clone.send(Task::Cancel).await.unwrap();
+            })
+        }))
+}
+```
+
+### Update the Navigation Template
+
+We need to remove the `disabled` and `icon` classes from the 'Download latest posts' anchor element and add an `href` tag. Once this change has been made, clicking on the download icon will fetch the latest posts for all the peers we're subscribed to.
+
+`templates/topbar.html.tera`
+
+```html
+<div class="nav">
+  <div class="flex-container">
+    <a href="/posts/download_latest" title="Download latest posts">
+      <img src="/icons/download.png">
+    </a>
+    <!-- ... -->
+  </div>
+</div>
+```
+
+### Conclusion
+
+That marks the conclusion of a relatively short installment in which we added the ability to keep our key-value database up-to-date with the latest posts and name assignments published by the peers we subscribe to. We updated the database to be able to track the latest sequence number for each peer and added a task to fetch all posts with a sequence number greater than that which is stored. We then added a route handler to invoke the task for each peer and wired it up to the download icon in the navigation bar of our UI.
+
+In the next installment we'll write more route handlers and update our templates in order to show a list of posts each peer has made. We'll also add the ability to display the content of each post. We are on the cusp of realising the fruits of our labour!
+
+## Funding
+
+This work has been funded by a Scuttlebutt Community Grant.
+
+## Contributions
+
+I would love to continue working on the Rust Scuttlebutt ecosystem, writing code and documentation, but I need your help. Please consider contributing to [my Liberapay account](https://liberapay.com/glyph) to support me in my coding and cultivation efforts.
+

part_7_latest_posts/src/db.rs

@@ -0,0 +1,216 @@
+use std::path::Path;
+
+use log::{debug, info};
+use serde::{Deserialize, Serialize};
+use sled::{Batch, Db, IVec, Result, Tree};
+
+/// Scuttlebutt peer data.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Peer {
+    pub public_key: String,
+    pub name: String,
+    pub latest_sequence: u64,
+}
+
+impl Peer {
+    /// Create a new instance of the Peer struct using the given public
+    /// key. A default value is set for name.
+    pub fn new(public_key: &str) -> Peer {
+        Peer {
+            public_key: public_key.to_string(),
+            name: "".to_string(),
+            latest_sequence: 0,
+        }
+    }
+
+    /// Modify the name field of an instance of the Peer struct, leaving
+    /// the other values unchanged.
+    pub fn set_name(self, name: &str) -> Peer {
+        Self {
+            name: name.to_string(),
+            ..self
+        }
+    }
+
+    /// Modify the latest_sequence field of an instance of the Peer struct,
+    /// leaving the other values unchanged.
+    pub fn set_latest_sequence(self, latest_sequence: u64) -> Peer {
+        Self {
+            latest_sequence,
+            ..self
+        }
+    }
+}
+
+/// The text and metadata of a Scuttlebutt root post.
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Post {
+    /// The key of the post-type message, also known as a message reference.
+    pub key: String,
+    /// The text of the post (may be formatted as markdown).
+    pub text: String,
+    /// The date the post was published (e.g. 17 May 2021).
+    pub date: String,
+    /// The sequence number of the post-type message.
+    pub sequence: u64,
+    /// The read state of the post; true if read, false if unread.
+    pub read: bool,
+    /// The timestamp representing the date the post was published.
+    pub timestamp: i64,
+    /// The subject of the post, represented as the first 53 characters of
+    /// the post text.
+    pub subject: Option<String>,
+}
+
+impl Post {
+    // Create a new instance of the Post struct. A default value of `false` is
+    // set for `read`.
+    pub fn new(
+        key: String,
+        text: String,
+        date: String,
+        sequence: u64,
+        timestamp: i64,
+        subject: Option<String>,
+    ) -> Post {
+        Post {
+            key,
+            text,
+            date,
+            sequence,
+            timestamp,
+            subject,
+            read: false,
+        }
+    }
+}
+
+/// An instance of the key-value database and relevant trees.
+#[allow(dead_code)]
+#[derive(Clone)]
+pub struct Database {
+    /// The sled database instance.
+    db: Db,
+    /// A database tree containing Peer struct instances for all the peers
+    /// we are subscribed to.
+    peer_tree: Tree,
+    /// A database tree containing Post struct instances for all of the posts
+    /// we have downloaded from the peer to whom we subscribe.
+    pub post_tree: Tree,
+}
+
+impl Database {
+    /// Initialise the database by opening the database file, loading the
+    /// peers tree and returning an instantiated Database struct.
+    pub fn init(path: &Path) -> Self {
+        // Open the database at the given path.
+        // The database will be created if it does not yet exist.
+        // This code will panic if an IO error is encountered.
+        info!("Initialising sled database");
+        let db = sled::open(path).expect("Failed to open database");
+        debug!("Opening 'peers' database tree");
+        let peer_tree = db
+            .open_tree("peers")
+            .expect("Failed to open 'peers' database tree");
+        debug!("Opening 'posts' database tree");
+        let post_tree = db
+            .open_tree("posts")
+            .expect("Failed to open 'posts' database tree");
+
+        Database {
+            db,
+            peer_tree,
+            post_tree,
+        }
+    }
+
+    /// Add a peer to the database by inserting the public key into the peer
+    /// tree.
+    pub fn add_peer(&self, peer: Peer) -> Result<Option<IVec>> {
+        debug!("Serializing peer data for {} to bincode", &peer.public_key);
+        let peer_bytes = bincode::serialize(&peer).unwrap();
+
+        debug!(
+            "Inserting peer {} into 'peers' database tree",
+            &peer.public_key
+        );
+        self.peer_tree.insert(&peer.public_key, peer_bytes)
+    }
+
+    /// Get a single peer from the peer tree, defined by the given public key.
+    /// The byte value for the matching entry, if found, is deserialized from
+    /// bincode into an instance of the Peer struct.
+    pub fn get_peer(&self, public_key: &str) -> Result<Option<Peer>> {
+        debug!(
+            "Retrieving peer data for {} from 'peers' database tree",
+            &public_key
+        );
+        let peer = self
+            .peer_tree
+            .get(public_key.as_bytes())
+            .unwrap()
+            .map(|peer| {
+                debug!("Deserializing peer data for {} from bincode", &public_key);
+                bincode::deserialize(&peer).unwrap()
+            });
+
+        Ok(peer)
+    }
+
+    /// Get a list of all peers in the peer tree. The byte value for each
+    /// peer entry is deserialized from bincode into an instance of the Peer
+    /// struct.
+    pub fn get_peers(&self) -> Vec<Peer> {
+        debug!("Retrieving data for all peers in the 'peers' database tree");
+        let mut peers = Vec::new();
+
+        self.peer_tree
+            .iter()
+            .map(|peer| peer.unwrap())
+            .for_each(|peer| {
+                debug!(
+                    "Deserializing peer data for {} from bincode",
+                    String::from_utf8_lossy(&peer.0).into_owned()
+                );
+                peers.push(bincode::deserialize(&peer.1).unwrap())
+            });
+
+        peers
+    }
+
+    /// Remove a peer from the database, as represented by the given public
+    /// key.
+    pub fn remove_peer(&self, public_key: &str) -> Result<()> {
+        debug!("Removing peer {} from 'peers' database tree", &public_key);
+        self.peer_tree.remove(&public_key).map(|_| ())
+    }
+
+    /// Add a post to the database by inserting an instance of the Post struct
+    /// into the post tree.
+    pub fn add_post(&self, public_key: &str, post: Post) -> Result<Option<IVec>> {
+        let post_key = format!("{}_{}", public_key, post.key);
+        debug!("Serializing post data for {} to bincode", &post_key);
+        let post_bytes = bincode::serialize(&post).unwrap();
+
+        debug!("Inserting post {} into 'posts' database tree", &post_key);
+        self.post_tree.insert(post_key.as_bytes(), post_bytes)
+    }
+
+    /// Add a batch of posts to the database by inserting a vector of instances
+    /// of the Post struct into the post tree.
+    pub fn add_post_batch(&self, public_key: &str, posts: Vec<Post>) -> Result<()> {
+        let mut post_batch = Batch::default();
+
+        for post in posts {
+            let post_key = format!("{}_{}", public_key, post.key);
+            debug!("Serializing post data for {} to bincode", &post_key);
+            let post_bytes = bincode::serialize(&post).unwrap();
+
+            debug!("Inserting post {} into 'posts' database tree", &post_key);
+            post_batch.insert(post_key.as_bytes(), post_bytes)
+        }
+
+        debug!("Applying batch insertion into 'posts' database tree");
+        self.post_tree.apply_batch(post_batch)
+    }
+}

part_7_latest_posts/src/main.rs

@@ -0,0 +1,56 @@
+mod db;
+mod routes;
+mod sbot;
+mod task_loop;
+mod utils;
+
+use async_std::channel;
+use log::info;
+use rocket::{
+    fairing::AdHoc,
+    fs::{relative, FileServer},
+    launch, routes,
+};
+use rocket_dyn_templates::Template;
+use xdg::BaseDirectories;
+
+use crate::{db::Database, routes::*, task_loop::Task};
+
+#[launch]
+async fn rocket() -> _ {
+    // Create the key-value database.
+    let xdg_dirs = BaseDirectories::with_prefix("lykin").unwrap();
+    let db_path = xdg_dirs
+        .place_config_file("database")
+        .expect("cannot create database directory");
+    let db = Database::init(&db_path);
+    let db_clone = db.clone();
+
+    // Create a message passing channel.
+    let (tx, rx) = channel::unbounded();
+    let tx_clone = tx.clone();
+
+    // Spawn the task loop, passing in the receiver half of the channel.
+    info!("Spawning task loop");
+    task_loop::spawn(db_clone, rx).await;
+
+    rocket::build()
+        .manage(db)
+        .manage(tx)
+        .attach(Template::fairing())
+        .mount(
+            "/",
+            routes![
+                home,
+                subscribe_form,
+                unsubscribe_form,
+                download_latest_posts
+            ],
+        )
+        .mount("/", FileServer::from(relative!("static")))
+        .attach(AdHoc::on_shutdown("cancel task loop", |_| {
+            Box::pin(async move {
+                tx_clone.send(Task::Cancel).await.unwrap();
+            })
+        }))
+}

part_7_latest_posts/src/routes.rs

@@ -0,0 +1,141 @@
+use async_std::channel::Sender;
+use log::{info, warn};
+use rocket::{
+    form::Form,
+    get, post,
+    request::FlashMessage,
+    response::{Flash, Redirect},
+    uri, FromForm, State,
+};
+use rocket_dyn_templates::{context, Template};
+
+use crate::{
+    db::{Database, Peer},
+    sbot,
+    task_loop::Task,
+    utils,
+};
+
+#[derive(FromForm)]
+pub struct PeerForm {
+    pub public_key: String,
+}
+
+#[get("/")]
+pub async fn home(db: &State<Database>, flash: Option<FlashMessage<'_>>) -> Template {
+    let peers = db.get_peers();
+
+    Template::render("base", context! { peers: peers, flash: flash })
+}
+
+#[post("/subscribe", data = "<peer>")]
+pub async fn subscribe_form(
+    db: &State<Database>,
+    tx: &State<Sender<Task>>,
+    peer: Form<PeerForm>,
+) -> Result<Redirect, Flash<Redirect>> {
+    if let Err(e) = utils::validate_public_key(&peer.public_key) {
+        let validation_err_msg = format!("Public key {} is invalid: {}", &peer.public_key, e);
+        warn!("{}", validation_err_msg);
+        return Err(Flash::error(Redirect::to(uri!(home)), validation_err_msg));
+    } else {
+        info!("Public key {} is valid", &peer.public_key);
+        // Retrieve the name of the peer to which we are subscribing.
+        let peer_name = match sbot::get_name(&peer.public_key).await {
+            Ok(name) => name,
+            Err(e) => {
+                warn!("Failed to fetch name for peer {}: {}", &peer.public_key, e);
+                // Return an empty string if an error occurs.
+                String::from("")
+            }
+        };
+        let peer_info = Peer::new(&peer.public_key).set_name(&peer_name);
+
+        match sbot::follow_if_not_following(&peer.public_key).await {
+            Ok(_) => {
+                // Add the peer to the database.
+                if db.add_peer(peer_info).is_ok() {
+                    info!("Added {} to 'peers' database tree", &peer.public_key);
+                    let peer_id = peer.public_key.to_string();
+
+                    // Fetch all root posts authored by the peer we're subscribing
+                    // to. Posts will be added to the key-value database.
+                    if let Err(e) = tx.send(Task::FetchAllPosts(peer_id)).await {
+                        warn!("Task loop error: {}", e)
+                    }
+                } else {
+                    let err_msg = format!(
+                        "Failed to add peer {} to 'peers' database tree",
+                        &peer.public_key
+                    );
+                    warn!("{}", err_msg);
+                    return Err(Flash::error(Redirect::to(uri!(home)), err_msg));
+                }
+            }
+            Err(e) => {
+                warn!("{}", e);
+                return Err(Flash::error(Redirect::to(uri!(home)), e));
+            }
+        }
+    }
+
+    Ok(Redirect::to(uri!(home)))
+}
+
+#[post("/unsubscribe", data = "<peer>")]
+pub async fn unsubscribe_form(
+    db: &State<Database>,
+    peer: Form<PeerForm>,
+) -> Result<Redirect, Flash<Redirect>> {
+    if let Err(e) = utils::validate_public_key(&peer.public_key) {
+        let validation_err_msg = format!("Public key {} is invalid: {}", &peer.public_key, e);
+        warn!("{}", validation_err_msg);
+        return Err(Flash::error(Redirect::to(uri!(home)), validation_err_msg));
+    } else {
+        info!("Public key {} is valid", &peer.public_key);
+        match sbot::unfollow_if_following(&peer.public_key).await {
+            Ok(_) => {
+                // Remove the peer from the database.
+                if db.remove_peer(&peer.public_key).is_ok() {
+                    info!(
+                        "Removed peer {} from 'peers' database tree",
+                        &peer.public_key
+                    );
+                } else {
+                    warn!(
+                        "Failed to remove peer {} from 'peers' database tree",
+                        &peer.public_key
+                    );
+                }
+            }
+            Err(e) => {
+                warn!("{}", e);
+                return Err(Flash::error(Redirect::to(uri!(home)), e));
+            }
+        }
+    }
+
+    Ok(Redirect::to(uri!(home)))
+}
+
+#[get("/posts/download_latest")]
+pub async fn download_latest_posts(db: &State<Database>, tx: &State<Sender<Task>>) -> Redirect {
+    for peer in db.get_peers() {
+        // Fetch the latest root posts authored by each peer we're
+        // subscribed to. Posts will be added to the key-value database.
+        if let Err(e) = tx
+            .send(Task::FetchLatestPosts(peer.public_key.clone()))
+            .await
+        {
+            warn!("Task loop error: {}", e)
+        }
+
+        // Fetch the latest name for each peer we're subscribed to and update
+        // the database.
+        if let Err(e) = tx.send(Task::FetchLatestName(peer.public_key)).await {
+            warn!("Task loop error: {}", e)
+        }
+    }
+
+    Redirect::to(uri!(home))
+}

part_7_latest_posts/src/sbot.rs

@@ -0,0 +1,213 @@
+use std::env;
+
+use async_std::stream::StreamExt;
+use chrono::NaiveDateTime;
+use golgi::{
+    api::{friends::RelationshipQuery, history_stream::CreateHistoryStream},
+    messages::{SsbMessageContentType, SsbMessageKVT},
+    sbot::Keystore,
+    GolgiError, Sbot,
+};
+use log::{info, warn};
+use serde_json::value::Value;
+
+use crate::db::Post;
+
+/// Initialise a connection to a Scuttlebutt server.
+pub async fn init_sbot() -> Result<Sbot, String> {
+    let go_sbot_port = env::var("GO_SBOT_PORT").unwrap_or_else(|_| "8021".to_string());
+
+    let keystore = Keystore::GoSbot;
+    let ip_port = Some(format!("127.0.0.1:{}", go_sbot_port));
+    let net_id = None;
+
+    Sbot::init(keystore, ip_port, net_id)
+        .await
+        .map_err(|e| e.to_string())
+}
+
+/// Return the public key of the local sbot instance.
+pub async fn whoami() -> Result<String, String> {
+    let mut sbot = init_sbot().await?;
+
+    sbot.whoami().await.map_err(|e| e.to_string())
+}
+
+/// Check follow status.
+///
+/// Is peer A (`public_key_a`) following peer B (`public_key_b`)?
+pub async fn is_following(public_key_a: &str, public_key_b: &str) -> Result<String, String> {
+    let mut sbot = init_sbot().await?;
+
+    let query = RelationshipQuery {
+        source: public_key_a.to_string(),
+        dest: public_key_b.to_string(),
+    };
+
+    sbot.friends_is_following(query)
+        .await
+        .map_err(|e| e.to_string())
+}
+
+/// Follow a peer.
+pub async fn follow_peer(public_key: &str) -> Result<String, String> {
+    let mut sbot = init_sbot().await?;
+
+    sbot.follow(public_key).await.map_err(|e| e.to_string())
+}
+
+/// Unfollow a peer.
+pub async fn unfollow_peer(public_key: &str) -> Result<String, String> {
+    let mut sbot = init_sbot().await?;
+
+    sbot.unfollow(public_key).await.map_err(|e| e.to_string())
+}
+
+/// Return the name (self-identifier) for the peer associated with the given
+/// public key.
+///
+/// The public key of the peer will be returned if a name is not found.
+pub async fn get_name(public_key: &str) -> Result<String, String> {
+    let mut sbot = init_sbot().await?;
+
+    sbot.get_name(public_key).await.map_err(|e| e.to_string())
+}
+
+/// Check the follow status of a remote peer and follow them if not already
+/// following.
+pub async fn follow_if_not_following(remote_peer: &str) -> Result<(), String> {
+    if let Ok(whoami) = whoami().await {
+        match is_following(&whoami, remote_peer).await {
+            Ok(status) if status.as_str() == "false" => match follow_peer(remote_peer).await {
+                Ok(_) => {
+                    info!("Followed peer {}", &remote_peer);
+                    Ok(())
+                }
+                Err(e) => {
+                    let err_msg = format!("Failed to follow peer {}: {}", &remote_peer, e);
+                    warn!("{}", err_msg);
+                    Err(err_msg)
+                }
+            },
+            Ok(status) if status.as_str() == "true" => {
+                info!(
+                    "Already following peer {}. No further action taken",
+                    &remote_peer
+                );
+                Ok(())
+            }
+            _ => Err(
+                "Failed to determine follow status: received unrecognised response from local sbot"
+                    .to_string(),
+            ),
+        }
+    } else {
+        let err_msg = String::from("Received an error during `whoami` RPC call. Please ensure the go-sbot is running and try again");
+        warn!("{}", err_msg);
+        Err(err_msg)
+    }
+}
+
+/// Check the follow status of a remote peer and unfollow them if already
+/// following.
+pub async fn unfollow_if_following(remote_peer: &str) -> Result<(), String> {
+    if let Ok(whoami) = whoami().await {
+        match is_following(&whoami, remote_peer).await {
+            Ok(status) if status.as_str() == "true" => {
+                info!("Unfollowing peer {}", &remote_peer);
+                match unfollow_peer(remote_peer).await {
+                    Ok(_) => {
+                        info!("Unfollowed peer {}", &remote_peer);
+                        Ok(())
+                    }
+                    Err(e) => {
+                        let err_msg = format!("Failed to unfollow peer {}: {}", &remote_peer, e);
+                        warn!("{}", err_msg);
+                        Err(err_msg)
+                    }
+                }
+            }
+            _ => Err(
+                "Failed to determine follow status: received unrecognised response from local sbot"
+                    .to_string(),
+            ),
+        }
+    } else {
+        let err_msg = String::from("Received an error during `whoami` RPC call. Please ensure the go-sbot is running and try again");
+        warn!("{}", err_msg);
+        Err(err_msg)
+    }
+}
+
+/// Return a stream of messages authored by the given public key.
+///
+/// This returns all messages regardless of type.
+pub async fn get_message_stream(
+    public_key: &str,
+    sequence_number: u64,
+) -> impl futures::Stream<Item = Result<SsbMessageKVT, GolgiError>> {
+    let mut sbot = init_sbot().await.unwrap();
+
+    let history_stream_args = CreateHistoryStream::new(public_key.to_string())
+        .keys_values(true, true)
+        .after_seq(sequence_number);
+
+    sbot.create_history_stream(history_stream_args)
+        .await
+        .unwrap()
+}
+
+/// Filter a stream of messages and return a vector of root posts.
+///
+/// Each returned vector element includes the key of the post, the content
+/// text, the date the post was published, the sequence number of the post
+/// and whether it is read or unread.
+pub async fn get_root_posts(
+    history_stream: impl futures::Stream<Item = Result<SsbMessageKVT, GolgiError>>,
+) -> (u64, Vec<Post>) {
+    let mut latest_sequence = 0;
+    let mut posts = Vec::new();
+
+    futures::pin_mut!(history_stream);
+
+    while let Some(res) = history_stream.next().await {
+        match res {
+            Ok(msg) => {
+                if msg.value.is_message_type(SsbMessageContentType::Post) {
+                    let content = msg.value.content.to_owned();
+                    if let Value::Object(content_map) = content {
+                        if !content_map.contains_key("root") {
+                            latest_sequence = msg.value.sequence;
+
+                            let text = match content_map.get_key_value("text") {
+                                Some(value) => value.1.to_string(),
+                                None => String::from(""),
+                            };
+                            let timestamp = msg.value.timestamp.round() as i64 / 1000;
+                            let datetime = NaiveDateTime::from_timestamp(timestamp, 0);
+                            let date = datetime.format("%d %b %Y").to_string();
+                            let subject = text.get(0..52).map(|s| s.to_string());
+
+                            let post = Post::new(
+                                msg.key.to_owned(),
+                                text,
+                                date,
+                                msg.value.sequence,
+                                timestamp,
+                                subject,
+                            );
+
+                            posts.push(post)
+                        }
+                    }
+                }
+            }
+            Err(err) => {
+                // Print the `GolgiError` of this element to `stderr`.
+                warn!("err: {:?}", err);
+            }
+        }
+    }
+
+    (latest_sequence, posts)
+}

part_7_latest_posts/src/task_loop.rs

@@ -0,0 +1,96 @@
+use async_std::{channel::Receiver, task};
+use log::{info, warn};
+
+use crate::{sbot, Database};
+
+async fn fetch_posts_and_update_db(db: &Database, peer_id: String, after_sequence: u64) {
+    let peer_msgs = sbot::get_message_stream(&peer_id, after_sequence).await;
+    let (latest_sequence, root_posts) = sbot::get_root_posts(peer_msgs).await;
+
+    match db.add_post_batch(&peer_id, root_posts) {
+        Ok(_) => {
+            info!(
+                "Inserted batch of posts into database post tree for peer: {}",
+                &peer_id
+            )
+        }
+        Err(e) => warn!(
+            "Failed to insert batch of posts into database post tree for peer: {}: {}",
+            &peer_id, e
+        ),
+    }
+
+    // Update the value of the latest sequence number for
+    // the peer (this is stored in the database).
+    if let Ok(Some(peer)) = db.get_peer(&peer_id) {
+        db.add_peer(peer.set_latest_sequence(latest_sequence))
+            .unwrap();
+    }
+}
+
+/// Request the name of the peer represented by the given public key (ID)
+/// and update the existing entry in the database.
+async fn fetch_name_and_update_db(db: &Database, peer_id: String) {
+    match sbot::get_name(&peer_id).await {
+        Ok(name) => {
+            if let Ok(Some(peer)) = db.get_peer(&peer_id) {
+                let updated_peer = peer.set_name(&name);
+                match db.add_peer(updated_peer) {
+                    Ok(_) => info!("Updated name for peer: {}", &peer_id),
+                    Err(e) => {
+                        warn!("Failed to update name for peer: {}: {}", &peer_id, e)
+                    }
+                }
+            }
+        }
+        Err(e) => warn!("Failed to fetch name for {}: {}", &peer_id, e),
+    }
+}
+
+pub enum Task {
+    Cancel,
+    FetchAllPosts(String),
+    FetchLatestPosts(String),
+    FetchLatestName(String),
+}
+
+/// Spawn an asynchronous loop which receives tasks over an unbounded channel
+/// and invokes task functions accordingly.
+pub async fn spawn(db: Database, rx: Receiver<Task>) {
+    task::spawn(async move {
+        while let Ok(task) = rx.recv().await {
+            match task {
+                // Fetch all messages authored by the given peer, filter
+                // the root posts and insert them into the posts tree of the
+                // database.
+                Task::FetchAllPosts(peer_id) => {
+                    info!("Fetching all posts for peer: {}", peer_id);
+                    fetch_posts_and_update_db(&db, peer_id, 0).await;
+                }
+                // Fetch only the latest messages authored by the given peer,
+                // ie. messages with sequence numbers greater than those
+                // which are already stored in the database.
+                //
+                // Retrieve the root posts from those messages and insert them
+                // into the posts tree of the database.
+                Task::FetchLatestPosts(peer_id) => {
+                    if let Ok(Some(peer)) = db.get_peer(&peer_id) {
+                        info!("Fetching latest posts for peer: {}", peer_id);
+                        fetch_posts_and_update_db(&db, peer_id, peer.latest_sequence).await;
+                    }
+                }
+                // Fetch the latest name for the given peer and update the
+                // peer entry in the peers tree of the database.
+                Task::FetchLatestName(peer_id) => {
+                    info!("Fetching latest name for peer: {}", peer_id);
+                    fetch_name_and_update_db(&db, peer_id).await;
+                }
+                // Break out of the task loop.
+                Task::Cancel => {
+                    info!("Exiting task loop...");
+                    break;
+                }
+            }
+        }
+    });
+}

part_7_latest_posts/src/utils.rs

@@ -0,0 +1,32 @@
+//! Public key validation.
+
+/// Ensure that the given public key is a valid ed25519 key.
+///
+/// Return an error string if the key is invalid.
+pub fn validate_public_key(public_key: &str) -> Result<(), String> {
+    // Ensure the ID starts with the correct sigil link.
+    if !public_key.starts_with('@') {
+        return Err("expected '@' sigil as first character".to_string());
+    }
+
+    // Find the dot index denoting the start of the algorithm definition tag.
+    let dot_index = match public_key.rfind('.') {
+        Some(index) => index,
+        None => return Err("no dot index was found".to_string()),
+    };
+
+    // Check the hashing algorithm (must end with ".ed25519").
+    if !&public_key.ends_with(".ed25519") {
+        return Err("hashing algorithm must be ed25519".to_string());
+    }
+
+    // Obtain the base64 portion (substring) of the public key.
+    let base64_str = &public_key[1..dot_index];
+
+    // Ensure the length of the base64 encoded ed25519 public key is correct.
+    if base64_str.len() != 44 {
+        return Err("base64 data length is incorrect".to_string());
+    }
+
+    Ok(())
+}

part_7_latest_posts/static/css/lykin.css

@@ -0,0 +1,153 @@
+.content {
+     background-color: lightyellow;
+     border: 5px solid #ffd700;
+     border-radius: 1rem;
+     grid-area: content;
+     padding: 1.5rem;
+     overflow-y: scroll;
+     word-wrap: anywhere;
+}
+
+.container {
+     height: 100%;
+     width: 100%;
+     margin: 0;
+}
+
+.disabled {
+     opacity: 0.4;
+     pointer-events: none;
+}
+
+.flash-message {
+     margin-left: auto;
+     margin-right: 0;
+     margin-top: 0;
+     margin-bottom: 0;
+     color: red;
+}
+
+.flex-container {
+     display: flex;
+     align-items: center;
+     flex-wrap: wrap;
+}
+
+.flex-container > input {
+     margin: 0.3rem;
+}
+
+.grid-container {
+     display: grid;
+     grid-template-columns: 1fr;
+     grid-template-rows: 1fr 1fr 1fr 3fr;
+     grid-template-areas: 'nav' 'peers' 'posts' 'content';
+     grid-gap: 0.5rem;
+     padding-left: 1rem;
+     padding-right: 1rem;
+     padding-top: 0.2rem;
+     overflow: hidden;
+     height: 85vh;
+}
+
+@media only screen and (min-width: 600px) {
+     .grid-container {
+         grid-template-columns: repeat(4, 1fr);
+         grid-template-rows: 1fr 3fr 4fr;
+         grid-template-areas:
+            'nav nav nav nav nav'
+            'peers posts posts posts posts'
+            'peers content content content content';
+    }
+}
+
+.icon {
+     margin-left: 1rem;
+}
+
+.nav {
+     background-color: lightgreen;
+     border: 5px solid #19a974;
+     border-radius: 1rem;
+     grid-area: nav;
+     padding: 1rem;
+}
+
+.peers {
+     background-color: lightblue;
+     border: 5px solid #357edd;
+     border-radius: 1rem;
+     grid-area: peers;
+     text-align: left;
+}
+
+.peers > ul {
+     padding-left: 1rem;
+}
+
+.peers > ul > li > a {
+     justify-content: space-between;
+}
+
+.peers > ul > li > a > p {
+     margin: 0;
+     font-weight: bold;
+     padding-right: 1rem;
+}
+
+.post > ul {
+     padding-left: 1rem;
+     padding-right: 1rem;
+}
+
+.posts {
+     background-color: bisque;
+     border: 5px solid #ff6300;
+     border-radius: 1rem;
+     grid-area: posts;
+     overflow-y: scroll;
+}
+
+.posts > ul {
+     padding-left: 1rem;
+     padding-right: 1rem;
+}
+
+.posts > ul > li > a {
+     justify-content: space-between;
+}
+
+.posts > ul > li > a > p {
+     margin: 0;
+}
+
+.selected {
+     background-color: #f9c587;
+}
+
+a {
+     text-decoration: none;
+     color: black;
+}
+
+code {
+     word-wrap: anywhere;
+}
+
+form {
+     margin-left: auto;
+     margin-right: 0.5rem;
+}
+
+h1 {
+     margin-left: 1rem;
+}
+
+img {
+     width: 3rem;
+}
+
+li {
+     list-style: none;
+     font-size: 12px;
+}

part_7_latest_posts/static/icons/icon_attributions

@@ -0,0 +1 @@
+<a href="https://www.flaticon.com/free-icons/download" title="download icons">Download icons created by Kiranshastry - Flaticon</a>

part_7_latest_posts/templates/base.html.tera

@@ -0,0 +1,23 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>lykin</title>
+    <meta name="description" content="lykin: an SSB tutorial application">
+    <meta name="author" content="glyph">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="stylesheet" href="/css/lykin.css">
+  </head>
+  <body class="container">
+    <h1>
+      <a href="/">lykin</a>
+    </h1>
+    </a>
+    <div class="grid-container">
+      {% include "topbar" %}
+      {% include "peer_list" %}
+      {% include "post_list" %}
+      {% include "post_content" %}
+    </div>
+  </body>
+</html>

part_7_latest_posts/templates/peer_list.html.tera

@@ -0,0 +1,14 @@
+<div class="peers">
+  <ul>
+  {% for peer in peers -%} 
+    <li>
+    {% if peer.name %}
+      {{ peer.name }}
+    {% else %}
+      {{ peer.public_key }}
+    {% endif %}
+    </li>
+  {%- endfor %}
+  </ul>
+</div>
+

part_7_latest_posts/templates/post_content.html.tera

@@ -0,0 +1,6 @@
+<div class="content">
+{% if post %}
+  {{ post.text }}
+{% endif %}
+</div>
+

part_7_latest_posts/templates/post_list.html.tera

@@ -0,0 +1,10 @@
+<div class="posts">
+  {% if posts %}
+  <ul>
+  {% for post in posts -%} 
+    Subject placeholder
+  {%- endfor %}
+  </ul>
+  {% endif %}
+</div>  
+

part_7_latest_posts/templates/topbar.html.tera

@@ -0,0 +1,28 @@
+<div class="nav">
+  <div class="flex-container">
+    <a href="/posts/download_latest" title="Download latest posts">
+      <img src="/icons/download.png">
+    </a>
+    <a class="disabled icon" title="Download latest posts">
+      <img src="/icons/download.png">
+    </a>
+    <a class="disabled icon" title="Mark as read">
+      <img src="/icons/read_post.png">
+    </a>
+    <a class="disabled icon" title="Mark as unread">
+      <img src="/icons/unread_post.png">
+    </a>
+    <a class="disabled icon" title="Delete post">
+      <img src="/icons/delete_post.png">
+    </a>
+    <form class="flex-container" action="/subscribe" method="post">
+      <label for="public_key">Public Key</label>
+      <input type="text" id="public_key" name="public_key" maxlength=53>
+      <input type="submit" value="Subscribe">
+      <input type="submit" value="Unsubscribe" formaction="/unsubscribe">
+    </form>
+    {% if flash and flash.kind == "error" %}
+    <p class="flash-message">[ {{ flash.message }} ]</p>
+    {% endif %}
+  </div>
+</div>