Dynamic post loading (#2)

Dynamically load the posts so that you can scroll a chronologically sorted timeline infinitely

Reviewed-on: #2
Co-authored-by: ari <ariadna@omg.lol>
Co-committed-by: ari <ariadna@omg.lol>

src/lib/PostComponent.svelte

@@ -113,7 +113,7 @@
           <div id="carouselControls">
             <button
               id="prevBtn"
-              on:click={prevImage}
+              onclick={prevImage}
               disabled={currentImageIndex === 0}>←</button
             >
             <div id="carouselIndicators">
@@ -125,7 +125,7 @@
             </div>
             <button
               id="nextBtn"
-              on:click={nextImage}
+              onclick={nextImage}
               disabled={currentImageIndex === post.imagesCid.length - 1}
               >→</button
             >

src/lib/pdsfetch.ts

@@ -18,11 +18,15 @@ import { Config } from "../../config";
 // import { AppBskyActorDefs } from "@atcute/client/lexicons";
 
 interface AccountMetadata {
-  did: string;
+  did: At.Did;
   displayName: string;
   handle: string;
   avatarCid: string | null;
+  currentCursor?: string;
 }
+
+let accountsMetadata: AccountMetadata[] = [];
+
 interface atUriObject {
   repo: string;
   collection: string;
@@ -45,7 +49,7 @@ class Post {
 
   constructor(
     record: ComAtprotoRepoListRecords.Record,
-    account: AccountMetadata
+    account: AccountMetadata,
   ) {
     this.postCid = record.cid;
     this.recordName = processAtUri(record.uri).rkey;
@@ -68,7 +72,7 @@ class Post {
     switch (post.embed?.$type) {
       case "app.bsky.embed.images":
         this.imagesCid = post.embed.images.map(
-          (imageRecord: any) => imageRecord.image.ref.$link
+          (imageRecord: any) => imageRecord.image.ref.$link,
         );
         break;
       case "app.bsky.embed.video":
@@ -82,7 +86,7 @@ class Post {
         switch (post.embed.media.$type) {
           case "app.bsky.embed.images":
             this.imagesCid = post.embed.media.images.map(
-              (imageRecord) => imageRecord.image.ref.$link
+              (imageRecord) => imageRecord.image.ref.$link,
             );
 
             break;
@@ -118,8 +122,8 @@ const getDidsFromPDS = async (): Promise<At.Did[]> => {
   return data.repos.map((repo: any) => repo.did) as At.Did[];
 };
 const getAccountMetadata = async (
-  did: `did:${string}:${string}`
-): Promise<AccountMetadata> => {
+  did: `did:${string}:${string}`,
+) => {
   // gonna assume self exists in the app.bsky.actor.profile
   try {
     const { data } = await rpc.get("com.atproto.repo.getRecord", {
@@ -143,12 +147,7 @@ const getAccountMetadata = async (
     return account;
   } catch (e) {
     console.error(`Error fetching metadata for ${did}:`, e);
-    return {
-      did: "error",
-      displayName: "",
-      avatarCid: null,
-      handle: "error",
-    };
+    return null;
   }
 };
 
@@ -157,33 +156,9 @@ const getAllMetadataFromPds = async (): Promise<AccountMetadata[]> => {
   const metadata = await Promise.all(
     dids.map(async (repo: `did:${string}:${string}`) => {
       return await getAccountMetadata(repo);
-    })
+    }),
   );
-  return metadata.filter((account) => account.did !== "error");
-};
-
-const fetchPosts = async (did: string) => {
-  try {
-    const { data } = await rpc.get("com.atproto.repo.listRecords", {
-      params: {
-        repo: did as At.Identifier,
-        collection: "app.bsky.feed.post",
-        limit: Config.MAX_POSTS,
-      },
-    });
-    return {
-      records: data.records as ComAtprotoRepoListRecords.Record[],
-      did: did,
-      error: false,
-    };
-  } catch (e) {
-    console.error(`Error fetching posts for ${did}:`, e);
-    return {
-      records: [],
-      did: did,
-      error: true,
-    };
-  }
+  return metadata.filter((account) => account !== null) as AccountMetadata[];
 };
 
 const identityResolve = async (did: At.Did) => {
@@ -196,7 +171,7 @@ const identityResolve = async (did: At.Did) => {
 
   if (did.startsWith("did:plc:") || did.startsWith("did:web:")) {
     const doc = await resolver.resolve(
-      did as `did:plc:${string}` | `did:web:${string}`
+      did as `did:plc:${string}` | `did:web:${string}`,
     );
     return doc;
   } else {
@@ -219,36 +194,146 @@ const blueskyHandleFromDid = async (did: At.Did) => {
   }
 };
 
-const fetchAllPosts = async () => {
-  const users: AccountMetadata[] = await getAllMetadataFromPds();
-  const postRecords = await Promise.all(
-    users.map(
-      async (metadata: AccountMetadata) => await fetchPosts(metadata.did)
-    )
-  );
-  const validPostRecords = postRecords.filter((record) => !record.error);
-  const posts: Post[] = validPostRecords.flatMap((userFetch) =>
-    userFetch.records.map((record) => {
-      const user = users.find(
-        (user: AccountMetadata) => user.did == userFetch.did
-      );
-      if (!user) {
-        throw new Error(`User with DID ${userFetch.did} not found`);
+interface PostsAcc {
+  posts: ComAtprotoRepoListRecords.Record[];
+  account: AccountMetadata;
+}
+const getCutoffDate = (postAccounts: PostsAcc[]) => {
+  const now = Date.now();
+  let cutoffDate: Date | null = null;
+  postAccounts.forEach((postAcc) => {
+    const latestPost = new Date(
+      (postAcc.posts[postAcc.posts.length - 1].value as AppBskyFeedPost.Record)
+        .createdAt,
+    );
+    if (!cutoffDate) {
+      cutoffDate = latestPost;
+    } else {
+      if (latestPost > cutoffDate) {
+        cutoffDate = latestPost;
       }
-      return new Post(record, user);
-    })
-  );
+    }
+  });
+  if (cutoffDate) {
+    return cutoffDate;
+  } else {
+    return new Date(now);
+  }
+};
 
-  posts.sort((a, b) => b.timestamp - a.timestamp);
-  
-  if(!Config.SHOW_FUTURE_POSTS) {
-    // Filter out posts that are in the future
-    const now = Date.now();
-    const filteredPosts = posts.filter((post) => post.timestamp <= now);
-    return filteredPosts.slice(0, Config.MAX_POSTS);
+const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => {
+  // filter posts for each account that are older than the cutoff date and save the cursor of the last post included
+  const filteredPosts: PostsAcc[] = posts.map((postAcc) => {
+    const filtered = postAcc.posts.filter((post) => {
+      const postDate = new Date(
+        (post.value as AppBskyFeedPost.Record).createdAt,
+      );
+      return postDate >= cutoffDate;
+    });
+    if (filtered.length > 0) {
+      postAcc.account.currentCursor = processAtUri(filtered[filtered.length - 1].uri).rkey;
+    }
+    return {
+      posts: filtered,
+      account: postAcc.account,
+    };
+  });
+  return filteredPosts;
+};
+// nightmare function. However it works so I am not touching it
+const getNextPosts = async () => {
+  if (!accountsMetadata.length) {
+    accountsMetadata = await getAllMetadataFromPds();
   }
 
-  return posts.slice(0, Config.MAX_POSTS);
+  const postsAcc: PostsAcc[] = await Promise.all(
+    accountsMetadata.map(async (account) => {
+      const posts = await fetchPostsForUser(
+        account.did,
+        account.currentCursor || null,
+      );
+      if (posts) {
+        return {
+          posts: posts,
+          account: account,
+        };
+      } else {
+        return {
+          posts: [],
+          account: account,
+        };
+      }
+    }),
+  );
+  const recordsFiltered = postsAcc.filter((postAcc) =>
+    postAcc.posts.length > 0
+  );
+  const cutoffDate = getCutoffDate(recordsFiltered);
+  const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate);
+  // update the accountMetadata with the new cursor
+  accountsMetadata = accountsMetadata.map((account) => {
+    const postAcc = recordsCutoff.find(
+      (postAcc) => postAcc.account.did == account.did,
+    );
+    if (postAcc) {
+      account.currentCursor = postAcc.account.currentCursor;
+    }
+    return account;
+  }
+  );
+  // throw the records in a big single array
+  let records = recordsCutoff.flatMap((postAcc) => postAcc.posts);
+  // sort the records by timestamp
+  records = records.sort((a, b) => {
+    const aDate = new Date(
+      (a.value as AppBskyFeedPost.Record).createdAt,
+    ).getTime();
+    const bDate = new Date(
+      (b.value as AppBskyFeedPost.Record).createdAt,
+    ).getTime();
+    return bDate - aDate;
+  });
+  // filter out posts that are in the future
+  if (!Config.SHOW_FUTURE_POSTS) {
+    const now = Date.now();
+    records = records.filter((post) => {
+      const postDate = new Date(
+        (post.value as AppBskyFeedPost.Record).createdAt,
+      ).getTime();
+      return postDate <= now;
+    });
+  }
+
+  const newPosts = records.map((record) => {
+    const account = accountsMetadata.find(
+      (account) => account.did == processAtUri(record.uri).repo,
+    );
+    if (!account) {
+      throw new Error(
+        `Account with DID ${processAtUri(record.uri).repo} not found`,
+      );
+    }
+    return new Post(record, account);
+  });
+  return newPosts;
 };
-export { fetchAllPosts, getAllMetadataFromPds, Post };
+
+const fetchPostsForUser = async (did: At.Did, cursor: string | null) => {
+  try {
+    const { data } = await rpc.get("com.atproto.repo.listRecords", {
+      params: {
+        repo: did as At.Identifier,
+        collection: "app.bsky.feed.post",
+        limit: Config.MAX_POSTS,
+        cursor: cursor || undefined,
+      },
+    });
+    return data.records as ComAtprotoRepoListRecords.Record[];
+  } catch (e) {
+    console.error(`Error fetching posts for ${did}:`, e);
+    return null;
+  }
+};
+
+export { getAllMetadataFromPds, getNextPosts, Post };
 export type { AccountMetadata };