config.ts

@@ -18,7 +18,7 @@ export class Config {
    * Maximum number of posts to show in the feed (across all users)
    * @default 100
    */
-  static readonly MAX_POSTS: number = 100;
+  static readonly MAX_POSTS: number = 5;
 
   /**
    * Footer text for the dashboard

src/App.svelte

@@ -1,9 +1,9 @@
 <script lang="ts">
   import PostComponent from "./lib/PostComponent.svelte";
   import AccountComponent from "./lib/AccountComponent.svelte";
-  import { fetchAllPosts, Post, getAllMetadataFromPds } from "./lib/pdsfetch";
+  import { getNextPosts, Post, getAllMetadataFromPds } from "./lib/pdsfetch";
   import { Config } from "../config";
-  const postsPromise = fetchAllPosts();
+  const postsPromise = getNextPosts();
   const accountsPromise = getAllMetadataFromPds();
 </script>
 
@@ -12,6 +12,7 @@
     {#await accountsPromise}
       <p>Loading...</p>
     {:then accountsData}
+      
       <div id="Account">
         <h1 id="Header">ATProto PDS</h1>
         <p>Home to {accountsData.length} accounts</p>
@@ -29,6 +30,9 @@
     {#await postsPromise}
       <p>Loading...</p>
     {:then postsData}
+      <button on:click={getNextPosts}>
+        Load more posts
+      </button>
       <div id="Feed">
         <div id="spacer"></div>
         {#each postsData as postObject}

src/lib/pdsfetch.ts

@@ -18,11 +18,17 @@ import { Config } from "../../config";
 // import { AppBskyActorDefs } from "@atcute/client/lexicons";
 
 interface AccountMetadata {
-  did: string;
+  did: At.Did;
   displayName: string;
   handle: string;
   avatarCid: string | null;
+  currentCursor?: string;
 }
+
+let accountsMetadata: AccountMetadata[] = [];
+// a chronologically sorted list of posts for all users, that will be shown by svelte
+// getNextPosts will populate this list with additional posts as needed
+let posts: Post[] = [];
 interface atUriObject {
   repo: string;
   collection: string;
@@ -45,7 +51,7 @@ class Post {
 
   constructor(
     record: ComAtprotoRepoListRecords.Record,
-    account: AccountMetadata
+    account: AccountMetadata,
   ) {
     this.postCid = record.cid;
     this.recordName = processAtUri(record.uri).rkey;
@@ -68,7 +74,7 @@ class Post {
     switch (post.embed?.$type) {
       case "app.bsky.embed.images":
         this.imagesCid = post.embed.images.map(
-          (imageRecord: any) => imageRecord.image.ref.$link
+          (imageRecord: any) => imageRecord.image.ref.$link,
         );
         break;
       case "app.bsky.embed.video":
@@ -82,7 +88,7 @@ class Post {
         switch (post.embed.media.$type) {
           case "app.bsky.embed.images":
             this.imagesCid = post.embed.media.images.map(
-              (imageRecord) => imageRecord.image.ref.$link
+              (imageRecord) => imageRecord.image.ref.$link,
             );
 
             break;
@@ -118,8 +124,8 @@ const getDidsFromPDS = async (): Promise<At.Did[]> => {
   return data.repos.map((repo: any) => repo.did) as At.Did[];
 };
 const getAccountMetadata = async (
-  did: `did:${string}:${string}`
+  did: `did:${string}:${string}`,
-): Promise<AccountMetadata> => {
+) => {
   // gonna assume self exists in the app.bsky.actor.profile
   try {
     const { data } = await rpc.get("com.atproto.repo.getRecord", {
@@ -143,12 +149,7 @@ const getAccountMetadata = async (
     return account;
   } catch (e) {
     console.error(`Error fetching metadata for ${did}:`, e);
-    return {
+    return null;
-      did: "error",
-      displayName: "",
-      avatarCid: null,
-      handle: "error",
-    };
   }
 };
 
@@ -157,11 +158,12 @@ const getAllMetadataFromPds = async (): Promise<AccountMetadata[]> => {
   const metadata = await Promise.all(
     dids.map(async (repo: `did:${string}:${string}`) => {
       return await getAccountMetadata(repo);
-    })
+    }),
   );
-  return metadata.filter((account) => account.did !== "error");
+  return metadata.filter((account) => account !== null) as AccountMetadata[];
 };
 
+// OLD
 const fetchPosts = async (did: string) => {
   try {
     const { data } = await rpc.get("com.atproto.repo.listRecords", {
@@ -196,7 +198,7 @@ const identityResolve = async (did: At.Did) => {
 
   if (did.startsWith("did:plc:") || did.startsWith("did:web:")) {
     const doc = await resolver.resolve(
-      did as `did:plc:${string}` | `did:web:${string}`
+      did as `did:plc:${string}` | `did:web:${string}`,
     );
     return doc;
   } else {
@@ -219,36 +221,172 @@ const blueskyHandleFromDid = async (did: At.Did) => {
   }
 };
 
-const fetchAllPosts = async () => {
+interface PostsAcc {
-  const users: AccountMetadata[] = await getAllMetadataFromPds();
+  posts: ComAtprotoRepoListRecords.Record[];
-  const postRecords = await Promise.all(
+  account: AccountMetadata;
-    users.map(
+}
-      async (metadata: AccountMetadata) => await fetchPosts(metadata.did)
+const getCutoffDate = (postAccounts: PostsAcc[]) => {
-    )
-  );
-  const validPostRecords = postRecords.filter((record) => !record.error);
-  const posts: Post[] = validPostRecords.flatMap((userFetch) =>
-    userFetch.records.map((record) => {
-      const user = users.find(
-        (user: AccountMetadata) => user.did == userFetch.did
-      );
-      if (!user) {
-        throw new Error(`User with DID ${userFetch.did} not found`);
-      }
-      return new Post(record, user);
-    })
-  );
-
-  posts.sort((a, b) => b.timestamp - a.timestamp);
-  
-  if(!Config.SHOW_FUTURE_POSTS) {
-    // Filter out posts that are in the future
   const now = Date.now();
-    const filteredPosts = posts.filter((post) => post.timestamp <= now);
+  let cutoffDate: Date | null = null;
-    return filteredPosts.slice(0, Config.MAX_POSTS);
+  postAccounts.forEach((postAcc) => {
+    const latestPost = new Date(
+      (postAcc.posts[postAcc.posts.length - 1].value as AppBskyFeedPost.Record)
+        .createdAt,
+    );
+    if (!cutoffDate) {
+      cutoffDate = latestPost;
+    } else {
+      if (latestPost > cutoffDate) {
+        cutoffDate = latestPost;
+      }
+    }
+  });
+  if (cutoffDate) {
+    console.log("Cutoff date:", cutoffDate);
+    return cutoffDate;
+  } else {
+    return new Date(now);
+  }
+};
+
+const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => {
+  // filter posts for each account that are older than the cutoff date and save the cursor of the last post included
+  const filteredPosts: PostsAcc[] = posts.map((postAcc) => {
+    const filtered = postAcc.posts.filter((post) => {
+      const postDate = new Date(
+        (post.value as AppBskyFeedPost.Record).createdAt,
+      );
+      return postDate >= cutoffDate;
+    });
+    if (filtered.length > 0) {
+      postAcc.account.currentCursor = filtered[filtered.length - 1].cid;
+    }
+    return {
+      posts: filtered,
+      account: postAcc.account,
+    };
+  });
+  return filteredPosts;
+};
+const getNextPosts = async () => {
+  if (!accountsMetadata.length) {
+    accountsMetadata = await getAllMetadataFromPds();
   }
 
-  return posts.slice(0, Config.MAX_POSTS);
+  const postsAcc: PostsAcc[] = await Promise.all(
+    accountsMetadata.map(async (account) => {
+      const posts = await fetchPostsForUser(
+        account.did,
+        account.currentCursor || null,
+      );
+      if (posts) {
+        return {
+          posts: posts,
+          account: account,
+        };
+      } else {
+        return {
+          posts: [],
+          account: account,
+        };
+      }
+    }),
+  );
+  const recordsFiltered = postsAcc.filter((postAcc) =>
+    postAcc.posts.length > 0
+  );
+  const cutoffDate = getCutoffDate(recordsFiltered);
+  const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate);
+  // update the accountMetadata with the new cursor
+  accountsMetadata = recordsCutoff.map((postAcc) => postAcc.account);
+  // throw the records in a big single array
+  let records = recordsCutoff.flatMap((postAcc) => postAcc.posts);
+  // sort the records by timestamp
+  records = records.sort((a, b) => {
+    const aDate = new Date(
+      (a.value as AppBskyFeedPost.Record).createdAt,
+    ).getTime();
+    const bDate = new Date(
+      (b.value as AppBskyFeedPost.Record).createdAt,
+    ).getTime();
+    return bDate - aDate;
+  }
+  );
+  // filter out posts that are in the future
+  if (!Config.SHOW_FUTURE_POSTS) {
+    const now = Date.now();
+    records = records.filter((post) => {
+      const postDate = new Date(
+        (post.value as AppBskyFeedPost.Record).createdAt,
+      ).getTime();
+      return postDate <= now;
+    });
+  }
+  // append the new posts to the existing posts
+  posts = posts.concat(
+    records.map((record) => {
+      const account = accountsMetadata.find(
+        (account) => account.did == processAtUri(record.uri).repo,
+      );
+      if (!account) {
+        throw new Error(`Account with DID ${processAtUri(record.uri).repo} not found`);
+      }
+      return new Post(record, account);
+    }),
+  );
+  console.log("Fetched posts:", posts);
+  return posts;
 };
-export { fetchAllPosts, getAllMetadataFromPds, Post };
+
+const fetchPostsForUser = async (did: At.Did, cursor: string | null) => {
+  try {
+    const { data } = await rpc.get("com.atproto.repo.listRecords", {
+      params: {
+        repo: did as At.Identifier,
+        collection: "app.bsky.feed.post",
+        limit: Config.MAX_POSTS,
+        cursor: cursor || undefined,
+      },
+    });
+    return data.records as ComAtprotoRepoListRecords.Record[];
+  } catch (e) {
+    console.error(`Error fetching posts for ${did}:`, e);
+    return null;
+  }
+};
+
+// const fetchAllPosts = async () => {
+//   const users: AccountMetadata[] = await getAllMetadataFromPds();
+//   const postRecords = await Promise.all(
+//     users.map(
+//       async (metadata: AccountMetadata) => await fetchPosts(metadata.did),
+//     ),
+//   );
+//   // Filter out any records that have an error
+//   const validPostRecords = postRecords.filter((record) => !record.error);
+
+//   const posts: Post[] = validPostRecords.flatMap((userFetch) =>
+//     userFetch.records.map((record) => {
+//       const user = users.find(
+//         (user: AccountMetadata) => user.did == userFetch.did,
+//       );
+//       if (!user) {
+//         throw new Error(`User with DID ${userFetch.did} not found`);
+//       }
+//       return new Post(record, user);
+//     })
+//   );
+
+//   posts.sort((a, b) => b.timestamp - a.timestamp);
+
+//   if (!Config.SHOW_FUTURE_POSTS) {
+//     // Filter out posts that are in the future
+//     const now = Date.now();
+//     const filteredPosts = posts.filter((post) => post.timestamp <= now);
+//     return filteredPosts.slice(0, Config.MAX_POSTS);
+//   }
+
+//   return posts.slice(0, Config.MAX_POSTS);
+// };
+export { getAllMetadataFromPds, getNextPosts, Post, posts };
 export type { AccountMetadata };