Loading...
{#each postsData as postObject}
diff --git a/src/lib/pdsfetch.ts b/src/lib/pdsfetch.ts
index 0d36e8d..d60718e 100644
--- a/src/lib/pdsfetch.ts
+++ b/src/lib/pdsfetch.ts
@@ -18,11 +18,17 @@ import { Config } from "../../config";
// import { AppBskyActorDefs } from "@atcute/client/lexicons";
interface AccountMetadata {
- did: string;
+ did: At.Did;
displayName: string;
handle: string;
avatarCid: string | null;
+ currentCursor?: string;
}
+
+let accountsMetadata: AccountMetadata[] = [];
+// a chronologically sorted list of posts for all users, that will be shown by svelte
+// getNextPosts will populate this list with additional posts as needed
+let posts: Post[] = [];
interface atUriObject {
repo: string;
collection: string;
@@ -45,7 +51,7 @@ class Post {
constructor(
record: ComAtprotoRepoListRecords.Record,
- account: AccountMetadata
+ account: AccountMetadata,
) {
this.postCid = record.cid;
this.recordName = processAtUri(record.uri).rkey;
@@ -68,7 +74,7 @@ class Post {
switch (post.embed?.$type) {
case "app.bsky.embed.images":
this.imagesCid = post.embed.images.map(
- (imageRecord: any) => imageRecord.image.ref.$link
+ (imageRecord: any) => imageRecord.image.ref.$link,
);
break;
case "app.bsky.embed.video":
@@ -82,7 +88,7 @@ class Post {
switch (post.embed.media.$type) {
case "app.bsky.embed.images":
this.imagesCid = post.embed.media.images.map(
- (imageRecord) => imageRecord.image.ref.$link
+ (imageRecord) => imageRecord.image.ref.$link,
);
break;
@@ -118,8 +124,8 @@ const getDidsFromPDS = async (): Promise
=> {
return data.repos.map((repo: any) => repo.did) as At.Did[];
};
const getAccountMetadata = async (
- did: `did:${string}:${string}`
-): Promise => {
+ did: `did:${string}:${string}`,
+) => {
// gonna assume self exists in the app.bsky.actor.profile
try {
const { data } = await rpc.get("com.atproto.repo.getRecord", {
@@ -143,12 +149,7 @@ const getAccountMetadata = async (
return account;
} catch (e) {
console.error(`Error fetching metadata for ${did}:`, e);
- return {
- did: "error",
- displayName: "",
- avatarCid: null,
- handle: "error",
- };
+ return null;
}
};
@@ -157,11 +158,12 @@ const getAllMetadataFromPds = async (): Promise => {
const metadata = await Promise.all(
dids.map(async (repo: `did:${string}:${string}`) => {
return await getAccountMetadata(repo);
- })
+ }),
);
- return metadata.filter((account) => account.did !== "error");
+ return metadata.filter((account) => account !== null) as AccountMetadata[];
};
+// OLD
const fetchPosts = async (did: string) => {
try {
const { data } = await rpc.get("com.atproto.repo.listRecords", {
@@ -196,7 +198,7 @@ const identityResolve = async (did: At.Did) => {
if (did.startsWith("did:plc:") || did.startsWith("did:web:")) {
const doc = await resolver.resolve(
- did as `did:plc:${string}` | `did:web:${string}`
+ did as `did:plc:${string}` | `did:web:${string}`,
);
return doc;
} else {
@@ -219,36 +221,172 @@ const blueskyHandleFromDid = async (did: At.Did) => {
}
};
-const fetchAllPosts = async () => {
- const users: AccountMetadata[] = await getAllMetadataFromPds();
- const postRecords = await Promise.all(
- users.map(
- async (metadata: AccountMetadata) => await fetchPosts(metadata.did)
- )
- );
- const validPostRecords = postRecords.filter((record) => !record.error);
- const posts: Post[] = validPostRecords.flatMap((userFetch) =>
- userFetch.records.map((record) => {
- const user = users.find(
- (user: AccountMetadata) => user.did == userFetch.did
- );
- if (!user) {
- throw new Error(`User with DID ${userFetch.did} not found`);
+interface PostsAcc {
+ posts: ComAtprotoRepoListRecords.Record[];
+ account: AccountMetadata;
+}
+const getCutoffDate = (postAccounts: PostsAcc[]) => {
+ const now = Date.now();
+ let cutoffDate: Date | null = null;
+ postAccounts.forEach((postAcc) => {
+ const latestPost = new Date(
+ (postAcc.posts[postAcc.posts.length - 1].value as AppBskyFeedPost.Record)
+ .createdAt,
+ );
+ if (!cutoffDate) {
+ cutoffDate = latestPost;
+ } else {
+ if (latestPost > cutoffDate) {
+ cutoffDate = latestPost;
}
- return new Post(record, user);
- })
- );
+ }
+ });
+ if (cutoffDate) {
+ console.log("Cutoff date:", cutoffDate);
+ return cutoffDate;
+ } else {
+ return new Date(now);
+ }
+};
- posts.sort((a, b) => b.timestamp - a.timestamp);
-
- if(!Config.SHOW_FUTURE_POSTS) {
- // Filter out posts that are in the future
- const now = Date.now();
- const filteredPosts = posts.filter((post) => post.timestamp <= now);
- return filteredPosts.slice(0, Config.MAX_POSTS);
+const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => {
+ // filter posts for each account that are older than the cutoff date and save the cursor of the last post included
+ const filteredPosts: PostsAcc[] = posts.map((postAcc) => {
+ const filtered = postAcc.posts.filter((post) => {
+ const postDate = new Date(
+ (post.value as AppBskyFeedPost.Record).createdAt,
+ );
+ return postDate >= cutoffDate;
+ });
+ if (filtered.length > 0) {
+ postAcc.account.currentCursor = filtered[filtered.length - 1].cid;
+ }
+ return {
+ posts: filtered,
+ account: postAcc.account,
+ };
+ });
+ return filteredPosts;
+};
+const getNextPosts = async () => {
+ if (!accountsMetadata.length) {
+ accountsMetadata = await getAllMetadataFromPds();
}
- return posts.slice(0, Config.MAX_POSTS);
+ const postsAcc: PostsAcc[] = await Promise.all(
+ accountsMetadata.map(async (account) => {
+ const posts = await fetchPostsForUser(
+ account.did,
+ account.currentCursor || null,
+ );
+ if (posts) {
+ return {
+ posts: posts,
+ account: account,
+ };
+ } else {
+ return {
+ posts: [],
+ account: account,
+ };
+ }
+ }),
+ );
+ const recordsFiltered = postsAcc.filter((postAcc) =>
+ postAcc.posts.length > 0
+ );
+ const cutoffDate = getCutoffDate(recordsFiltered);
+ const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate);
+ // update the accountMetadata with the new cursor
+ accountsMetadata = recordsCutoff.map((postAcc) => postAcc.account);
+ // throw the records in a big single array
+ let records = recordsCutoff.flatMap((postAcc) => postAcc.posts);
+ // sort the records by timestamp
+ records = records.sort((a, b) => {
+ const aDate = new Date(
+ (a.value as AppBskyFeedPost.Record).createdAt,
+ ).getTime();
+ const bDate = new Date(
+ (b.value as AppBskyFeedPost.Record).createdAt,
+ ).getTime();
+ return bDate - aDate;
+ }
+ );
+ // filter out posts that are in the future
+ if (!Config.SHOW_FUTURE_POSTS) {
+ const now = Date.now();
+ records = records.filter((post) => {
+ const postDate = new Date(
+ (post.value as AppBskyFeedPost.Record).createdAt,
+ ).getTime();
+ return postDate <= now;
+ });
+ }
+ // append the new posts to the existing posts
+ posts = posts.concat(
+ records.map((record) => {
+ const account = accountsMetadata.find(
+ (account) => account.did == processAtUri(record.uri).repo,
+ );
+ if (!account) {
+ throw new Error(`Account with DID ${processAtUri(record.uri).repo} not found`);
+ }
+ return new Post(record, account);
+ }),
+ );
+ console.log("Fetched posts:", posts);
+ return posts;
};
-export { fetchAllPosts, getAllMetadataFromPds, Post };
+
+const fetchPostsForUser = async (did: At.Did, cursor: string | null) => {
+ try {
+ const { data } = await rpc.get("com.atproto.repo.listRecords", {
+ params: {
+ repo: did as At.Identifier,
+ collection: "app.bsky.feed.post",
+ limit: Config.MAX_POSTS,
+ cursor: cursor || undefined,
+ },
+ });
+ return data.records as ComAtprotoRepoListRecords.Record[];
+ } catch (e) {
+ console.error(`Error fetching posts for ${did}:`, e);
+ return null;
+ }
+};
+
+// const fetchAllPosts = async () => {
+// const users: AccountMetadata[] = await getAllMetadataFromPds();
+// const postRecords = await Promise.all(
+// users.map(
+// async (metadata: AccountMetadata) => await fetchPosts(metadata.did),
+// ),
+// );
+// // Filter out any records that have an error
+// const validPostRecords = postRecords.filter((record) => !record.error);
+
+// const posts: Post[] = validPostRecords.flatMap((userFetch) =>
+// userFetch.records.map((record) => {
+// const user = users.find(
+// (user: AccountMetadata) => user.did == userFetch.did,
+// );
+// if (!user) {
+// throw new Error(`User with DID ${userFetch.did} not found`);
+// }
+// return new Post(record, user);
+// })
+// );
+
+// posts.sort((a, b) => b.timestamp - a.timestamp);
+
+// if (!Config.SHOW_FUTURE_POSTS) {
+// // Filter out posts that are in the future
+// const now = Date.now();
+// const filteredPosts = posts.filter((post) => post.timestamp <= now);
+// return filteredPosts.slice(0, Config.MAX_POSTS);
+// }
+
+// return posts.slice(0, Config.MAX_POSTS);
+// };
+export { getAllMetadataFromPds, getNextPosts, Post, posts };
export type { AccountMetadata };