From 6ca0a971f64dfcc5b502e09bf33fdbb2f435609c Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 21 Apr 2025 19:43:43 -0400 Subject: [PATCH 01/12] At a glance, the fetch mechanism works --- config.ts | 2 +- src/App.svelte | 8 +- src/lib/pdsfetch.ts | 222 +++++++++++++++++++++++++++++++++++--------- 3 files changed, 187 insertions(+), 45 deletions(-) diff --git a/config.ts b/config.ts index 2b1b511..989a249 100644 --- a/config.ts +++ b/config.ts @@ -18,7 +18,7 @@ export class Config { * Maximum number of posts to show in the feed (across all users) * @default 100 */ - static readonly MAX_POSTS: number = 100; + static readonly MAX_POSTS: number = 5; /** * Footer text for the dashboard diff --git a/src/App.svelte b/src/App.svelte index fa5a5c1..eb4ed82 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -1,9 +1,9 @@ @@ -12,6 +12,7 @@ {#await accountsPromise}

Loading...

{:then accountsData} +

ATProto PDS

Home to {accountsData.length} accounts

@@ -29,6 +30,9 @@ {#await postsPromise}

Loading...

{:then postsData} +
{#each postsData as postObject} diff --git a/src/lib/pdsfetch.ts b/src/lib/pdsfetch.ts index 0d36e8d..d60718e 100644 --- a/src/lib/pdsfetch.ts +++ b/src/lib/pdsfetch.ts @@ -18,11 +18,17 @@ import { Config } from "../../config"; // import { AppBskyActorDefs } from "@atcute/client/lexicons"; interface AccountMetadata { - did: string; + did: At.Did; displayName: string; handle: string; avatarCid: string | null; + currentCursor?: string; } + +let accountsMetadata: AccountMetadata[] = []; +// a chronologically sorted list of posts for all users, that will be shown by svelte +// getNextPosts will populate this list with additional posts as needed +let posts: Post[] = []; interface atUriObject { repo: string; collection: string; @@ -45,7 +51,7 @@ class Post { constructor( record: ComAtprotoRepoListRecords.Record, - account: AccountMetadata + account: AccountMetadata, ) { this.postCid = record.cid; this.recordName = processAtUri(record.uri).rkey; @@ -68,7 +74,7 @@ class Post { switch (post.embed?.$type) { case "app.bsky.embed.images": this.imagesCid = post.embed.images.map( - (imageRecord: any) => imageRecord.image.ref.$link + (imageRecord: any) => imageRecord.image.ref.$link, ); break; case "app.bsky.embed.video": @@ -82,7 +88,7 @@ class Post { switch (post.embed.media.$type) { case "app.bsky.embed.images": this.imagesCid = post.embed.media.images.map( - (imageRecord) => imageRecord.image.ref.$link + (imageRecord) => imageRecord.image.ref.$link, ); break; @@ -118,8 +124,8 @@ const getDidsFromPDS = async (): Promise => { return data.repos.map((repo: any) => repo.did) as At.Did[]; }; const getAccountMetadata = async ( - did: `did:${string}:${string}` -): Promise => { + did: `did:${string}:${string}`, +) => { // gonna assume self exists in the app.bsky.actor.profile try { const { data } = await rpc.get("com.atproto.repo.getRecord", { @@ -143,12 +149,7 @@ const getAccountMetadata = async ( return account; } catch (e) { console.error(`Error fetching metadata for ${did}:`, e); - return { - did: "error", - displayName: "", - avatarCid: null, - handle: "error", - }; + return null; } }; @@ -157,11 +158,12 @@ const getAllMetadataFromPds = async (): Promise => { const metadata = await Promise.all( dids.map(async (repo: `did:${string}:${string}`) => { return await getAccountMetadata(repo); - }) + }), ); - return metadata.filter((account) => account.did !== "error"); + return metadata.filter((account) => account !== null) as AccountMetadata[]; }; +// OLD const fetchPosts = async (did: string) => { try { const { data } = await rpc.get("com.atproto.repo.listRecords", { @@ -196,7 +198,7 @@ const identityResolve = async (did: At.Did) => { if (did.startsWith("did:plc:") || did.startsWith("did:web:")) { const doc = await resolver.resolve( - did as `did:plc:${string}` | `did:web:${string}` + did as `did:plc:${string}` | `did:web:${string}`, ); return doc; } else { @@ -219,36 +221,172 @@ const blueskyHandleFromDid = async (did: At.Did) => { } }; -const fetchAllPosts = async () => { - const users: AccountMetadata[] = await getAllMetadataFromPds(); - const postRecords = await Promise.all( - users.map( - async (metadata: AccountMetadata) => await fetchPosts(metadata.did) - ) - ); - const validPostRecords = postRecords.filter((record) => !record.error); - const posts: Post[] = validPostRecords.flatMap((userFetch) => - userFetch.records.map((record) => { - const user = users.find( - (user: AccountMetadata) => user.did == userFetch.did - ); - if (!user) { - throw new Error(`User with DID ${userFetch.did} not found`); +interface PostsAcc { + posts: ComAtprotoRepoListRecords.Record[]; + account: AccountMetadata; +} +const getCutoffDate = (postAccounts: PostsAcc[]) => { + const now = Date.now(); + let cutoffDate: Date | null = null; + postAccounts.forEach((postAcc) => { + const latestPost = new Date( + (postAcc.posts[postAcc.posts.length - 1].value as AppBskyFeedPost.Record) + .createdAt, + ); + if (!cutoffDate) { + cutoffDate = latestPost; + } else { + if (latestPost > cutoffDate) { + cutoffDate = latestPost; } - return new Post(record, user); - }) - ); + } + }); + if (cutoffDate) { + console.log("Cutoff date:", cutoffDate); + return cutoffDate; + } else { + return new Date(now); + } +}; - posts.sort((a, b) => b.timestamp - a.timestamp); - - if(!Config.SHOW_FUTURE_POSTS) { - // Filter out posts that are in the future - const now = Date.now(); - const filteredPosts = posts.filter((post) => post.timestamp <= now); - return filteredPosts.slice(0, Config.MAX_POSTS); +const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => { + // filter posts for each account that are older than the cutoff date and save the cursor of the last post included + const filteredPosts: PostsAcc[] = posts.map((postAcc) => { + const filtered = postAcc.posts.filter((post) => { + const postDate = new Date( + (post.value as AppBskyFeedPost.Record).createdAt, + ); + return postDate >= cutoffDate; + }); + if (filtered.length > 0) { + postAcc.account.currentCursor = filtered[filtered.length - 1].cid; + } + return { + posts: filtered, + account: postAcc.account, + }; + }); + return filteredPosts; +}; +const getNextPosts = async () => { + if (!accountsMetadata.length) { + accountsMetadata = await getAllMetadataFromPds(); } - return posts.slice(0, Config.MAX_POSTS); + const postsAcc: PostsAcc[] = await Promise.all( + accountsMetadata.map(async (account) => { + const posts = await fetchPostsForUser( + account.did, + account.currentCursor || null, + ); + if (posts) { + return { + posts: posts, + account: account, + }; + } else { + return { + posts: [], + account: account, + }; + } + }), + ); + const recordsFiltered = postsAcc.filter((postAcc) => + postAcc.posts.length > 0 + ); + const cutoffDate = getCutoffDate(recordsFiltered); + const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate); + // update the accountMetadata with the new cursor + accountsMetadata = recordsCutoff.map((postAcc) => postAcc.account); + // throw the records in a big single array + let records = recordsCutoff.flatMap((postAcc) => postAcc.posts); + // sort the records by timestamp + records = records.sort((a, b) => { + const aDate = new Date( + (a.value as AppBskyFeedPost.Record).createdAt, + ).getTime(); + const bDate = new Date( + (b.value as AppBskyFeedPost.Record).createdAt, + ).getTime(); + return bDate - aDate; + } + ); + // filter out posts that are in the future + if (!Config.SHOW_FUTURE_POSTS) { + const now = Date.now(); + records = records.filter((post) => { + const postDate = new Date( + (post.value as AppBskyFeedPost.Record).createdAt, + ).getTime(); + return postDate <= now; + }); + } + // append the new posts to the existing posts + posts = posts.concat( + records.map((record) => { + const account = accountsMetadata.find( + (account) => account.did == processAtUri(record.uri).repo, + ); + if (!account) { + throw new Error(`Account with DID ${processAtUri(record.uri).repo} not found`); + } + return new Post(record, account); + }), + ); + console.log("Fetched posts:", posts); + return posts; }; -export { fetchAllPosts, getAllMetadataFromPds, Post }; + +const fetchPostsForUser = async (did: At.Did, cursor: string | null) => { + try { + const { data } = await rpc.get("com.atproto.repo.listRecords", { + params: { + repo: did as At.Identifier, + collection: "app.bsky.feed.post", + limit: Config.MAX_POSTS, + cursor: cursor || undefined, + }, + }); + return data.records as ComAtprotoRepoListRecords.Record[]; + } catch (e) { + console.error(`Error fetching posts for ${did}:`, e); + return null; + } +}; + +// const fetchAllPosts = async () => { +// const users: AccountMetadata[] = await getAllMetadataFromPds(); +// const postRecords = await Promise.all( +// users.map( +// async (metadata: AccountMetadata) => await fetchPosts(metadata.did), +// ), +// ); +// // Filter out any records that have an error +// const validPostRecords = postRecords.filter((record) => !record.error); + +// const posts: Post[] = validPostRecords.flatMap((userFetch) => +// userFetch.records.map((record) => { +// const user = users.find( +// (user: AccountMetadata) => user.did == userFetch.did, +// ); +// if (!user) { +// throw new Error(`User with DID ${userFetch.did} not found`); +// } +// return new Post(record, user); +// }) +// ); + +// posts.sort((a, b) => b.timestamp - a.timestamp); + +// if (!Config.SHOW_FUTURE_POSTS) { +// // Filter out posts that are in the future +// const now = Date.now(); +// const filteredPosts = posts.filter((post) => post.timestamp <= now); +// return filteredPosts.slice(0, Config.MAX_POSTS); +// } + +// return posts.slice(0, Config.MAX_POSTS); +// }; +export { getAllMetadataFromPds, getNextPosts, Post, posts }; export type { AccountMetadata }; From f00e063861dc9118ee71cb19c90a9ea43bb4d1db Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 21 Apr 2025 20:41:01 -0400 Subject: [PATCH 02/12] Frontend works --- deno.lock | 5 +++++ package.json | 3 ++- src/App.svelte | 52 +++++++++++++++++++++++++++++++-------------- src/lib/pdsfetch.ts | 49 +++++++++++++++++++++++++----------------- 4 files changed, 72 insertions(+), 37 deletions(-) diff --git a/deno.lock b/deno.lock index 0616852..724a5c0 100644 --- a/deno.lock +++ b/deno.lock @@ -8,6 +8,7 @@ "npm:@tsconfig/svelte@^5.0.4": "5.0.4", "npm:moment@^2.30.1": "2.30.1", "npm:svelte-check@^4.1.5": "4.1.6_svelte@5.28.1__acorn@8.14.1_typescript@5.7.3", + "npm:svelte-infinite-loading@^1.4.0": "1.4.0", "npm:svelte@^5.23.1": "5.28.1_acorn@8.14.1", "npm:typescript@~5.7.2": "5.7.3", "npm:vite@^6.3.1": "6.3.2_picomatch@4.0.2" @@ -415,6 +416,9 @@ "typescript" ] }, + "svelte-infinite-loading@1.4.0": { + "integrity": "sha512-Jo+f/yr/HmZQuIiiKKzAHVFXdAUWHW2RBbrcQTil8JVk1sCm/riy7KTJVzjBgQvHasrFQYKF84zvtc9/Y4lFYg==" + }, "svelte@5.28.1_acorn@8.14.1": { "integrity": "sha512-iOa9WmfNG95lSOSJdMhdjJ4Afok7IRAQYXpbnxhd5EINnXseG0GVa9j6WPght4eX78XfFez45Fi+uRglGKPV/Q==", "dependencies": [ @@ -476,6 +480,7 @@ "npm:@tsconfig/svelte@^5.0.4", "npm:moment@^2.30.1", "npm:svelte-check@^4.1.5", + "npm:svelte-infinite-loading@^1.4.0", "npm:svelte@^5.23.1", "npm:typescript@~5.7.2", "npm:vite@^6.3.1" diff --git a/package.json b/package.json index 9f84465..1db6461 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "@atcute/bluesky": "^2.0.2", "@atcute/client": "^3.0.1", "@atcute/identity-resolver": "^0.1.2", - "moment": "^2.30.1" + "moment": "^2.30.1", + "svelte-infinite-loading": "^1.4.0" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^5.0.3", diff --git a/src/App.svelte b/src/App.svelte index eb4ed82..e87c087 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -1,10 +1,31 @@
@@ -12,7 +33,6 @@ {#await accountsPromise}

Loading...

{:then accountsData} -

ATProto PDS

Home to {accountsData.length} accounts

@@ -27,20 +47,20 @@

Error: {error.message}

{/await} - {#await postsPromise} -

Loading...

- {:then postsData} - -
-
- {#each postsData as postObject} - - {/each} -
-
- {/await} +
+
+ {#each posts as postObject} + + {/each} + +
+
diff --git a/src/lib/pdsfetch.ts b/src/lib/pdsfetch.ts index d60718e..adfee48 100644 --- a/src/lib/pdsfetch.ts +++ b/src/lib/pdsfetch.ts @@ -26,9 +26,7 @@ interface AccountMetadata { } let accountsMetadata: AccountMetadata[] = []; -// a chronologically sorted list of posts for all users, that will be shown by svelte -// getNextPosts will populate this list with additional posts as needed -let posts: Post[] = []; + interface atUriObject { repo: string; collection: string; @@ -259,7 +257,7 @@ const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => { return postDate >= cutoffDate; }); if (filtered.length > 0) { - postAcc.account.currentCursor = filtered[filtered.length - 1].cid; + postAcc.account.currentCursor = processAtUri(filtered[filtered.length - 1].uri).rkey; } return { posts: filtered, @@ -298,7 +296,16 @@ const getNextPosts = async () => { const cutoffDate = getCutoffDate(recordsFiltered); const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate); // update the accountMetadata with the new cursor - accountsMetadata = recordsCutoff.map((postAcc) => postAcc.account); + accountsMetadata = accountsMetadata.map((account) => { + const postAcc = recordsCutoff.find( + (postAcc) => postAcc.account.did == account.did, + ); + if (postAcc) { + account.currentCursor = postAcc.account.currentCursor; + } + return account; + } + ); // throw the records in a big single array let records = recordsCutoff.flatMap((postAcc) => postAcc.posts); // sort the records by timestamp @@ -310,8 +317,7 @@ const getNextPosts = async () => { (b.value as AppBskyFeedPost.Record).createdAt, ).getTime(); return bDate - aDate; - } - ); + }); // filter out posts that are in the future if (!Config.SHOW_FUTURE_POSTS) { const now = Date.now(); @@ -323,23 +329,26 @@ const getNextPosts = async () => { }); } // append the new posts to the existing posts - posts = posts.concat( - records.map((record) => { - const account = accountsMetadata.find( - (account) => account.did == processAtUri(record.uri).repo, + + const newPosts = records.map((record) => { + const account = accountsMetadata.find( + (account) => account.did == processAtUri(record.uri).repo, + ); + if (!account) { + throw new Error( + `Account with DID ${processAtUri(record.uri).repo} not found`, ); - if (!account) { - throw new Error(`Account with DID ${processAtUri(record.uri).repo} not found`); - } - return new Post(record, account); - }), - ); - console.log("Fetched posts:", posts); - return posts; + } + return new Post(record, account); + }); + console.log("Fetched posts:", newPosts); + console.log("Metadata:", accountsMetadata); + return newPosts; }; const fetchPostsForUser = async (did: At.Did, cursor: string | null) => { try { + console.log("Fetching posts for user:", did, "with Cursor: ", cursor); const { data } = await rpc.get("com.atproto.repo.listRecords", { params: { repo: did as At.Identifier, @@ -388,5 +397,5 @@ const fetchPostsForUser = async (did: At.Did, cursor: string | null) => { // return posts.slice(0, Config.MAX_POSTS); // }; -export { getAllMetadataFromPds, getNextPosts, Post, posts }; +export { getAllMetadataFromPds, getNextPosts, Post }; export type { AccountMetadata }; From eeea3063033cede48b9e7e28cbff2c774f27bf5d Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 21 Apr 2025 21:15:44 -0400 Subject: [PATCH 03/12] remove legacy stuff --- config.ts | 2 +- src/lib/pdsfetch.ts | 58 --------------------------------------------- 2 files changed, 1 insertion(+), 59 deletions(-) diff --git a/config.ts b/config.ts index 989a249..6ff3c9f 100644 --- a/config.ts +++ b/config.ts @@ -18,7 +18,7 @@ export class Config { * Maximum number of posts to show in the feed (across all users) * @default 100 */ - static readonly MAX_POSTS: number = 5; + static readonly MAX_POSTS: number = 15; /** * Footer text for the dashboard diff --git a/src/lib/pdsfetch.ts b/src/lib/pdsfetch.ts index adfee48..c4e2bdc 100644 --- a/src/lib/pdsfetch.ts +++ b/src/lib/pdsfetch.ts @@ -161,31 +161,6 @@ const getAllMetadataFromPds = async (): Promise => { return metadata.filter((account) => account !== null) as AccountMetadata[]; }; -// OLD -const fetchPosts = async (did: string) => { - try { - const { data } = await rpc.get("com.atproto.repo.listRecords", { - params: { - repo: did as At.Identifier, - collection: "app.bsky.feed.post", - limit: Config.MAX_POSTS, - }, - }); - return { - records: data.records as ComAtprotoRepoListRecords.Record[], - did: did, - error: false, - }; - } catch (e) { - console.error(`Error fetching posts for ${did}:`, e); - return { - records: [], - did: did, - error: true, - }; - } -}; - const identityResolve = async (did: At.Did) => { const resolver = new CompositeDidDocumentResolver({ methods: { @@ -364,38 +339,5 @@ const fetchPostsForUser = async (did: At.Did, cursor: string | null) => { } }; -// const fetchAllPosts = async () => { -// const users: AccountMetadata[] = await getAllMetadataFromPds(); -// const postRecords = await Promise.all( -// users.map( -// async (metadata: AccountMetadata) => await fetchPosts(metadata.did), -// ), -// ); -// // Filter out any records that have an error -// const validPostRecords = postRecords.filter((record) => !record.error); - -// const posts: Post[] = validPostRecords.flatMap((userFetch) => -// userFetch.records.map((record) => { -// const user = users.find( -// (user: AccountMetadata) => user.did == userFetch.did, -// ); -// if (!user) { -// throw new Error(`User with DID ${userFetch.did} not found`); -// } -// return new Post(record, user); -// }) -// ); - -// posts.sort((a, b) => b.timestamp - a.timestamp); - -// if (!Config.SHOW_FUTURE_POSTS) { -// // Filter out posts that are in the future -// const now = Date.now(); -// const filteredPosts = posts.filter((post) => post.timestamp <= now); -// return filteredPosts.slice(0, Config.MAX_POSTS); -// } - -// return posts.slice(0, Config.MAX_POSTS); -// }; export { getAllMetadataFromPds, getNextPosts, Post }; export type { AccountMetadata }; From bba230920b982ab21c99d28571c6978d2ac7c05f Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 21 Apr 2025 21:21:50 -0400 Subject: [PATCH 04/12] Change post load amount --- config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.ts b/config.ts index 6ff3c9f..d0a19ca 100644 --- a/config.ts +++ b/config.ts @@ -18,7 +18,7 @@ export class Config { * Maximum number of posts to show in the feed (across all users) * @default 100 */ - static readonly MAX_POSTS: number = 15; + static readonly MAX_POSTS: number = 20; /** * Footer text for the dashboard From 9a2ea50420f35dda36b4bc33a9e600c9e36ab307 Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 21 Apr 2025 22:56:12 -0400 Subject: [PATCH 05/12] Fix typescript and svelte warnings --- src/App.svelte | 6 +----- src/lib/PostComponent.svelte | 4 ++-- src/lib/pdsfetch.ts | 5 ----- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/src/App.svelte b/src/App.svelte index e87c087..94367b1 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -16,7 +16,7 @@ }); }); // Infinite loading function - const onInfinite = ({ detail: { loaded, complete } }) => { + const onInfinite = ({ detail: { loaded, complete } } : { detail : { loaded : () => void, complete : () => void}}) => { getNextPosts().then((newPosts) => { if (newPosts.length > 0) { posts = [...posts, ...newPosts]; @@ -53,11 +53,7 @@ {/each}
diff --git a/src/lib/PostComponent.svelte b/src/lib/PostComponent.svelte index c7c59ac..3c4178c 100644 --- a/src/lib/PostComponent.svelte +++ b/src/lib/PostComponent.svelte @@ -113,7 +113,7 @@
@@ -125,7 +125,7 @@
diff --git a/src/lib/pdsfetch.ts b/src/lib/pdsfetch.ts index c4e2bdc..f60e409 100644 --- a/src/lib/pdsfetch.ts +++ b/src/lib/pdsfetch.ts @@ -215,7 +215,6 @@ const getCutoffDate = (postAccounts: PostsAcc[]) => { } }); if (cutoffDate) { - console.log("Cutoff date:", cutoffDate); return cutoffDate; } else { return new Date(now); @@ -303,7 +302,6 @@ const getNextPosts = async () => { return postDate <= now; }); } - // append the new posts to the existing posts const newPosts = records.map((record) => { const account = accountsMetadata.find( @@ -316,14 +314,11 @@ const getNextPosts = async () => { } return new Post(record, account); }); - console.log("Fetched posts:", newPosts); - console.log("Metadata:", accountsMetadata); return newPosts; }; const fetchPostsForUser = async (did: At.Did, cursor: string | null) => { try { - console.log("Fetching posts for user:", did, "with Cursor: ", cursor); const { data } = await rpc.get("com.atproto.repo.listRecords", { params: { repo: did as At.Identifier, From dc3039ef8b4e6302641dd4a397dfc3dc64ce5b8d Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 21 Apr 2025 23:11:17 -0400 Subject: [PATCH 06/12] Updated config documentation --- config.ts | 19 +++++++++++-------- src/lib/pdsfetch.ts | 1 + 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/config.ts b/config.ts index d0a19ca..0dc01b0 100644 --- a/config.ts +++ b/config.ts @@ -9,14 +9,17 @@ export class Config { static readonly PDS_URL: string = "https://pds.witchcraft.systems"; /** - * The base URL of the frontend service for linking to replies + * The base URL of the frontend service for linking to replies/quotes/accounts etc. * @default "https://deer.social" */ static readonly FRONTEND_URL: string = "https://deer.social"; /** - * Maximum number of posts to show in the feed (across all users) - * @default 100 + * Maximum number of posts to fetch from the PDS per request + * Should be around 20 for about 10 users on the pds + * The more users you have, the lower the number should be + * since sorting is slow and is done on the frontend + * @default 20 */ static readonly MAX_POSTS: number = 20; @@ -27,9 +30,9 @@ export class Config { static readonly FOOTER_TEXT: string = "Astrally projected from witchcraft.systems"; - /** - * Whether to show the posts that are in the future - * @default false - */ - static readonly SHOW_FUTURE_POSTS: boolean = false; + /** + * Whether to show the posts that are in the future + * @default false + */ + static readonly SHOW_FUTURE_POSTS: boolean = false; } diff --git a/src/lib/pdsfetch.ts b/src/lib/pdsfetch.ts index f60e409..79edab0 100644 --- a/src/lib/pdsfetch.ts +++ b/src/lib/pdsfetch.ts @@ -240,6 +240,7 @@ const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => { }); return filteredPosts; }; +// nightmare function. However it works so I am not touching it const getNextPosts = async () => { if (!accountsMetadata.length) { accountsMetadata = await getAllMetadataFromPds(); From cff9eed1a43be8a40fc4d5fa63db99d6db8a4dfe Mon Sep 17 00:00:00 2001 From: ari Date: Tue, 22 Apr 2025 03:14:37 +0000 Subject: [PATCH 07/12] Dynamic post loading (#2) Dynamically load the posts so that you can scroll a chronologically sorted timeline infinitely Reviewed-on: https://git.witchcraft.systems/scientific-witchery/pds-dash/pulls/2 Co-authored-by: ari Co-committed-by: ari --- config.ts | 21 ++-- deno.lock | 5 + package.json | 3 +- src/App.svelte | 46 +++++--- src/lib/PostComponent.svelte | 4 +- src/lib/pdsfetch.ts | 217 ++++++++++++++++++++++++----------- 6 files changed, 205 insertions(+), 91 deletions(-) diff --git a/config.ts b/config.ts index 2b1b511..0dc01b0 100644 --- a/config.ts +++ b/config.ts @@ -9,16 +9,19 @@ export class Config { static readonly PDS_URL: string = "https://pds.witchcraft.systems"; /** - * The base URL of the frontend service for linking to replies + * The base URL of the frontend service for linking to replies/quotes/accounts etc. * @default "https://deer.social" */ static readonly FRONTEND_URL: string = "https://deer.social"; /** - * Maximum number of posts to show in the feed (across all users) - * @default 100 + * Maximum number of posts to fetch from the PDS per request + * Should be around 20 for about 10 users on the pds + * The more users you have, the lower the number should be + * since sorting is slow and is done on the frontend + * @default 20 */ - static readonly MAX_POSTS: number = 100; + static readonly MAX_POSTS: number = 20; /** * Footer text for the dashboard @@ -27,9 +30,9 @@ export class Config { static readonly FOOTER_TEXT: string = "Astrally projected from witchcraft.systems"; - /** - * Whether to show the posts that are in the future - * @default false - */ - static readonly SHOW_FUTURE_POSTS: boolean = false; + /** + * Whether to show the posts that are in the future + * @default false + */ + static readonly SHOW_FUTURE_POSTS: boolean = false; } diff --git a/deno.lock b/deno.lock index 0616852..724a5c0 100644 --- a/deno.lock +++ b/deno.lock @@ -8,6 +8,7 @@ "npm:@tsconfig/svelte@^5.0.4": "5.0.4", "npm:moment@^2.30.1": "2.30.1", "npm:svelte-check@^4.1.5": "4.1.6_svelte@5.28.1__acorn@8.14.1_typescript@5.7.3", + "npm:svelte-infinite-loading@^1.4.0": "1.4.0", "npm:svelte@^5.23.1": "5.28.1_acorn@8.14.1", "npm:typescript@~5.7.2": "5.7.3", "npm:vite@^6.3.1": "6.3.2_picomatch@4.0.2" @@ -415,6 +416,9 @@ "typescript" ] }, + "svelte-infinite-loading@1.4.0": { + "integrity": "sha512-Jo+f/yr/HmZQuIiiKKzAHVFXdAUWHW2RBbrcQTil8JVk1sCm/riy7KTJVzjBgQvHasrFQYKF84zvtc9/Y4lFYg==" + }, "svelte@5.28.1_acorn@8.14.1": { "integrity": "sha512-iOa9WmfNG95lSOSJdMhdjJ4Afok7IRAQYXpbnxhd5EINnXseG0GVa9j6WPght4eX78XfFez45Fi+uRglGKPV/Q==", "dependencies": [ @@ -476,6 +480,7 @@ "npm:@tsconfig/svelte@^5.0.4", "npm:moment@^2.30.1", "npm:svelte-check@^4.1.5", + "npm:svelte-infinite-loading@^1.4.0", "npm:svelte@^5.23.1", "npm:typescript@~5.7.2", "npm:vite@^6.3.1" diff --git a/package.json b/package.json index 9f84465..1db6461 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ "@atcute/bluesky": "^2.0.2", "@atcute/client": "^3.0.1", "@atcute/identity-resolver": "^0.1.2", - "moment": "^2.30.1" + "moment": "^2.30.1", + "svelte-infinite-loading": "^1.4.0" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^5.0.3", diff --git a/src/App.svelte b/src/App.svelte index fa5a5c1..94367b1 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -1,10 +1,31 @@
@@ -26,17 +47,16 @@

Error: {error.message}

{/await} - {#await postsPromise} -

Loading...

- {:then postsData} -
-
- {#each postsData as postObject} - - {/each} -
-
- {/await} +
+
+ {#each posts as postObject} + + {/each} + +
+
diff --git a/src/lib/PostComponent.svelte b/src/lib/PostComponent.svelte index c7c59ac..3c4178c 100644 --- a/src/lib/PostComponent.svelte +++ b/src/lib/PostComponent.svelte @@ -113,7 +113,7 @@
@@ -125,7 +125,7 @@
diff --git a/src/lib/pdsfetch.ts b/src/lib/pdsfetch.ts index 0d36e8d..79edab0 100644 --- a/src/lib/pdsfetch.ts +++ b/src/lib/pdsfetch.ts @@ -18,11 +18,15 @@ import { Config } from "../../config"; // import { AppBskyActorDefs } from "@atcute/client/lexicons"; interface AccountMetadata { - did: string; + did: At.Did; displayName: string; handle: string; avatarCid: string | null; + currentCursor?: string; } + +let accountsMetadata: AccountMetadata[] = []; + interface atUriObject { repo: string; collection: string; @@ -45,7 +49,7 @@ class Post { constructor( record: ComAtprotoRepoListRecords.Record, - account: AccountMetadata + account: AccountMetadata, ) { this.postCid = record.cid; this.recordName = processAtUri(record.uri).rkey; @@ -68,7 +72,7 @@ class Post { switch (post.embed?.$type) { case "app.bsky.embed.images": this.imagesCid = post.embed.images.map( - (imageRecord: any) => imageRecord.image.ref.$link + (imageRecord: any) => imageRecord.image.ref.$link, ); break; case "app.bsky.embed.video": @@ -82,7 +86,7 @@ class Post { switch (post.embed.media.$type) { case "app.bsky.embed.images": this.imagesCid = post.embed.media.images.map( - (imageRecord) => imageRecord.image.ref.$link + (imageRecord) => imageRecord.image.ref.$link, ); break; @@ -118,8 +122,8 @@ const getDidsFromPDS = async (): Promise => { return data.repos.map((repo: any) => repo.did) as At.Did[]; }; const getAccountMetadata = async ( - did: `did:${string}:${string}` -): Promise => { + did: `did:${string}:${string}`, +) => { // gonna assume self exists in the app.bsky.actor.profile try { const { data } = await rpc.get("com.atproto.repo.getRecord", { @@ -143,12 +147,7 @@ const getAccountMetadata = async ( return account; } catch (e) { console.error(`Error fetching metadata for ${did}:`, e); - return { - did: "error", - displayName: "", - avatarCid: null, - handle: "error", - }; + return null; } }; @@ -157,33 +156,9 @@ const getAllMetadataFromPds = async (): Promise => { const metadata = await Promise.all( dids.map(async (repo: `did:${string}:${string}`) => { return await getAccountMetadata(repo); - }) + }), ); - return metadata.filter((account) => account.did !== "error"); -}; - -const fetchPosts = async (did: string) => { - try { - const { data } = await rpc.get("com.atproto.repo.listRecords", { - params: { - repo: did as At.Identifier, - collection: "app.bsky.feed.post", - limit: Config.MAX_POSTS, - }, - }); - return { - records: data.records as ComAtprotoRepoListRecords.Record[], - did: did, - error: false, - }; - } catch (e) { - console.error(`Error fetching posts for ${did}:`, e); - return { - records: [], - did: did, - error: true, - }; - } + return metadata.filter((account) => account !== null) as AccountMetadata[]; }; const identityResolve = async (did: At.Did) => { @@ -196,7 +171,7 @@ const identityResolve = async (did: At.Did) => { if (did.startsWith("did:plc:") || did.startsWith("did:web:")) { const doc = await resolver.resolve( - did as `did:plc:${string}` | `did:web:${string}` + did as `did:plc:${string}` | `did:web:${string}`, ); return doc; } else { @@ -219,36 +194,146 @@ const blueskyHandleFromDid = async (did: At.Did) => { } }; -const fetchAllPosts = async () => { - const users: AccountMetadata[] = await getAllMetadataFromPds(); - const postRecords = await Promise.all( - users.map( - async (metadata: AccountMetadata) => await fetchPosts(metadata.did) - ) - ); - const validPostRecords = postRecords.filter((record) => !record.error); - const posts: Post[] = validPostRecords.flatMap((userFetch) => - userFetch.records.map((record) => { - const user = users.find( - (user: AccountMetadata) => user.did == userFetch.did - ); - if (!user) { - throw new Error(`User with DID ${userFetch.did} not found`); +interface PostsAcc { + posts: ComAtprotoRepoListRecords.Record[]; + account: AccountMetadata; +} +const getCutoffDate = (postAccounts: PostsAcc[]) => { + const now = Date.now(); + let cutoffDate: Date | null = null; + postAccounts.forEach((postAcc) => { + const latestPost = new Date( + (postAcc.posts[postAcc.posts.length - 1].value as AppBskyFeedPost.Record) + .createdAt, + ); + if (!cutoffDate) { + cutoffDate = latestPost; + } else { + if (latestPost > cutoffDate) { + cutoffDate = latestPost; } - return new Post(record, user); - }) - ); + } + }); + if (cutoffDate) { + return cutoffDate; + } else { + return new Date(now); + } +}; - posts.sort((a, b) => b.timestamp - a.timestamp); - - if(!Config.SHOW_FUTURE_POSTS) { - // Filter out posts that are in the future - const now = Date.now(); - const filteredPosts = posts.filter((post) => post.timestamp <= now); - return filteredPosts.slice(0, Config.MAX_POSTS); +const filterPostsByDate = (posts: PostsAcc[], cutoffDate: Date) => { + // filter posts for each account that are older than the cutoff date and save the cursor of the last post included + const filteredPosts: PostsAcc[] = posts.map((postAcc) => { + const filtered = postAcc.posts.filter((post) => { + const postDate = new Date( + (post.value as AppBskyFeedPost.Record).createdAt, + ); + return postDate >= cutoffDate; + }); + if (filtered.length > 0) { + postAcc.account.currentCursor = processAtUri(filtered[filtered.length - 1].uri).rkey; + } + return { + posts: filtered, + account: postAcc.account, + }; + }); + return filteredPosts; +}; +// nightmare function. However it works so I am not touching it +const getNextPosts = async () => { + if (!accountsMetadata.length) { + accountsMetadata = await getAllMetadataFromPds(); } - return posts.slice(0, Config.MAX_POSTS); + const postsAcc: PostsAcc[] = await Promise.all( + accountsMetadata.map(async (account) => { + const posts = await fetchPostsForUser( + account.did, + account.currentCursor || null, + ); + if (posts) { + return { + posts: posts, + account: account, + }; + } else { + return { + posts: [], + account: account, + }; + } + }), + ); + const recordsFiltered = postsAcc.filter((postAcc) => + postAcc.posts.length > 0 + ); + const cutoffDate = getCutoffDate(recordsFiltered); + const recordsCutoff = filterPostsByDate(recordsFiltered, cutoffDate); + // update the accountMetadata with the new cursor + accountsMetadata = accountsMetadata.map((account) => { + const postAcc = recordsCutoff.find( + (postAcc) => postAcc.account.did == account.did, + ); + if (postAcc) { + account.currentCursor = postAcc.account.currentCursor; + } + return account; + } + ); + // throw the records in a big single array + let records = recordsCutoff.flatMap((postAcc) => postAcc.posts); + // sort the records by timestamp + records = records.sort((a, b) => { + const aDate = new Date( + (a.value as AppBskyFeedPost.Record).createdAt, + ).getTime(); + const bDate = new Date( + (b.value as AppBskyFeedPost.Record).createdAt, + ).getTime(); + return bDate - aDate; + }); + // filter out posts that are in the future + if (!Config.SHOW_FUTURE_POSTS) { + const now = Date.now(); + records = records.filter((post) => { + const postDate = new Date( + (post.value as AppBskyFeedPost.Record).createdAt, + ).getTime(); + return postDate <= now; + }); + } + + const newPosts = records.map((record) => { + const account = accountsMetadata.find( + (account) => account.did == processAtUri(record.uri).repo, + ); + if (!account) { + throw new Error( + `Account with DID ${processAtUri(record.uri).repo} not found`, + ); + } + return new Post(record, account); + }); + return newPosts; }; -export { fetchAllPosts, getAllMetadataFromPds, Post }; + +const fetchPostsForUser = async (did: At.Did, cursor: string | null) => { + try { + const { data } = await rpc.get("com.atproto.repo.listRecords", { + params: { + repo: did as At.Identifier, + collection: "app.bsky.feed.post", + limit: Config.MAX_POSTS, + cursor: cursor || undefined, + }, + }); + return data.records as ComAtprotoRepoListRecords.Record[]; + } catch (e) { + console.error(`Error fetching posts for ${did}:`, e); + return null; + } +}; + +export { getAllMetadataFromPds, getNextPosts, Post }; export type { AccountMetadata }; From 5fbb29f7fb9be8d4940551ce1e1e0c1703a363b6 Mon Sep 17 00:00:00 2001 From: ari Date: Mon, 21 Apr 2025 23:32:59 -0400 Subject: [PATCH 08/12] Distance tweaks for hidden loads --- src/App.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/App.svelte b/src/App.svelte index 94367b1..9e36d30 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -18,6 +18,7 @@ // Infinite loading function const onInfinite = ({ detail: { loaded, complete } } : { detail : { loaded : () => void, complete : () => void}}) => { getNextPosts().then((newPosts) => { + console.log("Loading next posts..."); if (newPosts.length > 0) { posts = [...posts, ...newPosts]; loaded(); @@ -53,7 +54,7 @@ {/each}
From 5eca07724e23c8b72b1afba10b9da29a45d05f5f Mon Sep 17 00:00:00 2001 From: astra Date: Tue, 22 Apr 2025 03:53:05 +0000 Subject: [PATCH 09/12] Distance tweaks for hidden loads --- src/App.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/App.svelte b/src/App.svelte index 94367b1..9e36d30 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -18,6 +18,7 @@ // Infinite loading function const onInfinite = ({ detail: { loaded, complete } } : { detail : { loaded : () => void, complete : () => void}}) => { getNextPosts().then((newPosts) => { + console.log("Loading next posts..."); if (newPosts.length > 0) { posts = [...posts, ...newPosts]; loaded(); @@ -53,7 +54,7 @@ {/each}
From 3d38e0f68cb236909315c883a401cb70540a8f13 Mon Sep 17 00:00:00 2001 From: ari Date: Tue, 22 Apr 2025 05:27:05 +0000 Subject: [PATCH 10/12] Update README.md --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 25b3cfb..d9eb2ea 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,9 @@ we use our own CI/CD workflow at [`.forgejo/workflows/deploy.yaml`](.forgejo/wor ## theming -currently the only way to theme the app is to edit css in the components directly, glhf +the colors are designated in [`src/app.css`](src/app.css) as variables, go crazy with them + +the rest is done by editing the css files and style tags directly, good luck relevant files: From 72ba5779507662049ba96b89c2224e1c45b42e35 Mon Sep 17 00:00:00 2001 From: Astra Date: Wed, 23 Apr 2025 04:10:14 +0000 Subject: [PATCH 11/12] Proper post word wrapping (#3) Co-authored-by: ari Reviewed-on: https://git.witchcraft.systems/scientific-witchery/pds-dash/pulls/3 --- src/App.svelte | 10 ++++++---- src/lib/PostComponent.svelte | 4 ++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/App.svelte b/src/App.svelte index 9e36d30..733320e 100644 --- a/src/App.svelte +++ b/src/App.svelte @@ -16,7 +16,11 @@ }); }); // Infinite loading function - const onInfinite = ({ detail: { loaded, complete } } : { detail : { loaded : () => void, complete : () => void}}) => { + const onInfinite = ({ + detail: { loaded, complete }, + }: { + detail: { loaded: () => void; complete: () => void }; + }) => { getNextPosts().then((newPosts) => { console.log("Loading next posts..."); if (newPosts.length > 0) { @@ -53,9 +57,7 @@ {#each posts as postObject} {/each} - +
diff --git a/src/lib/PostComponent.svelte b/src/lib/PostComponent.svelte index 3c4178c..dc0b874 100644 --- a/src/lib/PostComponent.svelte +++ b/src/lib/PostComponent.svelte @@ -213,6 +213,10 @@ #postText { margin: 0; padding: 0; + overflow-wrap: break-word; + word-wrap: normal; + word-break: break-word; + hyphens: none; } #headerText { margin-left: 10px; From 614f2b4c30e819c5134e48ae72bace7cebe1d769 Mon Sep 17 00:00:00 2001 From: Astra Date: Sat, 26 Apr 2025 14:46:30 +0900 Subject: [PATCH 12/12] Add links to the source code --- config.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/config.ts b/config.ts index 0dc01b0..8d09cf6 100644 --- a/config.ts +++ b/config.ts @@ -24,11 +24,10 @@ export class Config { static readonly MAX_POSTS: number = 20; /** - * Footer text for the dashboard - * @default "Astrally projected from witchcraft.systems" + * Footer text for the dashboard, you probably want to change this */ static readonly FOOTER_TEXT: string = - "Astrally projected from witchcraft.systems"; + "Astrally projected from witchcraft.systems

Source (github mirror)"; /** * Whether to show the posts that are in the future