MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Posts are now lazy

pull/73/head
MillenniumEarl 2021-03-02 17:12:39 +01:00
parent 4a16642540
commit 35d359b028
1 changed files with 14 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/scripts/classes/thread.ts
View File

@ -9,7 +9,7 @@ import Post from "./post.js";
import PlatformUser from "./platform-user.js"; import PlatformUser from "./platform-user.js";
import { TCategory, TRating } from "../interfaces.js"; import { TCategory, TRating } from "../interfaces.js";
import { urls } from "../constants/url.js"; import { urls } from "../constants/url.js";
import { THREAD } from "../constants/css-selector.js"; import { POST, THREAD } from "../constants/css-selector.js";
import { fetchHTML, fetchPOSTResponse } from "../network-helper.js"; import { fetchHTML, fetchPOSTResponse } from "../network-helper.js";
import Shared from "../shared.js"; import Shared from "../shared.js";
import { GenericAxiosError, UnexpectedResponseContentType } from "./errors.js"; import { GenericAxiosError, UnexpectedResponseContentType } from "./errors.js";
@ -63,6 +63,8 @@ export default class Thread {
public get prefixes() { return this._prefixes; } public get prefixes() { return this._prefixes; }
/** /**
* List of posts belonging to the thread. * List of posts belonging to the thread.
*
* Each element must be loaded with the `fetch` method before it can be used.
*/ */
public get posts() { return this._posts; } public get posts() { return this._posts; }
/** /**
@ -120,23 +122,20 @@ export default class Thread {
/** /**
* Gets all posts on a page. * Gets all posts on a page.
*/ */
private async parsePostsInPage(html: string): Promise<Post[]> { private parsePostsInPage(html: string): Post[] {
// Load the HTML // Load the HTML
const $ = cheerio.load(html); const $ = cheerio.load(html);
// Start parsing the posts // Start parsing the posts
const postPromises = $(THREAD.POSTS_IN_PAGE) const posts = $(THREAD.POSTS_IN_PAGE)
.toArray() .toArray()
.map(async (idx, el) => { .map((el, idx) => {
// Parse post data const id = $(el).find(POST.ID).attr("id").replace("post-", "");
const p = new Post(); return new Post(parseInt(id));
await p.fetchData($(el));
return p;
}); });
// Wait for the post to be fetched // Wait for the post to be fetched
return await Promise.all(postPromises); return posts;
} }
/** /**
@ -148,13 +147,10 @@ export default class Thread {
const htmlPromiseList: TFetchResult[] = []; const htmlPromiseList: TFetchResult[] = [];
const fetchedPosts: Post[] = []; const fetchedPosts: Post[] = [];
// Set the maximum number of post to 100
await this.setMaximumPostsForPage(100);
// Fetch posts for every page in the thread // Fetch posts for every page in the thread
for (let i = 1; i <= pages; i++) { for (let i = 1; i <= pages; i++) {
// Prepare the URL // Prepare the URL
const url = new URL(`page-${i}`, urls.F95_BASE_URL).toString(); const url = new URL(`page-${i}`, `${this.url}/`).toString();
// Fetch the HTML source // Fetch the HTML source
const htmlResponse = fetchHTML(url); const htmlResponse = fetchHTML(url);
@ -167,9 +163,7 @@ export default class Thread {
// Scrape the pages // Scrape the pages
for (const response of responses) { for (const response of responses) {
if (response.isSuccess()) { if (response.isSuccess()) {
// Parse the posts const posts = this.parsePostsInPage(response.value);
const posts = await this.parsePostsInPage(response.value);
fetchedPosts.push(...posts); fetchedPosts.push(...posts);
} else throw response.value; } else throw response.value;
} }
@ -220,6 +214,9 @@ export default class Thread {
// Prepare the url // Prepare the url
this._url = new URL(this.id.toString(), urls.F95_THREADS).toString(); this._url = new URL(this.id.toString(), urls.F95_THREADS).toString();
// Set the maximum number of post to 100
await this.setMaximumPostsForPage(100);
// Fetch the HTML source // Fetch the HTML source
const htmlResponse = await fetchHTML(this.url); const htmlResponse = await fetchHTML(this.url);