diff --git a/src/scripts/classes/query/handiwork-search-query.ts b/src/scripts/classes/query/handiwork-search-query.ts index e25e40c..5b4e59f 100644 --- a/src/scripts/classes/query/handiwork-search-query.ts +++ b/src/scripts/classes/query/handiwork-search-query.ts @@ -1,10 +1,13 @@ "use strict"; +import { AxiosResponse } from 'axios'; // Public modules from npm import validator from 'class-validator'; // Module from files import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js"; +import { GenericAxiosError, UnexpectedResponseContentType } from '../errors.js'; +import { Result } from '../result.js'; import LatestSearchQuery, { TLatestOrder } from './latest-search-query.js'; import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js'; @@ -28,6 +31,8 @@ import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js'; * `views`: Order based on the number of visits. Replacement: `replies`. */ type THandiworkOrder = "date" | "likes" | "relevance" | "replies" | "title" | "views"; +type TLatestResult = Result; +type TThreadResult = Result>; export default class HandiworkSearchQuery implements IQuery { @@ -38,6 +43,7 @@ export default class HandiworkSearchQuery implements IQuery { //#endregion Private fields //#region Properties + /** * Keywords to use in the search. */ @@ -69,9 +75,11 @@ export default class HandiworkSearchQuery implements IQuery { }) public page: number = 1; itype: TQueryInterface = "HandiworkSearchQuery"; + //#endregion Properties //#region Public methods + /** * Select what kind of search should be * performed based on the properties of @@ -90,13 +98,11 @@ export default class HandiworkSearchQuery implements IQuery { return DEFAULT_SEARCH_TYPE; } - public validate(): boolean { - return validator.validateSync(this).length === 0; - } + public validate(): boolean { return validator.validateSync(this).length === 0; } - public createURL(): URL { + public async execute(): Promise { // Local variables - let url = null; + let response: TLatestResult | TThreadResult = null; // Check if the query is valid if (!this.validate()) { @@ -104,10 +110,10 @@ export default class HandiworkSearchQuery implements IQuery { } // Convert the query - if (this.selectSearchType() === "latest") url = this.cast("LatestSearchQuery").createURL(); - else url = this.cast("ThreadSearchQuery").createURL(); + if (this.selectSearchType() === "latest") response = await this.cast("LatestSearchQuery").execute(); + else response = await this.cast("ThreadSearchQuery").execute(); - return url; + return response; } public cast(type: TQueryInterface): T { @@ -122,9 +128,11 @@ export default class HandiworkSearchQuery implements IQuery { // Cast the result to T return returnValue as T; } + //#endregion Public methods //#region Private methods + private castToLatest(): LatestSearchQuery { // Cast the basic query object and copy common values const query: LatestSearchQuery = new LatestSearchQuery; @@ -165,5 +173,6 @@ export default class HandiworkSearchQuery implements IQuery { return query; } + //#endregion } \ No newline at end of file diff --git a/src/scripts/classes/query/latest-search-query.ts b/src/scripts/classes/query/latest-search-query.ts index afeb42d..17904f2 100644 --- a/src/scripts/classes/query/latest-search-query.ts +++ b/src/scripts/classes/query/latest-search-query.ts @@ -7,6 +7,7 @@ import validator from 'class-validator'; import { urls } from "../../constants/url.js"; import PrefixParser from '../prefix-parser.js'; import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js"; +import { fetchHTML } from '../../network-helper.js'; // Type definitions export type TLatestOrder = "date" | "likes" | "views" | "title" | "rating"; @@ -18,11 +19,14 @@ type TDate = 365 | 180 | 90 | 30 | 14 | 7 | 3 | 1; export default class LatestSearchQuery implements IQuery { //#region Private fields + private static MAX_TAGS = 5; private static MIN_PAGE = 1; + //#endregion Private fields //#region Properties + public category: TCategory = 'games'; /** * Ordering type. @@ -52,20 +56,47 @@ export default class LatestSearchQuery implements IQuery { }) public page = LatestSearchQuery.MIN_PAGE; itype: TQueryInterface = "LatestSearchQuery"; + //#endregion Properties //#region Public methods - public validate(): boolean { - return validator.validateSync(this).length === 0; - } + public validate(): boolean { return validator.validateSync(this).length === 0; } - public createURL(): URL { + public async execute() { // Check if the query is valid if (!this.validate()) { throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`); } + // Prepare the URL + const url = this.prepareGETurl(); + const decoded = decodeURIComponent(url.toString()); + + // Fetch the result + return await fetchHTML(decoded); + } + + public findNearestDate(d: Date): TDate { + // Find the difference between today and the passed date + const diff = this.dateDiffInDays(new Date(), d); + + // Find the closest valid value in the array + const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) { + return (Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev); + }); + + return closest as TDate; + } + + //#endregion Public methods + + //#region Private methods + + /** + * Prepare the URL by filling out the GET parameters with the data in the query. + */ + private prepareGETurl(): URL { // Create the URL const url = new URL(urls.F95_LATEST_PHP); url.searchParams.set("cmd", "list"); @@ -92,20 +123,6 @@ export default class LatestSearchQuery implements IQuery { return url; } - public findNearestDate(d: Date): TDate { - // Find the difference between today and the passed date - const diff = this.dateDiffInDays(new Date(), d); - - // Find the closest valid value in the array - const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) { - return (Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev); - }); - - return closest as TDate; - } - //#endregion Public methods - - //#region Private methods /** * */ @@ -118,5 +135,7 @@ export default class LatestSearchQuery implements IQuery { return Math.floor((utc2 - utc1) / MS_PER_DAY); } + //#endregion Private methodss + } \ No newline at end of file diff --git a/src/scripts/classes/query/thread-search-query.ts b/src/scripts/classes/query/thread-search-query.ts index 824f8b9..715d90f 100644 --- a/src/scripts/classes/query/thread-search-query.ts +++ b/src/scripts/classes/query/thread-search-query.ts @@ -7,6 +7,11 @@ import validator from 'class-validator'; import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js"; import { urls } from "../../constants/url.js"; import PrefixParser from "./../prefix-parser.js"; +import { fetchPOSTResponse } from '../../network-helper.js'; +import { AxiosResponse } from 'axios'; +import { GenericAxiosError } from '../errors.js'; +import { Result } from '../result.js'; +import Shared from '../../shared.js'; // Type definitions export type TThreadOrder = "relevance" | "date" | "last_update" | "replies"; @@ -14,10 +19,13 @@ export type TThreadOrder = "relevance" | "date" | "last_update" | "replies"; export default class ThreadSearchQuery implements IQuery { //#region Private fields + static MIN_PAGE = 1; + //#endregion Private fields //#region Properties + /** * Keywords to use in the search. */ @@ -57,78 +65,89 @@ export default class ThreadSearchQuery implements IQuery { }) public page: number = 1; itype: TQueryInterface = "ThreadSearchQuery"; + //#endregion Properties //#region Public methods - public validate(): boolean { - return validator.validateSync(this).length === 0; - } + public validate(): boolean { return validator.validateSync(this).length === 0; } - public createURL(): URL { + public async execute(): Promise>> { // Check if the query is valid if (!this.validate()) { throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`); } - - // Create the URL - const url = new URL(urls.F95_SEARCH_URL); - // Specifiy if only the title should be searched - if (this.onlyTitles) url.searchParams.set("c[title_only]", "1"); + // Define the POST parameters + const params = this.preparePOSTParameters(); + + // Return the POST response + return await fetchPOSTResponse(urls.F95_SEARCH_URL, params); + } + + //#endregion Public methods + + //#region Private methods + + /** + * Prepare the parameters for post request with the data in the query. + */ + private preparePOSTParameters(): { [s: string]: string } { + // Local variables + const params = {}; + + // Ad the session token + params["_xfToken"] = Shared.session.token; + + // Specify if only the title should be searched + if (this.onlyTitles) params["c[title_only]"] = "1"; // Add keywords - const encodedKeywords = this.keywords ?? "*"; - url.searchParams.set("q", encodedKeywords); - + params["keywords"] = this.keywords ?? "*"; + // Specify the scope of the search (only "threads/post") - url.searchParams.set("t", "post"); - + params["search_type"] = "post"; + // Set the dates if (this.newerThan) { const date = this.convertShortDate(this.newerThan); - url.searchParams.set("c[newer_than]", date); + params["c[newer_than]"] = date; } if (this.olderThan) { const date = this.convertShortDate(this.olderThan); - url.searchParams.set("c[older_than]", date); + params["c[older_than]"] = date; } - // Set included and excluded tags - // The tags are first joined with a comma, then encoded to URI - const includedTags = encodeURIComponent(this.includedTags.join(",")); - const excludedTags = encodeURIComponent(this.excludedTags.join(",")); - if (includedTags) url.searchParams.set("c[tags]", includedTags); - if (excludedTags) url.searchParams.set("c[excludeTags]", excludedTags); + // Set included and excluded tags (joined with a comma) + if (this.includedTags) params["c[tags]"] = this.includedTags.join(","); + if (this.excludedTags) params["c[excludeTags]"] = this.excludedTags.join(","); // Set minimum reply number - if (this.minimumReplies > 0) url.searchParams.set("c[min_reply_count]", this.minimumReplies.toString()); + if (this.minimumReplies > 0) params["c[min_reply_count]"] = this.minimumReplies.toString(); // Add prefixes const parser = new PrefixParser(); const ids = parser.prefixesToIDs(this.includedPrefixes); for (let i = 0; i < ids.length; i++) { const name = `c[prefixes][${i}]`; - url.searchParams.set(name, ids[i].toString()); + params[name] = ids[i].toString(); } // Set the category - url.searchParams.set("c[child_nodes]", "1"); // Always set + params["c[child_nodes]"] = "1"; // Always set if (this.category) { const catID = this.categoryToID(this.category).toString(); - url.searchParams.set("c[nodes][0]", catID); + params["c[nodes][0]"] = catID; } // Set the other values - url.searchParams.set("o", this.order.toString()); - url.searchParams.set("page", this.page.toString()); + params["o"] = this.order.toString(); + params["page"] = this.page.toString(); - return url; + return params; } - //#endregion Public methods - //#region Private methods /** * Convert a date in the YYYY-MM-DD format taking into account the time zone. */ @@ -152,6 +171,7 @@ export default class ThreadSearchQuery implements IQuery { return catMap[category as string]; } + //#endregion Private methods } \ No newline at end of file diff --git a/src/scripts/constants/url.ts b/src/scripts/constants/url.ts index 1d76398..94e121f 100644 --- a/src/scripts/constants/url.ts +++ b/src/scripts/constants/url.ts @@ -1,6 +1,6 @@ export const urls = { F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search/105286576/", + F95_SEARCH_URL: "https://f95zone.to/search/search/", F95_LATEST_UPDATES: "https://f95zone.to/latest", F95_THREADS: "https://f95zone.to/threads/", F95_LOGIN_URL: "https://f95zone.to/login/login", diff --git a/src/scripts/fetch-data/fetch-latest.ts b/src/scripts/fetch-data/fetch-latest.ts index b530c80..d9dcdc5 100644 --- a/src/scripts/fetch-data/fetch-latest.ts +++ b/src/scripts/fetch-data/fetch-latest.ts @@ -1,7 +1,6 @@ "use strict"; // Modules from file -import { fetchGETResponse } from "../network-helper.js"; import LatestSearchQuery from "../classes/query/latest-search-query.js"; import { urls as f95url } from "../constants/url.js"; @@ -23,11 +22,8 @@ export default async function fetchLatestHandiworkURLs(query: LatestSearchQuery, let noMorePages = false; do { - // Prepare the URL - const url = query.createURL().toString(); - // Fetch the response (application/json) - const response = await fetchGETResponse(url); + const response = await query.execute(); // Save the URLs if (response.isSuccess()) { diff --git a/src/scripts/fetch-data/fetch-thread.ts b/src/scripts/fetch-data/fetch-thread.ts index 9a86b97..370b653 100644 --- a/src/scripts/fetch-data/fetch-thread.ts +++ b/src/scripts/fetch-data/fetch-thread.ts @@ -4,13 +4,13 @@ import cheerio from "cheerio"; // Modules from file -import { fetchHTML } from "../network-helper.js"; import shared from "../shared.js"; import { selectors as f95Selector } from "../constants/css-selector.js"; import { urls as f95urls } from "../constants/url.js"; import ThreadSearchQuery from "../classes/query/thread-search-query.js"; //#region Public methods + /** * Gets the URLs of the handiwork' threads that match the passed parameters. * You *must* be logged. @@ -21,42 +21,37 @@ import ThreadSearchQuery from "../classes/query/thread-search-query.js"; * @returns {Promise} URLs of the handiworks */ export default async function fetchThreadHandiworkURLs(query: ThreadSearchQuery, limit: number = 30): Promise { - // Get the query - const url = decodeURI(query.createURL().href); - + // Execute the query + const response = await query.execute(); + // Fetch the results from F95 and return the handiwork urls - return await fetchResultURLs(url, limit); + if (response.isSuccess()) return await fetchResultURLs(response.value.data as string, limit); + else throw response.value } //#endregion Public methods //#region Private methods + /** * Gets the URLs of the threads resulting from the F95Zone search. * @param {number} limit * Maximum number of items to get. Default: 30 - * @return {Promise} List of URLs */ -async function fetchResultURLs(url: string, limit: number = 30): Promise { - shared.logger.trace(`Fetching ${url}...`); +async function fetchResultURLs(html: string, limit: number = 30): Promise { + // Prepare cheerio + const $ = cheerio.load(html); - // Fetch HTML and prepare Cheerio - const html = await fetchHTML(url); + // Here we get all the DIV that are the body of the various query results + const results = $("body").find(f95Selector.GS_RESULT_BODY); - if (html.isSuccess()) { - const $ = cheerio.load(html.value); + // Than we extract the URLs + const urls = results.slice(0, limit).map((idx, el) => { + const elementSelector = $(el); + return extractLinkFromResult(elementSelector); + }).get(); - // Here we get all the DIV that are the body of the various query results - const results = $("body").find(f95Selector.GS_RESULT_BODY); - - // Than we extract the URLs - const urls = results.slice(0, limit).map((idx, el) => { - const elementSelector = $(el); - return extractLinkFromResult(elementSelector); - }).get(); - - return urls; - } else throw html.value; + return urls; } /** @@ -75,4 +70,5 @@ function extractLinkFromResult(selector: cheerio.Cheerio): string { // Compose and return the URL return new URL(partialLink, f95urls.F95_BASE_URL).toString(); } + //#endregion Private methods \ No newline at end of file diff --git a/src/scripts/interfaces.d.ts b/src/scripts/interfaces.d.ts index 5719b08..c5314be 100644 --- a/src/scripts/interfaces.d.ts +++ b/src/scripts/interfaces.d.ts @@ -288,8 +288,9 @@ export interface IQuery { */ validate(): boolean, /** - * From the query values it generates the corresponding URL for the platform. + * Search with the data in the query and returns the result. + * * If the query is invalid it throws an exception. */ - createURL(): URL, + execute(): any, } \ No newline at end of file