Change createURL to execute

src/scripts/classes/query/handiwork-search-query.ts

@@ -1,10 +1,13 @@
 "use strict";
 
+import { AxiosResponse } from 'axios';
 // Public modules from npm
 import validator from 'class-validator';
 
 // Module from files
 import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js";
+import { GenericAxiosError, UnexpectedResponseContentType } from '../errors.js';
+import { Result } from '../result.js';
 import LatestSearchQuery, { TLatestOrder } from './latest-search-query.js';
 import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js';
 
@@ -28,6 +31,8 @@ import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js';
  * `views`: Order based on the number of visits. Replacement: `replies`.
  */
 type THandiworkOrder = "date" | "likes" | "relevance" | "replies" | "title" | "views";
+type TLatestResult = Result<GenericAxiosError | UnexpectedResponseContentType, string>;
+type TThreadResult = Result<GenericAxiosError, AxiosResponse<any>>;
 
 export default class HandiworkSearchQuery implements IQuery {
     
@@ -38,6 +43,7 @@ export default class HandiworkSearchQuery implements IQuery {
     //#endregion Private fields
 
     //#region Properties
+
     /**
      * Keywords to use in the search.
      */
@@ -69,9 +75,11 @@ export default class HandiworkSearchQuery implements IQuery {
     })
     public page: number = 1;
     itype: TQueryInterface = "HandiworkSearchQuery";
+
     //#endregion Properties
 
     //#region Public methods
+
     /**
      * Select what kind of search should be 
      * performed based on the properties of 
@@ -90,13 +98,11 @@ export default class HandiworkSearchQuery implements IQuery {
         return DEFAULT_SEARCH_TYPE;
     }
 
-    public validate(): boolean {
+    public validate(): boolean { return validator.validateSync(this).length === 0; }
-        return validator.validateSync(this).length === 0;
-    }
 
-    public createURL(): URL {
+    public async execute(): Promise<TLatestResult | TThreadResult> {
         // Local variables
-        let url = null;
+        let response: TLatestResult | TThreadResult = null;
 
         // Check if the query is valid
         if (!this.validate()) {
@@ -104,10 +110,10 @@ export default class HandiworkSearchQuery implements IQuery {
         }
 
         // Convert the query
-        if (this.selectSearchType() === "latest") url = this.cast<LatestSearchQuery>("LatestSearchQuery").createURL();
+        if (this.selectSearchType() === "latest") response = await this.cast<LatestSearchQuery>("LatestSearchQuery").execute();
-        else url = this.cast<ThreadSearchQuery>("ThreadSearchQuery").createURL();
+        else response = await this.cast<ThreadSearchQuery>("ThreadSearchQuery").execute();
 
-        return url;
+        return response;
     }
 
     public cast<T extends IQuery>(type: TQueryInterface): T {
@@ -122,9 +128,11 @@ export default class HandiworkSearchQuery implements IQuery {
         // Cast the result to T
         return returnValue as T;
     }
+
     //#endregion Public methods
 
     //#region Private methods
+
     private castToLatest(): LatestSearchQuery {
         // Cast the basic query object and copy common values
         const query: LatestSearchQuery = new LatestSearchQuery;
@@ -165,5 +173,6 @@ export default class HandiworkSearchQuery implements IQuery {
 
         return query;
     }
+    
     //#endregion
 }

src/scripts/classes/query/latest-search-query.ts

@@ -7,6 +7,7 @@ import validator from 'class-validator';
 import { urls } from "../../constants/url.js";
 import PrefixParser from '../prefix-parser.js';
 import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js";
+import { fetchHTML } from '../../network-helper.js';
 
 // Type definitions
 export type TLatestOrder = "date" | "likes" | "views" | "title" | "rating";
@@ -18,11 +19,14 @@ type TDate = 365 | 180 | 90 | 30 | 14 | 7 | 3 | 1;
 export default class LatestSearchQuery implements IQuery {
 
     //#region Private fields
+
     private static MAX_TAGS = 5;
     private static MIN_PAGE = 1;
+
     //#endregion Private fields
 
     //#region Properties
+
     public category: TCategory = 'games';
     /**
      * Ordering type. 
@@ -52,20 +56,47 @@ export default class LatestSearchQuery implements IQuery {
     })
     public page = LatestSearchQuery.MIN_PAGE;
     itype: TQueryInterface = "LatestSearchQuery";
+
     //#endregion Properties
 
     //#region Public methods
 
-    public validate(): boolean {
+    public validate(): boolean { return validator.validateSync(this).length === 0; }
-        return validator.validateSync(this).length === 0;
-    }
 
-    public createURL(): URL {
+    public async execute() {
         // Check if the query is valid
         if (!this.validate()) {
             throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`);
         }
 
+        // Prepare the URL
+        const url = this.prepareGETurl();
+        const decoded = decodeURIComponent(url.toString());
+        
+        // Fetch the result
+        return await fetchHTML(decoded);
+    }
+
+    public findNearestDate(d: Date): TDate {
+        // Find the difference between today and the passed date
+        const diff = this.dateDiffInDays(new Date(), d);
+
+        // Find the closest valid value in the array
+        const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) {
+            return (Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev);
+        });
+
+        return closest as TDate;
+    }
+
+    //#endregion Public methods
+
+    //#region Private methods
+
+    /**
+     * Prepare the URL by filling out the GET parameters with the data in the query.
+     */
+    private prepareGETurl(): URL {
         // Create the URL
         const url = new URL(urls.F95_LATEST_PHP);
         url.searchParams.set("cmd", "list");
@@ -92,20 +123,6 @@ export default class LatestSearchQuery implements IQuery {
         return url;
     }
 
-    public findNearestDate(d: Date): TDate {
-        // Find the difference between today and the passed date
-        const diff = this.dateDiffInDays(new Date(), d);
-
-        // Find the closest valid value in the array
-        const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) {
-            return (Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev);
-        });
-
-        return closest as TDate;
-    }
-    //#endregion Public methods
-
-    //#region Private methods
     /**
      * 
      */
@@ -118,5 +135,7 @@ export default class LatestSearchQuery implements IQuery {
 
         return Math.floor((utc2 - utc1) / MS_PER_DAY);
     }
+
     //#endregion Private methodss
+
 }

src/scripts/classes/query/thread-search-query.ts

@@ -7,6 +7,11 @@ import validator from 'class-validator';
 import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js";
 import { urls } from "../../constants/url.js";
 import PrefixParser from "./../prefix-parser.js";
+import { fetchPOSTResponse } from '../../network-helper.js';
+import { AxiosResponse } from 'axios';
+import { GenericAxiosError } from '../errors.js';
+import { Result } from '../result.js';
+import Shared from '../../shared.js';
 
 // Type definitions
 export type TThreadOrder = "relevance" | "date" | "last_update" | "replies";
@@ -14,10 +19,13 @@ export type TThreadOrder = "relevance" | "date" | "last_update" | "replies";
 export default class ThreadSearchQuery implements IQuery {
 
     //#region Private fields
+
     static MIN_PAGE = 1;
+
     //#endregion Private fields
 
     //#region Properties
+
     /**
      * Keywords to use in the search.
      */
@@ -57,78 +65,89 @@ export default class ThreadSearchQuery implements IQuery {
     })
     public page: number = 1;
     itype: TQueryInterface = "ThreadSearchQuery";
+
     //#endregion Properties
 
     //#region Public methods
     
-    public validate(): boolean {
+    public validate(): boolean { return validator.validateSync(this).length === 0; }
-        return validator.validateSync(this).length === 0;
-    }
     
-    public createURL(): URL {
+    public async execute(): Promise<Result<GenericAxiosError, AxiosResponse<any>>> {
         // Check if the query is valid
         if (!this.validate()) {
             throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`);
         }
 
-        // Create the URL
+        // Define the POST parameters
-        const url = new URL(urls.F95_SEARCH_URL);
+        const params = this.preparePOSTParameters();
 
-        // Specifiy if only the title should be searched
+        // Return the POST response
-        if (this.onlyTitles) url.searchParams.set("c[title_only]", "1");
+        return await fetchPOSTResponse(urls.F95_SEARCH_URL, params);
+    }
+
+    //#endregion Public methods
+
+    //#region Private methods
+
+    /**
+     * Prepare the parameters for post request with the data in the query.
+     */
+    private preparePOSTParameters(): { [s: string]: string } {
+        // Local variables
+        const params = {};
+
+        // Ad the session token
+        params["_xfToken"] = Shared.session.token;
+
+        // Specify if only the title should be searched
+        if (this.onlyTitles) params["c[title_only]"] = "1";
 
         // Add keywords
-        const encodedKeywords = this.keywords ?? "*";
+        params["keywords"] = this.keywords ?? "*";
-        url.searchParams.set("q", encodedKeywords);
 
         // Specify the scope of the search (only "threads/post")
-        url.searchParams.set("t", "post");
+        params["search_type"] = "post";
 
         // Set the dates
         if (this.newerThan) {
             const date = this.convertShortDate(this.newerThan);
-            url.searchParams.set("c[newer_than]", date);
+            params["c[newer_than]"] = date;
         }
 
         if (this.olderThan) {
             const date = this.convertShortDate(this.olderThan);
-            url.searchParams.set("c[older_than]", date);
+            params["c[older_than]"] = date;
         }
 
-        // Set included and excluded tags
+        // Set included and excluded tags (joined with a comma)
-        // The tags are first joined with a comma, then encoded to URI
+        if (this.includedTags) params["c[tags]"] = this.includedTags.join(",");
-        const includedTags = encodeURIComponent(this.includedTags.join(","));
+        if (this.excludedTags) params["c[excludeTags]"] = this.excludedTags.join(",");
-        const excludedTags = encodeURIComponent(this.excludedTags.join(","));
-        if (includedTags) url.searchParams.set("c[tags]", includedTags);
-        if (excludedTags) url.searchParams.set("c[excludeTags]", excludedTags);
 
         // Set minimum reply number
-        if (this.minimumReplies > 0) url.searchParams.set("c[min_reply_count]", this.minimumReplies.toString());
+        if (this.minimumReplies > 0) params["c[min_reply_count]"] = this.minimumReplies.toString();
 
         // Add prefixes
         const parser = new PrefixParser();
         const ids = parser.prefixesToIDs(this.includedPrefixes);
         for (let i = 0; i < ids.length; i++) {
             const name = `c[prefixes][${i}]`;
-            url.searchParams.set(name, ids[i].toString());
+            params[name] = ids[i].toString();
         }
 
         // Set the category
-        url.searchParams.set("c[child_nodes]", "1"); // Always set
+        params["c[child_nodes]"] = "1"; // Always set
         if (this.category) {
             const catID = this.categoryToID(this.category).toString();
-            url.searchParams.set("c[nodes][0]", catID);
+            params["c[nodes][0]"] = catID;
         }
 
         // Set the other values
-        url.searchParams.set("o", this.order.toString());
+        params["o"] = this.order.toString();
-        url.searchParams.set("page", this.page.toString());
+        params["page"] = this.page.toString();
 
-        return url;
+        return params;
     }
-    //#endregion Public methods
 
-    //#region Private methods
     /**
      * Convert a date in the YYYY-MM-DD format taking into account the time zone.
      */
@@ -152,6 +171,7 @@ export default class ThreadSearchQuery implements IQuery {
 
         return catMap[category as string];
     }
+
     //#endregion Private methods
 
 }

src/scripts/constants/url.ts

@@ -1,6 +1,6 @@
 export const urls = {
     F95_BASE_URL: "https://f95zone.to",
-    F95_SEARCH_URL: "https://f95zone.to/search/105286576/",
+    F95_SEARCH_URL: "https://f95zone.to/search/search/",
     F95_LATEST_UPDATES: "https://f95zone.to/latest",
     F95_THREADS: "https://f95zone.to/threads/",
     F95_LOGIN_URL: "https://f95zone.to/login/login",

src/scripts/fetch-data/fetch-latest.ts

@@ -1,7 +1,6 @@
 "use strict";
 
 // Modules from file
-import { fetchGETResponse } from "../network-helper.js";
 import LatestSearchQuery from "../classes/query/latest-search-query.js";
 import { urls as f95url } from "../constants/url.js";
 
@@ -23,11 +22,8 @@ export default async function fetchLatestHandiworkURLs(query: LatestSearchQuery,
     let noMorePages = false;
 
     do {
-        // Prepare the URL
-        const url = query.createURL().toString();
-
         // Fetch the response (application/json)
-        const response = await fetchGETResponse(url);
+        const response = await query.execute();
 
         // Save the URLs
         if (response.isSuccess()) {

src/scripts/fetch-data/fetch-thread.ts

@@ -4,13 +4,13 @@
 import cheerio from "cheerio";
 
 // Modules from file
-import { fetchHTML } from "../network-helper.js";
 import shared from "../shared.js";
 import { selectors as f95Selector } from "../constants/css-selector.js";
 import { urls as f95urls } from "../constants/url.js";
 import ThreadSearchQuery from "../classes/query/thread-search-query.js";
 
 //#region Public methods
+
 /**
  * Gets the URLs of the handiwork' threads that match the passed parameters.
  * You *must* be logged.
@@ -21,30 +21,26 @@ import ThreadSearchQuery from "../classes/query/thread-search-query.js";
  * @returns {Promise<String[]>} URLs of the handiworks
  */
 export default async function fetchThreadHandiworkURLs(query: ThreadSearchQuery, limit: number = 30): Promise<string[]> {
-    // Get the query
+    // Execute the query
-    const url = decodeURI(query.createURL().href);
+    const response = await query.execute();
 
     // Fetch the results from F95 and return the handiwork urls
-    return await fetchResultURLs(url, limit); 
+    if (response.isSuccess()) return await fetchResultURLs(response.value.data as string, limit); 
+    else throw response.value
 }
 
 //#endregion Public methods
 
 //#region Private methods
+
 /**
  * Gets the URLs of the threads resulting from the F95Zone search.
  * @param {number} limit
  * Maximum number of items to get. Default: 30
- * @return {Promise<String[]>} List of URLs
  */
-async function fetchResultURLs(url: string, limit: number = 30): Promise<string[]> {
+async function fetchResultURLs(html: string, limit: number = 30): Promise<string[]> {
-    shared.logger.trace(`Fetching ${url}...`);
+    // Prepare cheerio
-
+    const $ = cheerio.load(html);
-    // Fetch HTML and prepare Cheerio
-    const html = await fetchHTML(url);
-
-    if (html.isSuccess()) {
-        const $ = cheerio.load(html.value);
 
     // Here we get all the DIV that are the body of the various query results
     const results = $("body").find(f95Selector.GS_RESULT_BODY);
@@ -56,7 +52,6 @@ async function fetchResultURLs(url: string, limit: number = 30): Promise<string[
     }).get();
 
     return urls;
-    } else throw html.value;
 }
 
 /**
@@ -75,4 +70,5 @@ function extractLinkFromResult(selector: cheerio.Cheerio): string {
     // Compose and return the URL
     return new URL(partialLink, f95urls.F95_BASE_URL).toString();
 }
+
 //#endregion Private methods

src/scripts/interfaces.d.ts

@@ -288,8 +288,9 @@ export interface IQuery {
      */
     validate(): boolean,
     /**
-     * From the query values it generates the corresponding URL for the platform.
+     * Search with the data in the query and returns the result.
+     * 
      * If the query is invalid it throws an exception.
      */
-    createURL(): URL,
+    execute(): any,
 }