MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Change createURL to execute

pull/73/head
MillenniumEarl 2021-03-04 10:02:23 +01:00
parent 839016daa3
commit 1d7e06da4c
7 changed files with 129 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/scripts/classes/query/handiwork-search-query.ts
View File

@ -1,10 +1,13 @@
"use strict";
import { AxiosResponse } from 'axios';
// Public modules from npm
import validator from 'class-validator';
// Module from files
import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js";
import { GenericAxiosError, UnexpectedResponseContentType } from '../errors.js';
import { Result } from '../result.js';
import LatestSearchQuery, { TLatestOrder } from './latest-search-query.js';
import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js';
@ -28,6 +31,8 @@ import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js';
* `views`: Order based on the number of visits. Replacement: `replies`.
*/
type THandiworkOrder = "date" | "likes" | "relevance" | "replies" | "title" | "views";
type TLatestResult = Result<GenericAxiosError | UnexpectedResponseContentType, string>;
type TThreadResult = Result<GenericAxiosError, AxiosResponse<any>>;
export default class HandiworkSearchQuery implements IQuery {
@ -38,6 +43,7 @@ export default class HandiworkSearchQuery implements IQuery {
//#endregion Private fields
//#region Properties
/**
* Keywords to use in the search.
*/
@ -69,9 +75,11 @@ export default class HandiworkSearchQuery implements IQuery {
})
public page: number = 1;
itype: TQueryInterface = "HandiworkSearchQuery";
//#endregion Properties
//#region Public methods
/**
* Select what kind of search should be
* performed based on the properties of
@ -90,13 +98,11 @@ export default class HandiworkSearchQuery implements IQuery {
return DEFAULT_SEARCH_TYPE;
}
public validate(): boolean {
return validator.validateSync(this).length === 0;
}
public validate(): boolean { return validator.validateSync(this).length === 0; }
public createURL(): URL {
public async execute(): Promise<TLatestResult | TThreadResult> {
// Local variables
let url = null;
let response: TLatestResult | TThreadResult = null;
// Check if the query is valid
if (!this.validate()) {
@ -104,10 +110,10 @@ export default class HandiworkSearchQuery implements IQuery {
}
// Convert the query
if (this.selectSearchType() === "latest") url = this.cast<LatestSearchQuery>("LatestSearchQuery").createURL();
else url = this.cast<ThreadSearchQuery>("ThreadSearchQuery").createURL();
if (this.selectSearchType() === "latest") response = await this.cast<LatestSearchQuery>("LatestSearchQuery").execute();
else response = await this.cast<ThreadSearchQuery>("ThreadSearchQuery").execute();
return url;
return response;
}
public cast<T extends IQuery>(type: TQueryInterface): T {
@ -122,9 +128,11 @@ export default class HandiworkSearchQuery implements IQuery {
// Cast the result to T
return returnValue as T;
}
//#endregion Public methods
//#region Private methods
private castToLatest(): LatestSearchQuery {
// Cast the basic query object and copy common values
const query: LatestSearchQuery = new LatestSearchQuery;
@ -165,5 +173,6 @@ export default class HandiworkSearchQuery implements IQuery {
return query;
}
//#endregion
}

55
src/scripts/classes/query/latest-search-query.ts
View File

@ -7,6 +7,7 @@ import validator from 'class-validator';
import { urls } from "../../constants/url.js";
import PrefixParser from '../prefix-parser.js';
import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js";
import { fetchHTML } from '../../network-helper.js';
// Type definitions
export type TLatestOrder = "date" | "likes" | "views" | "title" | "rating";
@ -18,11 +19,14 @@ type TDate = 365 | 180 | 90 | 30 | 14 | 7 | 3 | 1;
export default class LatestSearchQuery implements IQuery {
//#region Private fields
private static MAX_TAGS = 5;
private static MIN_PAGE = 1;
//#endregion Private fields
//#region Properties
public category: TCategory = 'games';
/**
* Ordering type.
@ -52,20 +56,47 @@ export default class LatestSearchQuery implements IQuery {
})
public page = LatestSearchQuery.MIN_PAGE;
itype: TQueryInterface = "LatestSearchQuery";
//#endregion Properties
//#region Public methods
public validate(): boolean {
return validator.validateSync(this).length === 0;
}
public validate(): boolean { return validator.validateSync(this).length === 0; }
public createURL(): URL {
public async execute() {
// Check if the query is valid
if (!this.validate()) {
throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`);
}
// Prepare the URL
const url = this.prepareGETurl();
const decoded = decodeURIComponent(url.toString());
// Fetch the result
return await fetchHTML(decoded);
}
public findNearestDate(d: Date): TDate {
// Find the difference between today and the passed date
const diff = this.dateDiffInDays(new Date(), d);
// Find the closest valid value in the array
const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) {
return (Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev);
});
return closest as TDate;
}
//#endregion Public methods
//#region Private methods
/**
* Prepare the URL by filling out the GET parameters with the data in the query.
*/
private prepareGETurl(): URL {
// Create the URL
const url = new URL(urls.F95_LATEST_PHP);
url.searchParams.set("cmd", "list");
@ -92,20 +123,6 @@ export default class LatestSearchQuery implements IQuery {
return url;
}
public findNearestDate(d: Date): TDate {
// Find the difference between today and the passed date
const diff = this.dateDiffInDays(new Date(), d);
// Find the closest valid value in the array
const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) {
return (Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev);
});
return closest as TDate;
}
//#endregion Public methods
//#region Private methods
/**
*
*/
@ -118,5 +135,7 @@ export default class LatestSearchQuery implements IQuery {
return Math.floor((utc2 - utc1) / MS_PER_DAY);
}
//#endregion Private methodss
}

76
src/scripts/classes/query/thread-search-query.ts
View File

@ -7,6 +7,11 @@ import validator from 'class-validator';
import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js";
import { urls } from "../../constants/url.js";
import PrefixParser from "./../prefix-parser.js";
import { fetchPOSTResponse } from '../../network-helper.js';
import { AxiosResponse } from 'axios';
import { GenericAxiosError } from '../errors.js';
import { Result } from '../result.js';
import Shared from '../../shared.js';
// Type definitions
export type TThreadOrder = "relevance" | "date" | "last_update" | "replies";
@ -14,10 +19,13 @@ export type TThreadOrder = "relevance" | "date" | "last_update" | "replies";
export default class ThreadSearchQuery implements IQuery {
//#region Private fields
static MIN_PAGE = 1;
//#endregion Private fields
//#region Properties
/**
* Keywords to use in the search.
*/
@ -57,78 +65,89 @@ export default class ThreadSearchQuery implements IQuery {
})
public page: number = 1;
itype: TQueryInterface = "ThreadSearchQuery";
//#endregion Properties
//#region Public methods
public validate(): boolean {
return validator.validateSync(this).length === 0;
}
public validate(): boolean { return validator.validateSync(this).length === 0; }
public createURL(): URL {
public async execute(): Promise<Result<GenericAxiosError, AxiosResponse<any>>> {
// Check if the query is valid
if (!this.validate()) {
throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`);
}
// Create the URL
const url = new URL(urls.F95_SEARCH_URL);
// Define the POST parameters
const params = this.preparePOSTParameters();
// Specifiy if only the title should be searched
if (this.onlyTitles) url.searchParams.set("c[title_only]", "1");
// Return the POST response
return await fetchPOSTResponse(urls.F95_SEARCH_URL, params);
}
//#endregion Public methods
//#region Private methods
/**
* Prepare the parameters for post request with the data in the query.
*/
private preparePOSTParameters(): { [s: string]: string } {
// Local variables
const params = {};
// Ad the session token
params["_xfToken"] = Shared.session.token;
// Specify if only the title should be searched
if (this.onlyTitles) params["c[title_only]"] = "1";
// Add keywords
const encodedKeywords = this.keywords ?? "*";
url.searchParams.set("q", encodedKeywords);
params["keywords"] = this.keywords ?? "*";
// Specify the scope of the search (only "threads/post")
url.searchParams.set("t", "post");
params["search_type"] = "post";
// Set the dates
if (this.newerThan) {
const date = this.convertShortDate(this.newerThan);
url.searchParams.set("c[newer_than]", date);
params["c[newer_than]"] = date;
}
if (this.olderThan) {
const date = this.convertShortDate(this.olderThan);
url.searchParams.set("c[older_than]", date);
params["c[older_than]"] = date;
}
// Set included and excluded tags
// The tags are first joined with a comma, then encoded to URI
const includedTags = encodeURIComponent(this.includedTags.join(","));
const excludedTags = encodeURIComponent(this.excludedTags.join(","));
if (includedTags) url.searchParams.set("c[tags]", includedTags);
if (excludedTags) url.searchParams.set("c[excludeTags]", excludedTags);
// Set included and excluded tags (joined with a comma)
if (this.includedTags) params["c[tags]"] = this.includedTags.join(",");
if (this.excludedTags) params["c[excludeTags]"] = this.excludedTags.join(",");
// Set minimum reply number
if (this.minimumReplies > 0) url.searchParams.set("c[min_reply_count]", this.minimumReplies.toString());
if (this.minimumReplies > 0) params["c[min_reply_count]"] = this.minimumReplies.toString();
// Add prefixes
const parser = new PrefixParser();
const ids = parser.prefixesToIDs(this.includedPrefixes);
for (let i = 0; i < ids.length; i++) {
const name = `c[prefixes][${i}]`;
url.searchParams.set(name, ids[i].toString());
params[name] = ids[i].toString();
}
// Set the category
url.searchParams.set("c[child_nodes]", "1"); // Always set
params["c[child_nodes]"] = "1"; // Always set
if (this.category) {
const catID = this.categoryToID(this.category).toString();
url.searchParams.set("c[nodes][0]", catID);
params["c[nodes][0]"] = catID;
}
// Set the other values
url.searchParams.set("o", this.order.toString());
url.searchParams.set("page", this.page.toString());
params["o"] = this.order.toString();
params["page"] = this.page.toString();
return url;
return params;
}
//#endregion Public methods
//#region Private methods
/**
* Convert a date in the YYYY-MM-DD format taking into account the time zone.
*/
@ -152,6 +171,7 @@ export default class ThreadSearchQuery implements IQuery {
return catMap[category as string];
}
//#endregion Private methods
}

2
src/scripts/constants/url.ts
View File

@ -1,6 +1,6 @@
export const urls = {
F95_BASE_URL: "https://f95zone.to",
F95_SEARCH_URL: "https://f95zone.to/search/105286576/",
F95_SEARCH_URL: "https://f95zone.to/search/search/",
F95_LATEST_UPDATES: "https://f95zone.to/latest",
F95_THREADS: "https://f95zone.to/threads/",
F95_LOGIN_URL: "https://f95zone.to/login/login",

6
src/scripts/fetch-data/fetch-latest.ts
View File

@ -1,7 +1,6 @@
"use strict";
// Modules from file
import { fetchGETResponse } from "../network-helper.js";
import LatestSearchQuery from "../classes/query/latest-search-query.js";
import { urls as f95url } from "../constants/url.js";
@ -23,11 +22,8 @@ export default async function fetchLatestHandiworkURLs(query: LatestSearchQuery,
let noMorePages = false;
do {
// Prepare the URL
const url = query.createURL().toString();
// Fetch the response (application/json)
const response = await fetchGETResponse(url);
const response = await query.execute();
// Save the URLs
if (response.isSuccess()) {

40
src/scripts/fetch-data/fetch-thread.ts
View File

@ -4,13 +4,13 @@
import cheerio from "cheerio";
// Modules from file
import { fetchHTML } from "../network-helper.js";
import shared from "../shared.js";
import { selectors as f95Selector } from "../constants/css-selector.js";
import { urls as f95urls } from "../constants/url.js";
import ThreadSearchQuery from "../classes/query/thread-search-query.js";
//#region Public methods
/**
* Gets the URLs of the handiwork' threads that match the passed parameters.
* You *must* be logged.
@ -21,42 +21,37 @@ import ThreadSearchQuery from "../classes/query/thread-search-query.js";
* @returns {Promise<String[]>} URLs of the handiworks
*/
export default async function fetchThreadHandiworkURLs(query: ThreadSearchQuery, limit: number = 30): Promise<string[]> {
// Get the query
const url = decodeURI(query.createURL().href);
// Execute the query
const response = await query.execute();
// Fetch the results from F95 and return the handiwork urls
return await fetchResultURLs(url, limit);
if (response.isSuccess()) return await fetchResultURLs(response.value.data as string, limit);
else throw response.value
}
//#endregion Public methods
//#region Private methods
/**
* Gets the URLs of the threads resulting from the F95Zone search.
* @param {number} limit
* Maximum number of items to get. Default: 30
* @return {Promise<String[]>} List of URLs
*/
async function fetchResultURLs(url: string, limit: number = 30): Promise<string[]> {
shared.logger.trace(`Fetching ${url}...`);
async function fetchResultURLs(html: string, limit: number = 30): Promise<string[]> {
// Prepare cheerio
const $ = cheerio.load(html);
// Fetch HTML and prepare Cheerio
const html = await fetchHTML(url);
// Here we get all the DIV that are the body of the various query results
const results = $("body").find(f95Selector.GS_RESULT_BODY);
if (html.isSuccess()) {
const $ = cheerio.load(html.value);
// Than we extract the URLs
const urls = results.slice(0, limit).map((idx, el) => {
const elementSelector = $(el);
return extractLinkFromResult(elementSelector);
}).get();
// Here we get all the DIV that are the body of the various query results
const results = $("body").find(f95Selector.GS_RESULT_BODY);
// Than we extract the URLs
const urls = results.slice(0, limit).map((idx, el) => {
const elementSelector = $(el);
return extractLinkFromResult(elementSelector);
}).get();
return urls;
} else throw html.value;
return urls;
}
/**
@ -75,4 +70,5 @@ function extractLinkFromResult(selector: cheerio.Cheerio): string {
// Compose and return the URL
return new URL(partialLink, f95urls.F95_BASE_URL).toString();
}
//#endregion Private methods

5
src/scripts/interfaces.d.ts vendored
View File

@ -288,8 +288,9 @@ export interface IQuery {
*/
validate(): boolean,
/**
* From the query values it generates the corresponding URL for the platform.
* Search with the data in the query and returns the result.
*
* If the query is invalid it throws an exception.
*/
createURL(): URL,
execute(): any,
}