From e0fd96ab782c87fd1ed44518a9318a72ad7ab39e Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Thu, 4 Mar 2021 12:26:45 +0100 Subject: [PATCH] Format with prettier --- src/example.ts | 105 ++-- src/index.ts | 173 +++--- src/scripts/classes/credentials.ts | 46 +- src/scripts/classes/errors.ts | 60 +-- src/scripts/classes/handiwork/animation.ts | 48 +- src/scripts/classes/handiwork/asset.ts | 46 +- src/scripts/classes/handiwork/comic.ts | 39 +- src/scripts/classes/handiwork/game.ts | 63 ++- src/scripts/classes/handiwork/handiwork.ts | 81 +-- src/scripts/classes/login-result.ts | 26 +- src/scripts/classes/mapping/platform-user.ts | 283 +++++----- src/scripts/classes/mapping/post.ts | 225 ++++---- src/scripts/classes/mapping/thread.ts | 510 +++++++++--------- src/scripts/classes/mapping/user-profile.ts | 292 +++++----- src/scripts/classes/prefix-parser.ts | 190 +++---- .../classes/query/handiwork-search-query.ts | 262 ++++----- .../classes/query/latest-search-query.ts | 217 ++++---- .../classes/query/thread-search-query.ts | 280 +++++----- src/scripts/classes/result.ts | 58 +- src/scripts/classes/session.ts | 349 ++++++------ src/scripts/constants/css-selector.ts | 398 +++++++------- src/scripts/constants/url.ts | 48 +- src/scripts/fetch-data/fetch-handiwork.ts | 40 +- src/scripts/fetch-data/fetch-latest.ts | 66 +-- src/scripts/fetch-data/fetch-platform-data.ts | 144 ++--- src/scripts/fetch-data/fetch-query.ts | 27 +- src/scripts/fetch-data/fetch-thread.ts | 57 +- src/scripts/interfaces.d.ts | 473 ++++++++-------- src/scripts/network-helper.ts | 347 ++++++------ src/scripts/scrape-data/handiwork-parse.ts | 310 ++++++----- src/scripts/scrape-data/json-ld.ts | 46 +- src/scripts/scrape-data/post-parse.ts | 408 +++++++------- src/scripts/search.ts | 19 +- src/scripts/shared.ts | 83 +-- 34 files changed, 3083 insertions(+), 2736 deletions(-) diff --git a/src/example.ts b/src/example.ts index 43bbda0..62d9a8c 100644 --- a/src/example.ts +++ b/src/example.ts @@ -12,13 +12,14 @@ F95_PASSWORD = YOUR_PASSWORD import dotenv from "dotenv"; // Modules from file -import { login, - getUserData, - getLatestUpdates, - LatestSearchQuery, - Game, - searchHandiwork, - HandiworkSearchQuery +import { + login, + getUserData, + getLatestUpdates, + LatestSearchQuery, + Game, + searchHandiwork, + HandiworkSearchQuery } from "./index.js"; // Configure the .env reader @@ -27,54 +28,62 @@ dotenv.config(); main(); async function main() { - // Local variables - const gameList = [ - "City of broken dreamers", - "Seeds of chaos", - "MIST" - ]; + // Local variables + const gameList = ["City of broken dreamers", "Seeds of chaos", "MIST"]; - // Log in the platform - console.log("Authenticating..."); - const result = await login(process.env.F95_USERNAME, process.env.F95_PASSWORD); - console.log(`Authentication result: ${result.message}\n`); + // Log in the platform + console.log("Authenticating..."); + const result = await login( + process.env.F95_USERNAME, + process.env.F95_PASSWORD + ); + console.log(`Authentication result: ${result.message}\n`); - // Get user data - console.log("Fetching user data..."); - const userdata = await getUserData(); - const gameThreads = userdata.watched.filter(e => e.forum === "Games").length; - console.log(`${userdata.name} follows ${userdata.watched.length} threads of which ${gameThreads} are games\n`); + // Get user data + console.log("Fetching user data..."); + const userdata = await getUserData(); + const gameThreads = userdata.watched.filter((e) => e.forum === "Games") + .length; + console.log( + `${userdata.name} follows ${userdata.watched.length} threads of which ${gameThreads} are games\n` + ); - // Get latest game update - const latestQuery: LatestSearchQuery = new LatestSearchQuery(); - latestQuery.category = "games"; - latestQuery.includedTags = ["3d game"]; + // Get latest game update + const latestQuery: LatestSearchQuery = new LatestSearchQuery(); + latestQuery.category = "games"; + latestQuery.includedTags = ["3d game"]; - const latestUpdates = await getLatestUpdates(latestQuery, 1); - console.log(`"${latestUpdates.shift().name}" was the last "3d game" tagged game to be updated\n`); + const latestUpdates = await getLatestUpdates(latestQuery, 1); + console.log( + `"${ + latestUpdates.shift().name + }" was the last "3d game" tagged game to be updated\n` + ); - // Get game data - for(const gamename of gameList) { - console.log(`Searching '${gamename}'...`); + // Get game data + for (const gamename of gameList) { + console.log(`Searching '${gamename}'...`); - // Prepare the query - const query: HandiworkSearchQuery = new HandiworkSearchQuery(); - query.category = "games"; - query.keywords = gamename; - query.order = "likes"; // To find the most popular games + // Prepare the query + const query: HandiworkSearchQuery = new HandiworkSearchQuery(); + query.category = "games"; + query.keywords = gamename; + query.order = "likes"; // To find the most popular games - // Fetch the first result - const searchResult = await searchHandiwork(query, 1); + // Fetch the first result + const searchResult = await searchHandiwork(query, 1); - // No game found - if (searchResult.length === 0) { - console.log(`No data found for '${gamename}'\n`); - continue; - } - - // Extract first game - const gamedata = searchResult.shift(); - const authors = gamedata.authors.map((a, idx) => a.name).join(", "); - console.log(`Found: ${gamedata.name} (${gamedata.version}) by ${authors}\n`); + // No game found + if (searchResult.length === 0) { + console.log(`No data found for '${gamename}'\n`); + continue; } + + // Extract first game + const gamedata = searchResult.shift(); + const authors = gamedata.authors.map((a, idx) => a.name).join(", "); + console.log( + `Found: ${gamedata.name} (${gamedata.version}) by ${authors}\n` + ); + } } diff --git a/src/index.ts b/src/index.ts index a669332..3373fde 100644 --- a/src/index.ts +++ b/src/index.ts @@ -52,7 +52,9 @@ shared.logger.level = "warn"; // By default log only the warn messages /** * Indicates whether a user is logged in to the F95Zone platform or not. */ -export function isLogged(): boolean { return shared.isLogged; }; +export function isLogged(): boolean { + return shared.isLogged; +} //#endregion Export properties //#region Export methods @@ -62,68 +64,73 @@ export function isLogged(): boolean { return shared.isLogged; }; * * This **must** be the first operation performed before accessing any other script functions. */ -export async function login(username: string, password: string): Promise { - // Try to load a previous session - await shared.session.load(); +export async function login( + username: string, + password: string +): Promise { + // Try to load a previous session + await shared.session.load(); - // If the session is valid, return - if (shared.session.isValid(username, password)) { - shared.logger.info(`Loading previous session for ${username}`); + // If the session is valid, return + if (shared.session.isValid(username, password)) { + shared.logger.info(`Loading previous session for ${username}`); - // Load platform data - await fetchPlatformData(); + // Load platform data + await fetchPlatformData(); - shared.setIsLogged(true); - return new LoginResult(true, `${username} already authenticated (session)`); - } + shared.setIsLogged(true); + return new LoginResult(true, `${username} already authenticated (session)`); + } - // Creating credentials and fetch unique platform token - shared.logger.trace("Fetching token..."); - const creds = new Credentials(username, password); - await creds.fetchToken(); + // Creating credentials and fetch unique platform token + shared.logger.trace("Fetching token..."); + const creds = new Credentials(username, password); + await creds.fetchToken(); - shared.logger.trace(`Authentication for ${username}`); - const result = await authenticate(creds); - shared.setIsLogged(result.success); + shared.logger.trace(`Authentication for ${username}`); + const result = await authenticate(creds); + shared.setIsLogged(result.success); - if (result.success) { - // Load platform data - await fetchPlatformData(); + if (result.success) { + // Load platform data + await fetchPlatformData(); - // Recreate the session, overwriting the old one - shared.session.create(username, password, creds.token); - await shared.session.save(); + // Recreate the session, overwriting the old one + shared.session.create(username, password, creds.token); + await shared.session.save(); - shared.logger.info("User logged in through the platform"); - } else shared.logger.warn(`Error during authentication: ${result.message}`); + shared.logger.info("User logged in through the platform"); + } else shared.logger.warn(`Error during authentication: ${result.message}`); - return result; -}; + return result; +} /** * Chek if exists a new version of the handiwork. * * You **must** be logged in to the portal before calling this method. */ -export async function checkIfHandiworkHasUpdate(hw: HandiWork): Promise { - // Local variables - let hasUpdate = false; +export async function checkIfHandiworkHasUpdate( + hw: HandiWork +): Promise { + // Local variables + let hasUpdate = false; - // Check if the user is logged - if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); + // Check if the user is logged + if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); - // F95 change URL at every game update, - // so if the URL is different an update is available - if (await urlExists(hw.url, true)) { - // Fetch the online handiwork - const onlineHw = await getHandiworkFromURL(hw.url); + // F95 change URL at every game update, + // so if the URL is different an update is available + if (await urlExists(hw.url, true)) { + // Fetch the online handiwork + const onlineHw = await getHandiworkFromURL(hw.url); - // Compare the versions - hasUpdate = onlineHw.version?.toUpperCase() !== hw.version?.toUpperCase(); - } + // Compare the versions + hasUpdate = onlineHw.version?.toUpperCase() !== hw.version?.toUpperCase(); + } - return hasUpdate; -}; + return hasUpdate; +} /** * Search for one or more handiworks identified by a specific query. @@ -133,30 +140,35 @@ export async function checkIfHandiworkHasUpdate(hw: HandiWork): Promise * @param {HandiworkSearchQuery} query Parameters used for the search. * @param {Number} limit Maximum number of results. Default: 10 */ -export async function searchHandiwork(query: HandiworkSearchQuery, limit: number = 10): Promise { - // Check if the user is logged - if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); +export async function searchHandiwork( + query: HandiworkSearchQuery, + limit = 10 +): Promise { + // Check if the user is logged + if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); - return search(query, limit); -}; + return search(query, limit); +} /** * Given the url, it gets all the information about the handiwork requested. * * You **must** be logged in to the portal before calling this method. */ -export async function getHandiworkFromURL(url: string): Promise { - // Check if the user is logged - if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); +export async function getHandiworkFromURL( + url: string +): Promise { + // Check if the user is logged + if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); - // Check URL validity - const exists = await urlExists(url); - if (!exists) throw new URIError(`${url} is not a valid URL`); - if (!isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`); - - // Get game data - return getHandiworkInformation(url); -}; + // Check URL validity + const exists = await urlExists(url); + if (!exists) throw new URIError(`${url} is not a valid URL`); + if (!isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`); + + // Get game data + return getHandiworkInformation(url); +} /** * Gets the data of the currently logged in user. @@ -166,15 +178,15 @@ export async function getHandiworkFromURL(url: string): Promis * @returns {Promise} Data of the user currently logged in */ export async function getUserData(): Promise { - // Check if the user is logged - if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); + // Check if the user is logged + if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); - // Create and fetch profile data - const profile = new UserProfile(); - await profile.fetch(); - - return profile; -}; + // Create and fetch profile data + const profile = new UserProfile(); + await profile.fetch(); + + return profile; +} /** * Gets the latest updated games that match the specified parameters. @@ -184,19 +196,22 @@ export async function getUserData(): Promise { * @param {LatestSearchQuery} query Parameters used for the search. * @param {Number} limit Maximum number of results. Default: 10 */ -export async function getLatestUpdates(query: LatestSearchQuery, limit: number = 10): Promise { - // Check limit value - if (limit <= 0) throw new Error("limit must be greater than 0"); +export async function getLatestUpdates( + query: LatestSearchQuery, + limit = 10 +): Promise { + // Check limit value + if (limit <= 0) throw new Error("limit must be greater than 0"); - // Check if the user is logged - if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); + // Check if the user is logged + if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED); - // Fetch the results - const urls = await fetchLatestHandiworkURLs(query, limit); + // Fetch the results + const urls = await fetchLatestHandiworkURLs(query, limit); - // Get the data from urls - const promiseList = urls.map((u: string) => getHandiworkInformation(u)); - return Promise.all(promiseList); -}; + // Get the data from urls + const promiseList = urls.map((u: string) => getHandiworkInformation(u)); + return Promise.all(promiseList); +} //#endregion diff --git a/src/scripts/classes/credentials.ts b/src/scripts/classes/credentials.ts index 749daf8..afa5967 100644 --- a/src/scripts/classes/credentials.ts +++ b/src/scripts/classes/credentials.ts @@ -7,28 +7,28 @@ import { getF95Token } from "../network-helper.js"; * Represents the credentials used to access the platform. */ export default class Credentials { - /** - * Username - */ - public username: string; - /** - * Password of the user. - */ - public password: string; - /** - * One time token used during login. - */ - public token: string = null; + /** + * Username + */ + public username: string; + /** + * Password of the user. + */ + public password: string; + /** + * One time token used during login. + */ + public token: string = null; - constructor(username: string, password: string) { - this.username = username; - this.password = password; - } + constructor(username: string, password: string) { + this.username = username; + this.password = password; + } - /** - * Fetch and save the token used to log in to F95Zone. - */ - async fetchToken(): Promise { - this.token = await getF95Token(); - } -} \ No newline at end of file + /** + * Fetch and save the token used to log in to F95Zone. + */ + async fetchToken(): Promise { + this.token = await getF95Token(); + } +} diff --git a/src/scripts/classes/errors.ts b/src/scripts/classes/errors.ts index 2957e31..5832f8b 100644 --- a/src/scripts/classes/errors.ts +++ b/src/scripts/classes/errors.ts @@ -1,44 +1,44 @@ "use strict"; interface IBaseError { - /** - * Unique identifier of the error. - */ - id: number, - /** - * Error message. - */ - message: string, - /** - * Error to report. - */ - error: Error, + /** + * Unique identifier of the error. + */ + id: number; + /** + * Error message. + */ + message: string; + /** + * Error to report. + */ + error: Error; } export class GenericAxiosError extends Error implements IBaseError { - id: number; - message: string; - error: Error; + id: number; + message: string; + error: Error; - constructor(args: IBaseError) { - super(); - this.id = args.id; - this.message = args.message; - this.error = args.error; - } + constructor(args: IBaseError) { + super(); + this.id = args.id; + this.message = args.message; + this.error = args.error; + } } export class UnexpectedResponseContentType extends Error implements IBaseError { - id: number; - message: string; - error: Error; + id: number; + message: string; + error: Error; - constructor(args: IBaseError) { - super(); - this.id = args.id; - this.message = args.message; - this.error = args.error; - } + constructor(args: IBaseError) { + super(); + this.id = args.id; + this.message = args.message; + this.error = args.error; + } } export class InvalidF95Token extends Error {} diff --git a/src/scripts/classes/handiwork/animation.ts b/src/scripts/classes/handiwork/animation.ts index c13d4a1..f503157 100644 --- a/src/scripts/classes/handiwork/animation.ts +++ b/src/scripts/classes/handiwork/animation.ts @@ -4,28 +4,26 @@ import { TAuthor, IAnimation, TRating, TCategory } from "../../interfaces"; export default class Animation implements IAnimation { - - //#region Properties - censored: boolean; - genre: string[]; - installation: string; - language: string[]; - lenght: string; - pages: string; - resolution: string[]; - authors: TAuthor[]; - category: TCategory; - changelog: string[]; - cover: string; - id: number; - lastThreadUpdate: Date; - name: string; - overview: string; - prefixes: string[]; - rating: TRating; - tags: string[]; - threadPublishingDate: Date; - url: string; - //#endregion Properties - -} \ No newline at end of file + //#region Properties + censored: boolean; + genre: string[]; + installation: string; + language: string[]; + lenght: string; + pages: string; + resolution: string[]; + authors: TAuthor[]; + category: TCategory; + changelog: string[]; + cover: string; + id: number; + lastThreadUpdate: Date; + name: string; + overview: string; + prefixes: string[]; + rating: TRating; + tags: string[]; + threadPublishingDate: Date; + url: string; + //#endregion Properties +} diff --git a/src/scripts/classes/handiwork/asset.ts b/src/scripts/classes/handiwork/asset.ts index f5c124d..98bff80 100644 --- a/src/scripts/classes/handiwork/asset.ts +++ b/src/scripts/classes/handiwork/asset.ts @@ -4,27 +4,25 @@ import { TAuthor, IAsset, TRating, TCategory } from "../../interfaces"; export default class Asset implements IAsset { - - //#region Properties - assetLink: string; - associatedAssets: string[]; - compatibleSoftware: string; - includedAssets: string[]; - officialLinks: string[]; - sku: string; - authors: TAuthor[]; - category: TCategory; - changelog: string[]; - cover: string; - id: number; - lastThreadUpdate: Date; - name: string; - overview: string; - prefixes: string[]; - rating: TRating; - tags: string[]; - threadPublishingDate: Date; - url: string; - //#endregion Properties - -} \ No newline at end of file + //#region Properties + assetLink: string; + associatedAssets: string[]; + compatibleSoftware: string; + includedAssets: string[]; + officialLinks: string[]; + sku: string; + authors: TAuthor[]; + category: TCategory; + changelog: string[]; + cover: string; + id: number; + lastThreadUpdate: Date; + name: string; + overview: string; + prefixes: string[]; + rating: TRating; + tags: string[]; + threadPublishingDate: Date; + url: string; + //#endregion Properties +} diff --git a/src/scripts/classes/handiwork/comic.ts b/src/scripts/classes/handiwork/comic.ts index 4f266a8..dfeaff9 100644 --- a/src/scripts/classes/handiwork/comic.ts +++ b/src/scripts/classes/handiwork/comic.ts @@ -4,23 +4,22 @@ import { TAuthor, IComic, TRating, TCategory } from "../../interfaces"; export default class Comic implements IComic { - - //#region Properties - genre: string[]; - pages: string; - resolution: string[]; - authors: TAuthor[]; - category: TCategory; - changelog: string[]; - cover: string; - id: number; - lastThreadUpdate: Date; - name: string; - overview: string; - prefixes: string[]; - rating: TRating; - tags: string[]; - threadPublishingDate: Date; - url: string; - //#endregion Properties -} \ No newline at end of file + //#region Properties + genre: string[]; + pages: string; + resolution: string[]; + authors: TAuthor[]; + category: TCategory; + changelog: string[]; + cover: string; + id: number; + lastThreadUpdate: Date; + name: string; + overview: string; + prefixes: string[]; + rating: TRating; + tags: string[]; + threadPublishingDate: Date; + url: string; + //#endregion Properties +} diff --git a/src/scripts/classes/handiwork/game.ts b/src/scripts/classes/handiwork/game.ts index d391953..881b2d0 100644 --- a/src/scripts/classes/handiwork/game.ts +++ b/src/scripts/classes/handiwork/game.ts @@ -1,34 +1,39 @@ "use strict"; // Modules from files -import { TAuthor, TEngine, IGame, TRating, TStatus, TCategory } from "../../interfaces"; +import { + TAuthor, + TEngine, + IGame, + TRating, + TStatus, + TCategory +} from "../../interfaces"; export default class Game implements IGame { - - //#region Properties - censored: boolean; - engine: TEngine; - genre: string[]; - installation: string; - language: string[]; - lastRelease: Date; - mod: boolean; - os: string[]; - status: TStatus; - version: string; - authors: TAuthor[]; - category: TCategory; - changelog: string[]; - cover: string; - id: number; - lastThreadUpdate: Date; - name: string; - overview: string; - prefixes: string[]; - rating: TRating; - tags: string[]; - threadPublishingDate: Date; - url: string; - //#endregion Properties - -} \ No newline at end of file + //#region Properties + censored: boolean; + engine: TEngine; + genre: string[]; + installation: string; + language: string[]; + lastRelease: Date; + mod: boolean; + os: string[]; + status: TStatus; + version: string; + authors: TAuthor[]; + category: TCategory; + changelog: string[]; + cover: string; + id: number; + lastThreadUpdate: Date; + name: string; + overview: string; + prefixes: string[]; + rating: TRating; + tags: string[]; + threadPublishingDate: Date; + url: string; + //#endregion Properties +} diff --git a/src/scripts/classes/handiwork/handiwork.ts b/src/scripts/classes/handiwork/handiwork.ts index e52c794..1bb817e 100644 --- a/src/scripts/classes/handiwork/handiwork.ts +++ b/src/scripts/classes/handiwork/handiwork.ts @@ -1,46 +1,51 @@ "use strict"; // Modules from files -import { TAuthor, TRating, IHandiwork, TEngine, TCategory, TStatus } from "../../interfaces"; +import { + TAuthor, + TRating, + IHandiwork, + TEngine, + TCategory, + TStatus +} from "../../interfaces"; /** * It represents a generic work, be it a game, a comic, an animation or an asset. */ export default class HandiWork implements IHandiwork { - - //#region Properties - censored: boolean; - engine: TEngine; - genre: string[]; - installation: string; - language: string[]; - lastRelease: Date; - mod: boolean; - os: string[]; - status: TStatus; - version: string; - authors: TAuthor[]; - category: TCategory; - changelog: string[]; - cover: string; - id: number; - lastThreadUpdate: Date; - name: string; - overview: string; - prefixes: string[]; - rating: TRating; - tags: string[]; - threadPublishingDate: Date; - url: string; - pages: string; - resolution: string[]; - lenght: string; - assetLink: string; - associatedAssets: string[]; - compatibleSoftware: string; - includedAssets: string[]; - officialLinks: string[]; - sku: string; - //#endregion Properties - -} \ No newline at end of file + //#region Properties + censored: boolean; + engine: TEngine; + genre: string[]; + installation: string; + language: string[]; + lastRelease: Date; + mod: boolean; + os: string[]; + status: TStatus; + version: string; + authors: TAuthor[]; + category: TCategory; + changelog: string[]; + cover: string; + id: number; + lastThreadUpdate: Date; + name: string; + overview: string; + prefixes: string[]; + rating: TRating; + tags: string[]; + threadPublishingDate: Date; + url: string; + pages: string; + resolution: string[]; + lenght: string; + assetLink: string; + associatedAssets: string[]; + compatibleSoftware: string; + includedAssets: string[]; + officialLinks: string[]; + sku: string; + //#endregion Properties +} diff --git a/src/scripts/classes/login-result.ts b/src/scripts/classes/login-result.ts index c05f755..4ec72ad 100644 --- a/src/scripts/classes/login-result.ts +++ b/src/scripts/classes/login-result.ts @@ -4,17 +4,17 @@ * Object obtained in response to an attempt to login to the portal. */ export default class LoginResult { - /** - * Result of the login operation - */ - success: boolean; - /** - * Login response message - */ - message: string; - - constructor(success: boolean, message: string) { - this.success = success; - this.message = message; - } + /** + * Result of the login operation + */ + success: boolean; + /** + * Login response message + */ + message: string; + + constructor(success: boolean, message: string) { + this.success = success; + this.message = message; + } } diff --git a/src/scripts/classes/mapping/platform-user.ts b/src/scripts/classes/mapping/platform-user.ts index 990f43c..6a5c8a5 100644 --- a/src/scripts/classes/mapping/platform-user.ts +++ b/src/scripts/classes/mapping/platform-user.ts @@ -13,144 +13,181 @@ import { GENERIC, MEMBER } from "../../constants/css-selector.js"; * Represents a generic user registered on the platform. */ export default class PlatformUser { + //#region Fields - //#region Fields + private _id: number; + private _name: string; + private _title: string; + private _banners: string[]; + private _messages: number; + private _reactionScore: number; + private _points: number; + private _ratingsReceived: number; + private _joined: Date; + private _lastSeen: Date; + private _followed: boolean; + private _ignored: boolean; + private _private: boolean; + private _avatar: string; + private _amountDonated: number; - private _id: number; - private _name: string; - private _title: string; - private _banners: string[]; - private _messages: number; - private _reactionScore: number; - private _points: number; - private _ratingsReceived: number; - private _joined: Date; - private _lastSeen: Date; - private _followed: boolean; - private _ignored: boolean; - private _private: boolean; - private _avatar: string; - private _amountDonated: number; + //#endregion Fields - //#endregion Fields + //#region Getters - //#region Getters - - /** - * Unique user ID. - */ - public get id() { return this._id; } - /** - * Username. - */ - public get name() { return this._name; } - /** - * Title assigned to the user by the platform. - */ - public get title() { return this._title; } - /** - * List of banners assigned by the platform. - */ - public get banners() { return this._banners; } - /** - * Number of messages written by the user. - */ - public get messages() { return this._messages; } - /** - * @todo Reaction score. - */ - public get reactionScore() { return this._reactionScore; } - /** - * @todo Points. - */ - public get points() { return this._points; } - /** - * Number of ratings received. - */ - public get ratingsReceived() { return this._ratingsReceived; } - /** - * Date of joining the platform. - */ - public get joined() { return this._joined; } - /** - * Date of the last connection to the platform. - */ - public get lastSeen() { return this._lastSeen; } - /** - * Indicates whether the user is followed by the currently logged in user. - */ - public get followed() { return this._followed; } - /** - * Indicates whether the user is ignored by the currently logged on user. - */ - public get ignored() { return this._ignored; } - /** - * Indicates that the profile is private and not viewable by the user. - */ - public get private() { return this._private; } - /** - * URL of the image used as the user's avatar. - */ - public get avatar() { return this._avatar; } - /** - * Value of donations made. - */ - public get donation() { return this._amountDonated; } + /** + * Unique user ID. + */ + public get id() { + return this._id; + } + /** + * Username. + */ + public get name() { + return this._name; + } + /** + * Title assigned to the user by the platform. + */ + public get title() { + return this._title; + } + /** + * List of banners assigned by the platform. + */ + public get banners() { + return this._banners; + } + /** + * Number of messages written by the user. + */ + public get messages() { + return this._messages; + } + /** + * @todo Reaction score. + */ + public get reactionScore() { + return this._reactionScore; + } + /** + * @todo Points. + */ + public get points() { + return this._points; + } + /** + * Number of ratings received. + */ + public get ratingsReceived() { + return this._ratingsReceived; + } + /** + * Date of joining the platform. + */ + public get joined() { + return this._joined; + } + /** + * Date of the last connection to the platform. + */ + public get lastSeen() { + return this._lastSeen; + } + /** + * Indicates whether the user is followed by the currently logged in user. + */ + public get followed() { + return this._followed; + } + /** + * Indicates whether the user is ignored by the currently logged on user. + */ + public get ignored() { + return this._ignored; + } + /** + * Indicates that the profile is private and not viewable by the user. + */ + public get private() { + return this._private; + } + /** + * URL of the image used as the user's avatar. + */ + public get avatar() { + return this._avatar; + } + /** + * Value of donations made. + */ + public get donation() { + return this._amountDonated; + } - //#endregion Getters + //#endregion Getters - constructor(id?: number) { this._id = id; } + constructor(id?: number) { + this._id = id; + } - //#region Public methods + //#region Public methods - public setID(id: number) { this._id = id; } + public setID(id: number) { + this._id = id; + } - public async fetch() { - // Check ID - if (!this.id && this.id < 1) throw new Error("Invalid user ID"); + public async fetch() { + // Check ID + if (!this.id && this.id < 1) throw new Error("Invalid user ID"); - // Prepare the URL - const url = new URL(this.id.toString(), `${urls.F95_MEMBERS}/`).toString(); + // Prepare the URL + const url = new URL(this.id.toString(), `${urls.F95_MEMBERS}/`).toString(); - // Fetch the page - const htmlResponse = await fetchHTML(url); + // Fetch the page + const htmlResponse = await fetchHTML(url); - if (htmlResponse.isSuccess()) { - // Prepare cheerio - const $ = cheerio.load(htmlResponse.value); - - // Check if the profile is private - this._private = $(GENERIC.ERROR_BANNER) - ?.text() - .trim() === "This member limits who may view their full profile."; + if (htmlResponse.isSuccess()) { + // Prepare cheerio + const $ = cheerio.load(htmlResponse.value); - if (!this._private) { - // Parse the elements - this._name = $(MEMBER.NAME).text(); - this._title = $(MEMBER.TITLE).text(); - this._banners = $(MEMBER.BANNERS).toArray().map((el, idx) => $(el).text().trim()).filter(el => el); - this._avatar = $(MEMBER.AVATAR).attr("src"); - this._followed = $(MEMBER.FOLLOWED).text() === "Unfollow"; - this._ignored = $(MEMBER.IGNORED).text() === "Unignore"; - this._messages = parseInt($(MEMBER.MESSAGES).text(), 10); - this._reactionScore = parseInt($(MEMBER.REACTION_SCORE).text(), 10); - this._points = parseInt($(MEMBER.POINTS).text(), 10); - this._ratingsReceived = parseInt($(MEMBER.RATINGS_RECEIVED).text(), 10); + // Check if the profile is private + this._private = + $(GENERIC.ERROR_BANNER)?.text().trim() === + "This member limits who may view their full profile."; - // Parse date - const joined = $(MEMBER.JOINED)?.attr("datetime"); - if (luxon.DateTime.fromISO(joined).isValid) this._joined = new Date(joined); + if (!this._private) { + // Parse the elements + this._name = $(MEMBER.NAME).text(); + this._title = $(MEMBER.TITLE).text(); + this._banners = $(MEMBER.BANNERS) + .toArray() + .map((el, idx) => $(el).text().trim()) + .filter((el) => el); + this._avatar = $(MEMBER.AVATAR).attr("src"); + this._followed = $(MEMBER.FOLLOWED).text() === "Unfollow"; + this._ignored = $(MEMBER.IGNORED).text() === "Unignore"; + this._messages = parseInt($(MEMBER.MESSAGES).text(), 10); + this._reactionScore = parseInt($(MEMBER.REACTION_SCORE).text(), 10); + this._points = parseInt($(MEMBER.POINTS).text(), 10); + this._ratingsReceived = parseInt($(MEMBER.RATINGS_RECEIVED).text(), 10); - const lastSeen = $(MEMBER.LAST_SEEN)?.attr("datetime"); - if (luxon.DateTime.fromISO(lastSeen).isValid) this._joined = new Date(lastSeen); + // Parse date + const joined = $(MEMBER.JOINED)?.attr("datetime"); + if (luxon.DateTime.fromISO(joined).isValid) + this._joined = new Date(joined); - // Parse donation - const donation = $(MEMBER.AMOUNT_DONATED)?.text().replace("$", ""); - this._amountDonated = donation ? parseInt(donation, 10) : 0; - } - } else throw htmlResponse.value; - } + const lastSeen = $(MEMBER.LAST_SEEN)?.attr("datetime"); + if (luxon.DateTime.fromISO(lastSeen).isValid) + this._joined = new Date(lastSeen); - //#endregion Public method + // Parse donation + const donation = $(MEMBER.AMOUNT_DONATED)?.text().replace("$", ""); + this._amountDonated = donation ? parseInt(donation, 10) : 0; + } + } else throw htmlResponse.value; + } -} \ No newline at end of file + //#endregion Public method +} diff --git a/src/scripts/classes/mapping/post.ts b/src/scripts/classes/mapping/post.ts index 10db347..ae64bd4 100644 --- a/src/scripts/classes/mapping/post.ts +++ b/src/scripts/classes/mapping/post.ts @@ -5,7 +5,10 @@ import cheerio from "cheerio"; // Modules from file import PlatformUser from "./platform-user.js"; -import { IPostElement, parseF95ThreadPost } from "../../scrape-data/post-parse.js"; +import { + IPostElement, + parseF95ThreadPost +} from "../../scrape-data/post-parse.js"; import { POST, THREAD } from "../../constants/css-selector.js"; import { urls } from "../../constants/url.js"; import { fetchHTML } from "../../network-helper.js"; @@ -14,122 +17,150 @@ import { fetchHTML } from "../../network-helper.js"; * Represents a post published by a user on the F95Zone platform. */ export default class Post { + //#region Fields - //#region Fields + private _id: number; + private _number: number; + private _published: Date; + private _lastEdit: Date; + private _owner: PlatformUser; + private _bookmarked: boolean; + private _message: string; + private _body: IPostElement[]; - private _id: number; - private _number: number; - private _published: Date; - private _lastEdit: Date; - private _owner: PlatformUser; - private _bookmarked: boolean; - private _message: string; - private _body: IPostElement[]; + //#endregion Fields - //#endregion Fields + //#region Getters - //#region Getters + /** + * Represents a post published by a user on the F95Zone platform. + */ + public get id() { + return this._id; + } + /** + * Unique ID of the post within the thread in which it is present. + */ + public get number() { + return this._number; + } + /** + * Date the post was first published. + */ + public get published() { + return this._published; + } + /** + * Date the post was last modified. + */ + public get lastEdit() { + return this._lastEdit; + } + /** + * User who owns the post. + */ + public get owner() { + return this._owner; + } + /** + * Indicates whether the post has been bookmarked. + */ + public get bookmarked() { + return this._bookmarked; + } + /** + * Post message text. + */ + public get message() { + return this._message; + } + /** + * Set of the elements that make up the body of the post. + */ + public get body() { + return this._body; + } - /** - * Represents a post published by a user on the F95Zone platform. - */ - public get id() { return this._id; } - /** - * Unique ID of the post within the thread in which it is present. - */ - public get number() { return this._number; } - /** - * Date the post was first published. - */ - public get published() { return this._published; } - /** - * Date the post was last modified. - */ - public get lastEdit() { return this._lastEdit; } - /** - * User who owns the post. - */ - public get owner() { return this._owner; } - /** - * Indicates whether the post has been bookmarked. - */ - public get bookmarked() { return this._bookmarked; } - /** - * Post message text. - */ - public get message() { return this._message; } - /** - * Set of the elements that make up the body of the post. - */ - public get body() { return this._body; } + //#endregion Getters - //#endregion Getters + constructor(id: number) { + this._id = id; + } - constructor(id: number) { this._id = id; } + //#region Public methods - //#region Public methods + /** + * Gets the post data starting from its unique ID for the entire platform. + */ + public async fetch() { + // Fetch HTML page containing the post + const url = new URL(this.id.toString(), urls.F95_POSTS).toString(); + const htmlResponse = await fetchHTML(url); - /** - * Gets the post data starting from its unique ID for the entire platform. - */ - public async fetch() { - // Fetch HTML page containing the post - const url = new URL(this.id.toString(), urls.F95_POSTS).toString(); - const htmlResponse = await fetchHTML(url); + if (htmlResponse.isSuccess()) { + // Load cheerio and find post + const $ = cheerio.load(htmlResponse.value); - if (htmlResponse.isSuccess()) { - // Load cheerio and find post - const $ = cheerio.load(htmlResponse.value); + const post = $(THREAD.POSTS_IN_PAGE) + .toArray() + .find((el, idx) => { + // Fetch the ID and check if it is what we are searching + const sid: string = $(el) + .find(POST.ID) + .attr("id") + .replace("post-", ""); + const id = parseInt(sid, 10); - const post = $(THREAD.POSTS_IN_PAGE).toArray().find((el, idx) => { - // Fetch the ID and check if it is what we are searching - const sid: string = $(el).find(POST.ID).attr("id").replace("post-", ""); - const id = parseInt(sid, 10); + if (id === this.id) return el; + }); - if (id === this.id) return el; - }); + // Finally parse the post + await this.parsePost($, $(post)); + } else throw htmlResponse.value; + } - // Finally parse the post - await this.parsePost($, $(post)); - } else throw htmlResponse.value; - } + //#endregion Public methods - //#endregion Public methods + //#region Private methods - //#region Private methods + private async parsePost( + $: cheerio.Root, + post: cheerio.Cheerio + ): Promise { + // Find post's ID + const sid: string = post.find(POST.ID).attr("id").replace("post-", ""); + this._id = parseInt(sid, 10); - private async parsePost($: cheerio.Root, post: cheerio.Cheerio): Promise { - // Find post's ID - const sid: string = post.find(POST.ID).attr("id").replace("post-", ""); - this._id = parseInt(sid, 10); + // Find post's number + const sNumber: string = post.find(POST.NUMBER).text().replace("#", ""); + this._number = parseInt(sNumber, 10); - // Find post's number - const sNumber: string = post.find(POST.NUMBER).text().replace("#", ""); - this._number = parseInt(sNumber, 10); + // Find post's publishing date + const sPublishing: string = post.find(POST.PUBLISH_DATE).attr("datetime"); + this._published = new Date(sPublishing); - // Find post's publishing date - const sPublishing: string = post.find(POST.PUBLISH_DATE).attr("datetime"); - this._published = new Date(sPublishing); + // Find post's last edit date + const sLastEdit: string = post.find(POST.LAST_EDIT).attr("datetime"); + this._lastEdit = new Date(sLastEdit); - // Find post's last edit date - const sLastEdit: string = post.find(POST.LAST_EDIT).attr("datetime"); - this._lastEdit = new Date(sLastEdit); + // Find post's owner + const sOwnerID: string = post + .find(POST.OWNER_ID) + .attr("data-user-id") + .trim(); + this._owner = new PlatformUser(parseInt(sOwnerID, 10)); + await this._owner.fetch(); - // Find post's owner - const sOwnerID: string = post.find(POST.OWNER_ID).attr("data-user-id").trim(); - this._owner = new PlatformUser(parseInt(sOwnerID, 10)); - await this._owner.fetch(); + // Find if the post is bookmarked + this._bookmarked = post.find(POST.BOOKMARKED).length !== 0; - // Find if the post is bookmarked - this._bookmarked = post.find(POST.BOOKMARKED).length !== 0; + // Find post's message + this._message = post.find(POST.BODY).text(); - // Find post's message - this._message = post.find(POST.BODY).text(); - - // Parse post's body - const body = post.find(POST.BODY); - this._body = parseF95ThreadPost($, body); - } + // Parse post's body + const body = post.find(POST.BODY); + this._body = parseF95ThreadPost($, body); + } - //#endregion -} \ No newline at end of file + //#endregion +} diff --git a/src/scripts/classes/mapping/thread.ts b/src/scripts/classes/mapping/thread.ts index 008934b..aafa378 100644 --- a/src/scripts/classes/mapping/thread.ts +++ b/src/scripts/classes/mapping/thread.ts @@ -12,7 +12,11 @@ import { urls } from "../../constants/url.js"; import { POST, THREAD } from "../../constants/css-selector.js"; import { fetchHTML, fetchPOSTResponse } from "../../network-helper.js"; import Shared from "../../shared.js"; -import { GenericAxiosError, ParameterError, UnexpectedResponseContentType } from "../errors.js"; +import { + GenericAxiosError, + ParameterError, + UnexpectedResponseContentType +} from "../errors.js"; import { Result } from "../result.js"; import { getJSONLD, TJsonLD } from "../../scrape-data/json-ld.js"; @@ -20,263 +24,289 @@ import { getJSONLD, TJsonLD } from "../../scrape-data/json-ld.js"; * Represents a generic F95Zone platform thread. */ export default class Thread { + //#region Fields - //#region Fields + private POST_FOR_PAGE = 20; + private _id: number; + private _url: string; + private _title: string; + private _tags: string[]; + private _prefixes: string[]; + private _rating: TRating; + private _owner: PlatformUser; + private _publication: Date; + private _modified: Date; + private _category: TCategory; - private POST_FOR_PAGE: number = 20; - private _id: number; - private _url: string; - private _title: string; - private _tags: string[]; - private _prefixes: string[]; - private _rating: TRating; - private _owner: PlatformUser; - private _publication: Date; - private _modified: Date; - private _category: TCategory; + //#endregion Fields - //#endregion Fields + //#region Getters - //#region Getters + /** + * Unique ID of the thread on the platform. + */ + public get id() { + return this._id; + } + /** + * URL of the thread. + * + * It may vary depending on any versions of the contained product. + */ + public get url() { + return this._url; + } + /** + * Thread title. + */ + public get title() { + return this._title; + } + /** + * Tags associated with the thread. + */ + public get tags() { + return this._tags; + } + /** + * Prefixes associated with the thread + */ + public get prefixes() { + return this._prefixes; + } + /** + * Rating assigned to the thread. + */ + public get rating() { + return this._rating; + } + /** + * Owner of the thread. + */ + public get owner() { + return this._owner; + } + /** + * Date the thread was first published. + */ + public get publication() { + return this._publication; + } + /** + * Date the thread was last modified. + */ + public get modified() { + return this._modified; + } + /** + * Category to which the content of the thread belongs. + */ + public get category() { + return this._category; + } - /** - * Unique ID of the thread on the platform. - */ - public get id() { return this._id; } - /** - * URL of the thread. - * - * It may vary depending on any versions of the contained product. - */ - public get url() { return this._url; } - /** - * Thread title. - */ - public get title() { return this._title; } - /** - * Tags associated with the thread. - */ - public get tags() { return this._tags; } - /** - * Prefixes associated with the thread - */ - public get prefixes() { return this._prefixes; } - /** - * Rating assigned to the thread. - */ - public get rating() { return this._rating; } - /** - * Owner of the thread. - */ - public get owner() { return this._owner; } - /** - * Date the thread was first published. - */ - public get publication() { return this._publication; } - /** - * Date the thread was last modified. - */ - public get modified() { return this._modified; } - /** - * Category to which the content of the thread belongs. - */ - public get category() { return this._category; } + //#endregion Getters - //#endregion Getters + /** + * Initializes an object for mapping a thread. + * + * The unique ID of the thread must be specified. + */ + constructor(id: number) { + this._id = id; + } - /** - * Initializes an object for mapping a thread. - * - * The unique ID of the thread must be specified. - */ - constructor(id: number) { this._id = id; } + //#region Private methods - //#region Private methods + /** + * Set the number of posts to display for the current thread. + */ + private async setMaximumPostsForPage(n: 20 | 40 | 60 | 100): Promise { + // Prepare the parameters to send via POST request + const params = { + _xfResponseType: "json", + _xfRequestUri: `/account/dpp-update?content_type=thread&content_id=${this.id}`, + _xfToken: Shared.session.token, + _xfWithData: "1", + content_id: this.id.toString(), + content_type: "thread", + "dpp_custom_config[posts]": n.toString() + }; - /** - * Set the number of posts to display for the current thread. - */ - private async setMaximumPostsForPage(n: 20 | 40 | 60 | 100): Promise { - // Prepare the parameters to send via POST request - const params = { - "_xfResponseType": "json", - "_xfRequestUri": `/account/dpp-update?content_type=thread&content_id=${this.id}`, - "_xfToken": Shared.session.token, - "_xfWithData": "1", - "content_id": this.id.toString(), - "content_type": "thread", - "dpp_custom_config[posts]": n.toString(), - }; + // Send POST request + const response = await fetchPOSTResponse(urls.F95_POSTS_NUMBER, params); + if (response.isFailure()) throw response.value; + } - // Send POST request - const response = await fetchPOSTResponse(urls.F95_POSTS_NUMBER, params); - if (response.isFailure()) throw response.value; + /** + * Gets all posts on a page. + */ + private parsePostsInPage(html: string): Post[] { + // Load the HTML + const $ = cheerio.load(html); + + // Start parsing the posts + const posts = $(THREAD.POSTS_IN_PAGE) + .toArray() + .map((el, idx) => { + const id = $(el).find(POST.ID).attr("id").replace("post-", ""); + return new Post(parseInt(id, 10)); + }); + + // Wait for the post to be fetched + return posts; + } + + /** + * Gets all posts in the thread. + */ + private async fetchPosts(pages: number): Promise { + // Local variables + type TFetchResult = Promise< + Result + >; + const htmlPromiseList: TFetchResult[] = []; + const fetchedPosts: Post[] = []; + + // Fetch posts for every page in the thread + for (let i = 1; i <= pages; i++) { + // Prepare the URL + const url = new URL(`page-${i}`, `${this.url}/`).toString(); + + // Fetch the HTML source + const htmlResponse = fetchHTML(url); + htmlPromiseList.push(htmlResponse); } - /** - * Gets all posts on a page. - */ - private parsePostsInPage(html: string): Post[] { - // Load the HTML - const $ = cheerio.load(html); + // Wait for all the pages to load + const responses = await Promise.all(htmlPromiseList); - // Start parsing the posts - const posts = $(THREAD.POSTS_IN_PAGE) - .toArray() - .map((el, idx) => { - const id = $(el).find(POST.ID).attr("id").replace("post-", ""); - return new Post(parseInt(id, 10)); - }); - - // Wait for the post to be fetched - return posts; + // Scrape the pages + for (const response of responses) { + if (response.isSuccess()) { + const posts = this.parsePostsInPage(response.value); + fetchedPosts.push(...posts); + } else throw response.value; } - /** - * Gets all posts in the thread. - */ - private async fetchPosts(pages: number): Promise { - // Local variables - type TFetchResult = Promise>; - const htmlPromiseList: TFetchResult[] = []; - const fetchedPosts: Post[] = []; + // Sorts the list of posts + return fetchedPosts.sort((a, b) => + a.id > b.id ? 1 : b.id > a.id ? -1 : 0 + ); + } - // Fetch posts for every page in the thread - for (let i = 1; i <= pages; i++) { - // Prepare the URL - const url = new URL(`page-${i}`, `${this.url}/`).toString(); + /** + * It processes the rating of the thread + * starting from the data contained in the JSON+LD tag. + */ + private parseRating(data: TJsonLD): TRating { + const ratingTree = data["aggregateRating"] as TJsonLD; + const rating: TRating = { + average: ratingTree ? parseFloat(ratingTree["ratingValue"] as string) : 0, + best: ratingTree ? parseInt(ratingTree["bestRating"] as string, 10) : 0, + count: ratingTree ? parseInt(ratingTree["ratingCount"] as string, 10) : 0 + }; - // Fetch the HTML source - const htmlResponse = fetchHTML(url); - htmlPromiseList.push(htmlResponse); + return rating; + } + + /** + * Clean the title of a thread, removing prefixes + * and generic elements between square brackets, and + * returns the clean title of the work. + */ + private cleanHeadline(headline: string): string { + // From the title we can extract: Name, author and version + // [PREFIXES] TITLE [VERSION] [AUTHOR] + const matches = headline.match(/\[(.*?)\]/g); + + // Get the title name + let name = headline; + if (matches) matches.forEach((e) => (name = name.replace(e, ""))); + return name.trim(); + } + + //#endregion Private methods + + //#region Public methods + + /** + * Gets information about this thread. + */ + public async fetch() { + // Prepare the url + this._url = new URL(this.id.toString(), urls.F95_THREADS).toString(); + + // Fetch the HTML source + const htmlResponse = await fetchHTML(this.url); + + if (htmlResponse.isSuccess()) { + // Load the HTML + const $ = cheerio.load(htmlResponse.value); + + // Fetch data from selectors + const ownerID = $(THREAD.OWNER_ID).attr("data-user-id"); + const tagArray = $(THREAD.TAGS).toArray(); + const prefixArray = $(THREAD.PREFIXES).toArray(); + const JSONLD = getJSONLD($("body")); + const published = JSONLD["datePublished"] as string; + const modified = JSONLD["dateModified"] as string; + + // Parse the thread's data + this._title = this.cleanHeadline(JSONLD["headline"] as string); + this._tags = tagArray.map((el) => $(el).text().trim()); + this._prefixes = prefixArray.map((el) => $(el).text().trim()); + this._owner = new PlatformUser(parseInt(ownerID, 10)); + await this._owner.fetch(); + this._rating = this.parseRating(JSONLD); + this._category = JSONLD["articleSection"] as TCategory; + + // Validate the dates + if (luxon.DateTime.fromISO(modified).isValid) + this._modified = new Date(modified); + if (luxon.DateTime.fromISO(published).isValid) + this._publication = new Date(published); + } else throw htmlResponse.value; + } + + /** + * Gets the post in the `index` position with respect to the posts in the thread. + * + * `index` must be greater or equal to 1. + * If the post is not found, `null` is returned. + */ + public async getPost(index: number): Promise { + // Validate parameters + if (index < 1) + throw new ParameterError("Index must be greater or equal than 1"); + + // Local variables + let returnValue = null; + + // Get the page number of the post + const page = Math.ceil(index / this.POST_FOR_PAGE); + + // Fetch the page + const url = new URL(`page-${page}`, `${this.url}/`).toString(); + const htmlResponse = await fetchHTML(url); + + if (htmlResponse.isSuccess()) { + // Parse the post + const posts = this.parsePostsInPage(htmlResponse.value); + + // Find the searched post + for (const p of posts) { + await p.fetch(); + + if (p.number === index) { + returnValue = p; + break; } + } - // Wait for all the pages to load - const responses = await Promise.all(htmlPromiseList); - - // Scrape the pages - for (const response of responses) { - if (response.isSuccess()) { - const posts = this.parsePostsInPage(response.value); - fetchedPosts.push(...posts); - } else throw response.value; - } - - // Sorts the list of posts - return fetchedPosts.sort((a, b) => (a.id > b.id) ? 1 : ((b.id > a.id) ? -1 : 0)); - } - - /** - * It processes the rating of the thread - * starting from the data contained in the JSON+LD tag. - */ - private parseRating(data: TJsonLD): TRating { - const ratingTree = data["aggregateRating"] as TJsonLD; - const rating: TRating = { - average: ratingTree ? parseFloat(ratingTree["ratingValue"] as string) : 0, - best: ratingTree ? parseInt(ratingTree["bestRating"] as string, 10) : 0, - count: ratingTree ? parseInt(ratingTree["ratingCount"] as string, 10) : 0, - }; - - return rating; - } - - /** - * Clean the title of a thread, removing prefixes - * and generic elements between square brackets, and - * returns the clean title of the work. - */ - private cleanHeadline(headline: string): string { - // From the title we can extract: Name, author and version - // [PREFIXES] TITLE [VERSION] [AUTHOR] - const matches = headline.match(/\[(.*?)\]/g); - - // Get the title name - let name = headline; - if (matches) matches.forEach(e => name = name.replace(e, "")); - return name.trim(); - } - - //#endregion Private methods - - //#region Public methods - - /** - * Gets information about this thread. - */ - public async fetch() { - // Prepare the url - this._url = new URL(this.id.toString(), urls.F95_THREADS).toString(); - - // Fetch the HTML source - const htmlResponse = await fetchHTML(this.url); - - if (htmlResponse.isSuccess()) { - // Load the HTML - const $ = cheerio.load(htmlResponse.value); - - // Fetch data from selectors - const ownerID = $(THREAD.OWNER_ID).attr("data-user-id"); - const tagArray = $(THREAD.TAGS).toArray(); - const prefixArray = $(THREAD.PREFIXES).toArray(); - const JSONLD = getJSONLD($("body")); - const published = JSONLD["datePublished"] as string; - const modified = JSONLD["dateModified"] as string; - - // Parse the thread's data - this._title = this.cleanHeadline(JSONLD["headline"] as string); - this._tags = tagArray.map(el => $(el).text().trim()); - this._prefixes = prefixArray.map(el => $(el).text().trim()); - this._owner = new PlatformUser(parseInt(ownerID, 10)); - await this._owner.fetch(); - this._rating = this.parseRating(JSONLD); - this._category = JSONLD["articleSection"] as TCategory; - - // Validate the dates - if (luxon.DateTime.fromISO(modified).isValid) this._modified = new Date(modified); - if (luxon.DateTime.fromISO(published).isValid) this._publication = new Date(published); - - } else throw htmlResponse.value; - } - - /** - * Gets the post in the `index` position with respect to the posts in the thread. - * - * `index` must be greater or equal to 1. - * If the post is not found, `null` is returned. - */ - public async getPost(index: number): Promise { - // Validate parameters - if (index < 1) throw new ParameterError("Index must be greater or equal than 1"); - - // Local variables - let returnValue = null; - - // Get the page number of the post - const page = Math.ceil(index / this.POST_FOR_PAGE); - - // Fetch the page - const url = new URL(`page-${page}`, `${this.url}/`).toString(); - const htmlResponse = await fetchHTML(url); - - if (htmlResponse.isSuccess()) { - // Parse the post - const posts = this.parsePostsInPage(htmlResponse.value); - - // Find the searched post - for (const p of posts) { - await p.fetch(); - - if (p.number === index) { - returnValue = p; - break; - } - } - - return returnValue; - } else throw htmlResponse.value; - } - - //#endregion Public methods + return returnValue; + } else throw htmlResponse.value; + } + //#endregion Public methods } diff --git a/src/scripts/classes/mapping/user-profile.ts b/src/scripts/classes/mapping/user-profile.ts index ed4514a..ee33f96 100644 --- a/src/scripts/classes/mapping/user-profile.ts +++ b/src/scripts/classes/mapping/user-profile.ts @@ -14,164 +14,184 @@ import { Result } from "../result.js"; // Interfaces interface IWatchedThread { - /** - * URL of the thread - */ - url: string; - /** - * Indicates whether the thread has any unread posts. - */ - unread: boolean, - /** - * Specifies the forum to which the thread belongs. - */ - forum: string, + /** + * URL of the thread + */ + url: string; + /** + * Indicates whether the thread has any unread posts. + */ + unread: boolean; + /** + * Specifies the forum to which the thread belongs. + */ + forum: string; } // Types -type TFetchResult = Result; +type TFetchResult = Result< + GenericAxiosError | UnexpectedResponseContentType, + string +>; /** * Class containing the data of the user currently connected to the F95Zone platform. */ export default class UserProfile extends PlatformUser { + //#region Fields - //#region Fields + private _watched: IWatchedThread[] = []; + private _bookmarks: Post[] = []; + private _alerts: string[] = []; + private _conversations: string[]; - private _watched: IWatchedThread[] = []; - private _bookmarks: Post[] = []; - private _alerts: string[] = []; - private _conversations: string[]; + //#endregion Fields - //#endregion Fields - - //#region Getters + //#region Getters - /** - * List of followed thread data. - */ - public get watched() { return this._watched; } - /** - * List of bookmarked posts. - * @todo - */ - public get bookmarks() { return this._bookmarks; } - /** - * List of alerts. - * @todo - */ - public get alerts() { return this._alerts; } - /** - * List of conversations. - * @todo - */ - public get conversation() { return this._conversations; } + /** + * List of followed thread data. + */ + public get watched() { + return this._watched; + } + /** + * List of bookmarked posts. + * @todo + */ + public get bookmarks() { + return this._bookmarks; + } + /** + * List of alerts. + * @todo + */ + public get alerts() { + return this._alerts; + } + /** + * List of conversations. + * @todo + */ + public get conversation() { + return this._conversations; + } - //#endregion Getters + //#endregion Getters - constructor() { super(); } + constructor() { + super(); + } - //#region Public methods + //#region Public methods - public async fetch() { - // First get the user ID and set it - const id = await this.fetchUserID(); - super.setID(id); + public async fetch() { + // First get the user ID and set it + const id = await this.fetchUserID(); + super.setID(id); - // Than fetch the basic data - await super.fetch(); + // Than fetch the basic data + await super.fetch(); - // Now fetch the watched threads - this._watched = await this.fetchWatchedThread(); + // Now fetch the watched threads + this._watched = await this.fetchWatchedThread(); + } + + //#endregion Public methods + + //#region Private methods + + private async fetchUserID(): Promise { + // Local variables + const url = new URL(urls.F95_BASE_URL).toString(); + + // fetch and parse page + const htmlResponse = await fetchHTML(url); + if (htmlResponse.isSuccess()) { + // Load page with cheerio + const $ = cheerio.load(htmlResponse.value); + + const sid = $(GENERIC.CURRENT_USER_ID).attr("data-user-id").trim(); + return parseInt(sid, 10); + } else throw htmlResponse.value; + } + + private async fetchWatchedThread(): Promise { + // Prepare and fetch URL + const url = new URL(urls.F95_WATCHED_THREADS); + url.searchParams.set("unread", "0"); + + const htmlResponse = await fetchHTML(url.toString()); + + if (htmlResponse.isSuccess()) { + // Load page in cheerio + const $ = cheerio.load(htmlResponse.value); + + // Fetch the pages + const lastPage = parseInt($(WATCHED_THREAD.LAST_PAGE).text().trim(), 10); + const pages = await this.fetchPages(url, lastPage); + + const watchedThreads = pages.map((r, idx) => { + const elements = r.applyOnSuccess(this.fetchPageThreadElements); + if (elements.isSuccess()) return elements.value; + }); + + return [].concat(...watchedThreads); + } else throw htmlResponse.value; + } + + /** + * Gets the pages containing the thread data. + * @param url Base URL to use for scraping a page + * @param n Total number of pages + * @param s Page to start from + */ + private async fetchPages( + url: URL, + n: number, + s = 1 + ): Promise { + // Local variables + const responsePromiseList: Promise[] = []; + + // Fetch the page' HTML + for (let page = s; page <= n; page++) { + // Set the page URL + url.searchParams.set("page", page.toString()); + + // Fetch HTML but not wait for it + const promise = fetchHTML(url.toString()); + responsePromiseList.push(promise); } - //#endregion Public methods + // Wait for the promises to resolve + return Promise.all(responsePromiseList); + } - //#region Private methods + /** + * Gets thread data starting from the source code of the page passed by parameter. + */ + private fetchPageThreadElements(html: string): IWatchedThread[] { + // Local variables + const $ = cheerio.load(html); - private async fetchUserID(): Promise { - // Local variables - const url = new URL(urls.F95_BASE_URL).toString(); + return $(WATCHED_THREAD.BODIES) + .map((idx, el) => { + // Parse the URL + const partialURL = $(el).find(WATCHED_THREAD.URL).attr("href"); + const url = new URL( + partialURL.replace("unread", ""), + `${urls.F95_BASE_URL}` + ).toString(); - // fetch and parse page - const htmlResponse = await fetchHTML(url); - if (htmlResponse.isSuccess()) { - // Load page with cheerio - const $ = cheerio.load(htmlResponse.value); - - const sid = $(GENERIC.CURRENT_USER_ID).attr("data-user-id").trim(); - return parseInt(sid, 10); - } else throw htmlResponse.value; - } - - private async fetchWatchedThread(): Promise { - // Prepare and fetch URL - const url = new URL(urls.F95_WATCHED_THREADS); - url.searchParams.set("unread", "0"); - - const htmlResponse = await fetchHTML(url.toString()); - - if (htmlResponse.isSuccess()) { - // Load page in cheerio - const $ = cheerio.load(htmlResponse.value); - - // Fetch the pages - const lastPage = parseInt($(WATCHED_THREAD.LAST_PAGE).text().trim(), 10); - const pages = await this.fetchPages(url, lastPage); - - const watchedThreads = pages.map((r, idx) => { - const elements = r.applyOnSuccess(this.fetchPageThreadElements); - if (elements.isSuccess()) return elements.value; - }); - - return [].concat(...watchedThreads); - } else throw htmlResponse.value; - } - - /** - * Gets the pages containing the thread data. - * @param url Base URL to use for scraping a page - * @param n Total number of pages - * @param s Page to start from - */ - private async fetchPages(url: URL, n: number, s: number = 1): Promise { - // Local variables - const responsePromiseList: Promise[] = []; - - // Fetch the page' HTML - for (let page = s; page <= n; page++) { - // Set the page URL - url.searchParams.set("page", page.toString()); - - // Fetch HTML but not wait for it - const promise = fetchHTML(url.toString()) - responsePromiseList.push(promise); - } - - // Wait for the promises to resolve - return Promise.all(responsePromiseList); - } - - /** - * Gets thread data starting from the source code of the page passed by parameter. - */ - private fetchPageThreadElements(html: string): IWatchedThread[] { - // Local variables - const $ = cheerio.load(html); - - return $(WATCHED_THREAD.BODIES).map((idx, el) => { - // Parse the URL - const partialURL = $(el).find(WATCHED_THREAD.URL).attr("href"); - const url = new URL(partialURL.replace("unread", ""), `${urls.F95_BASE_URL}`).toString(); - - return { - url: url.toString(), - unread: partialURL.endsWith("unread"), - forum: $(el).find(WATCHED_THREAD.FORUM).text().trim() - } as IWatchedThread; - }).get(); - } - - //#endregion Private methods + return { + url: url.toString(), + unread: partialURL.endsWith("unread"), + forum: $(el).find(WATCHED_THREAD.FORUM).text().trim() + } as IWatchedThread; + }) + .get(); + } + //#endregion Private methods } diff --git a/src/scripts/classes/prefix-parser.ts b/src/scripts/classes/prefix-parser.ts index 5cb0779..aea0d98 100644 --- a/src/scripts/classes/prefix-parser.ts +++ b/src/scripts/classes/prefix-parser.ts @@ -7,102 +7,110 @@ import shared, { TPrefixDict } from "../shared.js"; * Convert prefixes and platform tags from string to ID and vice versa. */ export default class PrefixParser { + //#region Private methods + /** + * Gets the key associated with a given value from a dictionary. + * @param {Object} object Dictionary to search + * @param {Any} value Value associated with the key + * @returns {String|undefined} Key found or undefined + */ + private getKeyByValue( + object: TPrefixDict, + value: string + ): string | undefined { + return Object.keys(object).find((key) => object[key] === value); + } - //#region Private methods + /** + * Makes an array of strings uppercase. + */ + private toUpperCaseArray(a: string[]): string[] { /** - * Gets the key associated with a given value from a dictionary. - * @param {Object} object Dictionary to search - * @param {Any} value Value associated with the key - * @returns {String|undefined} Key found or undefined + * Makes a string uppercase. */ - private getKeyByValue(object: TPrefixDict, value: string): string | undefined { - return Object.keys(object).find(key => object[key] === value); + function toUpper(s: string): string { + return s.toUpperCase(); + } + return a.map(toUpper); + } + + /** + * Check if `dict` contains `value` as a value. + */ + private valueInDict(dict: TPrefixDict, value: string): boolean { + const array = Object.values(dict); + const upperArr = this.toUpperCaseArray(array); + const element = value.toUpperCase(); + return upperArr.includes(element); + } + + /** + * Search within the platform prefixes for the + * desired element and return the dictionary that contains it. + * @param element Element to search in the prefixes as a key or as a value + */ + private searchElementInPrefixes( + element: string | number + ): TPrefixDict | null { + // Local variables + let dictName = null; + + // Iterate the key/value pairs in order to find the element + for (const [key, subdict] of Object.entries(shared.prefixes)) { + // Check if the element is a value in the sub-dict + const valueInDict = + typeof element === "string" && + this.valueInDict(subdict, element as string); + + // Check if the element is a key in the subdict + const keyInDict = + typeof element === "number" && + Object.keys(subdict).includes(element.toString()); + + if (valueInDict || keyInDict) { + dictName = key; + break; + } } - /** - * Makes an array of strings uppercase. - */ - private toUpperCaseArray(a: string[]): string[] { - /** - * Makes a string uppercase. - */ - function toUpper(s: string): string { - return s.toUpperCase(); - } - return a.map(toUpper); + return shared.prefixes[dictName] ?? null; + } + //#endregion Private methods + + /** + * Convert a list of prefixes to their respective IDs. + */ + public prefixesToIDs(prefixes: string[]): number[] { + const ids: number[] = []; + + for (const p of prefixes) { + // Check what dict contains the value + const dict = this.searchElementInPrefixes(p); + + if (dict) { + // Extract the key from the dict + const key = this.getKeyByValue(dict, p); + ids.push(parseInt(key, 10)); + } } + return ids; + } - /** - * Check if `dict` contains `value` as a value. - */ - private valueInDict(dict: TPrefixDict, value: string): boolean { - const array = Object.values(dict); - const upperArr = this.toUpperCaseArray(array); - const element = value.toUpperCase(); - return upperArr.includes(element); + /** + * It converts a list of IDs into their respective prefixes. + */ + public idsToPrefixes(ids: number[]): string[] { + const prefixes: string[] = []; + + for (const id of ids) { + // Check what dict contains the key + const dict = this.searchElementInPrefixes(id); + + // Add the key to the list + if (dict) { + prefixes.push(dict[id]); + } } - - /** - * Search within the platform prefixes for the - * desired element and return the dictionary that contains it. - * @param element Element to search in the prefixes as a key or as a value - */ - private searchElementInPrefixes(element: string | number): TPrefixDict | null { - // Local variables - let dictName = null; - - // Iterate the key/value pairs in order to find the element - for (const [key, subdict] of Object.entries(shared.prefixes)) { - // Check if the element is a value in the sub-dict - const valueInDict = typeof element === "string" && this.valueInDict(subdict, element as string); - - // Check if the element is a key in the subdict - const keyInDict = typeof element === "number" && Object.keys(subdict).includes(element.toString()); - - if (valueInDict || keyInDict) { - dictName = key; - break; - } - } - - return shared.prefixes[dictName] ?? null; - } - //#endregion Private methods - - /** - * Convert a list of prefixes to their respective IDs. - */ - public prefixesToIDs(prefixes: string[]) : number[] { - const ids: number[] = []; - - for(const p of prefixes) { - // Check what dict contains the value - const dict = this.searchElementInPrefixes(p); - - if (dict) { - // Extract the key from the dict - const key = this.getKeyByValue(dict, p); - ids.push(parseInt(key, 10)); - } - } - return ids; - } - - /** - * It converts a list of IDs into their respective prefixes. - */ - public idsToPrefixes(ids: number[]): string[] { - const prefixes:string[] = []; - - for(const id of ids) { - // Check what dict contains the key - const dict = this.searchElementInPrefixes(id); - - // Add the key to the list - if (dict) { - prefixes.push(dict[id]); - } - } - return prefixes; - } -} \ No newline at end of file + return prefixes; + } +} diff --git a/src/scripts/classes/query/handiwork-search-query.ts b/src/scripts/classes/query/handiwork-search-query.ts index b63a635..e38b22b 100644 --- a/src/scripts/classes/query/handiwork-search-query.ts +++ b/src/scripts/classes/query/handiwork-search-query.ts @@ -1,15 +1,15 @@ "use strict"; -import { AxiosResponse } from 'axios'; +import { AxiosResponse } from "axios"; // Public modules from npm -import validator from 'class-validator'; +import validator from "class-validator"; // Module from files import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js"; -import { GenericAxiosError, UnexpectedResponseContentType } from '../errors.js'; -import { Result } from '../result.js'; -import LatestSearchQuery, { TLatestOrder } from './latest-search-query.js'; -import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js'; +import { GenericAxiosError, UnexpectedResponseContentType } from "../errors.js"; +import { Result } from "../result.js"; +import LatestSearchQuery, { TLatestOrder } from "./latest-search-query.js"; +import ThreadSearchQuery, { TThreadOrder } from "./thread-search-query.js"; // Type definitions /** @@ -30,149 +30,165 @@ import ThreadSearchQuery, { TThreadOrder } from './thread-search-query.js'; * * `views`: Order based on the number of visits. Replacement: `replies`. */ -type THandiworkOrder = "date" | "likes" | "relevance" | "replies" | "title" | "views"; +type THandiworkOrder = + | "date" + | "likes" + | "relevance" + | "replies" + | "title" + | "views"; type TExecuteResult = Result>; export default class HandiworkSearchQuery implements IQuery { - - //#region Private fields + //#region Private fields - static MIN_PAGE = 1; + static MIN_PAGE = 1; - //#endregion Private fields + //#endregion Private fields - //#region Properties + //#region Properties - /** - * Keywords to use in the search. - */ - public keywords: string = ""; - /** - * The results must be more recent than the date indicated. - */ - public newerThan: Date = null; - /** - * The results must be older than the date indicated. - */ - public olderThan: Date = null; - public includedTags: string[] = []; - /** - * Tags to exclude from the search. - */ - public excludedTags: string[] = []; - public includedPrefixes: string[] = []; - public category: TCategory = null; - /** - * Results presentation order. - */ - public order: THandiworkOrder = "relevance"; - @validator.IsInt({ - message: "$property expect an integer, received $value" - }) - @validator.Min(HandiworkSearchQuery.MIN_PAGE, { - message: "The minimum $property value must be $constraint1, received $value" - }) - public page: number = 1; - itype: TQueryInterface = "HandiworkSearchQuery"; + /** + * Keywords to use in the search. + */ + public keywords = ""; + /** + * The results must be more recent than the date indicated. + */ + public newerThan: Date = null; + /** + * The results must be older than the date indicated. + */ + public olderThan: Date = null; + public includedTags: string[] = []; + /** + * Tags to exclude from the search. + */ + public excludedTags: string[] = []; + public includedPrefixes: string[] = []; + public category: TCategory = null; + /** + * Results presentation order. + */ + public order: THandiworkOrder = "relevance"; + @validator.IsInt({ + message: "$property expect an integer, received $value" + }) + @validator.Min(HandiworkSearchQuery.MIN_PAGE, { + message: "The minimum $property value must be $constraint1, received $value" + }) + public page = 1; + itype: TQueryInterface = "HandiworkSearchQuery"; - //#endregion Properties + //#endregion Properties - //#region Public methods + //#region Public methods - /** - * Select what kind of search should be - * performed based on the properties of - * the query. - */ - public selectSearchType(): "latest" | "thread" { - // Local variables - const MAX_TAGS_LATEST_SEARCH = 5; - const DEFAULT_SEARCH_TYPE = "latest"; + /** + * Select what kind of search should be + * performed based on the properties of + * the query. + */ + public selectSearchType(): "latest" | "thread" { + // Local variables + const MAX_TAGS_LATEST_SEARCH = 5; + const DEFAULT_SEARCH_TYPE = "latest"; - // If the keywords are set or the number - // of included tags is greather than 5, - // we must perform a thread search - if (this.keywords || this.includedTags.length > MAX_TAGS_LATEST_SEARCH) return "thread"; + // If the keywords are set or the number + // of included tags is greather than 5, + // we must perform a thread search + if (this.keywords || this.includedTags.length > MAX_TAGS_LATEST_SEARCH) + return "thread"; - return DEFAULT_SEARCH_TYPE; + return DEFAULT_SEARCH_TYPE; + } + + public validate(): boolean { + return validator.validateSync(this).length === 0; + } + + public async execute(): Promise { + // Local variables + let response: TExecuteResult = null; + + // Check if the query is valid + if (!this.validate()) { + throw new Error( + `Invalid query: ${validator.validateSync(this).join("\n")}` + ); } - public validate(): boolean { return validator.validateSync(this).length === 0; } + // Convert the query + if (this.selectSearchType() === "latest") + response = await this.cast( + "LatestSearchQuery" + ).execute(); + else + response = await this.cast( + "ThreadSearchQuery" + ).execute(); - public async execute(): Promise { - // Local variables - let response: TExecuteResult = null; + return response; + } - // Check if the query is valid - if (!this.validate()) { - throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`); - } + public cast(type: TQueryInterface): T { + // Local variables + let returnValue = null; - // Convert the query - if (this.selectSearchType() === "latest") response = await this.cast("LatestSearchQuery").execute(); - else response = await this.cast("ThreadSearchQuery").execute(); + // Convert the query + if (type === "LatestSearchQuery") returnValue = this.castToLatest(); + else if (type === "ThreadSearchQuery") returnValue = this.castToThread(); + else returnValue = this as HandiworkSearchQuery; - return response; - } + // Cast the result to T + return returnValue as T; + } - public cast(type: TQueryInterface): T { - // Local variables - let returnValue = null; - - // Convert the query - if (type === "LatestSearchQuery") returnValue = this.castToLatest(); - else if (type === "ThreadSearchQuery") returnValue = this.castToThread(); - else returnValue = this as HandiworkSearchQuery; + //#endregion Public methods - // Cast the result to T - return returnValue as T; - } + //#region Private methods - //#endregion Public methods + private castToLatest(): LatestSearchQuery { + // Cast the basic query object and copy common values + const query: LatestSearchQuery = new LatestSearchQuery(); + Object.keys(this).forEach((key) => { + if (query.hasOwnProperty(key)) { + query[key] = this[key]; + } + }); - //#region Private methods + // Adapt order filter + let orderFilter = this.order as string; + if (orderFilter === "relevance") orderFilter = "rating"; + else if (orderFilter === "replies") orderFilter = "views"; + query.order = orderFilter as TLatestOrder; - private castToLatest(): LatestSearchQuery { - // Cast the basic query object and copy common values - const query: LatestSearchQuery = new LatestSearchQuery(); - Object.keys(this).forEach(key => { - if (query.hasOwnProperty(key)) { - query[key] = this[key]; - } - }); + // Adapt date + if (this.newerThan) query.date = query.findNearestDate(this.newerThan); - // Adapt order filter - let orderFilter = this.order as string; - if (orderFilter === "relevance") orderFilter = "rating"; - else if (orderFilter === "replies") orderFilter = "views"; - query.order = orderFilter as TLatestOrder; + return query; + } - // Adapt date - if (this.newerThan) query.date = query.findNearestDate(this.newerThan); + private castToThread(): ThreadSearchQuery { + // Cast the basic query object and copy common values + const query: ThreadSearchQuery = new ThreadSearchQuery(); + Object.keys(this).forEach((key) => { + if (query.hasOwnProperty(key)) { + query[key] = this[key]; + } + }); - return query; - } + // Set uncommon values + query.onlyTitles = true; - private castToThread(): ThreadSearchQuery { - // Cast the basic query object and copy common values - const query: ThreadSearchQuery = new ThreadSearchQuery(); - Object.keys(this).forEach(key => { - if (query.hasOwnProperty(key)) { - query[key] = this[key]; - } - }); + // Adapt order filter + let orderFilter = this.order as string; + if (orderFilter === "title") orderFilter = "relevance"; + else if (orderFilter === "likes") orderFilter = "replies"; + query.order = orderFilter as TThreadOrder; - // Set uncommon values - query.onlyTitles = true; - - // Adapt order filter - let orderFilter = this.order as string; - if (orderFilter === "title") orderFilter = "relevance"; - else if (orderFilter === "likes") orderFilter = "replies"; - query.order = orderFilter as TThreadOrder; + return query; + } - return query; - } - - //#endregion -} \ No newline at end of file + //#endregion +} diff --git a/src/scripts/classes/query/latest-search-query.ts b/src/scripts/classes/query/latest-search-query.ts index 3259db6..f0848ff 100644 --- a/src/scripts/classes/query/latest-search-query.ts +++ b/src/scripts/classes/query/latest-search-query.ts @@ -1,13 +1,13 @@ "use strict"; // Public modules from npm -import validator from 'class-validator'; +import validator from "class-validator"; // Modules from file import { urls } from "../../constants/url.js"; -import PrefixParser from '../prefix-parser.js'; +import PrefixParser from "../prefix-parser.js"; import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js"; -import { fetchGETResponse } from '../../network-helper.js'; +import { fetchGETResponse } from "../../network-helper.js"; // Type definitions export type TLatestOrder = "date" | "likes" | "views" | "title" | "rating"; @@ -17,128 +17,133 @@ type TDate = 365 | 180 | 90 | 30 | 14 | 7 | 3 | 1; * Query used to search handiwork in the "Latest" tab. */ export default class LatestSearchQuery implements IQuery { + //#region Private fields - //#region Private fields + private static MAX_TAGS = 5; + private static MIN_PAGE = 1; - private static MAX_TAGS = 5; - private static MIN_PAGE = 1; + //#endregion Private fields - //#endregion Private fields + //#region Properties - //#region Properties + public category: TCategory = "games"; + /** + * Ordering type. + * + * Default: `date`. + */ + public order: TLatestOrder = "date"; + /** + * Date limit in days, to be understood as "less than". + * Use `1` to indicate "today" or `null` to indicate "anytime". + * + * Default: `null` + */ + public date: TDate = null; - public category: TCategory = 'games'; - /** - * Ordering type. - * - * Default: `date`. - */ - public order: TLatestOrder = 'date'; - /** - * Date limit in days, to be understood as "less than". - * Use `1` to indicate "today" or `null` to indicate "anytime". - * - * Default: `null` - */ - public date: TDate = null; - - @validator.ArrayMaxSize(LatestSearchQuery.MAX_TAGS, { - message: "Too many tags: $value instead of $constraint1" - }) - public includedTags: string[] = []; - public includedPrefixes: string[] = []; + @validator.ArrayMaxSize(LatestSearchQuery.MAX_TAGS, { + message: "Too many tags: $value instead of $constraint1" + }) + public includedTags: string[] = []; + public includedPrefixes: string[] = []; - @validator.IsInt({ - message: "$property expect an integer, received $value" - }) - @validator.Min(LatestSearchQuery.MIN_PAGE, { - message: "The minimum $property value must be $constraint1, received $value" - }) - public page = LatestSearchQuery.MIN_PAGE; - itype: TQueryInterface = "LatestSearchQuery"; + @validator.IsInt({ + message: "$property expect an integer, received $value" + }) + @validator.Min(LatestSearchQuery.MIN_PAGE, { + message: "The minimum $property value must be $constraint1, received $value" + }) + public page = LatestSearchQuery.MIN_PAGE; + itype: TQueryInterface = "LatestSearchQuery"; - //#endregion Properties + //#endregion Properties - //#region Public methods + //#region Public methods - public validate(): boolean { return validator.validateSync(this).length === 0; } + public validate(): boolean { + return validator.validateSync(this).length === 0; + } - public async execute() { - // Check if the query is valid - if (!this.validate()) { - throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`); - } - - // Prepare the URL - const url = this.prepareGETurl(); - const decoded = decodeURIComponent(url.toString()); - - // Fetch the result - return fetchGETResponse(decoded); + public async execute() { + // Check if the query is valid + if (!this.validate()) { + throw new Error( + `Invalid query: ${validator.validateSync(this).join("\n")}` + ); } - /** - * Gets the value (in days) acceptable in the query starting from a generic date. - */ - public findNearestDate(d: Date): TDate { - // Find the difference between today and the passed date - const diff = this.dateDiffInDays(new Date(), d); + // Prepare the URL + const url = this.prepareGETurl(); + const decoded = decodeURIComponent(url.toString()); - // Find the closest valid value in the array - const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) { - return (Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev); - }); + // Fetch the result + return fetchGETResponse(decoded); + } - return closest as TDate; + /** + * Gets the value (in days) acceptable in the query starting from a generic date. + */ + public findNearestDate(d: Date): TDate { + // Find the difference between today and the passed date + const diff = this.dateDiffInDays(new Date(), d); + + // Find the closest valid value in the array + const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function ( + prev, + curr + ) { + return Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev; + }); + + return closest as TDate; + } + + //#endregion Public methods + + //#region Private methods + + /** + * Prepare the URL by filling out the GET parameters with the data in the query. + */ + private prepareGETurl(): URL { + // Create the URL + const url = new URL(urls.F95_LATEST_PHP); + url.searchParams.set("cmd", "list"); + + // Set the category + const cat: TCategory = this.category === "mods" ? "games" : this.category; + url.searchParams.set("cat", cat); + + // Add tags and prefixes + const parser = new PrefixParser(); + for (const tag of parser.prefixesToIDs(this.includedTags)) { + url.searchParams.append("tags[]", tag.toString()); } - //#endregion Public methods - - //#region Private methods - - /** - * Prepare the URL by filling out the GET parameters with the data in the query. - */ - private prepareGETurl(): URL { - // Create the URL - const url = new URL(urls.F95_LATEST_PHP); - url.searchParams.set("cmd", "list"); - - // Set the category - const cat: TCategory = this.category === "mods" ? "games" : this.category; - url.searchParams.set("cat", cat); - - // Add tags and prefixes - const parser = new PrefixParser(); - for (const tag of parser.prefixesToIDs(this.includedTags)) { - url.searchParams.append("tags[]", tag.toString()); - } - - for (const p of parser.prefixesToIDs(this.includedPrefixes)) { - url.searchParams.append("prefixes[]", p.toString()); - } - - // Set the other values - url.searchParams.set("sort", this.order.toString()); - url.searchParams.set("page", this.page.toString()); - if (this.date) url.searchParams.set("date", this.date.toString()); - - return url; + for (const p of parser.prefixesToIDs(this.includedPrefixes)) { + url.searchParams.append("prefixes[]", p.toString()); } - /** - * - */ - private dateDiffInDays(a: Date, b: Date) { - const MS_PER_DAY = 1000 * 60 * 60 * 24; + // Set the other values + url.searchParams.set("sort", this.order.toString()); + url.searchParams.set("page", this.page.toString()); + if (this.date) url.searchParams.set("date", this.date.toString()); - // Discard the time and time-zone information. - const utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate()); - const utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate()); + return url; + } - return Math.floor((utc2 - utc1) / MS_PER_DAY); - } + /** + * + */ + private dateDiffInDays(a: Date, b: Date) { + const MS_PER_DAY = 1000 * 60 * 60 * 24; - //#endregion Private methodss + // Discard the time and time-zone information. + const utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate()); + const utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate()); -} \ No newline at end of file + return Math.floor((utc2 - utc1) / MS_PER_DAY); + } + + //#endregion Private methodss +} diff --git a/src/scripts/classes/query/thread-search-query.ts b/src/scripts/classes/query/thread-search-query.ts index 83bff06..7336c09 100644 --- a/src/scripts/classes/query/thread-search-query.ts +++ b/src/scripts/classes/query/thread-search-query.ts @@ -1,177 +1,183 @@ "use strict"; // Public modules from npm -import validator from 'class-validator'; +import validator from "class-validator"; // Module from files import { IQuery, TCategory, TQueryInterface } from "../../interfaces.js"; import { urls } from "../../constants/url.js"; import PrefixParser from "./../prefix-parser.js"; -import { fetchPOSTResponse } from '../../network-helper.js'; -import { AxiosResponse } from 'axios'; -import { GenericAxiosError } from '../errors.js'; -import { Result } from '../result.js'; -import Shared from '../../shared.js'; +import { fetchPOSTResponse } from "../../network-helper.js"; +import { AxiosResponse } from "axios"; +import { GenericAxiosError } from "../errors.js"; +import { Result } from "../result.js"; +import Shared from "../../shared.js"; // Type definitions export type TThreadOrder = "relevance" | "date" | "last_update" | "replies"; export default class ThreadSearchQuery implements IQuery { + //#region Private fields - //#region Private fields + static MIN_PAGE = 1; - static MIN_PAGE = 1; + //#endregion Private fields - //#endregion Private fields + //#region Properties - //#region Properties + /** + * Keywords to use in the search. + */ + public keywords = ""; + /** + * Indicates to search by checking only the thread titles and not the content. + */ + public onlyTitles = false; + /** + * The results must be more recent than the date indicated. + */ + public newerThan: Date = null; + /** + * The results must be older than the date indicated. + */ + public olderThan: Date = null; + public includedTags: string[] = []; + /** + * Tags to exclude from the search. + */ + public excludedTags: string[] = []; + /** + * Minimum number of answers that the thread must possess. + */ + public minimumReplies = 0; + public includedPrefixes: string[] = []; + public category: TCategory = null; + /** + * Results presentation order. + */ + public order: TThreadOrder = "relevance"; + @validator.IsInt({ + message: "$property expect an integer, received $value" + }) + @validator.Min(ThreadSearchQuery.MIN_PAGE, { + message: "The minimum $property value must be $constraint1, received $value" + }) + public page = 1; + itype: TQueryInterface = "ThreadSearchQuery"; - /** - * Keywords to use in the search. - */ - public keywords: string = ""; - /** - * Indicates to search by checking only the thread titles and not the content. - */ - public onlyTitles: boolean = false; - /** - * The results must be more recent than the date indicated. - */ - public newerThan: Date = null; - /** - * The results must be older than the date indicated. - */ - public olderThan: Date = null; - public includedTags: string[] = []; - /** - * Tags to exclude from the search. - */ - public excludedTags: string[] = []; - /** - * Minimum number of answers that the thread must possess. - */ - public minimumReplies: number = 0; - public includedPrefixes: string[] = []; - public category: TCategory = null; - /** - * Results presentation order. - */ - public order: TThreadOrder = "relevance"; - @validator.IsInt({ - message: "$property expect an integer, received $value" - }) - @validator.Min(ThreadSearchQuery.MIN_PAGE, { - message: "The minimum $property value must be $constraint1, received $value" - }) - public page: number = 1; - itype: TQueryInterface = "ThreadSearchQuery"; + //#endregion Properties - //#endregion Properties + //#region Public methods - //#region Public methods - - public validate(): boolean { return validator.validateSync(this).length === 0; } - - public async execute(): Promise>> { - // Check if the query is valid - if (!this.validate()) { - throw new Error(`Invalid query: ${validator.validateSync(this).join("\n")}`); - } + public validate(): boolean { + return validator.validateSync(this).length === 0; + } - // Define the POST parameters - const params = this.preparePOSTParameters(); - - // Return the POST response - return fetchPOSTResponse(urls.F95_SEARCH_URL, params); + public async execute(): Promise< + Result> + > { + // Check if the query is valid + if (!this.validate()) { + throw new Error( + `Invalid query: ${validator.validateSync(this).join("\n")}` + ); } - //#endregion Public methods + // Define the POST parameters + const params = this.preparePOSTParameters(); - //#region Private methods + // Return the POST response + return fetchPOSTResponse(urls.F95_SEARCH_URL, params); + } - /** - * Prepare the parameters for post request with the data in the query. - */ - private preparePOSTParameters(): { [s: string]: string } { - // Local variables - const params = {}; + //#endregion Public methods - // Ad the session token - params["_xfToken"] = Shared.session.token; + //#region Private methods - // Specify if only the title should be searched - if (this.onlyTitles) params["c[title_only]"] = "1"; + /** + * Prepare the parameters for post request with the data in the query. + */ + private preparePOSTParameters(): { [s: string]: string } { + // Local variables + const params = {}; - // Add keywords - params["keywords"] = this.keywords ?? "*"; + // Ad the session token + params["_xfToken"] = Shared.session.token; - // Specify the scope of the search (only "threads/post") - params["search_type"] = "post"; + // Specify if only the title should be searched + if (this.onlyTitles) params["c[title_only]"] = "1"; - // Set the dates - if (this.newerThan) { - const date = this.convertShortDate(this.newerThan); - params["c[newer_than]"] = date; - } + // Add keywords + params["keywords"] = this.keywords ?? "*"; - if (this.olderThan) { - const date = this.convertShortDate(this.olderThan); - params["c[older_than]"] = date; - } + // Specify the scope of the search (only "threads/post") + params["search_type"] = "post"; - // Set included and excluded tags (joined with a comma) - if (this.includedTags) params["c[tags]"] = this.includedTags.join(","); - if (this.excludedTags) params["c[excludeTags]"] = this.excludedTags.join(","); - - // Set minimum reply number - if (this.minimumReplies > 0) params["c[min_reply_count]"] = this.minimumReplies.toString(); - - // Add prefixes - const parser = new PrefixParser(); - const ids = parser.prefixesToIDs(this.includedPrefixes); - for (let i = 0; i < ids.length; i++) { - const name = `c[prefixes][${i}]`; - params[name] = ids[i].toString(); - } - - // Set the category - params["c[child_nodes]"] = "1"; // Always set - if (this.category) { - const catID = this.categoryToID(this.category).toString(); - params["c[nodes][0]"] = catID; - } - - // Set the other values - params["order"] = this.order.toString(); - params["page"] = this.page.toString(); - - return params; + // Set the dates + if (this.newerThan) { + const date = this.convertShortDate(this.newerThan); + params["c[newer_than]"] = date; } - /** - * Convert a date in the YYYY-MM-DD format taking into account the time zone. - */ - private convertShortDate(d: Date): string { - const offset = d.getTimezoneOffset() - d = new Date(d.getTime() - (offset * 60 * 1000)) - return d.toISOString().split('T')[0] + if (this.olderThan) { + const date = this.convertShortDate(this.olderThan); + params["c[older_than]"] = date; } - /** - * Gets the unique ID of the selected category. - */ - private categoryToID(category: TCategory): number { - const catMap = { - "games": 2, - "mods": 41, - "comics": 40, - "animations": 94, - "assets": 95, - } + // Set included and excluded tags (joined with a comma) + if (this.includedTags) params["c[tags]"] = this.includedTags.join(","); + if (this.excludedTags) + params["c[excludeTags]"] = this.excludedTags.join(","); - return catMap[category as string]; + // Set minimum reply number + if (this.minimumReplies > 0) + params["c[min_reply_count]"] = this.minimumReplies.toString(); + + // Add prefixes + const parser = new PrefixParser(); + const ids = parser.prefixesToIDs(this.includedPrefixes); + for (let i = 0; i < ids.length; i++) { + const name = `c[prefixes][${i}]`; + params[name] = ids[i].toString(); } - //#endregion Private methods + // Set the category + params["c[child_nodes]"] = "1"; // Always set + if (this.category) { + const catID = this.categoryToID(this.category).toString(); + params["c[nodes][0]"] = catID; + } -} \ No newline at end of file + // Set the other values + params["order"] = this.order.toString(); + params["page"] = this.page.toString(); + + return params; + } + + /** + * Convert a date in the YYYY-MM-DD format taking into account the time zone. + */ + private convertShortDate(d: Date): string { + const offset = d.getTimezoneOffset(); + d = new Date(d.getTime() - offset * 60 * 1000); + return d.toISOString().split("T")[0]; + } + + /** + * Gets the unique ID of the selected category. + */ + private categoryToID(category: TCategory): number { + const catMap = { + games: 2, + mods: 41, + comics: 40, + animations: 94, + assets: 95 + }; + + return catMap[category as string]; + } + + //#endregion Private methods +} diff --git a/src/scripts/classes/result.ts b/src/scripts/classes/result.ts index de86728..a242714 100644 --- a/src/scripts/classes/result.ts +++ b/src/scripts/classes/result.ts @@ -1,49 +1,49 @@ export type Result = Failure | Success; export class Failure { - readonly value: L; + readonly value: L; - constructor(value: L) { - this.value = value; - } + constructor(value: L) { + this.value = value; + } - isFailure(): this is Failure { - return true; - } + isFailure(): this is Failure { + return true; + } - isSuccess(): this is Success { - return false; - } + isSuccess(): this is Success { + return false; + } - applyOnSuccess(_: (a: A) => B): Result { - return this as any; - } + applyOnSuccess(_: (a: A) => B): Result { + return this as any; + } } export class Success { - readonly value: A; + readonly value: A; - constructor(value: A) { - this.value = value; - } + constructor(value: A) { + this.value = value; + } - isFailure(): this is Failure { - return false; - } + isFailure(): this is Failure { + return false; + } - isSuccess(): this is Success { - return true; - } + isSuccess(): this is Success { + return true; + } - applyOnSuccess(func: (a: A) => B): Result { - return new Success(func(this.value)); - } + applyOnSuccess(func: (a: A) => B): Result { + return new Success(func(this.value)); + } } export const failure = (l: L): Result => { - return new Failure(l); + return new Failure(l); }; export const success = (a: A): Result => { - return new Success(a); -}; \ No newline at end of file + return new Success(a); +}; diff --git a/src/scripts/classes/session.ts b/src/scripts/classes/session.ts index fbf25e7..ff5fb02 100644 --- a/src/scripts/classes/session.ts +++ b/src/scripts/classes/session.ts @@ -15,189 +15,202 @@ const awritefile = promisify(fs.writeFile); const aunlinkfile = promisify(fs.unlink); export default class Session { + //#region Fields - //#region Fields + /** + * Max number of days the session is valid. + */ + private readonly SESSION_TIME: number = 3; + private readonly COOKIEJAR_FILENAME: string = "f95cookiejar.json"; + private _path: string; + private _isMapped: boolean; + private _created: Date; + private _hash: string; + private _token: string; + private _cookieJar: CookieJar; + private _cookieJarPath: string; - /** - * Max number of days the session is valid. - */ - private readonly SESSION_TIME: number = 3; - private readonly COOKIEJAR_FILENAME: string = "f95cookiejar.json"; - private _path: string; - private _isMapped: boolean; - private _created: Date; - private _hash: string; - private _token: string; - private _cookieJar: CookieJar; - private _cookieJarPath: string; + //#endregion Fields - //#endregion Fields + //#region Getters - //#region Getters + /** + * Path of the session map file on disk. + */ + public get path() { + return this._path; + } + /** + * Indicates if the session is mapped on disk. + */ + public get isMapped() { + return this._isMapped; + } + /** + * Date of creation of the session. + */ + public get created() { + return this._created; + } + /** + * MD5 hash of the username and the password. + */ + public get hash() { + return this._hash; + } + /** + * Token used to login to F95Zone. + */ + public get token() { + return this._token; + } + /** + * Cookie holder. + */ + public get cookieJar() { + return this._cookieJar; + } - /** - * Path of the session map file on disk. - */ - public get path() { return this._path; } - /** - * Indicates if the session is mapped on disk. - */ - public get isMapped() { return this._isMapped; } - /** - * Date of creation of the session. - */ - public get created() { return this._created; } - /** - * MD5 hash of the username and the password. - */ - public get hash() { return this._hash; } - /** - * Token used to login to F95Zone. - */ - public get token() { return this._token; } - /** - * Cookie holder. - */ - public get cookieJar() { return this._cookieJar; } + //#endregion Getters - //#endregion Getters + /** + * Initializes the session by setting the path for saving information to disk. + */ + constructor(p: string) { + this._path = p; + this._isMapped = fs.existsSync(this.path); + this._created = new Date(Date.now()); + this._hash = null; + this._token = null; + this._cookieJar = new tough.CookieJar(); - /** - * Initializes the session by setting the path for saving information to disk. - */ - constructor(p: string) { - this._path = p; - this._isMapped = fs.existsSync(this.path); - this._created = new Date(Date.now()); - this._hash = null; - this._token = null; - this._cookieJar = new tough.CookieJar(); + // Define the path for the cookiejar + const basedir = path.dirname(p); + this._cookieJarPath = path.join(basedir, this.COOKIEJAR_FILENAME); + } - // Define the path for the cookiejar - const basedir = path.dirname(p); - this._cookieJarPath = path.join(basedir, this.COOKIEJAR_FILENAME); + //#region Private Methods + + /** + * Get the difference in days between two dates. + */ + private dateDiffInDays(a: Date, b: Date) { + const MS_PER_DAY = 1000 * 60 * 60 * 24; + + // Discard the time and time-zone information. + const utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate()); + const utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate()); + + return Math.floor((utc2 - utc1) / MS_PER_DAY); + } + + /** + * Convert the object to a dictionary serializable in JSON. + */ + private toJSON(): Record { + return { + _created: this._created, + _hash: this._hash, + _token: this._token + }; + } + + //#endregion Private Methods + + //#region Public Methods + + /** + * Create a new session. + */ + create(username: string, password: string, token: string): void { + // First, create the _hash of the credentials + const value = `${username}%%%${password}`; + this._hash = sha256(value); + + // Set the token + this._token = token; + + // Update the creation date + this._created = new Date(Date.now()); + } + + /** + * Save the session to disk. + */ + async save(): Promise { + // Update the creation date + this._created = new Date(Date.now()); + + // Convert data + const json = this.toJSON(); + const data = JSON.stringify(json); + + // Write data + await awritefile(this.path, data); + + // Write cookiejar + const serializedJar = await this._cookieJar.serialize(); + await awritefile(this._cookieJarPath, JSON.stringify(serializedJar)); + } + + /** + * Load the session from disk. + */ + async load(): Promise { + if (this.isMapped) { + // Read data + const data = await areadfile(this.path, { encoding: "utf-8", flag: "r" }); + const json = JSON.parse(data); + + // Assign values + this._created = new Date(json._created); + this._hash = json._hash; + this._token = json._token; + + // Load cookiejar + const serializedJar = await areadfile(this._cookieJarPath, { + encoding: "utf-8", + flag: "r" + }); + this._cookieJar = await CookieJar.deserialize(JSON.parse(serializedJar)); } + } - //#region Private Methods + /** + * Delete the session from disk. + */ + async delete(): Promise { + if (this.isMapped) { + // Delete the session data + await aunlinkfile(this.path); - /** - * Get the difference in days between two dates. - */ - private dateDiffInDays(a: Date, b: Date) { - const MS_PER_DAY = 1000 * 60 * 60 * 24; - - // Discard the time and time-zone information. - const utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate()); - const utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate()); - - return Math.floor((utc2 - utc1) / MS_PER_DAY); + // Delete the cookiejar + await aunlinkfile(this._cookieJarPath); } + } - /** - * Convert the object to a dictionary serializable in JSON. - */ - private toJSON(): Record { - return { - _created: this._created, - _hash: this._hash, - _token: this._token, - }; - } + /** + * Check if the session is valid. + */ + isValid(username: string, password: string): boolean { + // Get the number of days from the file creation + const diff = this.dateDiffInDays(new Date(Date.now()), this.created); - //#endregion Private Methods + // The session is valid if the number of days is minor than SESSION_TIME + const dateValid = diff < this.SESSION_TIME; - //#region Public Methods + // Check the hash + const value = `${username}%%%${password}`; + const hashValid = sha256(value) === this._hash; - /** - * Create a new session. - */ - create(username: string, password: string, token: string): void { - // First, create the _hash of the credentials - const value = `${username}%%%${password}`; - this._hash = sha256(value); + // Search for expired cookies + const jarValid = + this._cookieJar + .getCookiesSync("https://f95zone.to") + .filter((el) => el.TTL() === 0).length === 0; - // Set the token - this._token = token; + return dateValid && hashValid && jarValid; + } - // Update the creation date - this._created = new Date(Date.now()); - } - - /** - * Save the session to disk. - */ - async save(): Promise { - // Update the creation date - this._created = new Date(Date.now()); - - // Convert data - const json = this.toJSON(); - const data = JSON.stringify(json); - - // Write data - await awritefile(this.path, data); - - // Write cookiejar - const serializedJar = await this._cookieJar.serialize(); - await awritefile(this._cookieJarPath, JSON.stringify(serializedJar)); - } - - /** - * Load the session from disk. - */ - async load(): Promise { - if (this.isMapped) { - // Read data - const data = await areadfile(this.path, { encoding: 'utf-8', flag: 'r' }); - const json = JSON.parse(data); - - // Assign values - this._created = new Date(json._created); - this._hash = json._hash; - this._token = json._token; - - // Load cookiejar - const serializedJar = await areadfile(this._cookieJarPath, { encoding: 'utf-8', flag: 'r' }); - this._cookieJar = await CookieJar.deserialize(JSON.parse(serializedJar)); - } - } - - /** - * Delete the session from disk. - */ - async delete(): Promise { - if (this.isMapped) { - // Delete the session data - await aunlinkfile(this.path); - - // Delete the cookiejar - await aunlinkfile(this._cookieJarPath); - } - } - - /** - * Check if the session is valid. - */ - isValid(username: string, password: string): boolean { - // Get the number of days from the file creation - const diff = this.dateDiffInDays(new Date(Date.now()), this.created); - - // The session is valid if the number of days is minor than SESSION_TIME - const dateValid = diff < this.SESSION_TIME; - - // Check the hash - const value = `${username}%%%${password}`; - const hashValid = sha256(value) === this._hash; - - // Search for expired cookies - const jarValid = this._cookieJar - .getCookiesSync("https://f95zone.to") - .filter(el => el.TTL() === 0) - .length === 0; - - return dateValid && hashValid && jarValid; - } - - //#endregion Public Methods - -} \ No newline at end of file + //#endregion Public Methods +} diff --git a/src/scripts/constants/css-selector.ts b/src/scripts/constants/css-selector.ts index 670232e..f6a62a7 100644 --- a/src/scripts/constants/css-selector.ts +++ b/src/scripts/constants/css-selector.ts @@ -1,208 +1,216 @@ export const selectors = { - WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WT_NEXT_PAGE: "a.pageNav-jump--next", - WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]", - WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", - GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", - GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", - GS_RESULT_BODY: "div.contentRow-main", - GS_MEMBERSHIP: "li > a:not(.username)", - GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]", - UD_USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", - UD_AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", - LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", - LU_TAGS_SCRIPT: "script:contains('latestUpdates')", - BK_RESULTS: "ol.listPlain > * div.contentRow-main", - BK_POST_URL: "div.contentRow-title > a", - BK_DESCRIPTION: "div.contentRow-snippet", - BK_POST_OWNER: "div.contentRow-minor > * a.username", - BK_TAGS: "div.contentRow-minor > * a.tagItem", - /** - * Attribute `datetime` contains an ISO date. - */ - BK_TIME: "div.contentRow-minor > * time", + WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WT_NEXT_PAGE: "a.pageNav-jump--next", + WT_URLS: 'a[href^="/threads/"][data-tp-primary]', + WT_UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]', + GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", + GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", + GS_RESULT_BODY: "div.contentRow-main", + GS_MEMBERSHIP: "li > a:not(.username)", + GET_REQUEST_TOKEN: 'input[name="_xfToken"]', + UD_USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText', + UD_AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]', + LOGIN_MESSAGE_ERROR: + "div.blockMessage.blockMessage--error.blockMessage--iconic", + LU_TAGS_SCRIPT: "script:contains('latestUpdates')", + BK_RESULTS: "ol.listPlain > * div.contentRow-main", + BK_POST_URL: "div.contentRow-title > a", + BK_DESCRIPTION: "div.contentRow-snippet", + BK_POST_OWNER: "div.contentRow-minor > * a.username", + BK_TAGS: "div.contentRow-minor > * a.tagItem", + /** + * Attribute `datetime` contains an ISO date. + */ + BK_TIME: "div.contentRow-minor > * time" }; export const GENERIC = { - /** - * The ID of the user currently logged into - * the platform in the attribute `data-user-id`. - */ - CURRENT_USER_ID: "span.avatar[data-user-id]", - /** - * Banner containing any error messages as text. - */ - ERROR_BANNER: "div.p-body-pageContent > div.blockMessage", -} + /** + * The ID of the user currently logged into + * the platform in the attribute `data-user-id`. + */ + CURRENT_USER_ID: "span.avatar[data-user-id]", + /** + * Banner containing any error messages as text. + */ + ERROR_BANNER: "div.p-body-pageContent > div.blockMessage" +}; export const WATCHED_THREAD = { - /** - * List of elements containing the data of the watched threads. - */ - BODIES: "div.structItem-cell--main", - /** - * Link element containing the partial URL - * of the thread in the `href` attribute. - * - * It may be followed by the `/unread` segment. - * - * For use within a `WATCHED_THREAD.BODIES` selector. - */ - URL: "div > a[data-tp-primary]", - /** - * Name of the forum to which the thread belongs as text. - * - * For use within a `WATCHED_THREAD.BODIES` selector. - */ - FORUM: "div.structItem-cell--main > div.structItem-minor > ul.structItem-parts > li:last-of-type > a", - /** - * Index of the last page available as text. - */ - LAST_PAGE: "ul.pageNav-main > li:last-child > a" -} + /** + * List of elements containing the data of the watched threads. + */ + BODIES: "div.structItem-cell--main", + /** + * Link element containing the partial URL + * of the thread in the `href` attribute. + * + * It may be followed by the `/unread` segment. + * + * For use within a `WATCHED_THREAD.BODIES` selector. + */ + URL: "div > a[data-tp-primary]", + /** + * Name of the forum to which the thread belongs as text. + * + * For use within a `WATCHED_THREAD.BODIES` selector. + */ + FORUM: + "div.structItem-cell--main > div.structItem-minor > ul.structItem-parts > li:last-of-type > a", + /** + * Index of the last page available as text. + */ + LAST_PAGE: "ul.pageNav-main > li:last-child > a" +}; export const THREAD = { - /** - * Number of pages in the thread (as text of the element). - * - * Two identical elements are identified. - */ - LAST_PAGE: "ul.pageNav-main > li:last-child > a", - /** - * Identify the creator of the thread. - * - * The ID is contained in the `data-user-id` attribute. - */ - OWNER_ID: "div.uix_headerInner > * a.username[data-user-id]", - /** - * Contains the creation date of the thread. - * - * The date is contained in the `datetime` attribute as an ISO string. - */ - CREATION: "div.uix_headerInner > * time", - /** - * List of tags assigned to the thread. - */ - TAGS: "a.tagItem", - /** - * List of prefixes assigned to the thread. - */ - PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", - /** - * Thread title. - */ - TITLE: "h1.p-title-value", - /** - * JSON containing thread information. - * - * Two different elements are found. - */ - JSONLD: "script[type=\"application/ld+json\"]", - /** - * Posts on the current page. - */ - POSTS_IN_PAGE: "article.message", -} + /** + * Number of pages in the thread (as text of the element). + * + * Two identical elements are identified. + */ + LAST_PAGE: "ul.pageNav-main > li:last-child > a", + /** + * Identify the creator of the thread. + * + * The ID is contained in the `data-user-id` attribute. + */ + OWNER_ID: "div.uix_headerInner > * a.username[data-user-id]", + /** + * Contains the creation date of the thread. + * + * The date is contained in the `datetime` attribute as an ISO string. + */ + CREATION: "div.uix_headerInner > * time", + /** + * List of tags assigned to the thread. + */ + TAGS: "a.tagItem", + /** + * List of prefixes assigned to the thread. + */ + PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]', + /** + * Thread title. + */ + TITLE: "h1.p-title-value", + /** + * JSON containing thread information. + * + * Two different elements are found. + */ + JSONLD: 'script[type="application/ld+json"]', + /** + * Posts on the current page. + */ + POSTS_IN_PAGE: "article.message" +}; export const POST = { - /** - * Unique post number for the current thread. - * - * For use within a `THREAD.POSTS_IN_PAGE` selector. - */ - NUMBER: "* ul.message-attribution-opposite > li > a:not([id])[rel=\"nofollow\"]", - /** - * Unique ID of the post in the F95Zone platform in the `id` attribute. - * - * For use within a `THREAD.POSTS_IN_PAGE` selector. - */ - ID: "span[id^=\"post\"]", - /** - * Unique ID of the post author in the `data-user-id` attribute. - * - * For use within a `THREAD.POSTS_IN_PAGE` selector. - */ - OWNER_ID: "* div.message-cell--user > * a[data-user-id]", - /** - * Main body of the post where the message written by the user is contained. - * - * For use within a `THREAD.POSTS_IN_PAGE` selector. - */ - BODY: "* article.message-body > div.bbWrapper", - /** - * Publication date of the post contained in the `datetime` attribute as an ISO date. - * - * For use within a `THREAD.POSTS_IN_PAGE` selector. - */ - PUBLISH_DATE: "* div.message-attribution-main > a > time", - /** - * Last modified date of the post contained in the `datetime` attribute as the ISO date. - * - * For use within a `THREAD.POSTS_IN_PAGE` selector. - */ - LAST_EDIT: "* div.message-lastEdit > time", - /** - * Gets the element only if the post has been bookmarked. - * - * For use within a `THREAD.POSTS_IN_PAGE` selector. - */ - BOOKMARKED: "* ul.message-attribution-opposite >li > a[title=\"Bookmark\"].is-bookmarked", + /** + * Unique post number for the current thread. + * + * For use within a `THREAD.POSTS_IN_PAGE` selector. + */ + NUMBER: + '* ul.message-attribution-opposite > li > a:not([id])[rel="nofollow"]', + /** + * Unique ID of the post in the F95Zone platform in the `id` attribute. + * + * For use within a `THREAD.POSTS_IN_PAGE` selector. + */ + ID: 'span[id^="post"]', + /** + * Unique ID of the post author in the `data-user-id` attribute. + * + * For use within a `THREAD.POSTS_IN_PAGE` selector. + */ + OWNER_ID: "* div.message-cell--user > * a[data-user-id]", + /** + * Main body of the post where the message written by the user is contained. + * + * For use within a `THREAD.POSTS_IN_PAGE` selector. + */ + BODY: "* article.message-body > div.bbWrapper", + /** + * Publication date of the post contained in the `datetime` attribute as an ISO date. + * + * For use within a `THREAD.POSTS_IN_PAGE` selector. + */ + PUBLISH_DATE: "* div.message-attribution-main > a > time", + /** + * Last modified date of the post contained in the `datetime` attribute as the ISO date. + * + * For use within a `THREAD.POSTS_IN_PAGE` selector. + */ + LAST_EDIT: "* div.message-lastEdit > time", + /** + * Gets the element only if the post has been bookmarked. + * + * For use within a `THREAD.POSTS_IN_PAGE` selector. + */ + BOOKMARKED: + '* ul.message-attribution-opposite >li > a[title="Bookmark"].is-bookmarked' }; export const MEMBER = { - /** - * Name of the user. - * - * It also contains the unique ID of the user in the `data-user-id` attribute. - */ - NAME: "span[class^=\"username\"]", - /** - * Title of the user in the platform. - * - * i.e.: Member - */ - TITLE: "span.userTitle", - /** - * Avatar used by the user. - * - * Source in the attribute `src`. - */ - AVATAR: "span.avatarWrapper > a.avatar > img", - /** - * User assigned banners. - * - * The last element is always empty and can be ignored. - */ - BANNERS: "em.userBanner > strong", - /** - * Date the user joined the platform. - * - * The date is contained in the `datetime` attribute as an ISO string. - */ - JOINED: "div.uix_memberHeader__extra > div.memberHeader-blurb:nth-child(1) > * time", - /** - * Last time the user connected to the platform. - * - * The date is contained in the `datetime` attribute as an ISO string. - */ - LAST_SEEN: "div.uix_memberHeader__extra > div.memberHeader-blurb:nth-child(2) > * time", - MESSAGES: "div.pairJustifier > dl:nth-child(1) > * a", - REACTION_SCORE: "div.pairJustifier > dl:nth-child(2) > dd", - POINTS: "div.pairJustifier > dl:nth-child(3) > * a", - RATINGS_RECEIVED: "div.pairJustifier > dl:nth-child(4) > dd", - AMOUNT_DONATED: "div.pairJustifier > dl:nth-child(5) > dd", - /** - * Button used to follow/unfollow the user. - * - * If the text is `Unfollow` then the user is followed. - * If the text is `Follow` then the user is not followed. - */ - FOLLOWED: "div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-follow] > span", - /** - * Button used to ignore/unignore the user. - * - * If the text is `Unignore` then the user is ignored. - * If the text is `Ignore` then the user is not ignored. - */ - IGNORED: "div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-ignore]", -} + /** + * Name of the user. + * + * It also contains the unique ID of the user in the `data-user-id` attribute. + */ + NAME: 'span[class^="username"]', + /** + * Title of the user in the platform. + * + * i.e.: Member + */ + TITLE: "span.userTitle", + /** + * Avatar used by the user. + * + * Source in the attribute `src`. + */ + AVATAR: "span.avatarWrapper > a.avatar > img", + /** + * User assigned banners. + * + * The last element is always empty and can be ignored. + */ + BANNERS: "em.userBanner > strong", + /** + * Date the user joined the platform. + * + * The date is contained in the `datetime` attribute as an ISO string. + */ + JOINED: + "div.uix_memberHeader__extra > div.memberHeader-blurb:nth-child(1) > * time", + /** + * Last time the user connected to the platform. + * + * The date is contained in the `datetime` attribute as an ISO string. + */ + LAST_SEEN: + "div.uix_memberHeader__extra > div.memberHeader-blurb:nth-child(2) > * time", + MESSAGES: "div.pairJustifier > dl:nth-child(1) > * a", + REACTION_SCORE: "div.pairJustifier > dl:nth-child(2) > dd", + POINTS: "div.pairJustifier > dl:nth-child(3) > * a", + RATINGS_RECEIVED: "div.pairJustifier > dl:nth-child(4) > dd", + AMOUNT_DONATED: "div.pairJustifier > dl:nth-child(5) > dd", + /** + * Button used to follow/unfollow the user. + * + * If the text is `Unfollow` then the user is followed. + * If the text is `Follow` then the user is not followed. + */ + FOLLOWED: + "div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-follow] > span", + /** + * Button used to ignore/unignore the user. + * + * If the text is `Unignore` then the user is ignored. + * If the text is `Ignore` then the user is not ignored. + */ + IGNORED: + "div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-ignore]" +}; diff --git a/src/scripts/constants/url.ts b/src/scripts/constants/url.ts index 88c2de9..b08114e 100644 --- a/src/scripts/constants/url.ts +++ b/src/scripts/constants/url.ts @@ -1,26 +1,26 @@ export const urls = { - F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search/search/", - F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_THREADS: "https://f95zone.to/threads/", - F95_LOGIN_URL: "https://f95zone.to/login/login", - F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", - F95_LATEST_PHP: "https://f95zone.to/new_latest.php", - F95_BOOKMARKS: "https://f95zone.to/account/bookmarks", - /** - * Add the unique ID of the post to - * get the thread page where the post - * is present. - */ - F95_POSTS: "https://f95zone.to/posts/", - /** - * @todo - */ - F95_CONVERSATIONS: "https://f95zone.to/conversations/", - /** - * @todo - */ - F95_ALERTS: "https://f95zone.to/account/alerts", - F95_POSTS_NUMBER: "https://f95zone.to/account/dpp-update", - F95_MEMBERS: "https://f95zone.to/members", + F95_BASE_URL: "https://f95zone.to", + F95_SEARCH_URL: "https://f95zone.to/search/search/", + F95_LATEST_UPDATES: "https://f95zone.to/latest", + F95_THREADS: "https://f95zone.to/threads/", + F95_LOGIN_URL: "https://f95zone.to/login/login", + F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", + F95_LATEST_PHP: "https://f95zone.to/new_latest.php", + F95_BOOKMARKS: "https://f95zone.to/account/bookmarks", + /** + * Add the unique ID of the post to + * get the thread page where the post + * is present. + */ + F95_POSTS: "https://f95zone.to/posts/", + /** + * @todo + */ + F95_CONVERSATIONS: "https://f95zone.to/conversations/", + /** + * @todo + */ + F95_ALERTS: "https://f95zone.to/account/alerts", + F95_POSTS_NUMBER: "https://f95zone.to/account/dpp-update", + F95_MEMBERS: "https://f95zone.to/members" }; diff --git a/src/scripts/fetch-data/fetch-handiwork.ts b/src/scripts/fetch-data/fetch-handiwork.ts index 88903eb..8d9b22e 100644 --- a/src/scripts/fetch-data/fetch-handiwork.ts +++ b/src/scripts/fetch-data/fetch-handiwork.ts @@ -16,26 +16,28 @@ import fetchThreadHandiworkURLs from "./fetch-thread.js"; * Maximum number of items to get. Default: 30 * @returns {Promise} URLs of the handiworks */ -export default async function fetchHandiworkURLs(query: HandiworkSearchQuery, limit: number = 30): Promise { - // Local variables - let urls: string[] = null; - const searchType = query.selectSearchType(); +export default async function fetchHandiworkURLs( + query: HandiworkSearchQuery, + limit = 30 +): Promise { + // Local variables + let urls: string[] = null; + const searchType = query.selectSearchType(); - // Convert the query - if (searchType === "latest") { - // Cast the query - const castedQuery = query.cast("LatestSearchQuery"); + // Convert the query + if (searchType === "latest") { + // Cast the query + const castedQuery = query.cast("LatestSearchQuery"); - // Fetch the urls - urls = await fetchLatestHandiworkURLs(castedQuery, limit); - } - else { - // Cast the query - const castedQuery = query.cast("ThreadSearchQuery"); + // Fetch the urls + urls = await fetchLatestHandiworkURLs(castedQuery, limit); + } else { + // Cast the query + const castedQuery = query.cast("ThreadSearchQuery"); - // Fetch the urls - urls = await fetchThreadHandiworkURLs(castedQuery, limit); - } + // Fetch the urls + urls = await fetchThreadHandiworkURLs(castedQuery, limit); + } - return urls; -} \ No newline at end of file + return urls; +} diff --git a/src/scripts/fetch-data/fetch-latest.ts b/src/scripts/fetch-data/fetch-latest.ts index bb6a08a..bb47f91 100644 --- a/src/scripts/fetch-data/fetch-latest.ts +++ b/src/scripts/fetch-data/fetch-latest.ts @@ -4,7 +4,6 @@ import LatestSearchQuery from "../classes/query/latest-search-query.js"; import { urls } from "../constants/url.js"; - /** * Gets the URLs of the latest handiworks that match the passed parameters. * @@ -15,39 +14,44 @@ import { urls } from "../constants/url.js"; * Maximum number of items to get. Default: 30 * @returns {Promise} URLs of the handiworks */ -export default async function fetchLatestHandiworkURLs(query: LatestSearchQuery, limit: number = 30): Promise { - // Local variables - const shallowQuery: LatestSearchQuery = Object.assign(new LatestSearchQuery(), query); - const resultURLs = []; - let fetchedResults = 0; - let noMorePages = false; +export default async function fetchLatestHandiworkURLs( + query: LatestSearchQuery, + limit = 30 +): Promise { + // Local variables + const shallowQuery: LatestSearchQuery = Object.assign( + new LatestSearchQuery(), + query + ); + const resultURLs = []; + let fetchedResults = 0; + let noMorePages = false; - do { - // Fetch the response (application/json) - const response = await shallowQuery.execute(); + do { + // Fetch the response (application/json) + const response = await shallowQuery.execute(); - // Save the URLs - if (response.isSuccess()) { - // In-loop variables - const data: [{ thread_id: number}] = response.value.data.msg.data; - const totalPages: number = response.value.data.msg.pagination.total; - - data.map((e, idx) => { - if (fetchedResults < limit) { - - const gameURL = new URL(e.thread_id.toString(), urls.F95_THREADS).href; - resultURLs.push(gameURL); + // Save the URLs + if (response.isSuccess()) { + // In-loop variables + const data: [{ thread_id: number }] = response.value.data.msg.data; + const totalPages: number = response.value.data.msg.pagination.total; - fetchedResults += 1; - } - }); + data.map((e, idx) => { + if (fetchedResults < limit) { + const gameURL = new URL(e.thread_id.toString(), urls.F95_THREADS) + .href; + resultURLs.push(gameURL); - // Increment page and check for it's existence - shallowQuery.page += 1; - noMorePages = shallowQuery.page > totalPages; - } else throw response.value; - } - while (fetchedResults < limit && !noMorePages); + fetchedResults += 1; + } + }); - return resultURLs; + // Increment page and check for it's existence + shallowQuery.page += 1; + noMorePages = shallowQuery.page > totalPages; + } else throw response.value; + } while (fetchedResults < limit && !noMorePages); + + return resultURLs; } diff --git a/src/scripts/fetch-data/fetch-platform-data.ts b/src/scripts/fetch-data/fetch-platform-data.ts index 701958c..956570c 100644 --- a/src/scripts/fetch-data/fetch-platform-data.ts +++ b/src/scripts/fetch-data/fetch-platform-data.ts @@ -9,7 +9,7 @@ import cheerio from "cheerio"; // Modules from file import shared, { TPrefixDict } from "../shared.js"; import { urls as f95url } from "../constants/url.js"; -import { selectors as f95selector} from "../constants/css-selector.js"; +import { selectors as f95selector } from "../constants/css-selector.js"; import { fetchHTML } from "../network-helper.js"; //#region Interface definitions @@ -17,27 +17,27 @@ import { fetchHTML } from "../network-helper.js"; * Represents the single element contained in the data categories. */ interface ISingleOption { - id: number, - name: string, - class: string + id: number; + name: string; + class: string; } /** * Represents the set of values associated with a specific category of data. */ interface ICategoryResource { - id: number, - name: string, - prefixes: ISingleOption[] + id: number; + name: string; + prefixes: ISingleOption[]; } /** * Represents the set of tags present on the platform- */ interface ILatestResource { - prefixes: { [s: string]: ICategoryResource[] }, - tags: TPrefixDict, - options: string + prefixes: { [s: string]: ICategoryResource[] }; + tags: TPrefixDict; + options: string; } //#endregion Interface definitions @@ -47,22 +47,22 @@ interface ILatestResource { * (such as graphics engines and progress statuses) */ export default async function fetchPlatformData(): Promise { - // Check if the data are cached - if (!readCache(shared.cachePath)) { - // Load the HTML - const html = await fetchHTML(f95url.F95_LATEST_UPDATES); - - // Parse data - if (html.isSuccess()) { - const data = parseLatestPlatformHTML(html.value); + // Check if the data are cached + if (!readCache(shared.cachePath)) { + // Load the HTML + const html = await fetchHTML(f95url.F95_LATEST_UPDATES); - // Assign data - assignLatestPlatformData(data); + // Parse data + if (html.isSuccess()) { + const data = parseLatestPlatformHTML(html.value); - // Cache data - saveCache(shared.cachePath); - } else throw html.value; - } + // Assign data + assignLatestPlatformData(data); + + // Cache data + saveCache(shared.cachePath); + } else throw html.value; + } } //#endregion Public methods @@ -72,21 +72,21 @@ export default async function fetchPlatformData(): Promise { * Read the platform cache (if available) */ function readCache(path: string) { - // Local variables - let returnValue = false; + // Local variables + let returnValue = false; - if (existsSync(path)) { - const data = readFileSync(path, {encoding: "utf-8", flag: "r"}); - const json: { [s: string]: TPrefixDict } = JSON.parse(data); + if (existsSync(path)) { + const data = readFileSync(path, { encoding: "utf-8", flag: "r" }); + const json: { [s: string]: TPrefixDict } = JSON.parse(data); - shared.setPrefixPair("engines", json.engines); - shared.setPrefixPair("statuses", json.statuses); - shared.setPrefixPair("tags", json.tags); - shared.setPrefixPair("others", json.others); - - returnValue = true; - } - return returnValue; + shared.setPrefixPair("engines", json.engines); + shared.setPrefixPair("statuses", json.statuses); + shared.setPrefixPair("tags", json.tags); + shared.setPrefixPair("others", json.others); + + returnValue = true; + } + return returnValue; } /** @@ -94,14 +94,14 @@ function readCache(path: string) { * Save the current platform variables to disk. */ function saveCache(path: string): void { - const saveDict = { - engines: shared.prefixes["engines"], - statuses: shared.prefixes["statuses"], - tags: shared.prefixes["tags"], - others: shared.prefixes["others"], - }; - const json = JSON.stringify(saveDict); - writeFileSync(path, json); + const saveDict = { + engines: shared.prefixes["engines"], + statuses: shared.prefixes["statuses"], + tags: shared.prefixes["tags"], + others: shared.prefixes["others"] + }; + const json = JSON.stringify(saveDict); + writeFileSync(path, json); } /** @@ -109,15 +109,15 @@ function saveCache(path: string): void { * Given the HTML code of the response from the F95Zone, * parse it and return the result. */ -function parseLatestPlatformHTML(html: string): ILatestResource{ - const $ = cheerio.load(html); +function parseLatestPlatformHTML(html: string): ILatestResource { + const $ = cheerio.load(html); - // Clean the JSON string - const unparsedText = $(f95selector.LU_TAGS_SCRIPT).html().trim(); - const startIndex = unparsedText.indexOf("{"); - const endIndex = unparsedText.lastIndexOf("}"); - const parsedText = unparsedText.substring(startIndex, endIndex + 1); - return JSON.parse(parsedText); + // Clean the JSON string + const unparsedText = $(f95selector.LU_TAGS_SCRIPT).html().trim(); + const startIndex = unparsedText.indexOf("{"); + const endIndex = unparsedText.lastIndexOf("}"); + const parsedText = unparsedText.substring(startIndex, endIndex + 1); + return JSON.parse(parsedText); } /** @@ -125,28 +125,28 @@ function parseLatestPlatformHTML(html: string): ILatestResource{ * Assign to the local variables the values from the F95Zone. */ function assignLatestPlatformData(data: ILatestResource): void { - // Local variables - const scrapedData = {}; + // Local variables + const scrapedData = {}; - // Parse and assign the values that are NOT tags - for (const [key, value] of Object.entries(data.prefixes)) { - for (const res of value) { - // Prepare the dict - const dict: TPrefixDict = {}; + // Parse and assign the values that are NOT tags + for (const [key, value] of Object.entries(data.prefixes)) { + for (const res of value) { + // Prepare the dict + const dict: TPrefixDict = {}; - for (const e of res.prefixes) { - dict[e.id] = e.name.replace("'", "'"); - } + for (const e of res.prefixes) { + dict[e.id] = e.name.replace("'", "'"); + } - // Save the property - scrapedData[res.name] = dict; - } + // Save the property + scrapedData[res.name] = dict; } + } - // Save the values - shared.setPrefixPair("engines", Object.assign({}, scrapedData["Engine"])); - shared.setPrefixPair("statuses", Object.assign({}, scrapedData["Status"])); - shared.setPrefixPair("others", Object.assign({}, scrapedData["Other"])); - shared.setPrefixPair("tags", data.tags); + // Save the values + shared.setPrefixPair("engines", Object.assign({}, scrapedData["Engine"])); + shared.setPrefixPair("statuses", Object.assign({}, scrapedData["Status"])); + shared.setPrefixPair("others", Object.assign({}, scrapedData["Other"])); + shared.setPrefixPair("tags", data.tags); } -//#endregion \ No newline at end of file +//#endregion diff --git a/src/scripts/fetch-data/fetch-query.ts b/src/scripts/fetch-data/fetch-query.ts index 15a6b6f..e14a71b 100644 --- a/src/scripts/fetch-data/fetch-query.ts +++ b/src/scripts/fetch-data/fetch-query.ts @@ -16,17 +16,20 @@ import { IQuery } from "../interfaces.js"; * @param limit Maximum number of items to get. Default: 30 * @returns URLs of the fetched games */ -export default async function getURLsFromQuery(query: IQuery, limit: number = 30): Promise { - switch (query.itype) { - case "HandiworkSearchQuery": - return fetchHandiworkURLs(query as HandiworkSearchQuery, limit); - case "LatestSearchQuery": - return fetchLatestHandiworkURLs(query as LatestSearchQuery, limit); - case "ThreadSearchQuery": - return fetchThreadHandiworkURLs(query as ThreadSearchQuery, limit); - default: - throw Error(`Invalid query type: ${query.itype}`); - } +export default async function getURLsFromQuery( + query: IQuery, + limit = 30 +): Promise { + switch (query.itype) { + case "HandiworkSearchQuery": + return fetchHandiworkURLs(query as HandiworkSearchQuery, limit); + case "LatestSearchQuery": + return fetchLatestHandiworkURLs(query as LatestSearchQuery, limit); + case "ThreadSearchQuery": + return fetchThreadHandiworkURLs(query as ThreadSearchQuery, limit); + default: + throw Error(`Invalid query type: ${query.itype}`); + } } -//#endregion \ No newline at end of file +//#endregion diff --git a/src/scripts/fetch-data/fetch-thread.ts b/src/scripts/fetch-data/fetch-thread.ts index 214df73..0a47936 100644 --- a/src/scripts/fetch-data/fetch-thread.ts +++ b/src/scripts/fetch-data/fetch-thread.ts @@ -21,13 +21,17 @@ import ThreadSearchQuery from "../classes/query/thread-search-query.js"; * Maximum number of items to get. Default: 30 * @returns {Promise} URLs of the handiworks */ -export default async function fetchThreadHandiworkURLs(query: ThreadSearchQuery, limit: number = 30): Promise { - // Execute the query - const response = await query.execute(); +export default async function fetchThreadHandiworkURLs( + query: ThreadSearchQuery, + limit = 30 +): Promise { + // Execute the query + const response = await query.execute(); - // Fetch the results from F95 and return the handiwork urls - if (response.isSuccess()) return fetchResultURLs(response.value.data as string, limit); - else throw response.value + // Fetch the results from F95 and return the handiwork urls + if (response.isSuccess()) + return fetchResultURLs(response.value.data as string, limit); + else throw response.value; } //#endregion Public methods @@ -39,20 +43,23 @@ export default async function fetchThreadHandiworkURLs(query: ThreadSearchQuery, * @param {number} limit * Maximum number of items to get. Default: 30 */ -async function fetchResultURLs(html: string, limit: number = 30): Promise { - // Prepare cheerio - const $ = cheerio.load(html); +async function fetchResultURLs(html: string, limit = 30): Promise { + // Prepare cheerio + const $ = cheerio.load(html); - // Here we get all the DIV that are the body of the various query results - const results = $("body").find(f95Selector.GS_RESULT_BODY); + // Here we get all the DIV that are the body of the various query results + const results = $("body").find(f95Selector.GS_RESULT_BODY); - // Than we extract the URLs - const urls = results.slice(0, limit).map((idx, el) => { - const elementSelector = $(el); - return extractLinkFromResult(elementSelector); - }).get(); + // Than we extract the URLs + const urls = results + .slice(0, limit) + .map((idx, el) => { + const elementSelector = $(el); + return extractLinkFromResult(elementSelector); + }) + .get(); - return urls; + return urls; } /** @@ -61,15 +68,15 @@ async function fetchResultURLs(html: string, limit: number = 30): Promise> { - // Fetch the response of the platform - const response = await fetchGETResponse(url); +export async function fetchHTML( + url: string +): Promise> { + // Fetch the response of the platform + const response = await fetchGETResponse(url); - if (response.isSuccess()) { - // Check if the response is a HTML source code - const isHTML = response.value.headers["content-type"].includes("text/html"); + if (response.isSuccess()) { + // Check if the response is a HTML source code + const isHTML = response.value.headers["content-type"].includes("text/html"); - const unexpectedResponseError = new UnexpectedResponseContentType({ - id: 2, - message: `Expected HTML but received ${response.value["content-type"]}`, - error: null - }); + const unexpectedResponseError = new UnexpectedResponseContentType({ + id: 2, + message: `Expected HTML but received ${response.value["content-type"]}`, + error: null + }); - return isHTML ? - success(response.value.data as string) : - failure(unexpectedResponseError); - } else return failure(response.value as GenericAxiosError); + return isHTML + ? success(response.value.data as string) + : failure(unexpectedResponseError); + } else return failure(response.value as GenericAxiosError); } /** @@ -75,102 +82,118 @@ export async function fetchHTML(url: string): Promise} Result of the operation */ -export async function authenticate(credentials: credentials, force: boolean = false): Promise { - shared.logger.info(`Authenticating with user ${credentials.username}`); - if (!credentials.token) throw new InvalidF95Token(`Invalid token for auth: ${credentials.token}`); +export async function authenticate( + credentials: credentials, + force = false +): Promise { + shared.logger.info(`Authenticating with user ${credentials.username}`); + if (!credentials.token) + throw new InvalidF95Token(`Invalid token for auth: ${credentials.token}`); - // Secure the URL - const secureURL = enforceHttpsUrl(f95url.F95_LOGIN_URL); + // Secure the URL + const secureURL = enforceHttpsUrl(f95url.F95_LOGIN_URL); - // Prepare the parameters to send to the platform to authenticate - const params = { - "login": credentials.username, - "url": "", - "password": credentials.password, - "password_confirm": "", - "additional_security": "", - "remember": "1", - "_xfRedirect": "https://f95zone.to/", - "website_code": "", - "_xfToken": credentials.token, - }; + // Prepare the parameters to send to the platform to authenticate + const params = { + login: credentials.username, + url: "", + password: credentials.password, + password_confirm: "", + additional_security: "", + remember: "1", + _xfRedirect: "https://f95zone.to/", + website_code: "", + _xfToken: credentials.token + }; - try { - // Try to log-in - const response = await fetchPOSTResponse(f95url.F95_LOGIN_URL, params, force); + try { + // Try to log-in + const response = await fetchPOSTResponse( + f95url.F95_LOGIN_URL, + params, + force + ); - if (response.isSuccess()) { - // Parse the response HTML - const $ = cheerio.load(response.value.data as string); + if (response.isSuccess()) { + // Parse the response HTML + const $ = cheerio.load(response.value.data as string); - // Get the error message (if any) and remove the new line chars - const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, ""); + // Get the error message (if any) and remove the new line chars + const errorMessage = $("body") + .find(f95selector.LOGIN_MESSAGE_ERROR) + .text() + .replace(/\n/g, ""); - // Return the result of the authentication - const result = errorMessage.trim() === ""; - const message = result ? "Authentication successful" : errorMessage; - return new LoginResult(result, message); - } - else throw response.value; - } catch (e) { - shared.logger.error(`Error ${e.message} occurred while authenticating to ${secureURL}`); - return new LoginResult(false, `Error ${e.message} while authenticating`); - } -}; + // Return the result of the authentication + const result = errorMessage.trim() === ""; + const message = result ? "Authentication successful" : errorMessage; + return new LoginResult(result, message); + } else throw response.value; + } catch (e) { + shared.logger.error( + `Error ${e.message} occurred while authenticating to ${secureURL}` + ); + return new LoginResult(false, `Error ${e.message} while authenticating`); + } +} /** * Obtain the token used to authenticate the user to the platform. */ export async function getF95Token() { - // Fetch the response of the platform - const response = await fetchGETResponse(f95url.F95_LOGIN_URL); + // Fetch the response of the platform + const response = await fetchGETResponse(f95url.F95_LOGIN_URL); - if (response.isSuccess()) { - // The response is a HTML page, we need to find the with name "_xfToken" - const $ = cheerio.load(response.value.data as string); - return $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value"); - } else throw response.value; + if (response.isSuccess()) { + // The response is a HTML page, we need to find the with name "_xfToken" + const $ = cheerio.load(response.value.data as string); + return $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value"); + } else throw response.value; } //#region Utility methods /** * Performs a GET request to a specific URL and returns the response. */ -export async function fetchGETResponse(url: string): Promise>> { - // Secure the URL - const secureURL = enforceHttpsUrl(url); +export async function fetchGETResponse( + url: string +): Promise>> { + // Secure the URL + const secureURL = enforceHttpsUrl(url); - try { - // Fetch and return the response - commonConfig.jar = shared.session.cookieJar; - const response = await axios.get(secureURL, commonConfig); - return success(response); - } catch (e) { - console.log(e.response); - shared.logger.error(`(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`); - const genericError = new GenericAxiosError({ - id: 1, - message:`(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`, - error: e - }); - return failure(genericError); - } + try { + // Fetch and return the response + commonConfig.jar = shared.session.cookieJar; + const response = await axios.get(secureURL, commonConfig); + return success(response); + } catch (e) { + console.log(e.response); + shared.logger.error( + `(GET) Error ${e.message} occurred while trying to fetch ${secureURL}` + ); + const genericError = new GenericAxiosError({ + id: 1, + message: `(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`, + error: e + }); + return failure(genericError); + } } /** * Enforces the scheme of the URL is https and returns the new URL. */ export function enforceHttpsUrl(url: string): string { - if (isStringAValidURL(url)) return url.replace(/^(https?:)?\/\//, "https://"); - else throw new Error(`${url} is not a valid URL`); -}; + if (isStringAValidURL(url)) return url.replace(/^(https?:)?\/\//, "https://"); + else throw new Error(`${url} is not a valid URL`); +} /** * Check if the url belongs to the domain of the F95 platform. */ export function isF95URL(url: string): boolean { - return url.toString().startsWith(f95url.F95_BASE_URL); -}; + return url.toString().startsWith(f95url.F95_BASE_URL); +} /** * Checks if the string passed by parameter has a @@ -178,11 +201,11 @@ export function isF95URL(url: string): boolean { * @param {String} url String to check for correctness */ export function isStringAValidURL(url: string): boolean { - // Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc) - const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/; - const regex = new RegExp(expression); - return url.match(regex).length > 0; -}; + // Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc) + const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/; + const regex = new RegExp(expression); + return url.match(regex).length > 0; +} /** * Check if a particular URL is valid and reachable on the web. @@ -192,20 +215,23 @@ export function isStringAValidURL(url: string): boolean { * Default: false * @returns {Promise} true if the URL exists, false otherwise */ -export async function urlExists(url: string, checkRedirect: boolean = false): Promise { - // Local variables - let valid = false; +export async function urlExists( + url: string, + checkRedirect = false +): Promise { + // Local variables + let valid = false; - if (isStringAValidURL(url)) { - valid = await axiosUrlExists(url); + if (isStringAValidURL(url)) { + valid = await axiosUrlExists(url); - if (valid && checkRedirect) { - const redirectUrl = await getUrlRedirect(url); - valid = redirectUrl === url; - } + if (valid && checkRedirect) { + const redirectUrl = await getUrlRedirect(url); + valid = redirectUrl === url; } + } - return valid; + return valid; } /** @@ -214,8 +240,8 @@ export async function urlExists(url: string, checkRedirect: boolean = false): Pr * @returns {Promise} Redirect URL or the passed URL */ export async function getUrlRedirect(url: string): Promise { - const response = await axios.head(url); - return response.config.url; + const response = await axios.head(url); + return response.config.url; } /** @@ -224,34 +250,41 @@ export async function getUrlRedirect(url: string): Promise { * @param params List of value pairs to send with the request * @param force If `true`, the request ignores the sending of cookies already present on the device. */ -export async function fetchPOSTResponse(url: string, params: { [s: string]: string }, force: boolean = false): Promise>> { - // Secure the URL - const secureURL = enforceHttpsUrl(url); - - // Prepare the parameters for the POST request - const urlParams = new URLSearchParams(); - for (const [key, value] of Object.entries(params)) urlParams.append(key, value); +export async function fetchPOSTResponse( + url: string, + params: { [s: string]: string }, + force = false +): Promise>> { + // Secure the URL + const secureURL = enforceHttpsUrl(url); - // Shallow copy of the common configuration object - commonConfig.jar = shared.session.cookieJar; - const config = Object.assign({}, commonConfig); + // Prepare the parameters for the POST request + const urlParams = new URLSearchParams(); + for (const [key, value] of Object.entries(params)) + urlParams.append(key, value); - // Remove the cookies if forced - if (force) delete config.jar; + // Shallow copy of the common configuration object + commonConfig.jar = shared.session.cookieJar; + const config = Object.assign({}, commonConfig); - // Send the POST request and await the response - try { - const response = await axios.post(secureURL, urlParams, config); - return success(response); - } catch (e) { - shared.logger.error(`(POST) Error ${e.message} occurred while trying to fetch ${secureURL}`); - const genericError = new GenericAxiosError({ - id: 3, - message: `(POST) Error ${e.message} occurred while trying to fetch ${secureURL}`, - error: e - }); - return failure(genericError); - } + // Remove the cookies if forced + if (force) delete config.jar; + + // Send the POST request and await the response + try { + const response = await axios.post(secureURL, urlParams, config); + return success(response); + } catch (e) { + shared.logger.error( + `(POST) Error ${e.message} occurred while trying to fetch ${secureURL}` + ); + const genericError = new GenericAxiosError({ + id: 3, + message: `(POST) Error ${e.message} occurred while trying to fetch ${secureURL}`, + error: e + }); + return failure(genericError); + } } //#endregion Utility methods @@ -261,20 +294,20 @@ export async function fetchPOSTResponse(url: string, params: { [s: string]: stri * Check with Axios if a URL exists. */ async function axiosUrlExists(url: string): Promise { - // Local variables - const ERROR_CODES = ["ENOTFOUND", "ETIMEDOUT"]; - let valid = false; + // Local variables + const ERROR_CODES = ["ENOTFOUND", "ETIMEDOUT"]; + let valid = false; - try { - const response = await axios.head(url, { - timeout: 3000 - }); - valid = response && !/4\d\d/.test(response.status.toString()); - } catch (error) { - // Throw error only if the error is unknown - if (!ERROR_CODES.includes(error.code)) throw error; - } + try { + const response = await axios.head(url, { + timeout: 3000 + }); + valid = response && !/4\d\d/.test(response.status.toString()); + } catch (error) { + // Throw error only if the error is unknown + if (!ERROR_CODES.includes(error.code)) throw error; + } - return valid; + return valid; } //#endregion diff --git a/src/scripts/scrape-data/handiwork-parse.ts b/src/scripts/scrape-data/handiwork-parse.ts index a333cb4..58de683 100644 --- a/src/scripts/scrape-data/handiwork-parse.ts +++ b/src/scripts/scrape-data/handiwork-parse.ts @@ -6,13 +6,23 @@ import luxon from "luxon"; // Modules from files import HandiWork from "../classes/handiwork/handiwork.js"; import Thread from "../classes/mapping/thread.js"; -import { IBasic, TAuthor, TEngine, TExternalPlatform, TStatus } from "../interfaces.js"; +import { + IBasic, + TAuthor, + TEngine, + TExternalPlatform, + TStatus +} from "../interfaces.js"; import shared, { TPrefixDict } from "../shared.js"; import { ILink, IPostElement } from "./post-parse.js"; -export async function getHandiworkInformation(url: string): Promise +export async function getHandiworkInformation( + url: string +): Promise; -export async function getHandiworkInformation(url: string): Promise +export async function getHandiworkInformation( + url: string +): Promise; /** * Gets information of a particular handiwork from its thread. @@ -21,36 +31,38 @@ export async function getHandiworkInformation(url: string): Pr * * @todo It does not currently support assets. */ -export default async function getHandiworkInformation(arg: string | Thread): Promise { - // Local variables - let thread: Thread = null; +export default async function getHandiworkInformation( + arg: string | Thread +): Promise { + // Local variables + let thread: Thread = null; - if (typeof arg === "string") { - // Fetch thread data - const id = extractIDFromURL(arg); - thread = new Thread(id); - await thread.fetch(); - } else thread = arg; + if (typeof arg === "string") { + // Fetch thread data + const id = extractIDFromURL(arg); + thread = new Thread(id); + await thread.fetch(); + } else thread = arg; - shared.logger.info(`Obtaining handiwork from ${thread.url}`); + shared.logger.info(`Obtaining handiwork from ${thread.url}`); - // Convert the info from thread to handiwork - const hw: HandiWork = {} as HandiWork; - hw.id = thread.id; - hw.url = thread.url; - hw.name = thread.title; - hw.category = thread.category; - hw.threadPublishingDate = thread.publication; - hw.lastThreadUpdate = thread.modified; - hw.tags = thread.tags; - hw.rating = thread.rating; - fillWithPrefixes(hw, thread.prefixes); + // Convert the info from thread to handiwork + const hw: HandiWork = {} as HandiWork; + hw.id = thread.id; + hw.url = thread.url; + hw.name = thread.title; + hw.category = thread.category; + hw.threadPublishingDate = thread.publication; + hw.lastThreadUpdate = thread.modified; + hw.tags = thread.tags; + hw.rating = thread.rating; + fillWithPrefixes(hw, thread.prefixes); - // Fetch info from first post - const post = await thread.getPost(1); - fillWithPostData(hw, post.body); + // Fetch info from first post + const post = await thread.getPost(1); + fillWithPostData(hw, post.body); - return hw; + return (hw); } //#region Private methods @@ -61,28 +73,28 @@ export default async function getHandiworkInformation(arg: str * Extracts the work's unique ID from its URL. */ function extractIDFromURL(url: string): number { - shared.logger.trace("Extracting ID from URL..."); + shared.logger.trace("Extracting ID from URL..."); - // URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/ - // or https://f95zone.to/threads/ID/ - const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/); - if (!match) return -1; + // URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/ + // or https://f95zone.to/threads/ID/ + const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/); + if (!match) return -1; - // Parse and return number - return parseInt(match[0], 10); + // Parse and return number + return parseInt(match[0], 10); } /** * Makes an array of strings uppercase. */ function toUpperCaseArray(a: string[]): string[] { - /** - * Makes a string uppercase. - */ - function toUpper(s: string): string { - return s.toUpperCase(); - } - return a.map(toUpper); + /** + * Makes a string uppercase. + */ + function toUpper(s: string): string { + return s.toUpperCase(); + } + return a.map(toUpper); } /** @@ -91,10 +103,10 @@ function toUpperCaseArray(a: string[]): string[] { * Case insensitive. */ function stringInDict(s: string, a: TPrefixDict): boolean { - // Make uppercase all the strings in the array - const values = toUpperCaseArray(Object.values(a)); + // Make uppercase all the strings in the array + const values = toUpperCaseArray(Object.values(a)); - return values.includes(s.toUpperCase()); + return values.includes(s.toUpperCase()); } /** @@ -103,15 +115,15 @@ function stringInDict(s: string, a: TPrefixDict): boolean { * Check also for `yes`/`no` and `1`/`0`. */ function stringToBoolean(s: string): boolean { - // Local variables - const positiveTerms = ["true", "yes", "1"]; - const negativeTerms = ["false", "no", "0"]; - const cleanString = s.toLowerCase().trim(); - let result = Boolean(s); + // Local variables + const positiveTerms = ["true", "yes", "1"]; + const negativeTerms = ["false", "no", "0"]; + const cleanString = s.toLowerCase().trim(); + let result = Boolean(s); - if (positiveTerms.includes(cleanString)) result = true; - else if (negativeTerms.includes(cleanString)) result = false; - return result; + if (positiveTerms.includes(cleanString)) result = true; + else if (negativeTerms.includes(cleanString)) result = false; + return result; } /** @@ -119,8 +131,11 @@ function stringToBoolean(s: string): boolean { * * Case-insensitive. */ -function getPostElementByName(elements: IPostElement[], name: string): IPostElement | undefined { - return elements.find(el => el.name.toUpperCase() === name.toUpperCase()); +function getPostElementByName( + elements: IPostElement[], + name: string +): IPostElement | undefined { + return elements.find((el) => el.name.toUpperCase() === name.toUpperCase()); } //#endregion Utilities @@ -132,43 +147,45 @@ function getPostElementByName(elements: IPostElement[], name: string): IPostElem * `Engine`, `Status`, `Mod`. */ function fillWithPrefixes(hw: HandiWork, prefixes: string[]) { - shared.logger.trace("Parsing prefixes..."); + shared.logger.trace("Parsing prefixes..."); - // Local variables - let mod = false; - let engine: TEngine = null; - let status: TStatus = null; + // Local variables + let mod = false; + let engine: TEngine = null; + let status: TStatus = null; - /** - * Emulated dictionary of mod prefixes. - */ - const fakeModDict: TPrefixDict = { - 0: "MOD", - 1: "CHEAT MOD", - } + /** + * Emulated dictionary of mod prefixes. + */ + const fakeModDict: TPrefixDict = { + 0: "MOD", + 1: "CHEAT MOD" + }; - // Initialize the array - hw.prefixes = []; + // Initialize the array + hw.prefixes = []; - prefixes.map((item, idx) => { - // Remove the square brackets - const prefix = item.replace("[", "").replace("]", ""); + prefixes.map((item, idx) => { + // Remove the square brackets + const prefix = item.replace("[", "").replace("]", ""); - // Check what the prefix indicates - if (stringInDict(prefix, shared.prefixes["engines"])) engine = prefix as TEngine; - else if (stringInDict(prefix, shared.prefixes["statuses"])) status = prefix as TStatus; - else if (stringInDict(prefix, fakeModDict)) mod = true; + // Check what the prefix indicates + if (stringInDict(prefix, shared.prefixes["engines"])) + engine = prefix as TEngine; + else if (stringInDict(prefix, shared.prefixes["statuses"])) + status = prefix as TStatus; + else if (stringInDict(prefix, fakeModDict)) mod = true; - // Anyway add the prefix to list - hw.prefixes.push(prefix); - }); + // Anyway add the prefix to list + hw.prefixes.push(prefix); + }); - // If the status is not set, then the game is in development (Ongoing) - status = (!status && hw.category === "games") ? status : "Ongoing"; + // If the status is not set, then the game is in development (Ongoing) + status = !status && hw.category === "games" ? status : "Ongoing"; - hw.engine = engine; - hw.status = status; - hw.mod = mod; + hw.engine = engine; + hw.status = status; + hw.mod = mod; } /** @@ -181,70 +198,87 @@ function fillWithPrefixes(hw: HandiWork, prefixes: string[]) { * `LastRelease`, `Authors`, `Changelog`, `Cover`. */ function fillWithPostData(hw: HandiWork, elements: IPostElement[]) { - // First fill the "simple" elements - hw.overview = getPostElementByName(elements, "overview")?.text; - hw.os = getPostElementByName(elements, "os")?.text?.split(",").map(s => s.trim()); - hw.language = getPostElementByName(elements, "language")?.text?.split(",").map(s => s.trim()); - hw.version = getPostElementByName(elements, "version")?.text; - hw.installation = getPostElementByName(elements, "installation")?.content.shift()?.text; - hw.pages = getPostElementByName(elements, "pages")?.text; - hw.resolution = getPostElementByName(elements, "resolution")?.text?.split(",").map(s => s.trim()); - hw.lenght = getPostElementByName(elements, "lenght")?.text; + // First fill the "simple" elements + hw.overview = getPostElementByName(elements, "overview")?.text; + hw.os = getPostElementByName(elements, "os") + ?.text?.split(",") + .map((s) => s.trim()); + hw.language = getPostElementByName(elements, "language") + ?.text?.split(",") + .map((s) => s.trim()); + hw.version = getPostElementByName(elements, "version")?.text; + hw.installation = getPostElementByName( + elements, + "installation" + )?.content.shift()?.text; + hw.pages = getPostElementByName(elements, "pages")?.text; + hw.resolution = getPostElementByName(elements, "resolution") + ?.text?.split(",") + .map((s) => s.trim()); + hw.lenght = getPostElementByName(elements, "lenght")?.text; - // Parse the censorship - const censored = getPostElementByName(elements, "censored") || getPostElementByName(elements, "censorship"); - if (censored) hw.censored = stringToBoolean(censored.text); + // Parse the censorship + const censored = + getPostElementByName(elements, "censored") || + getPostElementByName(elements, "censorship"); + if (censored) hw.censored = stringToBoolean(censored.text); - // Get the genres - const genre = getPostElementByName(elements, "genre")?.content.shift()?.text; - hw.genre = genre?.split(",").map(s => s.trim()); + // Get the genres + const genre = getPostElementByName(elements, "genre")?.content.shift()?.text; + hw.genre = genre?.split(",").map((s) => s.trim()); - // Get the cover - const cover = getPostElementByName(elements, "overview")?.content.find(el => el.type === "Image") as ILink; - hw.cover = cover?.href; + // Get the cover + const cover = getPostElementByName(elements, "overview")?.content.find( + (el) => el.type === "Image" + ) as ILink; + hw.cover = cover?.href; - // Fill the dates - const releaseDate = getPostElementByName(elements, "release date")?.text; - if (luxon.DateTime.fromISO(releaseDate).isValid) hw.lastRelease = new Date(releaseDate); + // Fill the dates + const releaseDate = getPostElementByName(elements, "release date")?.text; + if (luxon.DateTime.fromISO(releaseDate).isValid) + hw.lastRelease = new Date(releaseDate); - //#region Convert the author - const authorElement = getPostElementByName(elements, "developer") || - getPostElementByName(elements, "developer/publisher") || - getPostElementByName(elements, "artist"); - const author: TAuthor = { - name: authorElement?.text, - platforms: [] + //#region Convert the author + const authorElement = + getPostElementByName(elements, "developer") || + getPostElementByName(elements, "developer/publisher") || + getPostElementByName(elements, "artist"); + const author: TAuthor = { + name: authorElement?.text, + platforms: [] + }; + + // Add the found platforms + authorElement?.content.forEach((el: ILink, idx) => { + const platform: TExternalPlatform = { + name: el.text, + link: el.href }; - // Add the found platforms - authorElement?.content.forEach((el: ILink, idx) => { - const platform: TExternalPlatform = { - name: el.text, - link: el.href, - }; + author.platforms.push(platform); + }); + hw.authors = [author]; + //#endregion Convert the author - author.platforms.push(platform); + //#region Get the changelog + hw.changelog = []; + const changelogElement = + getPostElementByName(elements, "changelog") || + getPostElementByName(elements, "change-log"); + if (changelogElement) { + const changelogSpoiler = changelogElement?.content.find((el) => { + return el.type === "Spoiler" && el.content.length > 0; }); - hw.authors = [author]; - //#endregion Convert the author - //#region Get the changelog - hw.changelog = []; - const changelogElement = getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log"); - if (changelogElement) { - const changelogSpoiler = changelogElement?.content.find(el => { - return el.type === "Spoiler" && el.content.length > 0; - }); + // Add to the changelog the single spoilers + changelogSpoiler?.content.forEach((el) => { + if (el.text.trim()) hw.changelog.push(el.text); + }); - // Add to the changelog the single spoilers - changelogSpoiler?.content.forEach(el => { - if (el.text.trim()) hw.changelog.push(el.text); - }); - - // Add at the end also the text of the "changelog" element - hw.changelog.push(changelogSpoiler.text); - } - //#endregion Get the changelog + // Add at the end also the text of the "changelog" element + hw.changelog.push(changelogSpoiler.text); + } + //#endregion Get the changelog } -//#endregion Private methods \ No newline at end of file +//#endregion Private methods diff --git a/src/scripts/scrape-data/json-ld.ts b/src/scripts/scrape-data/json-ld.ts index f724a6d..27df6c5 100644 --- a/src/scripts/scrape-data/json-ld.ts +++ b/src/scripts/scrape-data/json-ld.ts @@ -10,7 +10,7 @@ import { THREAD } from "../constants/css-selector.js"; /** * Represents information contained in a JSON+LD tag. */ -export type TJsonLD = { [s: string]: string | TJsonLD } +export type TJsonLD = { [s: string]: string | TJsonLD }; /** * Extracts and processes the JSON-LD values of the page. @@ -18,16 +18,16 @@ export type TJsonLD = { [s: string]: string | TJsonLD } * @returns {TJsonLD[]} List of data obtained from the page */ export function getJSONLD(body: cheerio.Cheerio): TJsonLD { - shared.logger.trace("Extracting JSON-LD data..."); + shared.logger.trace("Extracting JSON-LD data..."); - // Fetch the JSON-LD data - const structuredDataElements = body.find(THREAD.JSONLD); + // Fetch the JSON-LD data + const structuredDataElements = body.find(THREAD.JSONLD); - // Parse the data - const values = structuredDataElements.map((idx, el) => parseJSONLD(el)).get(); + // Parse the data + const values = structuredDataElements.map((idx, el) => parseJSONLD(el)).get(); - // Merge the data and return a single value - return mergeJSONLD(values); + // Merge the data and return a single value + return mergeJSONLD(values); } //#region Private methods @@ -36,29 +36,29 @@ export function getJSONLD(body: cheerio.Cheerio): TJsonLD { * @param data List of JSON+LD tags */ function mergeJSONLD(data: TJsonLD[]): TJsonLD { - // Local variables - let merged: TJsonLD = {}; + // Local variables + let merged: TJsonLD = {}; - for (const value of data) { - merged = Object.assign(merged, value); - } + for (const value of data) { + merged = Object.assign(merged, value); + } - return merged; + return merged; } /** * Parse a JSON-LD element source code. */ function parseJSONLD(element: cheerio.Element): TJsonLD { - // Get the element HTML - const html = cheerio(element).html().trim(); + // Get the element HTML + const html = cheerio(element).html().trim(); - // Obtain the JSON-LD - const data = html - .replace("", ""); + // Obtain the JSON-LD + const data = html + .replace('", ""); - // Convert the string to an object - return JSON.parse(data); + // Convert the string to an object + return JSON.parse(data); } -//#endregion Private methods \ No newline at end of file +//#endregion Private methods diff --git a/src/scripts/scrape-data/post-parse.ts b/src/scripts/scrape-data/post-parse.ts index 52aaf80..b54c58b 100644 --- a/src/scripts/scrape-data/post-parse.ts +++ b/src/scripts/scrape-data/post-parse.ts @@ -3,15 +3,15 @@ //#region Interfaces export interface IPostElement { - type: "Empty" | "Text" | "Link" | "Image" | "Spoiler", - name: string, - text: string, - content: IPostElement[] + type: "Empty" | "Text" | "Link" | "Image" | "Spoiler"; + name: string; + text: string; + content: IPostElement[]; } export interface ILink extends IPostElement { - type: "Image" | "Link", - href: string, + type: "Image" | "Link"; + href: string; } //#endregion Interfaces @@ -20,24 +20,31 @@ export interface ILink extends IPostElement { /** * Given a post of a thread page it extracts the information contained in the body. */ -export function parseF95ThreadPost($: cheerio.Root, post: cheerio.Cheerio): IPostElement[] { - // The data is divided between "tag" and "text" elements. - // Simple data is composed of a "tag" element followed - // by a "text" element, while more complex data (contained - // in spoilers) is composed of a "tag" element, followed - // by a text containing only ":" and then by an additional - // "tag" element having as the first term "Spoiler" +export function parseF95ThreadPost( + $: cheerio.Root, + post: cheerio.Cheerio +): IPostElement[] { + // The data is divided between "tag" and "text" elements. + // Simple data is composed of a "tag" element followed + // by a "text" element, while more complex data (contained + // in spoilers) is composed of a "tag" element, followed + // by a text containing only ":" and then by an additional + // "tag" element having as the first term "Spoiler" - // First fetch all the elements in the post - const elements = post.contents().toArray().map(el => { - const node = parseCheerioNode($, el); - if (node.name || node.text || node.content.length != 0) { - return node; - } - }).filter(el => el); + // First fetch all the elements in the post + const elements = post + .contents() + .toArray() + .map((el) => { + const node = parseCheerioNode($, el); + if (node.name || node.text || node.content.length != 0) { + return node; + } + }) + .filter((el) => el); - // ... then parse the elements to create the pairs of title/data - return parsePostElements(elements); + // ... then parse the elements to create the pairs of title/data + return parsePostElements(elements); } //#endregion Public methods @@ -47,50 +54,57 @@ export function parseF95ThreadPost($: cheerio.Root, post: cheerio.Cheerio): IPos * Process a spoiler element by getting its text broken * down by any other spoiler elements present. */ -function parseCheerioSpoilerNode($: cheerio.Root, spoiler: cheerio.Cheerio): IPostElement { - // A spoiler block is composed of a div with class "bbCodeSpoiler", - // containing a div "bbCodeSpoiler-content" containing, in cascade, - // a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content". - // This last tag contains the required data. +function parseCheerioSpoilerNode( + $: cheerio.Root, + spoiler: cheerio.Cheerio +): IPostElement { + // A spoiler block is composed of a div with class "bbCodeSpoiler", + // containing a div "bbCodeSpoiler-content" containing, in cascade, + // a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content". + // This last tag contains the required data. - // Local variables - const BUTTON_CLASS = "button.bbCodeSpoiler-button"; - const SPOILER_CONTENT_CLASS = "div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content"; - const content: IPostElement = { - type: "Spoiler", - name: "", - text: "", - content: [] - }; + // Local variables + const BUTTON_CLASS = "button.bbCodeSpoiler-button"; + const SPOILER_CONTENT_CLASS = + "div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content"; + const content: IPostElement = { + type: "Spoiler", + name: "", + text: "", + content: [] + }; - // Find the title of the spoiler (contained in the button) - const button = spoiler.find(BUTTON_CLASS).toArray().shift(); - content.name = $(button).text().trim(); + // Find the title of the spoiler (contained in the button) + const button = spoiler.find(BUTTON_CLASS).toArray().shift(); + content.name = $(button).text().trim(); - // Parse the content of the spoiler - spoiler.find(SPOILER_CONTENT_CLASS).contents().map((idx, el) => { - // Convert the element - const element = $(el); + // Parse the content of the spoiler + spoiler + .find(SPOILER_CONTENT_CLASS) + .contents() + .map((idx, el) => { + // Convert the element + const element = $(el); - // Parse nested spoiler - if (element.attr("class") === "bbCodeSpoiler") { - const spoiler = parseCheerioSpoilerNode($, element); - content.content.push(spoiler); - } - //@ts-ignore - // else if (el.name === "br") { - // // Add new line - // content.text += "\n"; - // } - else if (el.type === "text") { - // Append text - content.text += element.text(); - } + // Parse nested spoiler + if (element.attr("class") === "bbCodeSpoiler") { + const spoiler = parseCheerioSpoilerNode($, element); + content.content.push(spoiler); + } + //@ts-ignore + // else if (el.name === "br") { + // // Add new line + // content.text += "\n"; + // } + else if (el.type === "text") { + // Append text + content.text += element.text(); + } }); - // Clean text - content.text = content.text.replace(/\s\s+/g, ' ').trim(); - return content; + // Clean text + content.text = content.text.replace(/\s\s+/g, " ").trim(); + return content; } /** @@ -98,11 +112,11 @@ function parseCheerioSpoilerNode($: cheerio.Root, spoiler: cheerio.Cheerio): IPo * This also includes formatted nodes (i.e. ``). */ function isTextNode(node: cheerio.Element): boolean { - const formattedTags = ["b", "i"] - const isText = node.type === "text"; - const isFormatted = node.type === "tag" && formattedTags.includes(node.name); + const formattedTags = ["b", "i"]; + const isText = node.type === "text"; + const isFormatted = node.type === "tag" && formattedTags.includes(node.name); - return isText || isFormatted; + return isText || isFormatted; } /** @@ -110,13 +124,17 @@ function isTextNode(node: cheerio.Element): boolean { * Also includes formatted text elements (i.e. ``). */ function getCheerioNonChildrenText(node: cheerio.Cheerio): string { - // Find all the text nodes in the node - const text = node.first().contents().filter((idx, el) => { - return isTextNode(el); - }).text(); + // Find all the text nodes in the node + const text = node + .first() + .contents() + .filter((idx, el) => { + return isTextNode(el); + }) + .text(); - // Clean and return the text - return text.replace(/\s\s+/g, ' ').trim(); + // Clean and return the text + return text.replace(/\s\s+/g, " ").trim(); } /** @@ -124,28 +142,27 @@ function getCheerioNonChildrenText(node: cheerio.Cheerio): string { * link or image. If not, it returns `null`. */ function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null { - //@ts-ignore - const name = element[0]?.name; - const link: ILink = { - name: "", - type: "Link", - text: "", - href: "", - content: [] - }; + //@ts-ignore + const name = element[0]?.name; + const link: ILink = { + name: "", + type: "Link", + text: "", + href: "", + content: [] + }; - if (name === "img") { - link.type = "Image"; - link.text = element.attr("alt"); - link.href = element.attr("data-src"); - } - else if (name === "a") { - link.type = "Link"; - link.text = element.text().replace(/\s\s+/g, ' ').trim(); - link.href = element.attr("href"); - } + if (name === "img") { + link.type = "Image"; + link.text = element.attr("alt"); + link.href = element.attr("data-src"); + } else if (name === "a") { + link.type = "Link"; + link.text = element.text().replace(/\s\s+/g, " ").trim(); + link.href = element.attr("href"); + } - return link.href ? link : null; + return link.href ? link : null; } /** @@ -153,84 +170,93 @@ function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null { * in the `Content` field in case it has no information. */ function reducePostElement(element: IPostElement): IPostElement { - if (element.content.length === 1) { - const content = element.content[0] as IPostElement; - const nullValues = (!element.name || !content.name) && (!element.text || !content.text); - const sameValues = (element.name === content.name) || (element.text === content.text) + if (element.content.length === 1) { + const content = element.content[0] as IPostElement; + const nullValues = + (!element.name || !content.name) && (!element.text || !content.text); + const sameValues = + element.name === content.name || element.text === content.text; - if (nullValues || sameValues) { - element.name = element.name || content.name; - element.text = element.text || content.text; - element.content.push(...content.content); - element.type = content.type; + if (nullValues || sameValues) { + element.name = element.name || content.name; + element.text = element.text || content.text; + element.content.push(...content.content); + element.type = content.type; - // If the content is a link, add the HREF to the element - const contentILink = content as ILink; - const elementILink = element as ILink; - if (contentILink.href) elementILink.href = contentILink.href; - } + // If the content is a link, add the HREF to the element + const contentILink = content as ILink; + const elementILink = element as ILink; + if (contentILink.href) elementILink.href = contentILink.href; } + } - return element; + return element; } /** * Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes. * @param reduce Compress subsequent subnodes if they contain no information. Default: `true`. */ -function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true): IPostElement { - // Local variables - let content: IPostElement = { - type: "Empty", - name: "", - text: "", - content: [] - }; - const cheerioNode = $(node); +function parseCheerioNode( + $: cheerio.Root, + node: cheerio.Element, + reduce = true +): IPostElement { + // Local variables + const content: IPostElement = { + type: "Empty", + name: "", + text: "", + content: [] + }; + const cheerioNode = $(node); - if (isTextNode(node)) { - content.text = cheerioNode.text().replace(/\s\s+/g, ' ').trim(); - content.type = "Text"; - } else { - // Get the number of children that the element own - const nChildren = cheerioNode.children().length; + if (isTextNode(node)) { + content.text = cheerioNode.text().replace(/\s\s+/g, " ").trim(); + content.type = "Text"; + } else { + // Get the number of children that the element own + const nChildren = cheerioNode.children().length; - // Get the text of the element without childrens - content.text = getCheerioNonChildrenText(cheerioNode); + // Get the text of the element without childrens + content.text = getCheerioNonChildrenText(cheerioNode); - // Parse spoilers - if (cheerioNode.attr("class") === "bbCodeSpoiler") { - const spoiler = parseCheerioSpoilerNode($, cheerioNode); + // Parse spoilers + if (cheerioNode.attr("class") === "bbCodeSpoiler") { + const spoiler = parseCheerioSpoilerNode($, cheerioNode); - // Add element if not null - if (spoiler) { - content.content.push(spoiler); - content.type = "Spoiler"; - } - } - // Parse links - else if (nChildren === 0 && cheerioNode.length != 0) { - const link = parseCheerioLinkNode(cheerioNode); - - // Add element if not null - if (link) { - content.content.push(link); - content.type = "Link"; - } - } else { - cheerioNode.children().map((idx, el) => { - // Parse the children of the element passed as parameter - const childElement = parseCheerioNode($, el); - - // If the children is valid (not empty) push it - if ((childElement.text || childElement.content.length !== 0) && !isTextNode(el)) { - content.content.push(childElement); - } - }); - } + // Add element if not null + if (spoiler) { + content.content.push(spoiler); + content.type = "Spoiler"; + } } + // Parse links + else if (nChildren === 0 && cheerioNode.length != 0) { + const link = parseCheerioLinkNode(cheerioNode); - return reduce ? reducePostElement(content) : content; + // Add element if not null + if (link) { + content.content.push(link); + content.type = "Link"; + } + } else { + cheerioNode.children().map((idx, el) => { + // Parse the children of the element passed as parameter + const childElement = parseCheerioNode($, el); + + // If the children is valid (not empty) push it + if ( + (childElement.text || childElement.content.length !== 0) && + !isTextNode(el) + ) { + content.content.push(childElement); + } + }); + } + } + + return reduce ? reducePostElement(content) : content; } /** @@ -238,49 +264,49 @@ function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true) * the corresponding value to each characterizing element (i.e. author). */ function parsePostElements(elements: IPostElement[]): IPostElement[] { - // Local variables - const pairs: IPostElement[] = []; - const specialCharsRegex = /^[-!$%^&*()_+|~=`{}\[\]:";'<>?,.\/]/; - const specialRegex = new RegExp(specialCharsRegex); + // Local variables + const pairs: IPostElement[] = []; + const specialCharsRegex = /^[-!$%^&*()_+|~=`{}\[\]:";'<>?,.\/]/; + const specialRegex = new RegExp(specialCharsRegex); - for (let i = 0; i < elements.length; i++) { - // If the text starts with a special char, clean it - const startWithSpecial = specialRegex.test(elements[i].text); + for (let i = 0; i < elements.length; i++) { + // If the text starts with a special char, clean it + const startWithSpecial = specialRegex.test(elements[i].text); - // Get the latest IPostElement in "pairs" - const lastIndex = pairs.length - 1; - const lastPair = pairs[lastIndex]; + // Get the latest IPostElement in "pairs" + const lastIndex = pairs.length - 1; + const lastPair = pairs[lastIndex]; - // If this statement is valid, we have a "data" - if (elements[i].type === "Text" && startWithSpecial && pairs.length > 0) { - // We merge this element with the last element appended to 'pairs' - const cleanText = elements[i].text.replace(specialCharsRegex, "").trim(); - lastPair.text = lastPair.text || cleanText; - lastPair.content.push(...elements[i].content); - } - // This is a special case - else if (elements[i].text.startsWith("Overview:\n")) { - // We add the overview to the pairs as a text element - elements[i].type = "Text"; - elements[i].name = "Overview"; - elements[i].text = elements[i].text.replace("Overview:\n", ""); - pairs.push(elements[i]); - } - // We have an element referred to the previous "title" - else if (elements[i].type != "Text" && pairs.length > 0) { - // We append this element to the content of the last title - lastPair.content.push(elements[i]); - } - // ... else we have a "title" (we need to swap the text to the name because it is a title) - else { - const swap: IPostElement = Object.assign({}, elements[i]); - swap.name = elements[i].text; - swap.text = ""; - pairs.push(swap); - } + // If this statement is valid, we have a "data" + if (elements[i].type === "Text" && startWithSpecial && pairs.length > 0) { + // We merge this element with the last element appended to 'pairs' + const cleanText = elements[i].text.replace(specialCharsRegex, "").trim(); + lastPair.text = lastPair.text || cleanText; + lastPair.content.push(...elements[i].content); } + // This is a special case + else if (elements[i].text.startsWith("Overview:\n")) { + // We add the overview to the pairs as a text element + elements[i].type = "Text"; + elements[i].name = "Overview"; + elements[i].text = elements[i].text.replace("Overview:\n", ""); + pairs.push(elements[i]); + } + // We have an element referred to the previous "title" + else if (elements[i].type != "Text" && pairs.length > 0) { + // We append this element to the content of the last title + lastPair.content.push(elements[i]); + } + // ... else we have a "title" (we need to swap the text to the name because it is a title) + else { + const swap: IPostElement = Object.assign({}, elements[i]); + swap.name = elements[i].text; + swap.text = ""; + pairs.push(swap); + } + } - return pairs; + return pairs; } -//#endregion Private methods \ No newline at end of file +//#endregion Private methods diff --git a/src/scripts/search.ts b/src/scripts/search.ts index bd72d34..a4102fd 100644 --- a/src/scripts/search.ts +++ b/src/scripts/search.ts @@ -11,14 +11,17 @@ import getURLsFromQuery from "./fetch-data/fetch-query.js"; * @param {Number} limit * Maximum number of items to get. Default: 30 */ -export default async function search(query: IQuery, limit: number = 30): Promise { - // Fetch the URLs - const urls: string[] = await getURLsFromQuery(query, limit); +export default async function search( + query: IQuery, + limit = 30 +): Promise { + // Fetch the URLs + const urls: string[] = await getURLsFromQuery(query, limit); - // Fetch the data - const results = urls.map((url, idx) => { - return getHandiworkInformation(url); - }); + // Fetch the data + const results = urls.map((url, idx) => { + return getHandiworkInformation(url); + }); - return Promise.all(results); + return Promise.all(results); } diff --git a/src/scripts/shared.ts b/src/scripts/shared.ts index 568acce..be9ce9a 100644 --- a/src/scripts/shared.ts +++ b/src/scripts/shared.ts @@ -12,53 +12,68 @@ import log4js from "log4js"; import Session from "./classes/session.js"; // Types declaration -export type TPrefixDict = { [n: number]: string; }; +export type TPrefixDict = { [n: number]: string }; type TPrefixKey = "engines" | "statuses" | "tags" | "others"; /** * Class containing variables shared between modules. */ export default abstract class Shared { - - //#region Fields + //#region Fields - private static _isLogged = false; - private static _prefixes: { [key in TPrefixKey]: TPrefixDict } = {} as { [key in TPrefixKey]: TPrefixDict }; - private static _logger: log4js.Logger = log4js.getLogger(); - private static _session = new Session(join(tmpdir(), "f95session.json")); + private static _isLogged = false; + private static _prefixes: { [key in TPrefixKey]: TPrefixDict } = {} as { + [key in TPrefixKey]: TPrefixDict; + }; + private static _logger: log4js.Logger = log4js.getLogger(); + private static _session = new Session(join(tmpdir(), "f95session.json")); - //#endregion Fields + //#endregion Fields - //#region Getters + //#region Getters - /** - * Indicates whether a user is logged in to the F95Zone platform or not. - */ - static get isLogged(): boolean { return this._isLogged; } - /** - * List of platform prefixes and tags. - */ - static get prefixes(): { [s: string]: TPrefixDict } { return this._prefixes; } - /** - * Logger object used to write to both file and console. - */ - static get logger(): log4js.Logger { return this._logger; } - /** - * Path to the cache used by this module wich contains engines, statuses, tags... - */ - static get cachePath(): string { return join(tmpdir(), "f95cache.json"); } - /** - * Session on the F95Zone platform. - */ - static get session(): Session { return this._session; } + /** + * Indicates whether a user is logged in to the F95Zone platform or not. + */ + static get isLogged(): boolean { + return this._isLogged; + } + /** + * List of platform prefixes and tags. + */ + static get prefixes(): { [s: string]: TPrefixDict } { + return this._prefixes; + } + /** + * Logger object used to write to both file and console. + */ + static get logger(): log4js.Logger { + return this._logger; + } + /** + * Path to the cache used by this module wich contains engines, statuses, tags... + */ + static get cachePath(): string { + return join(tmpdir(), "f95cache.json"); + } + /** + * Session on the F95Zone platform. + */ + static get session(): Session { + return this._session; + } - //#endregion Getters + //#endregion Getters - //#region Setters + //#region Setters - static setPrefixPair(key: TPrefixKey, val: TPrefixDict): void { this._prefixes[key] = val; } + static setPrefixPair(key: TPrefixKey, val: TPrefixDict): void { + this._prefixes[key] = val; + } - static setIsLogged(val: boolean): void { this._isLogged = val; } + static setIsLogged(val: boolean): void { + this._isLogged = val; + } - //#endregion Setters + //#endregion Setters }