Translate scripts
parent
52264a487a
commit
e508149257
|
@ -9,10 +9,10 @@ F95_PASSWORD = YOUR_PASSWORD
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Public modules from npm
|
// Public modules from npm
|
||||||
const dotenv = require("dotenv");
|
import dotenv from "dotenv";
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
const F95API = require("./index.js");
|
import { login, getUserData, getLatestUpdates, getGameData} from "./index";
|
||||||
|
|
||||||
// Configure the .env reader
|
// Configure the .env reader
|
||||||
dotenv.config();
|
dotenv.config();
|
||||||
|
@ -29,16 +29,16 @@ async function main() {
|
||||||
|
|
||||||
// Log in the platform
|
// Log in the platform
|
||||||
console.log("Authenticating...");
|
console.log("Authenticating...");
|
||||||
const result = await F95API.login(process.env.F95_USERNAME, process.env.F95_PASSWORD);
|
const result = await login(process.env.F95_USERNAME, process.env.F95_PASSWORD);
|
||||||
console.log(`Authentication result: ${result.message}\n`);
|
console.log(`Authentication result: ${result.message}\n`);
|
||||||
|
|
||||||
// Get user data
|
// Get user data
|
||||||
console.log("Fetching user data...");
|
console.log("Fetching user data...");
|
||||||
const userdata = await F95API.getUserData();
|
const userdata = await getUserData();
|
||||||
console.log(`${userdata.username} follows ${userdata.watchedGameThreads.length} threads\n`);
|
console.log(`${userdata.username} follows ${userdata.watchedGameThreads.length} threads\n`);
|
||||||
|
|
||||||
// Get latest game update
|
// Get latest game update
|
||||||
const latestUpdates = await F95API.getLatestUpdates({
|
const latestUpdates = await getLatestUpdates({
|
||||||
tags: ["3d game"]
|
tags: ["3d game"]
|
||||||
}, 1);
|
}, 1);
|
||||||
console.log(`"${latestUpdates[0].name}" was the last "3d game" tagged game to be updated\n`);
|
console.log(`"${latestUpdates[0].name}" was the last "3d game" tagged game to be updated\n`);
|
||||||
|
@ -46,7 +46,7 @@ async function main() {
|
||||||
// Get game data
|
// Get game data
|
||||||
for(const gamename of gameList) {
|
for(const gamename of gameList) {
|
||||||
console.log(`Searching '${gamename}'...`);
|
console.log(`Searching '${gamename}'...`);
|
||||||
const found = await F95API.getGameData(gamename, false);
|
const found = await getGameData(gamename, false);
|
||||||
|
|
||||||
// If no game is found
|
// If no game is found
|
||||||
if (found.length === 0) {
|
if (found.length === 0) {
|
||||||
|
|
77
src/index.ts
77
src/index.ts
|
@ -1,20 +1,20 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
const shared = require("./scripts/shared.js");
|
import shared from "./scripts/shared.js";
|
||||||
const networkHelper = require("./scripts/network-helper.js");
|
import { authenticate, urlExists, isF95URL } from "./scripts/network-helper.js";
|
||||||
const scraper = require("./scripts/scraper.js");
|
import { getGameInfo } from "./scripts/scraper.js";
|
||||||
const searcher = require("./scripts/searcher.js");
|
import { searchGame, searchMod } from "./scripts/searcher.js";
|
||||||
const uScraper = require("./scripts/user-scraper.js");
|
import { getUserData as retrieveUserData } from "./scripts/user-scraper.js";
|
||||||
const latestFetch = require("./scripts/latest-fetch.js");
|
import { fetchLatest } from "./scripts/latest-fetch.js";
|
||||||
const fetchPlatformData = require("./scripts/platform-data.js").fetchPlatformData;
|
const fetchPlatformData = require("./scripts/platform-data.js").fetchPlatformData;
|
||||||
|
|
||||||
// Classes from file
|
// Classes from file
|
||||||
const Credentials = require("./scripts/classes/credentials.js");
|
import Credentials from "./scripts/classes/credentials.js";
|
||||||
const GameInfo = require("./scripts/classes/game-info.js");
|
import GameInfo from "./scripts/classes/game-info.js";
|
||||||
const LoginResult = require("./scripts/classes/login-result.js");
|
import LoginResult from "./scripts/classes/login-result.js";
|
||||||
const UserData = require("./scripts/classes/user-data.js");
|
import UserData from "./scripts/classes/user-data.js";
|
||||||
const PrefixParser = require("./scripts/classes/prefix-parser.js");
|
import PrefixParser from "./scripts/classes/prefix-parser.js";
|
||||||
|
|
||||||
//#region Global variables
|
//#region Global variables
|
||||||
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
|
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
|
||||||
|
@ -35,26 +35,22 @@ module.exports.PrefixParser = PrefixParser;
|
||||||
/* istambul ignore next */
|
/* istambul ignore next */
|
||||||
module.exports.loggerLevel = shared.logger.level;
|
module.exports.loggerLevel = shared.logger.level;
|
||||||
exports.loggerLevel = "warn"; // By default log only the warn messages
|
exports.loggerLevel = "warn"; // By default log only the warn messages
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
|
||||||
* Indicates whether a user is logged in to the F95Zone platform or not.
|
* Indicates whether a user is logged in to the F95Zone platform or not.
|
||||||
* @returns {String}
|
|
||||||
*/
|
*/
|
||||||
module.exports.isLogged = function isLogged() {
|
export function isLogged(): boolean {
|
||||||
return shared.isLogged;
|
return shared.isLogged;
|
||||||
};
|
};
|
||||||
//#endregion Export properties
|
//#endregion Export properties
|
||||||
|
|
||||||
//#region Export methods
|
//#region Export methods
|
||||||
/**
|
/**
|
||||||
* @public
|
|
||||||
* Log in to the F95Zone platform.
|
* Log in to the F95Zone platform.
|
||||||
* This **must** be the first operation performed before accessing any other script functions.
|
* This **must** be the first operation performed before accessing any other script functions.
|
||||||
* @param {String} username Username used for login
|
|
||||||
* @param {String} password Password used for login
|
|
||||||
* @returns {Promise<LoginResult>} Result of the operation
|
* @returns {Promise<LoginResult>} Result of the operation
|
||||||
*/
|
*/
|
||||||
module.exports.login = async function (username, password) {
|
export async function login(username: string, password: string): Promise<LoginResult> {
|
||||||
/* istanbul ignore next */
|
/* istanbul ignore next */
|
||||||
if (shared.isLogged) {
|
if (shared.isLogged) {
|
||||||
shared.logger.info(`${username} already authenticated`);
|
shared.logger.info(`${username} already authenticated`);
|
||||||
|
@ -66,8 +62,8 @@ module.exports.login = async function (username, password) {
|
||||||
await creds.fetchToken();
|
await creds.fetchToken();
|
||||||
|
|
||||||
shared.logger.trace(`Authentication for ${username}`);
|
shared.logger.trace(`Authentication for ${username}`);
|
||||||
const result = await networkHelper.authenticate(creds);
|
const result = await authenticate(creds);
|
||||||
shared.isLogged = result.success;
|
shared.setIsLogged(result.success);
|
||||||
|
|
||||||
// Load platform data
|
// Load platform data
|
||||||
if (result.success) await fetchPlatformData();
|
if (result.success) await fetchPlatformData();
|
||||||
|
@ -80,13 +76,12 @@ module.exports.login = async function (username, password) {
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
|
||||||
* Chek if exists a new version of the game.
|
* Chek if exists a new version of the game.
|
||||||
* You **must** be logged in to the portal before calling this method.
|
* You **must** be logged in to the portal before calling this method.
|
||||||
* @param {GameInfo} info Information about the game to get the version for
|
* @param {GameInfo} info Information about the game to get the version for
|
||||||
* @returns {Promise<Boolean>} true if an update is available, false otherwise
|
* @returns {Promise<Boolean>} true if an update is available, false otherwise
|
||||||
*/
|
*/
|
||||||
module.exports.checkIfGameHasUpdate = async function (info) {
|
export async function checkIfGameHasUpdate(info: GameInfo): Promise<boolean> {
|
||||||
/* istanbul ignore next */
|
/* istanbul ignore next */
|
||||||
if (!shared.isLogged) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
|
@ -95,11 +90,11 @@ module.exports.checkIfGameHasUpdate = async function (info) {
|
||||||
|
|
||||||
// F95 change URL at every game update,
|
// F95 change URL at every game update,
|
||||||
// so if the URL is different an update is available
|
// so if the URL is different an update is available
|
||||||
const exists = await networkHelper.urlExists(info.url, true);
|
const exists = await urlExists(info.url, true);
|
||||||
if (!exists) return true;
|
if (!exists) return true;
|
||||||
|
|
||||||
// Parse version from title
|
// Parse version from title
|
||||||
const onlineInfo = await scraper.getGameInfo(info.url);
|
const onlineInfo = await getGameInfo(info.url);
|
||||||
const onlineVersion = onlineInfo.version;
|
const onlineVersion = onlineInfo.version;
|
||||||
|
|
||||||
// Compare the versions
|
// Compare the versions
|
||||||
|
@ -107,7 +102,6 @@ module.exports.checkIfGameHasUpdate = async function (info) {
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
|
||||||
* Starting from the name, it gets all the information about the game you are looking for.
|
* Starting from the name, it gets all the information about the game you are looking for.
|
||||||
* You **must** be logged in to the portal before calling this method.
|
* You **must** be logged in to the portal before calling this method.
|
||||||
* @param {String} name Name of the game searched
|
* @param {String} name Name of the game searched
|
||||||
|
@ -115,7 +109,7 @@ module.exports.checkIfGameHasUpdate = async function (info) {
|
||||||
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
|
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
|
||||||
* an identified game (in the case of homonymy of titles)
|
* an identified game (in the case of homonymy of titles)
|
||||||
*/
|
*/
|
||||||
module.exports.getGameData = async function (name, mod) {
|
export async function getGameData (name: string, mod: boolean): Promise<GameInfo[]> {
|
||||||
/* istanbul ignore next */
|
/* istanbul ignore next */
|
||||||
if (!shared.isLogged) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
|
@ -124,27 +118,26 @@ module.exports.getGameData = async function (name, mod) {
|
||||||
|
|
||||||
// Gets the search results of the game/mod being searched for
|
// Gets the search results of the game/mod being searched for
|
||||||
const urls = mod ?
|
const urls = mod ?
|
||||||
await searcher.searchMod(name) :
|
await searchMod(name) :
|
||||||
await searcher.searchGame(name);
|
await searchGame(name);
|
||||||
|
|
||||||
// Process previous partial results
|
// Process previous partial results
|
||||||
const results = [];
|
const results = [];
|
||||||
for (const url of urls) {
|
for (const url of urls) {
|
||||||
// Start looking for information
|
// Start looking for information
|
||||||
const info = await scraper.getGameInfo(url);
|
const info = await getGameInfo(url);
|
||||||
if (info) results.push(info);
|
if (info) results.push(info);
|
||||||
}
|
}
|
||||||
return results;
|
return results;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
|
||||||
* Starting from the url, it gets all the information about the game you are looking for.
|
* Starting from the url, it gets all the information about the game you are looking for.
|
||||||
* You **must** be logged in to the portal before calling this method.
|
* You **must** be logged in to the portal before calling this method.
|
||||||
* @param {String} url URL of the game to obtain information of
|
* @param {String} url URL of the game to obtain information of
|
||||||
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
|
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
|
||||||
*/
|
*/
|
||||||
module.exports.getGameDataFromURL = async function (url) {
|
export async function getGameDataFromURL(url: string): Promise<GameInfo> {
|
||||||
/* istanbul ignore next */
|
/* istanbul ignore next */
|
||||||
if (!shared.isLogged) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
|
@ -152,32 +145,30 @@ module.exports.getGameDataFromURL = async function (url) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check URL validity
|
// Check URL validity
|
||||||
const exists = await networkHelper.urlExists(url);
|
const exists = await urlExists(url);
|
||||||
if (!exists) throw new URIError(`${url} is not a valid URL`);
|
if (!exists) throw new URIError(`${url} is not a valid URL`);
|
||||||
if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
|
if (!isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
|
||||||
|
|
||||||
// Get game data
|
// Get game data
|
||||||
return await scraper.getGameInfo(url);
|
return await getGameInfo(url);
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
|
||||||
* Gets the data of the currently logged in user.
|
* Gets the data of the currently logged in user.
|
||||||
* You **must** be logged in to the portal before calling this method.
|
* You **must** be logged in to the portal before calling this method.
|
||||||
* @returns {Promise<UserData>} Data of the user currently logged in
|
* @returns {Promise<UserData>} Data of the user currently logged in
|
||||||
*/
|
*/
|
||||||
module.exports.getUserData = async function () {
|
export async function getUserData(): Promise<UserData> {
|
||||||
/* istanbul ignore next */
|
/* istanbul ignore next */
|
||||||
if (!shared.isLogged) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
return await uScraper.getUserData();
|
return await retrieveUserData();
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
|
||||||
* Gets the latest updated games that match the specified parameters.
|
* Gets the latest updated games that match the specified parameters.
|
||||||
* You **must** be logged in to the portal before calling this method.
|
* You **must** be logged in to the portal before calling this method.
|
||||||
* @param {Object} args
|
* @param {Object} args
|
||||||
|
@ -196,7 +187,7 @@ module.exports.getUserData = async function () {
|
||||||
* @param {Number} limit Maximum number of results
|
* @param {Number} limit Maximum number of results
|
||||||
* @returns {Promise<GameInfo[]>} List of games
|
* @returns {Promise<GameInfo[]>} List of games
|
||||||
*/
|
*/
|
||||||
module.exports.getLatestUpdates = async function(args, limit) {
|
export async function getLatestUpdates(args, limit: number): Promise<GameInfo[]> {
|
||||||
// Check limit value
|
// Check limit value
|
||||||
if(limit <= 0) throw new Error("limit must be greater than 0");
|
if(limit <= 0) throw new Error("limit must be greater than 0");
|
||||||
|
|
||||||
|
@ -217,22 +208,20 @@ module.exports.getLatestUpdates = async function(args, limit) {
|
||||||
sort: args.sorting ? args.sorting : "date",
|
sort: args.sorting ? args.sorting : "date",
|
||||||
date: filterDate,
|
date: filterDate,
|
||||||
};
|
};
|
||||||
const urls = await latestFetch.fetchLatest(query, limit);
|
const urls = await fetchLatest(query, limit);
|
||||||
|
|
||||||
// Get the gamedata from urls
|
// Get the gamedata from urls
|
||||||
const promiseList = urls.map(u => exports.getGameDataFromURL(u));
|
const promiseList = urls.map((u: string) => exports.getGameDataFromURL(u));
|
||||||
return await Promise.all(promiseList);
|
return await Promise.all(promiseList);
|
||||||
};
|
};
|
||||||
//#endregion
|
//#endregion
|
||||||
|
|
||||||
//#region Private Methods
|
//#region Private Methods
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* Given an array of numbers, get the nearest value for a given `value`.
|
* Given an array of numbers, get the nearest value for a given `value`.
|
||||||
* @param {Number[]} array List of default values
|
* @param {Number[]} array List of default values
|
||||||
* @param {Number} value Value to search
|
* @param {Number} value Value to search
|
||||||
*/
|
*/
|
||||||
function getNearestValueFromArray(array, value) {
|
function getNearestValueFromArray(array: number[], value: number) {
|
||||||
// Script taken from:
|
// Script taken from:
|
||||||
// https://www.gavsblog.com/blog/find-closest-number-in-array-javascript
|
// https://www.gavsblog.com/blog/find-closest-number-in-array-javascript
|
||||||
array.sort((a, b) => {
|
array.sort((a, b) => {
|
||||||
|
|
|
@ -1,8 +1,9 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
import { fetchGETResponse } from "./network-helper.js";
|
import { fetchGETResponse } from "./network-helper";
|
||||||
import SearchQuery from "./classes/search-query";
|
import SearchQuery from "./classes/search-query";
|
||||||
|
import { urls as f95url } from "./constants/url";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
* @public
|
||||||
|
|
|
@ -1,16 +1,17 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Public modules from npm
|
// Public modules from npm
|
||||||
const axios = require("axios").default;
|
import axios, { AxiosResponse } from "axios";
|
||||||
const cheerio = require("cheerio");
|
import cheerio from "cheerio";
|
||||||
const axiosCookieJarSupport = require("axios-cookiejar-support").default;
|
import axiosCookieJarSupport from "axios-cookiejar-support";
|
||||||
const tough = require("tough-cookie");
|
import tough from "tough-cookie";
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
const shared = require("./shared.js");
|
import shared from "./shared";
|
||||||
const f95url = require("./constants/url.js");
|
import { urls as f95url } from "./constants/url";
|
||||||
const f95selector = require("./constants/css-selector.js");
|
import { selectors as f95selector } from "./constants/css-selector";
|
||||||
const LoginResult = require("./classes/login-result.js");
|
import LoginResult from "./classes/login-result";
|
||||||
|
import credentials from "./classes/credentials";
|
||||||
|
|
||||||
// Global variables
|
// Global variables
|
||||||
const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " +
|
const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " +
|
||||||
|
@ -27,12 +28,10 @@ const commonConfig = {
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Gets the HTML code of a page.
|
* Gets the HTML code of a page.
|
||||||
* @param {String} url URL to fetch
|
|
||||||
* @returns {Promise<String>} HTML code or `null` if an error arise
|
* @returns {Promise<String>} HTML code or `null` if an error arise
|
||||||
*/
|
*/
|
||||||
module.exports.fetchHTML = async function (url) {
|
export async function fetchHTML(url: string): Promise<string|null> {
|
||||||
// Local variables
|
// Local variables
|
||||||
let returnValue = null;
|
let returnValue = null;
|
||||||
|
|
||||||
|
@ -52,18 +51,17 @@ module.exports.fetchHTML = async function (url) {
|
||||||
|
|
||||||
returnValue = response.data;
|
returnValue = response.data;
|
||||||
return returnValue;
|
return returnValue;
|
||||||
};
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* It authenticates to the platform using the credentials
|
* It authenticates to the platform using the credentials
|
||||||
* and token obtained previously. Save cookies on your
|
* and token obtained previously. Save cookies on your
|
||||||
* device after authentication.
|
* device after authentication.
|
||||||
* @param {Credentials} credentials Platform access credentials
|
* @param {module:./classes/credentials.ts:Credentials} credentials Platform access credentials
|
||||||
* @param {Boolean} force Specifies whether the request should be forced, ignoring any saved cookies
|
* @param {Boolean} force Specifies whether the request should be forced, ignoring any saved cookies
|
||||||
* @returns {Promise<LoginResult>} Result of the operation
|
* @returns {Promise<LoginResult>} Result of the operation
|
||||||
*/
|
*/
|
||||||
module.exports.authenticate = async function (credentials, force) {
|
export async function authenticate(credentials: credentials, force: boolean = false): Promise<LoginResult> {
|
||||||
shared.logger.info(`Authenticating with user ${credentials.username}`);
|
shared.logger.info(`Authenticating with user ${credentials.username}`);
|
||||||
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
|
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
|
||||||
|
|
||||||
|
@ -108,7 +106,7 @@ module.exports.authenticate = async function (credentials, force) {
|
||||||
* Obtain the token used to authenticate the user to the platform.
|
* Obtain the token used to authenticate the user to the platform.
|
||||||
* @returns {Promise<String>} Token or `null` if an error arise
|
* @returns {Promise<String>} Token or `null` if an error arise
|
||||||
*/
|
*/
|
||||||
module.exports.getF95Token = async function() {
|
export async function getF95Token(): Promise<string|null> {
|
||||||
// Fetch the response of the platform
|
// Fetch the response of the platform
|
||||||
const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL);
|
const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL);
|
||||||
/* istambul ignore next */
|
/* istambul ignore next */
|
||||||
|
@ -119,18 +117,15 @@ module.exports.getF95Token = async function() {
|
||||||
|
|
||||||
// The response is a HTML page, we need to find the <input> with name "_xfToken"
|
// The response is a HTML page, we need to find the <input> with name "_xfToken"
|
||||||
const $ = cheerio.load(response.data);
|
const $ = cheerio.load(response.data);
|
||||||
const token = $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value");
|
return $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value");
|
||||||
return token;
|
}
|
||||||
};
|
|
||||||
|
|
||||||
//#region Utility methods
|
//#region Utility methods
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Performs a GET request to a specific URL and returns the response.
|
* Performs a GET request to a specific URL and returns the response.
|
||||||
* If the request generates an error (for example 400) `null` is returned.
|
* If the request generates an error (for example 400) `null` is returned.
|
||||||
* @param {String} url
|
|
||||||
*/
|
*/
|
||||||
module.exports.fetchGETResponse = async function(url) {
|
export async function fetchGETResponse(url: string): Promise<AxiosResponse<unknown>> {
|
||||||
// Secure the URL
|
// Secure the URL
|
||||||
const secureURL = exports.enforceHttpsUrl(url);
|
const secureURL = exports.enforceHttpsUrl(url);
|
||||||
|
|
||||||
|
@ -141,37 +136,34 @@ module.exports.fetchGETResponse = async function(url) {
|
||||||
shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`);
|
shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Enforces the scheme of the URL is https and returns the new URL.
|
* Enforces the scheme of the URL is https and returns the new URL.
|
||||||
* @param {String} url
|
* @param {String} url
|
||||||
* @returns {String} Secure URL or `null` if the argument is not a string
|
* @returns {String} Secure URL or `null` if the argument is not a string
|
||||||
*/
|
*/
|
||||||
module.exports.enforceHttpsUrl = function (url) {
|
export function enforceHttpsUrl(url: string): string {
|
||||||
return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
|
return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Check if the url belongs to the domain of the F95 platform.
|
* Check if the url belongs to the domain of the F95 platform.
|
||||||
* @param {String} url URL to check
|
* @param {String} url URL to check
|
||||||
* @returns {Boolean} true if the url belongs to the domain, false otherwise
|
* @returns {Boolean} true if the url belongs to the domain, false otherwise
|
||||||
*/
|
*/
|
||||||
module.exports.isF95URL = function (url) {
|
export function isF95URL(url: string): boolean {
|
||||||
if (url.toString().startsWith(f95url.F95_BASE_URL)) return true;
|
if (url.toString().startsWith(f95url.F95_BASE_URL)) return true;
|
||||||
else return false;
|
else return false;
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Checks if the string passed by parameter has a
|
* Checks if the string passed by parameter has a
|
||||||
* properly formatted and valid path to a URL (HTTP/HTTPS).
|
* properly formatted and valid path to a URL (HTTP/HTTPS).
|
||||||
* @param {String} url String to check for correctness
|
* @param {String} url String to check for correctness
|
||||||
* @returns {Boolean} true if the string is a valid URL, false otherwise
|
* @returns {Boolean} true if the string is a valid URL, false otherwise
|
||||||
*/
|
*/
|
||||||
module.exports.isStringAValidURL = function (url) {
|
export function isStringAValidURL(url: string): boolean {
|
||||||
// Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc)
|
// Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc)
|
||||||
const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/;
|
const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/;
|
||||||
const regex = new RegExp(expression);
|
const regex = new RegExp(expression);
|
||||||
|
@ -180,20 +172,19 @@ module.exports.isStringAValidURL = function (url) {
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Check if a particular URL is valid and reachable on the web.
|
* Check if a particular URL is valid and reachable on the web.
|
||||||
* @param {String} url URL to check
|
* @param {string} url URL to check
|
||||||
* @param {Boolean} [checkRedirect]
|
* @param {boolean} [checkRedirect]
|
||||||
* If true, the function will consider redirects a violation and return false.
|
* If true, the function will consider redirects a violation and return false.
|
||||||
* Default: false
|
* Default: false
|
||||||
* @returns {Promise<Boolean>} true if the URL exists, false otherwise
|
* @returns {Promise<Boolean>} true if the URL exists, false otherwise
|
||||||
*/
|
*/
|
||||||
module.exports.urlExists = async function (url, checkRedirect = false) {
|
export async function urlExists(url: string, checkRedirect: boolean = false): Promise<boolean> {
|
||||||
// Local variables
|
// Local variables
|
||||||
let valid = false;
|
let valid = false;
|
||||||
|
|
||||||
if (exports.isStringAValidURL(url)) {
|
if (exports.isStringAValidURL(url)) {
|
||||||
valid = await _axiosUrlExists(url);
|
valid = await axiosUrlExists(url);
|
||||||
|
|
||||||
if (valid && checkRedirect) {
|
if (valid && checkRedirect) {
|
||||||
const redirectUrl = await exports.getUrlRedirect(url);
|
const redirectUrl = await exports.getUrlRedirect(url);
|
||||||
|
@ -202,18 +193,17 @@ module.exports.urlExists = async function (url, checkRedirect = false) {
|
||||||
}
|
}
|
||||||
|
|
||||||
return valid;
|
return valid;
|
||||||
};
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Check if the URL has a redirect to another page.
|
* Check if the URL has a redirect to another page.
|
||||||
* @param {String} url URL to check for redirect
|
* @param {String} url URL to check for redirect
|
||||||
* @returns {Promise<String>} Redirect URL or the passed URL
|
* @returns {Promise<String>} Redirect URL or the passed URL
|
||||||
*/
|
*/
|
||||||
module.exports.getUrlRedirect = async function (url) {
|
export async function getUrlRedirect(url: string): Promise<string> {
|
||||||
const response = await axios.head(url);
|
const response = await axios.head(url);
|
||||||
return response.config.url;
|
return response.config.url;
|
||||||
};
|
}
|
||||||
//#endregion Utility methods
|
//#endregion Utility methods
|
||||||
|
|
||||||
//#region Private methods
|
//#region Private methods
|
||||||
|
@ -222,12 +212,12 @@ module.exports.getUrlRedirect = async function (url) {
|
||||||
* Check with Axios if a URL exists.
|
* Check with Axios if a URL exists.
|
||||||
* @param {String} url
|
* @param {String} url
|
||||||
*/
|
*/
|
||||||
async function _axiosUrlExists(url) {
|
async function axiosUrlExists(url: string): Promise<boolean> {
|
||||||
// Local variables
|
// Local variables
|
||||||
let valid = false;
|
let valid = false;
|
||||||
try {
|
try {
|
||||||
const response = await axios.head(url, {timeout: 3000});
|
const response = await axios.head(url, {timeout: 3000});
|
||||||
valid = response && !/4\d\d/.test(response.status);
|
valid = response && !/4\d\d/.test(response.status.toString());
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (error.code === "ENOTFOUND") valid = false;
|
if (error.code === "ENOTFOUND") valid = false;
|
||||||
else if (error.code === "ETIMEDOUT") valid = false;
|
else if (error.code === "ETIMEDOUT") valid = false;
|
||||||
|
|
|
@ -1,23 +1,22 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Public modules from npm
|
// Public modules from npm
|
||||||
const cheerio = require("cheerio");
|
import cheerio from "cheerio";
|
||||||
const {DateTime} = require("luxon");
|
import { DateTime } from "luxon";
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
const { fetchHTML, getUrlRedirect } = require("./network-helper.js");
|
import { fetchHTML, getUrlRedirect } from "./network-helper.js";
|
||||||
const shared = require("./shared.js");
|
import shared from "./shared.js";
|
||||||
const GameInfo = require("./classes/game-info.js");
|
import GameInfo from "./classes/game-info.js";
|
||||||
const f95Selector = require("./constants/css-selector.js");
|
import { selectors as f95Selector} from "./constants/css-selector.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Get information from the game's main page.
|
* Get information from the game's main page.
|
||||||
* @param {String} url URL of the game/mod to extract data from
|
* @param {String} url URL of the game/mod to extract data from
|
||||||
* @return {Promise<GameInfo>} Complete information about the game you are
|
* @return {Promise<GameInfo>} Complete information about the game you are
|
||||||
* looking for or `null` if is impossible to parse information
|
* looking for or `null` if is impossible to parse information
|
||||||
*/
|
*/
|
||||||
module.exports.getGameInfo = async function (url) {
|
export async function getGameInfo(url: string): Promise<GameInfo|null> {
|
||||||
shared.logger.info("Obtaining game info");
|
shared.logger.info("Obtaining game info");
|
||||||
|
|
||||||
// Fetch HTML and prepare Cheerio
|
// Fetch HTML and prepare Cheerio
|
||||||
|
@ -39,7 +38,7 @@ module.exports.getGameInfo = async function (url) {
|
||||||
if(!structuredData) return null;
|
if(!structuredData) return null;
|
||||||
|
|
||||||
const parsedInfos = parseMainPostText(structuredData.description);
|
const parsedInfos = parseMainPostText(structuredData.description);
|
||||||
const overview = getOverview(structuredData.description, prefixesData.mod);
|
const overview = getOverview(structuredData.description, prefixesData.mod as boolean);
|
||||||
|
|
||||||
// Obtain the updated URL
|
// Obtain the updated URL
|
||||||
const redirectUrl = await getUrlRedirect(url);
|
const redirectUrl = await getUrlRedirect(url);
|
||||||
|
@ -49,23 +48,23 @@ module.exports.getGameInfo = async function (url) {
|
||||||
info.id = extractIDFromURL(url);
|
info.id = extractIDFromURL(url);
|
||||||
info.name = titleData.name;
|
info.name = titleData.name;
|
||||||
info.author = titleData.author;
|
info.author = titleData.author;
|
||||||
info.isMod = prefixesData.mod;
|
info.isMod = prefixesData.mod as boolean;
|
||||||
info.engine = prefixesData.engine;
|
info.engine = prefixesData.engine as string;
|
||||||
info.status = prefixesData.status;
|
info.status = prefixesData.status as string;
|
||||||
info.tags = tags;
|
info.tags = tags;
|
||||||
info.url = redirectUrl;
|
info.url = redirectUrl;
|
||||||
info.language = parsedInfos.Language;
|
info.language = parsedInfos.Language as unknown as string;
|
||||||
info.overview = overview;
|
info.overview = overview;
|
||||||
info.supportedOS = parsedInfos.SupportedOS;
|
info.supportedOS = parsedInfos.SupportedOS as string[];
|
||||||
info.censored = parsedInfos.Censored;
|
info.censored = parsedInfos.Censored as unknown as boolean;
|
||||||
info.lastUpdate = parsedInfos.LastUpdate;
|
info.lastUpdate = parsedInfos.LastUpdate as Date;
|
||||||
info.previewSrc = src;
|
info.previewSrc = src;
|
||||||
info.changelog = changelog;
|
info.changelog = changelog;
|
||||||
info.version = titleData.version;
|
info.version = titleData.version;
|
||||||
|
|
||||||
shared.logger.info(`Founded data for ${info.name}`);
|
shared.logger.info(`Founded data for ${info.name}`);
|
||||||
return info;
|
return info;
|
||||||
};
|
}
|
||||||
|
|
||||||
//#region Private methods
|
//#region Private methods
|
||||||
/**
|
/**
|
||||||
|
@ -75,7 +74,7 @@ module.exports.getGameInfo = async function (url) {
|
||||||
* @param {cheerio.Cheerio} body Page `body` selector
|
* @param {cheerio.Cheerio} body Page `body` selector
|
||||||
* @returns {Object.<string, object>} Dictionary of values with keys `engine`, `status`, `mod`
|
* @returns {Object.<string, object>} Dictionary of values with keys `engine`, `status`, `mod`
|
||||||
*/
|
*/
|
||||||
function parseGamePrefixes(body) {
|
function parseGamePrefixes(body: cheerio.Cheerio): { [s: string]: string | boolean; } {
|
||||||
shared.logger.trace("Parsing prefixes...");
|
shared.logger.trace("Parsing prefixes...");
|
||||||
|
|
||||||
// Local variables
|
// Local variables
|
||||||
|
@ -86,10 +85,9 @@ function parseGamePrefixes(body) {
|
||||||
// Obtain the title prefixes
|
// Obtain the title prefixes
|
||||||
const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES);
|
const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES);
|
||||||
|
|
||||||
const $ = cheerio.load([].concat(body));
|
|
||||||
prefixeElements.each(function parseGamePrefix(idx, el) {
|
prefixeElements.each(function parseGamePrefix(idx, el) {
|
||||||
// Obtain the prefix text
|
// Obtain the prefix text
|
||||||
let prefix = $(el).text().trim();
|
let prefix = cheerio(el).text().trim();
|
||||||
|
|
||||||
// Remove the square brackets
|
// Remove the square brackets
|
||||||
prefix = prefix.replace("[", "").replace("]", "");
|
prefix = prefix.replace("[", "").replace("]", "");
|
||||||
|
@ -100,8 +98,8 @@ function parseGamePrefixes(body) {
|
||||||
else if (isMod(prefix)) mod = true;
|
else if (isMod(prefix)) mod = true;
|
||||||
});
|
});
|
||||||
|
|
||||||
// If the status is not set, then the game in in development (Ongoing)
|
// If the status is not set, then the game is in development (Ongoing)
|
||||||
status = !status ? "Ongoing" : status; // status ?? "Ongoing";
|
status = status ?? "Ongoing";
|
||||||
|
|
||||||
return {
|
return {
|
||||||
engine,
|
engine,
|
||||||
|
@ -116,7 +114,7 @@ function parseGamePrefixes(body) {
|
||||||
* @param {cheerio.Cheerio} body Page `body` selector
|
* @param {cheerio.Cheerio} body Page `body` selector
|
||||||
* @returns {Object.<string, string>} Dictionary of values with keys `name`, `author`, `version`
|
* @returns {Object.<string, string>} Dictionary of values with keys `name`, `author`, `version`
|
||||||
*/
|
*/
|
||||||
function extractInfoFromTitle(body) {
|
function extractInfoFromTitle(body: cheerio.Cheerio): { [s: string]: string; } {
|
||||||
shared.logger.trace("Extracting information from title...");
|
shared.logger.trace("Extracting information from title...");
|
||||||
const title = body
|
const title = body
|
||||||
.find(f95Selector.GT_TITLE)
|
.find(f95Selector.GT_TITLE)
|
||||||
|
@ -163,14 +161,14 @@ function extractInfoFromTitle(body) {
|
||||||
* @param {cheerio.Cheerio} body Page `body` selector
|
* @param {cheerio.Cheerio} body Page `body` selector
|
||||||
* @returns {String[]} List of tags
|
* @returns {String[]} List of tags
|
||||||
*/
|
*/
|
||||||
function extractTags(body) {
|
function extractTags(body: cheerio.Cheerio): string[] {
|
||||||
shared.logger.trace("Extracting tags...");
|
shared.logger.trace("Extracting tags...");
|
||||||
|
|
||||||
// Get the game tags
|
// Get the game tags
|
||||||
const tagResults = body.find(f95Selector.GT_TAGS);
|
const tagResults = body.find(f95Selector.GT_TAGS);
|
||||||
const $ = cheerio.load([].concat(body));
|
|
||||||
return tagResults.map(function parseGameTags(idx, el) {
|
return tagResults.map(function parseGameTags(idx, el) {
|
||||||
return $(el).text().trim();
|
return cheerio(el).text().trim();
|
||||||
}).get();
|
}).get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -180,7 +178,7 @@ function extractTags(body) {
|
||||||
* @param {cheerio.Cheerio} body Page `body` selector
|
* @param {cheerio.Cheerio} body Page `body` selector
|
||||||
* @returns {String} URL of the image
|
* @returns {String} URL of the image
|
||||||
*/
|
*/
|
||||||
function extractPreviewSource(body) {
|
function extractPreviewSource(body: cheerio.Cheerio): string {
|
||||||
shared.logger.trace("Extracting image preview source...");
|
shared.logger.trace("Extracting image preview source...");
|
||||||
const image = body.find(f95Selector.GT_IMAGES);
|
const image = body.find(f95Selector.GT_IMAGES);
|
||||||
|
|
||||||
|
@ -196,7 +194,7 @@ function extractPreviewSource(body) {
|
||||||
* @param {cheerio.Cheerio} mainPost main post selector
|
* @param {cheerio.Cheerio} mainPost main post selector
|
||||||
* @returns {String} Changelog of the last version or `null` if no changelog is fetched
|
* @returns {String} Changelog of the last version or `null` if no changelog is fetched
|
||||||
*/
|
*/
|
||||||
function extractChangelog(mainPost) {
|
function extractChangelog(mainPost: cheerio.Cheerio): string|null {
|
||||||
shared.logger.trace("Extracting last changelog...");
|
shared.logger.trace("Extracting last changelog...");
|
||||||
|
|
||||||
// Obtain the changelog for ALL the versions
|
// Obtain the changelog for ALL the versions
|
||||||
|
@ -229,10 +227,17 @@ function extractChangelog(mainPost) {
|
||||||
* @param {String} text Structured text of the post
|
* @param {String} text Structured text of the post
|
||||||
* @returns {Object.<string, object>} Dictionary of information
|
* @returns {Object.<string, object>} Dictionary of information
|
||||||
*/
|
*/
|
||||||
function parseMainPostText(text) {
|
function parseMainPostText(text: string): { [s: string]: object; } {
|
||||||
shared.logger.trace("Parsing main post raw text...");
|
shared.logger.trace("Parsing main post raw text...");
|
||||||
|
|
||||||
const data = {};
|
interface DataFormat {
|
||||||
|
CENSORED: string,
|
||||||
|
UPDATED: string,
|
||||||
|
THREAD_UPDATED: string,
|
||||||
|
OS: string,
|
||||||
|
LANGUAGE: string
|
||||||
|
}
|
||||||
|
const data = {} as DataFormat;
|
||||||
|
|
||||||
// The information searched in the game post are one per line
|
// The information searched in the game post are one per line
|
||||||
const splittedText = text.split("\n");
|
const splittedText = text.split("\n");
|
||||||
|
@ -275,7 +280,7 @@ function parseMainPostText(text) {
|
||||||
|
|
||||||
// Usually the string is something like "Windows, Linux, Mac"
|
// Usually the string is something like "Windows, Linux, Mac"
|
||||||
const splitted = data.OS.split(",");
|
const splitted = data.OS.split(",");
|
||||||
splitted.forEach(function (os) {
|
splitted.forEach(function (os: string) {
|
||||||
listOS.push(os.trim());
|
listOS.push(os.trim());
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -296,13 +301,11 @@ function parseMainPostText(text) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* Parse a JSON-LD element.
|
* Parse a JSON-LD element.
|
||||||
* @param {cheerio.Element} element
|
|
||||||
*/
|
*/
|
||||||
function parseScriptTag(element) {
|
function parseJSONLD(element: cheerio.Element) {
|
||||||
// Get the element HTML
|
// Get the element HTML
|
||||||
const html = cheerio.load([].concat(element)).html().trim();
|
const html = cheerio.load(element).html().trim();
|
||||||
|
|
||||||
// Obtain the JSON-LD
|
// Obtain the JSON-LD
|
||||||
const data = html
|
const data = html
|
||||||
|
@ -310,10 +313,7 @@ function parseScriptTag(element) {
|
||||||
.replace("</script>", "");
|
.replace("</script>", "");
|
||||||
|
|
||||||
// Convert the string to an object
|
// Convert the string to an object
|
||||||
const json = JSON.parse(data);
|
return JSON.parse(data);
|
||||||
|
|
||||||
// Return only the data of the game
|
|
||||||
if (json["@type"] === "Book") return json;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -322,15 +322,15 @@ function parseScriptTag(element) {
|
||||||
* @param {cheerio.Cheerio} body Page `body` selector
|
* @param {cheerio.Cheerio} body Page `body` selector
|
||||||
* @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
|
* @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
|
||||||
*/
|
*/
|
||||||
function extractStructuredData(body) {
|
function extractStructuredData(body: cheerio.Cheerio): { [s: string]: string; } {
|
||||||
shared.logger.trace("Extracting JSON-LD data...");
|
shared.logger.trace("Extracting JSON-LD data...");
|
||||||
|
|
||||||
// Fetch the JSON-LD data
|
// Fetch the JSON-LD data
|
||||||
const structuredDataElements = body.find(f95Selector.GT_JSONLD);
|
const structuredDataElements = body.find(f95Selector.GT_JSONLD);
|
||||||
|
|
||||||
// Parse the data
|
// Parse the data
|
||||||
const json = structuredDataElements.map((idx, el) => parseScriptTag(el)).get();
|
const json = structuredDataElements.map((idx, el) => parseJSONLD(el)).get();
|
||||||
return json.lenght !== 0 ? json[0] : null;
|
return json.length !== 0 ? json[0] : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -341,7 +341,7 @@ function extractStructuredData(body) {
|
||||||
* @param {Boolean} mod Specify if it is a game or a mod
|
* @param {Boolean} mod Specify if it is a game or a mod
|
||||||
* @returns {String} Game description
|
* @returns {String} Game description
|
||||||
*/
|
*/
|
||||||
function getOverview(text, mod) {
|
function getOverview(text: string, mod: boolean): string {
|
||||||
shared.logger.trace("Extracting game overview...");
|
shared.logger.trace("Extracting game overview...");
|
||||||
|
|
||||||
// Get overview (different parsing for game and mod)
|
// Get overview (different parsing for game and mod)
|
||||||
|
@ -355,7 +355,7 @@ function getOverview(text, mod) {
|
||||||
* @param {String} prefix Prefix to check
|
* @param {String} prefix Prefix to check
|
||||||
* @return {Boolean}
|
* @return {Boolean}
|
||||||
*/
|
*/
|
||||||
function isEngine(prefix) {
|
function isEngine(prefix: string): boolean {
|
||||||
const engines = toUpperCaseArray(Object.values(shared.engines));
|
const engines = toUpperCaseArray(Object.values(shared.engines));
|
||||||
return engines.includes(prefix.toUpperCase());
|
return engines.includes(prefix.toUpperCase());
|
||||||
}
|
}
|
||||||
|
@ -366,7 +366,7 @@ function isEngine(prefix) {
|
||||||
* @param {String} prefix Prefix to check
|
* @param {String} prefix Prefix to check
|
||||||
* @return {Boolean}
|
* @return {Boolean}
|
||||||
*/
|
*/
|
||||||
function isStatus(prefix) {
|
function isStatus(prefix: string): boolean {
|
||||||
const statuses = toUpperCaseArray(Object.values(shared.statuses));
|
const statuses = toUpperCaseArray(Object.values(shared.statuses));
|
||||||
return statuses.includes(prefix.toUpperCase());
|
return statuses.includes(prefix.toUpperCase());
|
||||||
}
|
}
|
||||||
|
@ -377,7 +377,7 @@ function isStatus(prefix) {
|
||||||
* @param {String} prefix Prefix to check
|
* @param {String} prefix Prefix to check
|
||||||
* @return {Boolean}
|
* @return {Boolean}
|
||||||
*/
|
*/
|
||||||
function isMod(prefix) {
|
function isMod(prefix: string): boolean {
|
||||||
const modPrefixes = ["MOD", "CHEAT MOD"];
|
const modPrefixes = ["MOD", "CHEAT MOD"];
|
||||||
return modPrefixes.includes(prefix.toUpperCase());
|
return modPrefixes.includes(prefix.toUpperCase());
|
||||||
}
|
}
|
||||||
|
@ -388,7 +388,7 @@ function isMod(prefix) {
|
||||||
* @param {String} url Game's URL
|
* @param {String} url Game's URL
|
||||||
* @return {Number} Game's ID
|
* @return {Number} Game's ID
|
||||||
*/
|
*/
|
||||||
function extractIDFromURL(url) {
|
function extractIDFromURL(url: string): number {
|
||||||
// URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/
|
// URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/
|
||||||
// or https://f95zone.to/threads/ID/
|
// or https://f95zone.to/threads/ID/
|
||||||
const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/);
|
const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/);
|
||||||
|
@ -403,13 +403,13 @@ function extractIDFromURL(url) {
|
||||||
* Makes an array of strings uppercase.
|
* Makes an array of strings uppercase.
|
||||||
* @param {String[]} a
|
* @param {String[]} a
|
||||||
*/
|
*/
|
||||||
function toUpperCaseArray(a) {
|
function toUpperCaseArray(a: string[]) {
|
||||||
/**
|
/**
|
||||||
* Makes a string uppercase.
|
* Makes a string uppercase.
|
||||||
* @param {String} s
|
* @param {String} s
|
||||||
* @returns {String}
|
* @returns {String}
|
||||||
*/
|
*/
|
||||||
function toUpper(s) {
|
function toUpper(s: string): string {
|
||||||
return s.toUpperCase();
|
return s.toUpperCase();
|
||||||
}
|
}
|
||||||
return a.map(toUpper);
|
return a.map(toUpper);
|
||||||
|
|
|
@ -1,22 +1,20 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Public modules from npm
|
// Public modules from npm
|
||||||
const cheerio = require("cheerio");
|
import cheerio from "cheerio";
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
const { fetchHTML } = require("./network-helper.js");
|
import { fetchHTML } from "./network-helper.js";
|
||||||
const shared = require("./shared.js");
|
import shared from "./shared.js";
|
||||||
const f95Selector = require("./constants/css-selector.js");
|
import { selectors as f95Selector } from "./constants/css-selector.js";
|
||||||
const { F95_BASE_URL } = require("./constants/url.js");
|
import { urls as f95urls } from "./constants/url.js";
|
||||||
|
|
||||||
//#region Public methods
|
//#region Public methods
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Search for a game on F95Zone and return a list of URLs, one for each search result.
|
* Search for a game on F95Zone and return a list of URLs, one for each search result.
|
||||||
* @param {String} name Game name
|
|
||||||
* @returns {Promise<String[]>} URLs of results
|
* @returns {Promise<String[]>} URLs of results
|
||||||
*/
|
*/
|
||||||
module.exports.searchGame = async function (name) {
|
export async function searchGame(name: string): Promise<string[]> {
|
||||||
shared.logger.info(`Searching games with name ${name}`);
|
shared.logger.info(`Searching games with name ${name}`);
|
||||||
|
|
||||||
// Replace the whitespaces with +
|
// Replace the whitespaces with +
|
||||||
|
@ -30,12 +28,10 @@ module.exports.searchGame = async function (name) {
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Search for a mod on F95Zone and return a list of URLs, one for each search result.
|
* Search for a mod on F95Zone and return a list of URLs, one for each search result.
|
||||||
* @param {String} name Mod name
|
|
||||||
* @returns {Promise<String[]>} URLs of results
|
* @returns {Promise<String[]>} URLs of results
|
||||||
*/
|
*/
|
||||||
module.exports.searchMod = async function (name) {
|
export async function searchMod(name: string): Promise<string[]> {
|
||||||
shared.logger.info(`Searching mods with name ${name}`);
|
shared.logger.info(`Searching mods with name ${name}`);
|
||||||
|
|
||||||
// Replace the whitespaces with +
|
// Replace the whitespaces with +
|
||||||
|
@ -51,12 +47,10 @@ module.exports.searchMod = async function (name) {
|
||||||
|
|
||||||
//#region Private methods
|
//#region Private methods
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* Gets the URLs of the threads resulting from the F95Zone search.
|
* Gets the URLs of the threads resulting from the F95Zone search.
|
||||||
* @param {String} url Search URL
|
|
||||||
* @return {Promise<String[]>} List of URLs
|
* @return {Promise<String[]>} List of URLs
|
||||||
*/
|
*/
|
||||||
async function fetchResultURLs(url) {
|
async function fetchResultURLs(url: string): Promise<string[]> {
|
||||||
shared.logger.trace(`Fetching ${url}...`);
|
shared.logger.trace(`Fetching ${url}...`);
|
||||||
|
|
||||||
// Fetch HTML and prepare Cheerio
|
// Fetch HTML and prepare Cheerio
|
||||||
|
@ -76,12 +70,11 @@ async function fetchResultURLs(url) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* Look for the URL to the thread referenced by the item.
|
* Look for the URL to the thread referenced by the item.
|
||||||
* @param {cheerio.Cheerio} selector Element to search
|
* @param {cheerio.Cheerio} selector Element to search
|
||||||
* @returns {String} URL to thread
|
* @returns {String} URL to thread
|
||||||
*/
|
*/
|
||||||
function extractLinkFromResult(selector) {
|
function extractLinkFromResult(selector: cheerio.Cheerio): string {
|
||||||
shared.logger.trace("Extracting thread link from result...");
|
shared.logger.trace("Extracting thread link from result...");
|
||||||
|
|
||||||
const partialLink = selector
|
const partialLink = selector
|
||||||
|
@ -90,6 +83,6 @@ function extractLinkFromResult(selector) {
|
||||||
.trim();
|
.trim();
|
||||||
|
|
||||||
// Compose and return the URL
|
// Compose and return the URL
|
||||||
return new URL(partialLink, F95_BASE_URL).toString();
|
return new URL(partialLink, f95urls.F95_BASE_URL).toString();
|
||||||
}
|
}
|
||||||
//#endregion Private methods
|
//#endregion Private methods
|
||||||
|
|
|
@ -1,20 +1,18 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Public modules from npm
|
// Public modules from npm
|
||||||
const cheerio = require("cheerio");
|
import cheerio from "cheerio";
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
const networkHelper = require("./network-helper.js");
|
import { fetchHTML } from "./network-helper.js";
|
||||||
const f95Selector = require("./constants/css-selector.js");
|
import { selectors as f95Selector } from "./constants/css-selector.js";
|
||||||
const f95url = require("./constants/url.js");
|
import { urls as f95url } from "./constants/url.js";
|
||||||
const UserData = require("./classes/user-data.js");
|
import UserData from "./classes/user-data.js";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @protected
|
|
||||||
* Gets user data, such as username, url of watched threads, and profile picture url.
|
* Gets user data, such as username, url of watched threads, and profile picture url.
|
||||||
* @return {Promise<UserData>} User data
|
|
||||||
*/
|
*/
|
||||||
module.exports.getUserData = async function() {
|
export async function getUserData(): Promise<UserData> {
|
||||||
// Fetch data
|
// Fetch data
|
||||||
const data = await fetchUsernameAndAvatar();
|
const data = await fetchUsernameAndAvatar();
|
||||||
const urls = await fetchWatchedGameThreadURLs();
|
const urls = await fetchWatchedGameThreadURLs();
|
||||||
|
@ -30,15 +28,13 @@ module.exports.getUserData = async function() {
|
||||||
|
|
||||||
//#region Private methods
|
//#region Private methods
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* It connects to the page and extracts the name
|
* It connects to the page and extracts the name
|
||||||
* of the currently logged in user and the URL
|
* of the currently logged in user and the URL
|
||||||
* of their profile picture.
|
* of their profile picture.
|
||||||
* @return {Promise<Object.<string, string>>}
|
|
||||||
*/
|
*/
|
||||||
async function fetchUsernameAndAvatar() {
|
async function fetchUsernameAndAvatar(): Promise<{ [s: string]: string; }> {
|
||||||
// Fetch page
|
// Fetch page
|
||||||
const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL);
|
const html = await fetchHTML(f95url.F95_BASE_URL);
|
||||||
|
|
||||||
// Load HTML response
|
// Load HTML response
|
||||||
const $ = cheerio.load(html);
|
const $ = cheerio.load(html);
|
||||||
|
@ -57,11 +53,10 @@ async function fetchUsernameAndAvatar() {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* Gets the list of URLs of game threads watched by the user.
|
* Gets the list of URLs of game threads watched by the user.
|
||||||
* @returns {Promise<String[]>} List of URLs
|
* @returns {Promise<String[]>} List of URLs
|
||||||
*/
|
*/
|
||||||
async function fetchWatchedGameThreadURLs() {
|
async function fetchWatchedGameThreadURLs(): Promise<string[]> {
|
||||||
// Local variables
|
// Local variables
|
||||||
const watchedGameThreadURLs = [];
|
const watchedGameThreadURLs = [];
|
||||||
|
|
||||||
|
@ -76,7 +71,7 @@ async function fetchWatchedGameThreadURLs() {
|
||||||
|
|
||||||
do {
|
do {
|
||||||
// Fetch page
|
// Fetch page
|
||||||
const html = await networkHelper.fetchHTML(currentURL);
|
const html = await fetchHTML(currentURL);
|
||||||
|
|
||||||
// Load HTML response
|
// Load HTML response
|
||||||
const $ = cheerio.load(html);
|
const $ = cheerio.load(html);
|
||||||
|
@ -95,17 +90,15 @@ async function fetchWatchedGameThreadURLs() {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* Gets the URLs of the watched threads on the page.
|
* Gets the URLs of the watched threads on the page.
|
||||||
* @param {cheerio.Cheerio} body Page `body` selector
|
* @param {cheerio.Cheerio} body Page `body` selector
|
||||||
* @returns {String[]}
|
|
||||||
*/
|
*/
|
||||||
function fetchPageURLs(body) {
|
function fetchPageURLs(body: cheerio.Cheerio): string[] {
|
||||||
const elements = body.find(f95Selector.WT_URLS);
|
const elements = body.find(f95Selector.WT_URLS);
|
||||||
|
|
||||||
return elements.map(function extractURLs(idx, e) {
|
return elements.map(function extractURLs(idx, e) {
|
||||||
// Obtain the link (replace "unread" only for the unread threads)
|
// Obtain the link (replace "unread" only for the unread threads)
|
||||||
const partialLink = e.attribs.href.replace("unread", "");
|
const partialLink = cheerio(e).attr("href").replace("unread", "");
|
||||||
|
|
||||||
// Compose and return the URL
|
// Compose and return the URL
|
||||||
return new URL(partialLink, f95url.F95_BASE_URL).toString();
|
return new URL(partialLink, f95url.F95_BASE_URL).toString();
|
||||||
|
@ -113,13 +106,11 @@ function fetchPageURLs(body) {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @private
|
|
||||||
* Gets the URL of the next page containing the watched threads
|
* Gets the URL of the next page containing the watched threads
|
||||||
* or `null` if that page does not exist.
|
* or `null` if that page does not exist.
|
||||||
* @param {cheerio.Cheerio} body Page `body` selector
|
* @param {cheerio.Cheerio} body Page `body` selector
|
||||||
* @returns {String}
|
|
||||||
*/
|
*/
|
||||||
function fetchNextPageURL(body) {
|
function fetchNextPageURL(body: cheerio.Cheerio): string {
|
||||||
const element = body.find(f95Selector.WT_NEXT_PAGE).first();
|
const element = body.find(f95Selector.WT_NEXT_PAGE).first();
|
||||||
|
|
||||||
// No element found
|
// No element found
|
||||||
|
|
Loading…
Reference in New Issue