Translate scripts

pull/73/head
MillenniumEarl 2021-02-15 22:26:18 +01:00
parent 52264a487a
commit e508149257
7 changed files with 144 additions and 180 deletions

View File

@ -9,10 +9,10 @@ F95_PASSWORD = YOUR_PASSWORD
"use strict"; "use strict";
// Public modules from npm // Public modules from npm
const dotenv = require("dotenv"); import dotenv from "dotenv";
// Modules from file // Modules from file
const F95API = require("./index.js"); import { login, getUserData, getLatestUpdates, getGameData} from "./index";
// Configure the .env reader // Configure the .env reader
dotenv.config(); dotenv.config();
@ -29,16 +29,16 @@ async function main() {
// Log in the platform // Log in the platform
console.log("Authenticating..."); console.log("Authenticating...");
const result = await F95API.login(process.env.F95_USERNAME, process.env.F95_PASSWORD); const result = await login(process.env.F95_USERNAME, process.env.F95_PASSWORD);
console.log(`Authentication result: ${result.message}\n`); console.log(`Authentication result: ${result.message}\n`);
// Get user data // Get user data
console.log("Fetching user data..."); console.log("Fetching user data...");
const userdata = await F95API.getUserData(); const userdata = await getUserData();
console.log(`${userdata.username} follows ${userdata.watchedGameThreads.length} threads\n`); console.log(`${userdata.username} follows ${userdata.watchedGameThreads.length} threads\n`);
// Get latest game update // Get latest game update
const latestUpdates = await F95API.getLatestUpdates({ const latestUpdates = await getLatestUpdates({
tags: ["3d game"] tags: ["3d game"]
}, 1); }, 1);
console.log(`"${latestUpdates[0].name}" was the last "3d game" tagged game to be updated\n`); console.log(`"${latestUpdates[0].name}" was the last "3d game" tagged game to be updated\n`);
@ -46,7 +46,7 @@ async function main() {
// Get game data // Get game data
for(const gamename of gameList) { for(const gamename of gameList) {
console.log(`Searching '${gamename}'...`); console.log(`Searching '${gamename}'...`);
const found = await F95API.getGameData(gamename, false); const found = await getGameData(gamename, false);
// If no game is found // If no game is found
if (found.length === 0) { if (found.length === 0) {

View File

@ -1,20 +1,20 @@
"use strict"; "use strict";
// Modules from file // Modules from file
const shared = require("./scripts/shared.js"); import shared from "./scripts/shared.js";
const networkHelper = require("./scripts/network-helper.js"); import { authenticate, urlExists, isF95URL } from "./scripts/network-helper.js";
const scraper = require("./scripts/scraper.js"); import { getGameInfo } from "./scripts/scraper.js";
const searcher = require("./scripts/searcher.js"); import { searchGame, searchMod } from "./scripts/searcher.js";
const uScraper = require("./scripts/user-scraper.js"); import { getUserData as retrieveUserData } from "./scripts/user-scraper.js";
const latestFetch = require("./scripts/latest-fetch.js"); import { fetchLatest } from "./scripts/latest-fetch.js";
const fetchPlatformData = require("./scripts/platform-data.js").fetchPlatformData; const fetchPlatformData = require("./scripts/platform-data.js").fetchPlatformData;
// Classes from file // Classes from file
const Credentials = require("./scripts/classes/credentials.js"); import Credentials from "./scripts/classes/credentials.js";
const GameInfo = require("./scripts/classes/game-info.js"); import GameInfo from "./scripts/classes/game-info.js";
const LoginResult = require("./scripts/classes/login-result.js"); import LoginResult from "./scripts/classes/login-result.js";
const UserData = require("./scripts/classes/user-data.js"); import UserData from "./scripts/classes/user-data.js";
const PrefixParser = require("./scripts/classes/prefix-parser.js"); import PrefixParser from "./scripts/classes/prefix-parser.js";
//#region Global variables //#region Global variables
const USER_NOT_LOGGED = "User not authenticated, unable to continue"; const USER_NOT_LOGGED = "User not authenticated, unable to continue";
@ -35,26 +35,22 @@ module.exports.PrefixParser = PrefixParser;
/* istambul ignore next */ /* istambul ignore next */
module.exports.loggerLevel = shared.logger.level; module.exports.loggerLevel = shared.logger.level;
exports.loggerLevel = "warn"; // By default log only the warn messages exports.loggerLevel = "warn"; // By default log only the warn messages
/** /**
* @public
* Indicates whether a user is logged in to the F95Zone platform or not. * Indicates whether a user is logged in to the F95Zone platform or not.
* @returns {String}
*/ */
module.exports.isLogged = function isLogged() { export function isLogged(): boolean {
return shared.isLogged; return shared.isLogged;
}; };
//#endregion Export properties //#endregion Export properties
//#region Export methods //#region Export methods
/** /**
* @public
* Log in to the F95Zone platform. * Log in to the F95Zone platform.
* This **must** be the first operation performed before accessing any other script functions. * This **must** be the first operation performed before accessing any other script functions.
* @param {String} username Username used for login
* @param {String} password Password used for login
* @returns {Promise<LoginResult>} Result of the operation * @returns {Promise<LoginResult>} Result of the operation
*/ */
module.exports.login = async function (username, password) { export async function login(username: string, password: string): Promise<LoginResult> {
/* istanbul ignore next */ /* istanbul ignore next */
if (shared.isLogged) { if (shared.isLogged) {
shared.logger.info(`${username} already authenticated`); shared.logger.info(`${username} already authenticated`);
@ -66,8 +62,8 @@ module.exports.login = async function (username, password) {
await creds.fetchToken(); await creds.fetchToken();
shared.logger.trace(`Authentication for ${username}`); shared.logger.trace(`Authentication for ${username}`);
const result = await networkHelper.authenticate(creds); const result = await authenticate(creds);
shared.isLogged = result.success; shared.setIsLogged(result.success);
// Load platform data // Load platform data
if (result.success) await fetchPlatformData(); if (result.success) await fetchPlatformData();
@ -80,13 +76,12 @@ module.exports.login = async function (username, password) {
}; };
/** /**
* @public
* Chek if exists a new version of the game. * Chek if exists a new version of the game.
* You **must** be logged in to the portal before calling this method. * You **must** be logged in to the portal before calling this method.
* @param {GameInfo} info Information about the game to get the version for * @param {GameInfo} info Information about the game to get the version for
* @returns {Promise<Boolean>} true if an update is available, false otherwise * @returns {Promise<Boolean>} true if an update is available, false otherwise
*/ */
module.exports.checkIfGameHasUpdate = async function (info) { export async function checkIfGameHasUpdate(info: GameInfo): Promise<boolean> {
/* istanbul ignore next */ /* istanbul ignore next */
if (!shared.isLogged) { if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED); shared.logger.warn(USER_NOT_LOGGED);
@ -95,11 +90,11 @@ module.exports.checkIfGameHasUpdate = async function (info) {
// F95 change URL at every game update, // F95 change URL at every game update,
// so if the URL is different an update is available // so if the URL is different an update is available
const exists = await networkHelper.urlExists(info.url, true); const exists = await urlExists(info.url, true);
if (!exists) return true; if (!exists) return true;
// Parse version from title // Parse version from title
const onlineInfo = await scraper.getGameInfo(info.url); const onlineInfo = await getGameInfo(info.url);
const onlineVersion = onlineInfo.version; const onlineVersion = onlineInfo.version;
// Compare the versions // Compare the versions
@ -107,7 +102,6 @@ module.exports.checkIfGameHasUpdate = async function (info) {
}; };
/** /**
* @public
* Starting from the name, it gets all the information about the game you are looking for. * Starting from the name, it gets all the information about the game you are looking for.
* You **must** be logged in to the portal before calling this method. * You **must** be logged in to the portal before calling this method.
* @param {String} name Name of the game searched * @param {String} name Name of the game searched
@ -115,7 +109,7 @@ module.exports.checkIfGameHasUpdate = async function (info) {
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to * @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
* an identified game (in the case of homonymy of titles) * an identified game (in the case of homonymy of titles)
*/ */
module.exports.getGameData = async function (name, mod) { export async function getGameData (name: string, mod: boolean): Promise<GameInfo[]> {
/* istanbul ignore next */ /* istanbul ignore next */
if (!shared.isLogged) { if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED); shared.logger.warn(USER_NOT_LOGGED);
@ -124,27 +118,26 @@ module.exports.getGameData = async function (name, mod) {
// Gets the search results of the game/mod being searched for // Gets the search results of the game/mod being searched for
const urls = mod ? const urls = mod ?
await searcher.searchMod(name) : await searchMod(name) :
await searcher.searchGame(name); await searchGame(name);
// Process previous partial results // Process previous partial results
const results = []; const results = [];
for (const url of urls) { for (const url of urls) {
// Start looking for information // Start looking for information
const info = await scraper.getGameInfo(url); const info = await getGameInfo(url);
if (info) results.push(info); if (info) results.push(info);
} }
return results; return results;
}; };
/** /**
* @public
* Starting from the url, it gets all the information about the game you are looking for. * Starting from the url, it gets all the information about the game you are looking for.
* You **must** be logged in to the portal before calling this method. * You **must** be logged in to the portal before calling this method.
* @param {String} url URL of the game to obtain information of * @param {String} url URL of the game to obtain information of
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned * @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
*/ */
module.exports.getGameDataFromURL = async function (url) { export async function getGameDataFromURL(url: string): Promise<GameInfo> {
/* istanbul ignore next */ /* istanbul ignore next */
if (!shared.isLogged) { if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED); shared.logger.warn(USER_NOT_LOGGED);
@ -152,32 +145,30 @@ module.exports.getGameDataFromURL = async function (url) {
} }
// Check URL validity // Check URL validity
const exists = await networkHelper.urlExists(url); const exists = await urlExists(url);
if (!exists) throw new URIError(`${url} is not a valid URL`); if (!exists) throw new URIError(`${url} is not a valid URL`);
if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`); if (!isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
// Get game data // Get game data
return await scraper.getGameInfo(url); return await getGameInfo(url);
}; };
/** /**
* @public
* Gets the data of the currently logged in user. * Gets the data of the currently logged in user.
* You **must** be logged in to the portal before calling this method. * You **must** be logged in to the portal before calling this method.
* @returns {Promise<UserData>} Data of the user currently logged in * @returns {Promise<UserData>} Data of the user currently logged in
*/ */
module.exports.getUserData = async function () { export async function getUserData(): Promise<UserData> {
/* istanbul ignore next */ /* istanbul ignore next */
if (!shared.isLogged) { if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED); shared.logger.warn(USER_NOT_LOGGED);
return null; return null;
} }
return await uScraper.getUserData(); return await retrieveUserData();
}; };
/** /**
* @public
* Gets the latest updated games that match the specified parameters. * Gets the latest updated games that match the specified parameters.
* You **must** be logged in to the portal before calling this method. * You **must** be logged in to the portal before calling this method.
* @param {Object} args * @param {Object} args
@ -196,7 +187,7 @@ module.exports.getUserData = async function () {
* @param {Number} limit Maximum number of results * @param {Number} limit Maximum number of results
* @returns {Promise<GameInfo[]>} List of games * @returns {Promise<GameInfo[]>} List of games
*/ */
module.exports.getLatestUpdates = async function(args, limit) { export async function getLatestUpdates(args, limit: number): Promise<GameInfo[]> {
// Check limit value // Check limit value
if(limit <= 0) throw new Error("limit must be greater than 0"); if(limit <= 0) throw new Error("limit must be greater than 0");
@ -217,22 +208,20 @@ module.exports.getLatestUpdates = async function(args, limit) {
sort: args.sorting ? args.sorting : "date", sort: args.sorting ? args.sorting : "date",
date: filterDate, date: filterDate,
}; };
const urls = await latestFetch.fetchLatest(query, limit); const urls = await fetchLatest(query, limit);
// Get the gamedata from urls // Get the gamedata from urls
const promiseList = urls.map(u => exports.getGameDataFromURL(u)); const promiseList = urls.map((u: string) => exports.getGameDataFromURL(u));
return await Promise.all(promiseList); return await Promise.all(promiseList);
}; };
//#endregion //#endregion
//#region Private Methods //#region Private Methods
/** /**
* @private
* Given an array of numbers, get the nearest value for a given `value`. * Given an array of numbers, get the nearest value for a given `value`.
* @param {Number[]} array List of default values * @param {Number[]} array List of default values
* @param {Number} value Value to search * @param {Number} value Value to search
*/ */
function getNearestValueFromArray(array, value) { function getNearestValueFromArray(array: number[], value: number) {
// Script taken from: // Script taken from:
// https://www.gavsblog.com/blog/find-closest-number-in-array-javascript // https://www.gavsblog.com/blog/find-closest-number-in-array-javascript
array.sort((a, b) => { array.sort((a, b) => {

View File

@ -1,8 +1,9 @@
"use strict"; "use strict";
// Modules from file // Modules from file
import { fetchGETResponse } from "./network-helper.js"; import { fetchGETResponse } from "./network-helper";
import SearchQuery from "./classes/search-query"; import SearchQuery from "./classes/search-query";
import { urls as f95url } from "./constants/url";
/** /**
* @public * @public

View File

@ -1,16 +1,17 @@
"use strict"; "use strict";
// Public modules from npm // Public modules from npm
const axios = require("axios").default; import axios, { AxiosResponse } from "axios";
const cheerio = require("cheerio"); import cheerio from "cheerio";
const axiosCookieJarSupport = require("axios-cookiejar-support").default; import axiosCookieJarSupport from "axios-cookiejar-support";
const tough = require("tough-cookie"); import tough from "tough-cookie";
// Modules from file // Modules from file
const shared = require("./shared.js"); import shared from "./shared";
const f95url = require("./constants/url.js"); import { urls as f95url } from "./constants/url";
const f95selector = require("./constants/css-selector.js"); import { selectors as f95selector } from "./constants/css-selector";
const LoginResult = require("./classes/login-result.js"); import LoginResult from "./classes/login-result";
import credentials from "./classes/credentials";
// Global variables // Global variables
const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " +
@ -27,12 +28,10 @@ const commonConfig = {
}; };
/** /**
* @protected
* Gets the HTML code of a page. * Gets the HTML code of a page.
* @param {String} url URL to fetch
* @returns {Promise<String>} HTML code or `null` if an error arise * @returns {Promise<String>} HTML code or `null` if an error arise
*/ */
module.exports.fetchHTML = async function (url) { export async function fetchHTML(url: string): Promise<string|null> {
// Local variables // Local variables
let returnValue = null; let returnValue = null;
@ -52,18 +51,17 @@ module.exports.fetchHTML = async function (url) {
returnValue = response.data; returnValue = response.data;
return returnValue; return returnValue;
}; }
/** /**
* @protected
* It authenticates to the platform using the credentials * It authenticates to the platform using the credentials
* and token obtained previously. Save cookies on your * and token obtained previously. Save cookies on your
* device after authentication. * device after authentication.
* @param {Credentials} credentials Platform access credentials * @param {module:./classes/credentials.ts:Credentials} credentials Platform access credentials
* @param {Boolean} force Specifies whether the request should be forced, ignoring any saved cookies * @param {Boolean} force Specifies whether the request should be forced, ignoring any saved cookies
* @returns {Promise<LoginResult>} Result of the operation * @returns {Promise<LoginResult>} Result of the operation
*/ */
module.exports.authenticate = async function (credentials, force) { export async function authenticate(credentials: credentials, force: boolean = false): Promise<LoginResult> {
shared.logger.info(`Authenticating with user ${credentials.username}`); shared.logger.info(`Authenticating with user ${credentials.username}`);
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
@ -108,7 +106,7 @@ module.exports.authenticate = async function (credentials, force) {
* Obtain the token used to authenticate the user to the platform. * Obtain the token used to authenticate the user to the platform.
* @returns {Promise<String>} Token or `null` if an error arise * @returns {Promise<String>} Token or `null` if an error arise
*/ */
module.exports.getF95Token = async function() { export async function getF95Token(): Promise<string|null> {
// Fetch the response of the platform // Fetch the response of the platform
const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL); const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL);
/* istambul ignore next */ /* istambul ignore next */
@ -119,18 +117,15 @@ module.exports.getF95Token = async function() {
// The response is a HTML page, we need to find the <input> with name "_xfToken" // The response is a HTML page, we need to find the <input> with name "_xfToken"
const $ = cheerio.load(response.data); const $ = cheerio.load(response.data);
const token = $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value"); return $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value");
return token; }
};
//#region Utility methods //#region Utility methods
/** /**
* @protected
* Performs a GET request to a specific URL and returns the response. * Performs a GET request to a specific URL and returns the response.
* If the request generates an error (for example 400) `null` is returned. * If the request generates an error (for example 400) `null` is returned.
* @param {String} url
*/ */
module.exports.fetchGETResponse = async function(url) { export async function fetchGETResponse(url: string): Promise<AxiosResponse<unknown>> {
// Secure the URL // Secure the URL
const secureURL = exports.enforceHttpsUrl(url); const secureURL = exports.enforceHttpsUrl(url);
@ -141,37 +136,34 @@ module.exports.fetchGETResponse = async function(url) {
shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`); shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`);
return null; return null;
} }
}; }
/** /**
* @protected
* Enforces the scheme of the URL is https and returns the new URL. * Enforces the scheme of the URL is https and returns the new URL.
* @param {String} url * @param {String} url
* @returns {String} Secure URL or `null` if the argument is not a string * @returns {String} Secure URL or `null` if the argument is not a string
*/ */
module.exports.enforceHttpsUrl = function (url) { export function enforceHttpsUrl(url: string): string {
return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
}; };
/** /**
* @protected
* Check if the url belongs to the domain of the F95 platform. * Check if the url belongs to the domain of the F95 platform.
* @param {String} url URL to check * @param {String} url URL to check
* @returns {Boolean} true if the url belongs to the domain, false otherwise * @returns {Boolean} true if the url belongs to the domain, false otherwise
*/ */
module.exports.isF95URL = function (url) { export function isF95URL(url: string): boolean {
if (url.toString().startsWith(f95url.F95_BASE_URL)) return true; if (url.toString().startsWith(f95url.F95_BASE_URL)) return true;
else return false; else return false;
}; };
/** /**
* @protected
* Checks if the string passed by parameter has a * Checks if the string passed by parameter has a
* properly formatted and valid path to a URL (HTTP/HTTPS). * properly formatted and valid path to a URL (HTTP/HTTPS).
* @param {String} url String to check for correctness * @param {String} url String to check for correctness
* @returns {Boolean} true if the string is a valid URL, false otherwise * @returns {Boolean} true if the string is a valid URL, false otherwise
*/ */
module.exports.isStringAValidURL = function (url) { export function isStringAValidURL(url: string): boolean {
// Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc) // Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc)
const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/; const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/;
const regex = new RegExp(expression); const regex = new RegExp(expression);
@ -180,20 +172,19 @@ module.exports.isStringAValidURL = function (url) {
}; };
/** /**
* @protected
* Check if a particular URL is valid and reachable on the web. * Check if a particular URL is valid and reachable on the web.
* @param {String} url URL to check * @param {string} url URL to check
* @param {Boolean} [checkRedirect] * @param {boolean} [checkRedirect]
* If true, the function will consider redirects a violation and return false. * If true, the function will consider redirects a violation and return false.
* Default: false * Default: false
* @returns {Promise<Boolean>} true if the URL exists, false otherwise * @returns {Promise<Boolean>} true if the URL exists, false otherwise
*/ */
module.exports.urlExists = async function (url, checkRedirect = false) { export async function urlExists(url: string, checkRedirect: boolean = false): Promise<boolean> {
// Local variables // Local variables
let valid = false; let valid = false;
if (exports.isStringAValidURL(url)) { if (exports.isStringAValidURL(url)) {
valid = await _axiosUrlExists(url); valid = await axiosUrlExists(url);
if (valid && checkRedirect) { if (valid && checkRedirect) {
const redirectUrl = await exports.getUrlRedirect(url); const redirectUrl = await exports.getUrlRedirect(url);
@ -202,18 +193,17 @@ module.exports.urlExists = async function (url, checkRedirect = false) {
} }
return valid; return valid;
}; }
/** /**
* @protected
* Check if the URL has a redirect to another page. * Check if the URL has a redirect to another page.
* @param {String} url URL to check for redirect * @param {String} url URL to check for redirect
* @returns {Promise<String>} Redirect URL or the passed URL * @returns {Promise<String>} Redirect URL or the passed URL
*/ */
module.exports.getUrlRedirect = async function (url) { export async function getUrlRedirect(url: string): Promise<string> {
const response = await axios.head(url); const response = await axios.head(url);
return response.config.url; return response.config.url;
}; }
//#endregion Utility methods //#endregion Utility methods
//#region Private methods //#region Private methods
@ -222,12 +212,12 @@ module.exports.getUrlRedirect = async function (url) {
* Check with Axios if a URL exists. * Check with Axios if a URL exists.
* @param {String} url * @param {String} url
*/ */
async function _axiosUrlExists(url) { async function axiosUrlExists(url: string): Promise<boolean> {
// Local variables // Local variables
let valid = false; let valid = false;
try { try {
const response = await axios.head(url, {timeout: 3000}); const response = await axios.head(url, {timeout: 3000});
valid = response && !/4\d\d/.test(response.status); valid = response && !/4\d\d/.test(response.status.toString());
} catch (error) { } catch (error) {
if (error.code === "ENOTFOUND") valid = false; if (error.code === "ENOTFOUND") valid = false;
else if (error.code === "ETIMEDOUT") valid = false; else if (error.code === "ETIMEDOUT") valid = false;

View File

@ -1,23 +1,22 @@
"use strict"; "use strict";
// Public modules from npm // Public modules from npm
const cheerio = require("cheerio"); import cheerio from "cheerio";
const {DateTime} = require("luxon"); import { DateTime } from "luxon";
// Modules from file // Modules from file
const { fetchHTML, getUrlRedirect } = require("./network-helper.js"); import { fetchHTML, getUrlRedirect } from "./network-helper.js";
const shared = require("./shared.js"); import shared from "./shared.js";
const GameInfo = require("./classes/game-info.js"); import GameInfo from "./classes/game-info.js";
const f95Selector = require("./constants/css-selector.js"); import { selectors as f95Selector} from "./constants/css-selector.js";
/** /**
* @protected
* Get information from the game's main page. * Get information from the game's main page.
* @param {String} url URL of the game/mod to extract data from * @param {String} url URL of the game/mod to extract data from
* @return {Promise<GameInfo>} Complete information about the game you are * @return {Promise<GameInfo>} Complete information about the game you are
* looking for or `null` if is impossible to parse information * looking for or `null` if is impossible to parse information
*/ */
module.exports.getGameInfo = async function (url) { export async function getGameInfo(url: string): Promise<GameInfo|null> {
shared.logger.info("Obtaining game info"); shared.logger.info("Obtaining game info");
// Fetch HTML and prepare Cheerio // Fetch HTML and prepare Cheerio
@ -39,7 +38,7 @@ module.exports.getGameInfo = async function (url) {
if(!structuredData) return null; if(!structuredData) return null;
const parsedInfos = parseMainPostText(structuredData.description); const parsedInfos = parseMainPostText(structuredData.description);
const overview = getOverview(structuredData.description, prefixesData.mod); const overview = getOverview(structuredData.description, prefixesData.mod as boolean);
// Obtain the updated URL // Obtain the updated URL
const redirectUrl = await getUrlRedirect(url); const redirectUrl = await getUrlRedirect(url);
@ -49,23 +48,23 @@ module.exports.getGameInfo = async function (url) {
info.id = extractIDFromURL(url); info.id = extractIDFromURL(url);
info.name = titleData.name; info.name = titleData.name;
info.author = titleData.author; info.author = titleData.author;
info.isMod = prefixesData.mod; info.isMod = prefixesData.mod as boolean;
info.engine = prefixesData.engine; info.engine = prefixesData.engine as string;
info.status = prefixesData.status; info.status = prefixesData.status as string;
info.tags = tags; info.tags = tags;
info.url = redirectUrl; info.url = redirectUrl;
info.language = parsedInfos.Language; info.language = parsedInfos.Language as unknown as string;
info.overview = overview; info.overview = overview;
info.supportedOS = parsedInfos.SupportedOS; info.supportedOS = parsedInfos.SupportedOS as string[];
info.censored = parsedInfos.Censored; info.censored = parsedInfos.Censored as unknown as boolean;
info.lastUpdate = parsedInfos.LastUpdate; info.lastUpdate = parsedInfos.LastUpdate as Date;
info.previewSrc = src; info.previewSrc = src;
info.changelog = changelog; info.changelog = changelog;
info.version = titleData.version; info.version = titleData.version;
shared.logger.info(`Founded data for ${info.name}`); shared.logger.info(`Founded data for ${info.name}`);
return info; return info;
}; }
//#region Private methods //#region Private methods
/** /**
@ -75,7 +74,7 @@ module.exports.getGameInfo = async function (url) {
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, object>} Dictionary of values with keys `engine`, `status`, `mod` * @returns {Object.<string, object>} Dictionary of values with keys `engine`, `status`, `mod`
*/ */
function parseGamePrefixes(body) { function parseGamePrefixes(body: cheerio.Cheerio): { [s: string]: string | boolean; } {
shared.logger.trace("Parsing prefixes..."); shared.logger.trace("Parsing prefixes...");
// Local variables // Local variables
@ -86,10 +85,9 @@ function parseGamePrefixes(body) {
// Obtain the title prefixes // Obtain the title prefixes
const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES); const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES);
const $ = cheerio.load([].concat(body));
prefixeElements.each(function parseGamePrefix(idx, el) { prefixeElements.each(function parseGamePrefix(idx, el) {
// Obtain the prefix text // Obtain the prefix text
let prefix = $(el).text().trim(); let prefix = cheerio(el).text().trim();
// Remove the square brackets // Remove the square brackets
prefix = prefix.replace("[", "").replace("]", ""); prefix = prefix.replace("[", "").replace("]", "");
@ -100,8 +98,8 @@ function parseGamePrefixes(body) {
else if (isMod(prefix)) mod = true; else if (isMod(prefix)) mod = true;
}); });
// If the status is not set, then the game in in development (Ongoing) // If the status is not set, then the game is in development (Ongoing)
status = !status ? "Ongoing" : status; // status ?? "Ongoing"; status = status ?? "Ongoing";
return { return {
engine, engine,
@ -116,7 +114,7 @@ function parseGamePrefixes(body) {
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, string>} Dictionary of values with keys `name`, `author`, `version` * @returns {Object.<string, string>} Dictionary of values with keys `name`, `author`, `version`
*/ */
function extractInfoFromTitle(body) { function extractInfoFromTitle(body: cheerio.Cheerio): { [s: string]: string; } {
shared.logger.trace("Extracting information from title..."); shared.logger.trace("Extracting information from title...");
const title = body const title = body
.find(f95Selector.GT_TITLE) .find(f95Selector.GT_TITLE)
@ -163,14 +161,14 @@ function extractInfoFromTitle(body) {
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Cheerio} body Page `body` selector
* @returns {String[]} List of tags * @returns {String[]} List of tags
*/ */
function extractTags(body) { function extractTags(body: cheerio.Cheerio): string[] {
shared.logger.trace("Extracting tags..."); shared.logger.trace("Extracting tags...");
// Get the game tags // Get the game tags
const tagResults = body.find(f95Selector.GT_TAGS); const tagResults = body.find(f95Selector.GT_TAGS);
const $ = cheerio.load([].concat(body));
return tagResults.map(function parseGameTags(idx, el) { return tagResults.map(function parseGameTags(idx, el) {
return $(el).text().trim(); return cheerio(el).text().trim();
}).get(); }).get();
} }
@ -180,7 +178,7 @@ function extractTags(body) {
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Cheerio} body Page `body` selector
* @returns {String} URL of the image * @returns {String} URL of the image
*/ */
function extractPreviewSource(body) { function extractPreviewSource(body: cheerio.Cheerio): string {
shared.logger.trace("Extracting image preview source..."); shared.logger.trace("Extracting image preview source...");
const image = body.find(f95Selector.GT_IMAGES); const image = body.find(f95Selector.GT_IMAGES);
@ -196,7 +194,7 @@ function extractPreviewSource(body) {
* @param {cheerio.Cheerio} mainPost main post selector * @param {cheerio.Cheerio} mainPost main post selector
* @returns {String} Changelog of the last version or `null` if no changelog is fetched * @returns {String} Changelog of the last version or `null` if no changelog is fetched
*/ */
function extractChangelog(mainPost) { function extractChangelog(mainPost: cheerio.Cheerio): string|null {
shared.logger.trace("Extracting last changelog..."); shared.logger.trace("Extracting last changelog...");
// Obtain the changelog for ALL the versions // Obtain the changelog for ALL the versions
@ -229,10 +227,17 @@ function extractChangelog(mainPost) {
* @param {String} text Structured text of the post * @param {String} text Structured text of the post
* @returns {Object.<string, object>} Dictionary of information * @returns {Object.<string, object>} Dictionary of information
*/ */
function parseMainPostText(text) { function parseMainPostText(text: string): { [s: string]: object; } {
shared.logger.trace("Parsing main post raw text..."); shared.logger.trace("Parsing main post raw text...");
const data = {}; interface DataFormat {
CENSORED: string,
UPDATED: string,
THREAD_UPDATED: string,
OS: string,
LANGUAGE: string
}
const data = {} as DataFormat;
// The information searched in the game post are one per line // The information searched in the game post are one per line
const splittedText = text.split("\n"); const splittedText = text.split("\n");
@ -275,7 +280,7 @@ function parseMainPostText(text) {
// Usually the string is something like "Windows, Linux, Mac" // Usually the string is something like "Windows, Linux, Mac"
const splitted = data.OS.split(","); const splitted = data.OS.split(",");
splitted.forEach(function (os) { splitted.forEach(function (os: string) {
listOS.push(os.trim()); listOS.push(os.trim());
}); });
@ -296,13 +301,11 @@ function parseMainPostText(text) {
} }
/** /**
* @private
* Parse a JSON-LD element. * Parse a JSON-LD element.
* @param {cheerio.Element} element
*/ */
function parseScriptTag(element) { function parseJSONLD(element: cheerio.Element) {
// Get the element HTML // Get the element HTML
const html = cheerio.load([].concat(element)).html().trim(); const html = cheerio.load(element).html().trim();
// Obtain the JSON-LD // Obtain the JSON-LD
const data = html const data = html
@ -310,10 +313,7 @@ function parseScriptTag(element) {
.replace("</script>", ""); .replace("</script>", "");
// Convert the string to an object // Convert the string to an object
const json = JSON.parse(data); return JSON.parse(data);
// Return only the data of the game
if (json["@type"] === "Book") return json;
} }
/** /**
@ -322,15 +322,15 @@ function parseScriptTag(element) {
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found * @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
*/ */
function extractStructuredData(body) { function extractStructuredData(body: cheerio.Cheerio): { [s: string]: string; } {
shared.logger.trace("Extracting JSON-LD data..."); shared.logger.trace("Extracting JSON-LD data...");
// Fetch the JSON-LD data // Fetch the JSON-LD data
const structuredDataElements = body.find(f95Selector.GT_JSONLD); const structuredDataElements = body.find(f95Selector.GT_JSONLD);
// Parse the data // Parse the data
const json = structuredDataElements.map((idx, el) => parseScriptTag(el)).get(); const json = structuredDataElements.map((idx, el) => parseJSONLD(el)).get();
return json.lenght !== 0 ? json[0] : null; return json.length !== 0 ? json[0] : null;
} }
/** /**
@ -341,7 +341,7 @@ function extractStructuredData(body) {
* @param {Boolean} mod Specify if it is a game or a mod * @param {Boolean} mod Specify if it is a game or a mod
* @returns {String} Game description * @returns {String} Game description
*/ */
function getOverview(text, mod) { function getOverview(text: string, mod: boolean): string {
shared.logger.trace("Extracting game overview..."); shared.logger.trace("Extracting game overview...");
// Get overview (different parsing for game and mod) // Get overview (different parsing for game and mod)
@ -355,7 +355,7 @@ function getOverview(text, mod) {
* @param {String} prefix Prefix to check * @param {String} prefix Prefix to check
* @return {Boolean} * @return {Boolean}
*/ */
function isEngine(prefix) { function isEngine(prefix: string): boolean {
const engines = toUpperCaseArray(Object.values(shared.engines)); const engines = toUpperCaseArray(Object.values(shared.engines));
return engines.includes(prefix.toUpperCase()); return engines.includes(prefix.toUpperCase());
} }
@ -366,7 +366,7 @@ function isEngine(prefix) {
* @param {String} prefix Prefix to check * @param {String} prefix Prefix to check
* @return {Boolean} * @return {Boolean}
*/ */
function isStatus(prefix) { function isStatus(prefix: string): boolean {
const statuses = toUpperCaseArray(Object.values(shared.statuses)); const statuses = toUpperCaseArray(Object.values(shared.statuses));
return statuses.includes(prefix.toUpperCase()); return statuses.includes(prefix.toUpperCase());
} }
@ -377,7 +377,7 @@ function isStatus(prefix) {
* @param {String} prefix Prefix to check * @param {String} prefix Prefix to check
* @return {Boolean} * @return {Boolean}
*/ */
function isMod(prefix) { function isMod(prefix: string): boolean {
const modPrefixes = ["MOD", "CHEAT MOD"]; const modPrefixes = ["MOD", "CHEAT MOD"];
return modPrefixes.includes(prefix.toUpperCase()); return modPrefixes.includes(prefix.toUpperCase());
} }
@ -388,7 +388,7 @@ function isMod(prefix) {
* @param {String} url Game's URL * @param {String} url Game's URL
* @return {Number} Game's ID * @return {Number} Game's ID
*/ */
function extractIDFromURL(url) { function extractIDFromURL(url: string): number {
// URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/ // URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/
// or https://f95zone.to/threads/ID/ // or https://f95zone.to/threads/ID/
const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/); const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/);
@ -403,13 +403,13 @@ function extractIDFromURL(url) {
* Makes an array of strings uppercase. * Makes an array of strings uppercase.
* @param {String[]} a * @param {String[]} a
*/ */
function toUpperCaseArray(a) { function toUpperCaseArray(a: string[]) {
/** /**
* Makes a string uppercase. * Makes a string uppercase.
* @param {String} s * @param {String} s
* @returns {String} * @returns {String}
*/ */
function toUpper(s) { function toUpper(s: string): string {
return s.toUpperCase(); return s.toUpperCase();
} }
return a.map(toUpper); return a.map(toUpper);

View File

@ -1,22 +1,20 @@
"use strict"; "use strict";
// Public modules from npm // Public modules from npm
const cheerio = require("cheerio"); import cheerio from "cheerio";
// Modules from file // Modules from file
const { fetchHTML } = require("./network-helper.js"); import { fetchHTML } from "./network-helper.js";
const shared = require("./shared.js"); import shared from "./shared.js";
const f95Selector = require("./constants/css-selector.js"); import { selectors as f95Selector } from "./constants/css-selector.js";
const { F95_BASE_URL } = require("./constants/url.js"); import { urls as f95urls } from "./constants/url.js";
//#region Public methods //#region Public methods
/** /**
* @protected
* Search for a game on F95Zone and return a list of URLs, one for each search result. * Search for a game on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Game name
* @returns {Promise<String[]>} URLs of results * @returns {Promise<String[]>} URLs of results
*/ */
module.exports.searchGame = async function (name) { export async function searchGame(name: string): Promise<string[]> {
shared.logger.info(`Searching games with name ${name}`); shared.logger.info(`Searching games with name ${name}`);
// Replace the whitespaces with + // Replace the whitespaces with +
@ -30,12 +28,10 @@ module.exports.searchGame = async function (name) {
}; };
/** /**
* @protected
* Search for a mod on F95Zone and return a list of URLs, one for each search result. * Search for a mod on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Mod name
* @returns {Promise<String[]>} URLs of results * @returns {Promise<String[]>} URLs of results
*/ */
module.exports.searchMod = async function (name) { export async function searchMod(name: string): Promise<string[]> {
shared.logger.info(`Searching mods with name ${name}`); shared.logger.info(`Searching mods with name ${name}`);
// Replace the whitespaces with + // Replace the whitespaces with +
@ -51,12 +47,10 @@ module.exports.searchMod = async function (name) {
//#region Private methods //#region Private methods
/** /**
* @private
* Gets the URLs of the threads resulting from the F95Zone search. * Gets the URLs of the threads resulting from the F95Zone search.
* @param {String} url Search URL
* @return {Promise<String[]>} List of URLs * @return {Promise<String[]>} List of URLs
*/ */
async function fetchResultURLs(url) { async function fetchResultURLs(url: string): Promise<string[]> {
shared.logger.trace(`Fetching ${url}...`); shared.logger.trace(`Fetching ${url}...`);
// Fetch HTML and prepare Cheerio // Fetch HTML and prepare Cheerio
@ -76,12 +70,11 @@ async function fetchResultURLs(url) {
} }
/** /**
* @private
* Look for the URL to the thread referenced by the item. * Look for the URL to the thread referenced by the item.
* @param {cheerio.Cheerio} selector Element to search * @param {cheerio.Cheerio} selector Element to search
* @returns {String} URL to thread * @returns {String} URL to thread
*/ */
function extractLinkFromResult(selector) { function extractLinkFromResult(selector: cheerio.Cheerio): string {
shared.logger.trace("Extracting thread link from result..."); shared.logger.trace("Extracting thread link from result...");
const partialLink = selector const partialLink = selector
@ -90,6 +83,6 @@ function extractLinkFromResult(selector) {
.trim(); .trim();
// Compose and return the URL // Compose and return the URL
return new URL(partialLink, F95_BASE_URL).toString(); return new URL(partialLink, f95urls.F95_BASE_URL).toString();
} }
//#endregion Private methods //#endregion Private methods

View File

@ -1,20 +1,18 @@
"use strict"; "use strict";
// Public modules from npm // Public modules from npm
const cheerio = require("cheerio"); import cheerio from "cheerio";
// Modules from file // Modules from file
const networkHelper = require("./network-helper.js"); import { fetchHTML } from "./network-helper.js";
const f95Selector = require("./constants/css-selector.js"); import { selectors as f95Selector } from "./constants/css-selector.js";
const f95url = require("./constants/url.js"); import { urls as f95url } from "./constants/url.js";
const UserData = require("./classes/user-data.js"); import UserData from "./classes/user-data.js";
/** /**
* @protected
* Gets user data, such as username, url of watched threads, and profile picture url. * Gets user data, such as username, url of watched threads, and profile picture url.
* @return {Promise<UserData>} User data
*/ */
module.exports.getUserData = async function() { export async function getUserData(): Promise<UserData> {
// Fetch data // Fetch data
const data = await fetchUsernameAndAvatar(); const data = await fetchUsernameAndAvatar();
const urls = await fetchWatchedGameThreadURLs(); const urls = await fetchWatchedGameThreadURLs();
@ -30,15 +28,13 @@ module.exports.getUserData = async function() {
//#region Private methods //#region Private methods
/** /**
* @private
* It connects to the page and extracts the name * It connects to the page and extracts the name
* of the currently logged in user and the URL * of the currently logged in user and the URL
* of their profile picture. * of their profile picture.
* @return {Promise<Object.<string, string>>}
*/ */
async function fetchUsernameAndAvatar() { async function fetchUsernameAndAvatar(): Promise<{ [s: string]: string; }> {
// Fetch page // Fetch page
const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL); const html = await fetchHTML(f95url.F95_BASE_URL);
// Load HTML response // Load HTML response
const $ = cheerio.load(html); const $ = cheerio.load(html);
@ -57,11 +53,10 @@ async function fetchUsernameAndAvatar() {
} }
/** /**
* @private
* Gets the list of URLs of game threads watched by the user. * Gets the list of URLs of game threads watched by the user.
* @returns {Promise<String[]>} List of URLs * @returns {Promise<String[]>} List of URLs
*/ */
async function fetchWatchedGameThreadURLs() { async function fetchWatchedGameThreadURLs(): Promise<string[]> {
// Local variables // Local variables
const watchedGameThreadURLs = []; const watchedGameThreadURLs = [];
@ -76,7 +71,7 @@ async function fetchWatchedGameThreadURLs() {
do { do {
// Fetch page // Fetch page
const html = await networkHelper.fetchHTML(currentURL); const html = await fetchHTML(currentURL);
// Load HTML response // Load HTML response
const $ = cheerio.load(html); const $ = cheerio.load(html);
@ -95,17 +90,15 @@ async function fetchWatchedGameThreadURLs() {
} }
/** /**
* @private
* Gets the URLs of the watched threads on the page. * Gets the URLs of the watched threads on the page.
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Cheerio} body Page `body` selector
* @returns {String[]}
*/ */
function fetchPageURLs(body) { function fetchPageURLs(body: cheerio.Cheerio): string[] {
const elements = body.find(f95Selector.WT_URLS); const elements = body.find(f95Selector.WT_URLS);
return elements.map(function extractURLs(idx, e) { return elements.map(function extractURLs(idx, e) {
// Obtain the link (replace "unread" only for the unread threads) // Obtain the link (replace "unread" only for the unread threads)
const partialLink = e.attribs.href.replace("unread", ""); const partialLink = cheerio(e).attr("href").replace("unread", "");
// Compose and return the URL // Compose and return the URL
return new URL(partialLink, f95url.F95_BASE_URL).toString(); return new URL(partialLink, f95url.F95_BASE_URL).toString();
@ -113,13 +106,11 @@ function fetchPageURLs(body) {
} }
/** /**
* @private
* Gets the URL of the next page containing the watched threads * Gets the URL of the next page containing the watched threads
* or `null` if that page does not exist. * or `null` if that page does not exist.
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Cheerio} body Page `body` selector
* @returns {String}
*/ */
function fetchNextPageURL(body) { function fetchNextPageURL(body: cheerio.Cheerio): string {
const element = body.find(f95Selector.WT_NEXT_PAGE).first(); const element = body.find(f95Selector.WT_NEXT_PAGE).first();
// No element found // No element found