Refactored main file, removed old files
MillenniumEarl
parent
1391f28ad1
commit
49712b2337
496
app/index.js
496
app/index.js
|
|
@ -1,24 +1,18 @@
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
// Core modules
|
|
||||||
const fs = require("fs");
|
|
||||||
|
|
||||||
// Modules from file
|
// Modules from file
|
||||||
const shared = require("../app/scripts/shared.js");
|
const shared = require("./scripts/shared.js");
|
||||||
const urlK = require("../app/scripts/constants/url.js");
|
const f95url = require("./scripts/constants/url.js");
|
||||||
const selectorK = require("../app/scripts/constants/css-selector.js");
|
const f95selector = require("./scripts/constants/css-selector.js");
|
||||||
const urlHelper = require("../app/scripts/url-helper.js");
|
const networkHelper = require("./scripts/network-helper.js");
|
||||||
const scraper = require("../app/scripts/game-scraper.js");
|
const scraper = require("./scripts/scraper.js");
|
||||||
const {
|
const searcher = require("./scripts/searcher.js");
|
||||||
prepareBrowser,
|
|
||||||
preparePage,
|
|
||||||
} = require("../app/scripts/puppeteer-helper.js");
|
|
||||||
const searcher = require("../app/scripts/game-searcher.js");
|
|
||||||
|
|
||||||
// Classes from file
|
// Classes from file
|
||||||
const GameInfo = require("../app/scripts/classes/game-info.js");
|
const Credentials = require("./scripts/classes/credentials.js");
|
||||||
const LoginResult = require("../app/scripts/classes/login-result.js");
|
const GameInfo = require("./scripts/classes/game-info.js");
|
||||||
const UserData = require("../app/scripts/classes/user-data.js");
|
const LoginResult = require("./scripts/classes/login-result.js");
|
||||||
|
const UserData = require("./scripts/classes/user-data.js");
|
||||||
|
|
||||||
//#region Export classes
|
//#region Export classes
|
||||||
module.exports.GameInfo = GameInfo;
|
module.exports.GameInfo = GameInfo;
|
||||||
|
|
@ -35,7 +29,7 @@ module.exports.debug = function (value) {
|
||||||
shared.debug = value;
|
shared.debug = value;
|
||||||
|
|
||||||
// Configure logger
|
// Configure logger
|
||||||
shared.logger.level = value ? "debug" : "warn";
|
shared.logger.level = value ? "trace" : "warn";
|
||||||
};
|
};
|
||||||
/**
|
/**
|
||||||
* @public
|
* @public
|
||||||
|
|
@ -45,46 +39,9 @@ module.exports.debug = function (value) {
|
||||||
module.exports.isLogged = function () {
|
module.exports.isLogged = function () {
|
||||||
return shared.isLogged;
|
return shared.isLogged;
|
||||||
};
|
};
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* If true, it opens a new browser for each request
|
|
||||||
* to the F95Zone platform, otherwise it reuses the same.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.setIsolation = function (value) {
|
|
||||||
shared.isolation = value;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Path to the cache directory
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.getCacheDir = function () {
|
|
||||||
return shared.cacheDir;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Set path to the cache directory
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.setCacheDir = function (value) {
|
|
||||||
shared.cacheDir = value;
|
|
||||||
|
|
||||||
// Create directory if it doesn't exist
|
|
||||||
if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Set local chromium path.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.setChromiumPath = function (value) {
|
|
||||||
shared.chromiumLocalPath = value;
|
|
||||||
};
|
|
||||||
//#endregion Export properties
|
//#endregion Export properties
|
||||||
|
|
||||||
//#region Global variables
|
//#region Global variables
|
||||||
var _browser = null;
|
|
||||||
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
|
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
|
||||||
//#endregion
|
//#endregion
|
||||||
|
|
||||||
|
|
@ -99,91 +56,24 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue";
|
||||||
*/
|
*/
|
||||||
module.exports.login = async function (username, password) {
|
module.exports.login = async function (username, password) {
|
||||||
if (shared.isLogged) {
|
if (shared.isLogged) {
|
||||||
shared.logger.info("Already logged in");
|
shared.logger.info(`${username} already authenticated`);
|
||||||
const result = new LoginResult(true, "Already logged in");
|
return new LoginResult(true, `${username} already authenticated`);
|
||||||
return result;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// If cookies are loaded, use them to authenticate
|
shared.logger.trace("Fetching token...");
|
||||||
shared.cookies = loadCookies();
|
const creds = new Credentials(username, password);
|
||||||
if (shared.cookies !== null) {
|
await creds.fetchToken();
|
||||||
shared.logger.info("Valid session, no need to re-authenticate");
|
|
||||||
shared.isLogged = true;
|
|
||||||
const result = new LoginResult(true, "Logged with cookies");
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Else, log in throught browser
|
shared.logger.trace(`Authentication for ${username}`);
|
||||||
shared.logger.info(
|
const result = await networkHelper.autenticate(creds);
|
||||||
"No saved sessions or expired session, login on the platform"
|
|
||||||
);
|
|
||||||
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
|
||||||
const result = await loginF95(browser, username, password);
|
|
||||||
shared.isLogged = result.success;
|
shared.isLogged = result.success;
|
||||||
|
|
||||||
if (result.success) {
|
if (result.success) shared.logger.info("User logged in through the platform");
|
||||||
// Reload cookies
|
else shared.logger.warn(`Error during authentication: ${result.message}`);
|
||||||
shared.cookies = loadCookies();
|
|
||||||
shared.logger.info("User logged in through the platform");
|
|
||||||
} else {
|
|
||||||
shared.logger.warn("Error during authentication: " + result.message);
|
|
||||||
}
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
return result;
|
return result;
|
||||||
};
|
};
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* This method loads the main data from the F95 portal
|
|
||||||
* used to provide game information. You **must** be logged
|
|
||||||
* in to the portal before calling this method.
|
|
||||||
* @returns {Promise<Boolean>} Result of the operation
|
|
||||||
*/
|
|
||||||
module.exports.loadF95BaseData = async function () {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
shared.logger.info("Loading base data...");
|
|
||||||
|
|
||||||
// Prepare a new web page
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
|
|
||||||
// Go to latest update page and wait for it to load
|
|
||||||
await page.goto(urlK.F95_LATEST_UPDATES, {
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Obtain engines (disk/online)
|
|
||||||
await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR);
|
|
||||||
shared.engines = await loadValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
shared.enginesCachePath,
|
|
||||||
selectorK.ENGINE_ID_SELECTOR,
|
|
||||||
"engines"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Obtain statuses (disk/online)
|
|
||||||
await page.waitForSelector(selectorK.STATUS_ID_SELECTOR);
|
|
||||||
shared.statuses = await loadValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
shared.statusesCachePath,
|
|
||||||
selectorK.STATUS_ID_SELECTOR,
|
|
||||||
"statuses"
|
|
||||||
);
|
|
||||||
|
|
||||||
await page.close();
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
shared.logger.info("Base data loaded");
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
/**
|
/**
|
||||||
* @public
|
* @public
|
||||||
* Chek if exists a new version of the game.
|
* Chek if exists a new version of the game.
|
||||||
|
|
@ -191,68 +81,54 @@ module.exports.loadF95BaseData = async function () {
|
||||||
* @param {GameInfo} info Information about the game to get the version for
|
* @param {GameInfo} info Information about the game to get the version for
|
||||||
* @returns {Promise<Boolean>} true if an update is available, false otherwise
|
* @returns {Promise<Boolean>} true if an update is available, false otherwise
|
||||||
*/
|
*/
|
||||||
module.exports.chekIfGameHasUpdate = async function (info) {
|
module.exports.checkIfGameHasUpdate = async function (info) {
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
// F95 change URL at every game update,
|
// F95 change URL at every game update,
|
||||||
// so if the URL is different an update is available
|
// so if the URL is different an update is available
|
||||||
const exists = await urlHelper.urlExists(info.f95url, true);
|
const exists = await networkHelper.urlExists(info.url, true);
|
||||||
if (!exists) return true;
|
if (!exists) return true;
|
||||||
|
|
||||||
// Parse version from title
|
// Parse version from title
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
const onlineVersion = await scraper.getGameInfo(info.url).version;
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
// Compare the versions
|
||||||
const onlineVersion = await scraper.getGameVersionFromTitle(browser, info);
|
|
||||||
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
|
|
||||||
return onlineVersion.toUpperCase() !== info.version.toUpperCase();
|
return onlineVersion.toUpperCase() !== info.version.toUpperCase();
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
* @public
|
||||||
* Starting from the name, it gets all the information about the game you are looking for.
|
* Starting from the name, it gets all the information about the game you are looking for.
|
||||||
* You **must** be logged in to the portal before calling this method.
|
* You **must** be logged in to the portal before calling this method.
|
||||||
* @param {String} name Name of the game searched
|
* @param {String} name Name of the game searched
|
||||||
* @param {Boolean} includeMods Indicates whether to also take mods into account when searching
|
* @param {Boolean} mod Indicate if you are looking for mods or games
|
||||||
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
|
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
|
||||||
* an identified game (in the case of homonymy of titles)
|
* an identified game (in the case of homonymy of titles)
|
||||||
*/
|
*/
|
||||||
module.exports.getGameData = async function (name, includeMods) {
|
module.exports.getGameData = async function (name, mod) {
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Gets the search results of the game being searched for
|
// Gets the search results of the game/mod being searched for
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
let urls = [];
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
if(mod) urls = await searcher.searchMod(name);
|
||||||
const urlList = await searcher.getSearchGameResults(browser, name);
|
else urls = await searcher.searchGame(name);
|
||||||
|
|
||||||
// Process previous partial results
|
// Process previous partial results
|
||||||
const promiseList = [];
|
const results = [];
|
||||||
for (const url of urlList) {
|
for (const url of urls) {
|
||||||
// Start looking for information
|
// Start looking for information
|
||||||
promiseList.push(scraper.getGameInfo(browser, url));
|
const info = scraper.getGameInfo(url);
|
||||||
|
results.push(info);
|
||||||
}
|
}
|
||||||
|
return results;
|
||||||
// Filter for mods
|
|
||||||
const result = [];
|
|
||||||
for (const info of await Promise.all(promiseList)) {
|
|
||||||
// Ignore empty results
|
|
||||||
if (!info) continue;
|
|
||||||
// Skip mods if not required
|
|
||||||
if (info.isMod && !includeMods) continue;
|
|
||||||
// Else save data
|
|
||||||
result.push(info);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
return result;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
* @public
|
||||||
* Starting from the url, it gets all the information about the game you are looking for.
|
* Starting from the url, it gets all the information about the game you are looking for.
|
||||||
|
|
@ -261,27 +137,20 @@ module.exports.getGameData = async function (name, includeMods) {
|
||||||
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
|
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
|
||||||
*/
|
*/
|
||||||
module.exports.getGameDataFromURL = async function (url) {
|
module.exports.getGameDataFromURL = async function (url) {
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check URL
|
// Check URL validity
|
||||||
const exists = await urlHelper.urlExists(url);
|
const exists = await networkHelper.urlExists(url);
|
||||||
if (!exists) throw new URIError(url + " is not a valid URL");
|
if (!exists) throw new URIError(`${url} is not a valid URL`);
|
||||||
if (!urlHelper.isF95URL(url))
|
if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
|
||||||
throw new Error(url + " is not a valid F95Zone URL");
|
|
||||||
|
|
||||||
// Gets the search results of the game being searched for
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
|
||||||
// Get game data
|
// Get game data
|
||||||
const result = await scraper.getGameInfo(browser, url);
|
return await scraper.getGameInfo(url);
|
||||||
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
return result;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @public
|
* @public
|
||||||
* Gets the data of the currently logged in user.
|
* Gets the data of the currently logged in user.
|
||||||
|
|
@ -289,297 +158,76 @@ module.exports.getGameDataFromURL = async function (url) {
|
||||||
* @returns {Promise<UserData>} Data of the user currently logged in
|
* @returns {Promise<UserData>} Data of the user currently logged in
|
||||||
*/
|
*/
|
||||||
module.exports.getUserData = async function () {
|
module.exports.getUserData = async function () {
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
if (!shared.isLogged) {
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
shared.logger.warn(USER_NOT_LOGGED);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prepare a new web page
|
const threads = await getUserWatchedGameThreads(null);
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
await page.goto(urlK.F95_BASE_URL); // Go to base page
|
|
||||||
|
|
||||||
// Explicitly wait for the required items to load
|
|
||||||
await Promise.all([
|
|
||||||
page.waitForSelector(selectorK.USERNAME_ELEMENT),
|
|
||||||
page.waitForSelector(selectorK.AVATAR_PIC),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const threads = getUserWatchedGameThreads(browser);
|
|
||||||
|
|
||||||
const username = await page.evaluate(
|
const username = await page.evaluate(
|
||||||
/* istanbul ignore next */ (selector) =>
|
/* istanbul ignore next */ (selector) =>
|
||||||
document.querySelector(selector).innerText,
|
document.querySelector(selector).innerText,
|
||||||
selectorK.USERNAME_ELEMENT
|
f95selector.USERNAME_ELEMENT
|
||||||
);
|
);
|
||||||
|
|
||||||
const avatarSrc = await page.evaluate(
|
const avatarSrc = await page.evaluate(
|
||||||
/* istanbul ignore next */ (selector) =>
|
/* istanbul ignore next */ (selector) =>
|
||||||
document.querySelector(selector).getAttribute("src"),
|
document.querySelector(selector).getAttribute("src"),
|
||||||
selectorK.AVATAR_PIC
|
f95selector.AVATAR_PIC
|
||||||
);
|
);
|
||||||
|
|
||||||
const ud = new UserData();
|
const ud = new UserData();
|
||||||
ud.username = username;
|
ud.username = username;
|
||||||
ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null;
|
ud.avatarSrc = networkHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null;
|
||||||
ud.watchedThreads = await threads;
|
ud.watchedThreads = threads;
|
||||||
|
|
||||||
await page.close();
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
|
|
||||||
return ud;
|
return ud;
|
||||||
};
|
};
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Logout from the current user and gracefully close shared browser.
|
|
||||||
* You **must** be logged in to the portal before calling this method.
|
|
||||||
*/
|
|
||||||
module.exports.logout = async function () {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Logout
|
|
||||||
shared.isLogged = false;
|
|
||||||
|
|
||||||
// Gracefully close shared browser
|
|
||||||
if (!shared.isolation && _browser !== null) {
|
|
||||||
await _browser.close();
|
|
||||||
_browser = null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
//#endregion
|
//#endregion
|
||||||
|
|
||||||
//#region Private methods
|
//#region Private methods
|
||||||
|
|
||||||
//#region Cookies functions
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Loads and verifies the expiration of previously stored cookies from disk
|
|
||||||
* if they exist, otherwise it returns null.
|
|
||||||
* @return {object[]} List of dictionaries or null if cookies don't exist
|
|
||||||
*/
|
|
||||||
function loadCookies() {
|
|
||||||
// Check the existence of the cookie file
|
|
||||||
if (fs.existsSync(shared.cookiesCachePath)) {
|
|
||||||
// Read cookies
|
|
||||||
const cookiesJSON = fs.readFileSync(shared.cookiesCachePath);
|
|
||||||
const cookies = JSON.parse(cookiesJSON);
|
|
||||||
|
|
||||||
// Check if the cookies have expired
|
|
||||||
for (const cookie of cookies) {
|
|
||||||
if (isCookieExpired(cookie)) return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cookies loaded and verified
|
|
||||||
return cookies;
|
|
||||||
} else return null;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Check the validity of a cookie.
|
|
||||||
* @param {object} cookie Cookies to verify the validity. It's a dictionary
|
|
||||||
* @returns {Boolean} true if the cookie has expired, false otherwise
|
|
||||||
*/
|
|
||||||
function isCookieExpired(cookie) {
|
|
||||||
// Local variables
|
|
||||||
let expiredCookies = false;
|
|
||||||
|
|
||||||
// Ignore cookies that never expire
|
|
||||||
const expirationUnixTimestamp = cookie.expire;
|
|
||||||
|
|
||||||
if (expirationUnixTimestamp !== "-1") {
|
|
||||||
// Convert UNIX epoch timestamp to normal Date
|
|
||||||
const expirationDate = new Date(expirationUnixTimestamp * 1000);
|
|
||||||
|
|
||||||
if (expirationDate < Date.now()) {
|
|
||||||
shared.logger.warn(
|
|
||||||
"Cookie " + cookie.name + " expired, you need to re-authenticate"
|
|
||||||
);
|
|
||||||
expiredCookies = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return expiredCookies;
|
|
||||||
}
|
|
||||||
//#endregion Cookies functions
|
|
||||||
|
|
||||||
//#region Latest Updates page parserer
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* If present, it reads the file containing the searched values (engines or states)
|
|
||||||
* from the disk, otherwise it connects to the F95 portal (at the page
|
|
||||||
* https://f95zone.to/latest) and downloads them.
|
|
||||||
* @param {puppeteer.Page} page Page used to locate the required elements
|
|
||||||
* @param {String} path Path to disk of the JSON file containing the data to read / write
|
|
||||||
* @param {String} selector CSS selector of the required elements
|
|
||||||
* @param {String} elementRequested Required element (engines or states) used to detail log messages
|
|
||||||
* @returns {Promise<String[]>} List of required values in uppercase
|
|
||||||
*/
|
|
||||||
async function loadValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
path,
|
|
||||||
selector,
|
|
||||||
elementRequested
|
|
||||||
) {
|
|
||||||
// If the values already exist they are loaded from disk without having to connect to F95
|
|
||||||
shared.logger.info("Load " + elementRequested + " from disk...");
|
|
||||||
if (fs.existsSync(path)) {
|
|
||||||
const valueJSON = fs.readFileSync(path);
|
|
||||||
return JSON.parse(valueJSON);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Otherwise, connect and download the data from the portal
|
|
||||||
shared.logger.info("No " + elementRequested + " cached, downloading...");
|
|
||||||
const values = await getValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
selector,
|
|
||||||
"Getting " + elementRequested + " from page"
|
|
||||||
);
|
|
||||||
fs.writeFileSync(path, JSON.stringify(values));
|
|
||||||
return values;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Gets all the textual values of the elements present
|
|
||||||
* in the F95 portal page and identified by the selector
|
|
||||||
* passed by parameter
|
|
||||||
* @param {puppeteer.Page} page Page used to locate items specified by the selector
|
|
||||||
* @param {String} selector CSS selector
|
|
||||||
* @param {String} logMessage Log message indicating which items the selector is requesting
|
|
||||||
* @return {Promise<String[]>} List of uppercase strings indicating the textual values of the elements identified by the selector
|
|
||||||
*/
|
|
||||||
async function getValuesFromLatestPage(page, selector, logMessage) {
|
|
||||||
shared.logger.info(logMessage);
|
|
||||||
|
|
||||||
const result = [];
|
|
||||||
const elements = await page.$$(selector);
|
|
||||||
|
|
||||||
for (const element of elements) {
|
|
||||||
const text = await element.evaluate(
|
|
||||||
/* istanbul ignore next */ (e) => e.innerText
|
|
||||||
);
|
|
||||||
|
|
||||||
// Save as upper text for better match if used in query
|
|
||||||
result.push(text.toUpperCase());
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
//#endregion
|
|
||||||
|
|
||||||
//#region User
|
//#region User
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Log in to the F95Zone portal and, if successful, save the cookies.
|
|
||||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
|
||||||
* @param {String} username Username to use during login
|
|
||||||
* @param {String} password Password to use during login
|
|
||||||
* @returns {Promise<LoginResult>} Result of the operation
|
|
||||||
*/
|
|
||||||
async function loginF95(browser, username, password) {
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.goto(urlK.F95_LOGIN_URL); // Go to login page
|
|
||||||
|
|
||||||
// Explicitly wait for the required items to load
|
|
||||||
await Promise.all([
|
|
||||||
page.waitForSelector(selectorK.USERNAME_INPUT),
|
|
||||||
page.waitForSelector(selectorK.PASSWORD_INPUT),
|
|
||||||
page.waitForSelector(selectorK.LOGIN_BUTTON),
|
|
||||||
]);
|
|
||||||
|
|
||||||
await page.type(selectorK.USERNAME_INPUT, username); // Insert username
|
|
||||||
await page.type(selectorK.PASSWORD_INPUT, password); // Insert password
|
|
||||||
await Promise.all([
|
|
||||||
page.click(selectorK.LOGIN_BUTTON), // Click on the login button
|
|
||||||
page.waitForNavigation({
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
}), // Wait for page to load
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Prepare result
|
|
||||||
let message = "";
|
|
||||||
|
|
||||||
// Check if the user is logged in
|
|
||||||
const success = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) =>
|
|
||||||
document.querySelector(selector) !== null,
|
|
||||||
selectorK.AVATAR_INFO
|
|
||||||
);
|
|
||||||
|
|
||||||
const errorMessageExists = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(selector) => document.querySelector(selector) !== null,
|
|
||||||
selectorK.LOGIN_MESSAGE_ERROR
|
|
||||||
);
|
|
||||||
|
|
||||||
// Save cookies to avoid re-auth
|
|
||||||
if (success) {
|
|
||||||
const c = await page.cookies();
|
|
||||||
fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
|
|
||||||
message = "Authentication successful";
|
|
||||||
} else if (errorMessageExists) {
|
|
||||||
const errorMessage = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) =>
|
|
||||||
document.querySelector(selector).innerText,
|
|
||||||
selectorK.LOGIN_MESSAGE_ERROR
|
|
||||||
);
|
|
||||||
|
|
||||||
if (errorMessage === "Incorrect password. Please try again.") {
|
|
||||||
message = "Incorrect password";
|
|
||||||
} else if (
|
|
||||||
errorMessage ===
|
|
||||||
"The requested user '" + username + "' could not be found."
|
|
||||||
) {
|
|
||||||
// The escaped quotes are important!
|
|
||||||
message = "Incorrect username";
|
|
||||||
} else message = errorMessage;
|
|
||||||
} else message = "Unknown error";
|
|
||||||
|
|
||||||
await page.close(); // Close the page
|
|
||||||
return new LoginResult(success, message);
|
|
||||||
}
|
|
||||||
/**
|
/**
|
||||||
* @private
|
* @private
|
||||||
* Gets the list of URLs of threads the user follows.
|
* Gets the list of URLs of threads the user follows.
|
||||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
* @param {puppeteer.Browser} browser Browser object used for navigation
|
||||||
* @returns {Promise<String[]>} URL list
|
* @returns {Promise<String[]>} URL list
|
||||||
*/
|
*/
|
||||||
async function getUserWatchedGameThreads(browser) {
|
async function getUserWatchedGameThreads() {
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
const page = null;
|
||||||
await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page
|
await page.goto(f95url.F95_WATCHED_THREADS); // Go to the thread page
|
||||||
|
|
||||||
// Explicitly wait for the required items to load
|
// Explicitly wait for the required items to load
|
||||||
await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON);
|
await page.waitForSelector(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON);
|
||||||
|
|
||||||
// Show the popup
|
// Show the popup
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON),
|
page.click(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON),
|
||||||
page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX),
|
page.waitForSelector(f95selector.UNREAD_THREAD_CHECKBOX),
|
||||||
page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION),
|
page.waitForSelector(f95selector.ONLY_GAMES_THREAD_OPTION),
|
||||||
page.waitForSelector(selectorK.FILTER_THREADS_BUTTON),
|
page.waitForSelector(f95selector.FILTER_THREADS_BUTTON),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Set the filters
|
// Set the filters
|
||||||
await page.evaluate(
|
await page.evaluate(
|
||||||
/* istanbul ignore next */ (selector) =>
|
/* istanbul ignore next */ (selector) =>
|
||||||
document.querySelector(selector).removeAttribute("checked"),
|
document.querySelector(selector).removeAttribute("checked"),
|
||||||
selectorK.UNREAD_THREAD_CHECKBOX
|
f95selector.UNREAD_THREAD_CHECKBOX
|
||||||
); // Also read the threads already read
|
); // Also read the threads already read
|
||||||
|
|
||||||
// Filter the threads
|
// Filter the threads
|
||||||
await page.click(selectorK.ONLY_GAMES_THREAD_OPTION);
|
await page.click(f95selector.ONLY_GAMES_THREAD_OPTION);
|
||||||
await page.click(selectorK.FILTER_THREADS_BUTTON);
|
await page.click(f95selector.FILTER_THREADS_BUTTON);
|
||||||
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
|
await page.waitForSelector(f95selector.WATCHED_THREAD_URLS);
|
||||||
|
|
||||||
// Get the threads urls
|
// Get the threads urls
|
||||||
const urls = [];
|
const urls = [];
|
||||||
let nextPageExists = false;
|
let nextPageExists = false;
|
||||||
do {
|
do {
|
||||||
// Get all the URLs
|
// Get all the URLs
|
||||||
for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) {
|
for (const handle of await page.$$(f95selector.WATCHED_THREAD_URLS)) {
|
||||||
const src = await page.evaluate(
|
const src = await page.evaluate(
|
||||||
/* istanbul ignore next */ (element) => element.href,
|
/* istanbul ignore next */ (element) => element.href,
|
||||||
handle
|
handle
|
||||||
|
|
@ -591,13 +239,13 @@ async function getUserWatchedGameThreads(browser) {
|
||||||
|
|
||||||
nextPageExists = await page.evaluate(
|
nextPageExists = await page.evaluate(
|
||||||
/* istanbul ignore next */ (selector) => document.querySelector(selector),
|
/* istanbul ignore next */ (selector) => document.querySelector(selector),
|
||||||
selectorK.WATCHED_THREAD_NEXT_PAGE
|
f95selector.WATCHED_THREAD_NEXT_PAGE
|
||||||
);
|
);
|
||||||
|
|
||||||
// Click to next page
|
// Click to next page
|
||||||
if (nextPageExists) {
|
if (nextPageExists) {
|
||||||
await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE);
|
await page.click(f95selector.WATCHED_THREAD_NEXT_PAGE);
|
||||||
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
|
await page.waitForSelector(f95selector.WATCHED_THREAD_URLS);
|
||||||
}
|
}
|
||||||
} while (nextPageExists);
|
} while (nextPageExists);
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -59,12 +59,6 @@ module.exports.autenticate = async function (credentials) {
|
||||||
shared.logger.info(`Authenticating with user ${credentials.username}`);
|
shared.logger.info(`Authenticating with user ${credentials.username}`);
|
||||||
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
|
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
|
||||||
|
|
||||||
// If the user is already logged, return
|
|
||||||
if(shared.isLogged) {
|
|
||||||
shared.logger.warn(`${credentials.username} already authenticated`);
|
|
||||||
return new LoginResult(true, "Already authenticated");
|
|
||||||
}
|
|
||||||
|
|
||||||
// Secure the URL
|
// Secure the URL
|
||||||
const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL);
|
const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL);
|
||||||
|
|
||||||
|
|
@ -91,7 +85,6 @@ module.exports.autenticate = async function (credentials) {
|
||||||
const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, "");
|
const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, "");
|
||||||
|
|
||||||
// Return the result of the authentication
|
// Return the result of the authentication
|
||||||
shared.isLogged = errorMessage === "";
|
|
||||||
if (errorMessage === "") return new LoginResult(true, "Authentication successful");
|
if (errorMessage === "") return new LoginResult(true, "Authentication successful");
|
||||||
else return new LoginResult(false, errorMessage);
|
else return new LoginResult(false, errorMessage);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
"use strict";
|
||||||
609
legacy/index.js
609
legacy/index.js
|
|
@ -1,609 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
// Core modules
|
|
||||||
const fs = require("fs");
|
|
||||||
|
|
||||||
// Modules from file
|
|
||||||
const shared = require("../app/scripts/shared.js");
|
|
||||||
const urlK = require("../app/scripts/constants/url.js");
|
|
||||||
const selectorK = require("../app/scripts/constants/css-selector.js");
|
|
||||||
const urlHelper = require("../app/scripts/url-helper.js");
|
|
||||||
const scraper = require("../app/scripts/game-scraper.js");
|
|
||||||
const {
|
|
||||||
prepareBrowser,
|
|
||||||
preparePage,
|
|
||||||
} = require("../app/scripts/puppeteer-helper.js");
|
|
||||||
const searcher = require("../app/scripts/game-searcher.js");
|
|
||||||
|
|
||||||
// Classes from file
|
|
||||||
const GameInfo = require("../app/scripts/classes/game-info.js");
|
|
||||||
const LoginResult = require("../app/scripts/classes/login-result.js");
|
|
||||||
const UserData = require("../app/scripts/classes/user-data.js");
|
|
||||||
|
|
||||||
//#region Export classes
|
|
||||||
module.exports.GameInfo = GameInfo;
|
|
||||||
module.exports.LoginResult = LoginResult;
|
|
||||||
module.exports.UserData = UserData;
|
|
||||||
//#endregion Export classes
|
|
||||||
|
|
||||||
//#region Export properties
|
|
||||||
/**
|
|
||||||
* Shows log messages and other useful functions for module debugging.
|
|
||||||
* @param {Boolean} value
|
|
||||||
*/
|
|
||||||
module.exports.debug = function (value) {
|
|
||||||
shared.debug = value;
|
|
||||||
|
|
||||||
// Configure logger
|
|
||||||
shared.logger.level = value ? "debug" : "warn";
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Indicates whether a user is logged in to the F95Zone platform or not.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.isLogged = function () {
|
|
||||||
return shared.isLogged;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* If true, it opens a new browser for each request
|
|
||||||
* to the F95Zone platform, otherwise it reuses the same.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.setIsolation = function (value) {
|
|
||||||
shared.isolation = value;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Path to the cache directory
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.getCacheDir = function () {
|
|
||||||
return shared.cacheDir;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Set path to the cache directory
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.setCacheDir = function (value) {
|
|
||||||
shared.cacheDir = value;
|
|
||||||
|
|
||||||
// Create directory if it doesn't exist
|
|
||||||
if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Set local chromium path.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
module.exports.setChromiumPath = function (value) {
|
|
||||||
shared.chromiumLocalPath = value;
|
|
||||||
};
|
|
||||||
//#endregion Export properties
|
|
||||||
|
|
||||||
//#region Global variables
|
|
||||||
var _browser = null;
|
|
||||||
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
|
|
||||||
//#endregion
|
|
||||||
|
|
||||||
//#region Export methods
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Log in to the F95Zone platform.
|
|
||||||
* This **must** be the first operation performed before accessing any other script functions.
|
|
||||||
* @param {String} username Username used for login
|
|
||||||
* @param {String} password Password used for login
|
|
||||||
* @returns {Promise<LoginResult>} Result of the operation
|
|
||||||
*/
|
|
||||||
module.exports.login = async function (username, password) {
|
|
||||||
if (shared.isLogged) {
|
|
||||||
shared.logger.info("Already logged in");
|
|
||||||
const result = new LoginResult(true, "Already logged in");
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// If cookies are loaded, use them to authenticate
|
|
||||||
shared.cookies = loadCookies();
|
|
||||||
if (shared.cookies !== null) {
|
|
||||||
shared.logger.info("Valid session, no need to re-authenticate");
|
|
||||||
shared.isLogged = true;
|
|
||||||
const result = new LoginResult(true, "Logged with cookies");
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Else, log in throught browser
|
|
||||||
shared.logger.info(
|
|
||||||
"No saved sessions or expired session, login on the platform"
|
|
||||||
);
|
|
||||||
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
|
||||||
const result = await loginF95(browser, username, password);
|
|
||||||
shared.isLogged = result.success;
|
|
||||||
|
|
||||||
if (result.success) {
|
|
||||||
// Reload cookies
|
|
||||||
shared.cookies = loadCookies();
|
|
||||||
shared.logger.info("User logged in through the platform");
|
|
||||||
} else {
|
|
||||||
shared.logger.warn("Error during authentication: " + result.message);
|
|
||||||
}
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* This method loads the main data from the F95 portal
|
|
||||||
* used to provide game information. You **must** be logged
|
|
||||||
* in to the portal before calling this method.
|
|
||||||
* @returns {Promise<Boolean>} Result of the operation
|
|
||||||
*/
|
|
||||||
module.exports.loadF95BaseData = async function () {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
shared.logger.info("Loading base data...");
|
|
||||||
|
|
||||||
// Prepare a new web page
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
|
|
||||||
// Go to latest update page and wait for it to load
|
|
||||||
await page.goto(urlK.F95_LATEST_UPDATES, {
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Obtain engines (disk/online)
|
|
||||||
await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR);
|
|
||||||
shared.engines = await loadValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
shared.enginesCachePath,
|
|
||||||
selectorK.ENGINE_ID_SELECTOR,
|
|
||||||
"engines"
|
|
||||||
);
|
|
||||||
|
|
||||||
// Obtain statuses (disk/online)
|
|
||||||
await page.waitForSelector(selectorK.STATUS_ID_SELECTOR);
|
|
||||||
shared.statuses = await loadValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
shared.statusesCachePath,
|
|
||||||
selectorK.STATUS_ID_SELECTOR,
|
|
||||||
"statuses"
|
|
||||||
);
|
|
||||||
|
|
||||||
await page.close();
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
shared.logger.info("Base data loaded");
|
|
||||||
return true;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Chek if exists a new version of the game.
|
|
||||||
* You **must** be logged in to the portal before calling this method.
|
|
||||||
* @param {GameInfo} info Information about the game to get the version for
|
|
||||||
* @returns {Promise<Boolean>} true if an update is available, false otherwise
|
|
||||||
*/
|
|
||||||
module.exports.chekIfGameHasUpdate = async function (info) {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
// F95 change URL at every game update,
|
|
||||||
// so if the URL is different an update is available
|
|
||||||
const exists = await urlHelper.urlExists(info.f95url, true);
|
|
||||||
if (!exists) return true;
|
|
||||||
|
|
||||||
// Parse version from title
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
|
||||||
const onlineVersion = await scraper.getGameVersionFromTitle(browser, info);
|
|
||||||
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
|
|
||||||
return onlineVersion.toUpperCase() !== info.version.toUpperCase();
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Starting from the name, it gets all the information about the game you are looking for.
|
|
||||||
* You **must** be logged in to the portal before calling this method.
|
|
||||||
* @param {String} name Name of the game searched
|
|
||||||
* @param {Boolean} includeMods Indicates whether to also take mods into account when searching
|
|
||||||
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
|
|
||||||
* an identified game (in the case of homonymy of titles)
|
|
||||||
*/
|
|
||||||
module.exports.getGameData = async function (name, includeMods) {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Gets the search results of the game being searched for
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
const urlList = await searcher.getSearchGameResults(browser, name);
|
|
||||||
|
|
||||||
// Process previous partial results
|
|
||||||
const promiseList = [];
|
|
||||||
for (const url of urlList) {
|
|
||||||
// Start looking for information
|
|
||||||
promiseList.push(scraper.getGameInfo(browser, url));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Filter for mods
|
|
||||||
const result = [];
|
|
||||||
for (const info of await Promise.all(promiseList)) {
|
|
||||||
// Ignore empty results
|
|
||||||
if (!info) continue;
|
|
||||||
// Skip mods if not required
|
|
||||||
if (info.isMod && !includeMods) continue;
|
|
||||||
// Else save data
|
|
||||||
result.push(info);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Starting from the url, it gets all the information about the game you are looking for.
|
|
||||||
* You **must** be logged in to the portal before calling this method.
|
|
||||||
* @param {String} url URL of the game to obtain information of
|
|
||||||
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
|
|
||||||
*/
|
|
||||||
module.exports.getGameDataFromURL = async function (url) {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check URL
|
|
||||||
const exists = await urlHelper.urlExists(url);
|
|
||||||
if (!exists) throw new URIError(url + " is not a valid URL");
|
|
||||||
if (!urlHelper.isF95URL(url))
|
|
||||||
throw new Error(url + " is not a valid F95Zone URL");
|
|
||||||
|
|
||||||
// Gets the search results of the game being searched for
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
|
|
||||||
// Get game data
|
|
||||||
const result = await scraper.getGameInfo(browser, url);
|
|
||||||
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Gets the data of the currently logged in user.
|
|
||||||
* You **must** be logged in to the portal before calling this method.
|
|
||||||
* @returns {Promise<UserData>} Data of the user currently logged in
|
|
||||||
*/
|
|
||||||
module.exports.getUserData = async function () {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Prepare a new web page
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
await page.goto(urlK.F95_BASE_URL); // Go to base page
|
|
||||||
|
|
||||||
// Explicitly wait for the required items to load
|
|
||||||
await Promise.all([
|
|
||||||
page.waitForSelector(selectorK.USERNAME_ELEMENT),
|
|
||||||
page.waitForSelector(selectorK.AVATAR_PIC),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const threads = getUserWatchedGameThreads(browser);
|
|
||||||
|
|
||||||
const username = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) =>
|
|
||||||
document.querySelector(selector).innerText,
|
|
||||||
selectorK.USERNAME_ELEMENT
|
|
||||||
);
|
|
||||||
|
|
||||||
const avatarSrc = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) =>
|
|
||||||
document.querySelector(selector).getAttribute("src"),
|
|
||||||
selectorK.AVATAR_PIC
|
|
||||||
);
|
|
||||||
|
|
||||||
const ud = new UserData();
|
|
||||||
ud.username = username;
|
|
||||||
ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null;
|
|
||||||
ud.watchedThreads = await threads;
|
|
||||||
|
|
||||||
await page.close();
|
|
||||||
if (shared.isolation) await browser.close();
|
|
||||||
|
|
||||||
return ud;
|
|
||||||
};
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Logout from the current user and gracefully close shared browser.
|
|
||||||
* You **must** be logged in to the portal before calling this method.
|
|
||||||
*/
|
|
||||||
module.exports.logout = async function () {
|
|
||||||
if (!shared.isLogged || !shared.cookies) {
|
|
||||||
shared.logger.warn(USER_NOT_LOGGED);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Logout
|
|
||||||
shared.isLogged = false;
|
|
||||||
|
|
||||||
// Gracefully close shared browser
|
|
||||||
if (!shared.isolation && _browser !== null) {
|
|
||||||
await _browser.close();
|
|
||||||
_browser = null;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
//#endregion
|
|
||||||
|
|
||||||
//#region Private methods
|
|
||||||
|
|
||||||
//#region Cookies functions
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Loads and verifies the expiration of previously stored cookies from disk
|
|
||||||
* if they exist, otherwise it returns null.
|
|
||||||
* @return {object[]} List of dictionaries or null if cookies don't exist
|
|
||||||
*/
|
|
||||||
function loadCookies() {
|
|
||||||
// Check the existence of the cookie file
|
|
||||||
if (fs.existsSync(shared.cookiesCachePath)) {
|
|
||||||
// Read cookies
|
|
||||||
const cookiesJSON = fs.readFileSync(shared.cookiesCachePath);
|
|
||||||
const cookies = JSON.parse(cookiesJSON);
|
|
||||||
|
|
||||||
// Check if the cookies have expired
|
|
||||||
for (const cookie of cookies) {
|
|
||||||
if (isCookieExpired(cookie)) return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Cookies loaded and verified
|
|
||||||
return cookies;
|
|
||||||
} else return null;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Check the validity of a cookie.
|
|
||||||
* @param {object} cookie Cookies to verify the validity. It's a dictionary
|
|
||||||
* @returns {Boolean} true if the cookie has expired, false otherwise
|
|
||||||
*/
|
|
||||||
function isCookieExpired(cookie) {
|
|
||||||
// Local variables
|
|
||||||
let expiredCookies = false;
|
|
||||||
|
|
||||||
// Ignore cookies that never expire
|
|
||||||
const expirationUnixTimestamp = cookie.expire;
|
|
||||||
|
|
||||||
if (expirationUnixTimestamp !== "-1") {
|
|
||||||
// Convert UNIX epoch timestamp to normal Date
|
|
||||||
const expirationDate = new Date(expirationUnixTimestamp * 1000);
|
|
||||||
|
|
||||||
if (expirationDate < Date.now()) {
|
|
||||||
shared.logger.warn(
|
|
||||||
"Cookie " + cookie.name + " expired, you need to re-authenticate"
|
|
||||||
);
|
|
||||||
expiredCookies = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return expiredCookies;
|
|
||||||
}
|
|
||||||
//#endregion Cookies functions
|
|
||||||
|
|
||||||
//#region Latest Updates page parserer
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* If present, it reads the file containing the searched values (engines or states)
|
|
||||||
* from the disk, otherwise it connects to the F95 portal (at the page
|
|
||||||
* https://f95zone.to/latest) and downloads them.
|
|
||||||
* @param {puppeteer.Page} page Page used to locate the required elements
|
|
||||||
* @param {String} path Path to disk of the JSON file containing the data to read / write
|
|
||||||
* @param {String} selector CSS selector of the required elements
|
|
||||||
* @param {String} elementRequested Required element (engines or states) used to detail log messages
|
|
||||||
* @returns {Promise<String[]>} List of required values in uppercase
|
|
||||||
*/
|
|
||||||
async function loadValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
path,
|
|
||||||
selector,
|
|
||||||
elementRequested
|
|
||||||
) {
|
|
||||||
// If the values already exist they are loaded from disk without having to connect to F95
|
|
||||||
shared.logger.info("Load " + elementRequested + " from disk...");
|
|
||||||
if (fs.existsSync(path)) {
|
|
||||||
const valueJSON = fs.readFileSync(path);
|
|
||||||
return JSON.parse(valueJSON);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Otherwise, connect and download the data from the portal
|
|
||||||
shared.logger.info("No " + elementRequested + " cached, downloading...");
|
|
||||||
const values = await getValuesFromLatestPage(
|
|
||||||
page,
|
|
||||||
selector,
|
|
||||||
"Getting " + elementRequested + " from page"
|
|
||||||
);
|
|
||||||
fs.writeFileSync(path, JSON.stringify(values));
|
|
||||||
return values;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Gets all the textual values of the elements present
|
|
||||||
* in the F95 portal page and identified by the selector
|
|
||||||
* passed by parameter
|
|
||||||
* @param {puppeteer.Page} page Page used to locate items specified by the selector
|
|
||||||
* @param {String} selector CSS selector
|
|
||||||
* @param {String} logMessage Log message indicating which items the selector is requesting
|
|
||||||
* @return {Promise<String[]>} List of uppercase strings indicating the textual values of the elements identified by the selector
|
|
||||||
*/
|
|
||||||
async function getValuesFromLatestPage(page, selector, logMessage) {
|
|
||||||
shared.logger.info(logMessage);
|
|
||||||
|
|
||||||
const result = [];
|
|
||||||
const elements = await page.$$(selector);
|
|
||||||
|
|
||||||
for (const element of elements) {
|
|
||||||
const text = await element.evaluate(
|
|
||||||
/* istanbul ignore next */ (e) => e.innerText
|
|
||||||
);
|
|
||||||
|
|
||||||
// Save as upper text for better match if used in query
|
|
||||||
result.push(text.toUpperCase());
|
|
||||||
}
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
//#endregion
|
|
||||||
|
|
||||||
//#region User
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Log in to the F95Zone portal and, if successful, save the cookies.
|
|
||||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
|
||||||
* @param {String} username Username to use during login
|
|
||||||
* @param {String} password Password to use during login
|
|
||||||
* @returns {Promise<LoginResult>} Result of the operation
|
|
||||||
*/
|
|
||||||
async function loginF95(browser, username, password) {
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.goto(urlK.F95_LOGIN_URL); // Go to login page
|
|
||||||
|
|
||||||
// Explicitly wait for the required items to load
|
|
||||||
await Promise.all([
|
|
||||||
page.waitForSelector(selectorK.USERNAME_INPUT),
|
|
||||||
page.waitForSelector(selectorK.PASSWORD_INPUT),
|
|
||||||
page.waitForSelector(selectorK.LOGIN_BUTTON),
|
|
||||||
]);
|
|
||||||
|
|
||||||
await page.type(selectorK.USERNAME_INPUT, username); // Insert username
|
|
||||||
await page.type(selectorK.PASSWORD_INPUT, password); // Insert password
|
|
||||||
await Promise.all([
|
|
||||||
page.click(selectorK.LOGIN_BUTTON), // Click on the login button
|
|
||||||
page.waitForNavigation({
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
}), // Wait for page to load
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Prepare result
|
|
||||||
let message = "";
|
|
||||||
|
|
||||||
// Check if the user is logged in
|
|
||||||
const success = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) =>
|
|
||||||
document.querySelector(selector) !== null,
|
|
||||||
selectorK.AVATAR_INFO
|
|
||||||
);
|
|
||||||
|
|
||||||
const errorMessageExists = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(selector) => document.querySelector(selector) !== null,
|
|
||||||
selectorK.LOGIN_MESSAGE_ERROR
|
|
||||||
);
|
|
||||||
|
|
||||||
// Save cookies to avoid re-auth
|
|
||||||
if (success) {
|
|
||||||
const c = await page.cookies();
|
|
||||||
fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
|
|
||||||
message = "Authentication successful";
|
|
||||||
} else if (errorMessageExists) {
|
|
||||||
const errorMessage = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) =>
|
|
||||||
document.querySelector(selector).innerText,
|
|
||||||
selectorK.LOGIN_MESSAGE_ERROR
|
|
||||||
);
|
|
||||||
|
|
||||||
if (errorMessage === "Incorrect password. Please try again.") {
|
|
||||||
message = "Incorrect password";
|
|
||||||
} else if (
|
|
||||||
errorMessage ===
|
|
||||||
"The requested user '" + username + "' could not be found."
|
|
||||||
) {
|
|
||||||
// The escaped quotes are important!
|
|
||||||
message = "Incorrect username";
|
|
||||||
} else message = errorMessage;
|
|
||||||
} else message = "Unknown error";
|
|
||||||
|
|
||||||
await page.close(); // Close the page
|
|
||||||
return new LoginResult(success, message);
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Gets the list of URLs of threads the user follows.
|
|
||||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
|
||||||
* @returns {Promise<String[]>} URL list
|
|
||||||
*/
|
|
||||||
async function getUserWatchedGameThreads(browser) {
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page
|
|
||||||
|
|
||||||
// Explicitly wait for the required items to load
|
|
||||||
await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON);
|
|
||||||
|
|
||||||
// Show the popup
|
|
||||||
await Promise.all([
|
|
||||||
page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON),
|
|
||||||
page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX),
|
|
||||||
page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION),
|
|
||||||
page.waitForSelector(selectorK.FILTER_THREADS_BUTTON),
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Set the filters
|
|
||||||
await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) =>
|
|
||||||
document.querySelector(selector).removeAttribute("checked"),
|
|
||||||
selectorK.UNREAD_THREAD_CHECKBOX
|
|
||||||
); // Also read the threads already read
|
|
||||||
|
|
||||||
// Filter the threads
|
|
||||||
await page.click(selectorK.ONLY_GAMES_THREAD_OPTION);
|
|
||||||
await page.click(selectorK.FILTER_THREADS_BUTTON);
|
|
||||||
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
|
|
||||||
|
|
||||||
// Get the threads urls
|
|
||||||
const urls = [];
|
|
||||||
let nextPageExists = false;
|
|
||||||
do {
|
|
||||||
// Get all the URLs
|
|
||||||
for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) {
|
|
||||||
const src = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (element) => element.href,
|
|
||||||
handle
|
|
||||||
);
|
|
||||||
// If 'unread' is left, it will redirect to the last unread post
|
|
||||||
const url = src.replace("/unread", "");
|
|
||||||
urls.push(url);
|
|
||||||
}
|
|
||||||
|
|
||||||
nextPageExists = await page.evaluate(
|
|
||||||
/* istanbul ignore next */ (selector) => document.querySelector(selector),
|
|
||||||
selectorK.WATCHED_THREAD_NEXT_PAGE
|
|
||||||
);
|
|
||||||
|
|
||||||
// Click to next page
|
|
||||||
if (nextPageExists) {
|
|
||||||
await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE);
|
|
||||||
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
|
|
||||||
}
|
|
||||||
} while (nextPageExists);
|
|
||||||
|
|
||||||
await page.close();
|
|
||||||
return urls;
|
|
||||||
}
|
|
||||||
//#endregion User
|
|
||||||
|
|
||||||
//#endregion Private methods
|
|
||||||
|
|
@ -1,111 +0,0 @@
|
||||||
/* istanbul ignore file */
|
|
||||||
|
|
||||||
"use strict";
|
|
||||||
|
|
||||||
// Core modules
|
|
||||||
const fs = require("fs");
|
|
||||||
const path = require("path");
|
|
||||||
|
|
||||||
// Public modules from npm
|
|
||||||
// const { File } = require('megajs');
|
|
||||||
|
|
||||||
// Modules from file
|
|
||||||
const { prepareBrowser, preparePage } = require("../puppeteer-helper.js");
|
|
||||||
const shared = require("../shared.js");
|
|
||||||
|
|
||||||
class GameDownload {
|
|
||||||
constructor() {
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Platform that hosts game files
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.hosting = "";
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Link to game files
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.link = null;
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Operating systems supported by the game version indicated in this class.
|
|
||||||
* Can be *WINDOWS/LINUX/MACOS*
|
|
||||||
* @type String[]
|
|
||||||
*/
|
|
||||||
this.supportedOS = [];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @public
|
|
||||||
* Download the game data in the indicated path.
|
|
||||||
* Supported hosting platforms: MEGA, NOPY
|
|
||||||
* @param {String} path Save path
|
|
||||||
* @return {Promise<Boolean>} Result of the operation
|
|
||||||
*/
|
|
||||||
async download(path) {
|
|
||||||
if (this.link.includes("mega.nz"))
|
|
||||||
return await downloadMEGA(this.link, path);
|
|
||||||
else if (this.link.includes("nopy.to"))
|
|
||||||
return await downloadNOPY(this.link, path);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
module.exports = GameDownload;
|
|
||||||
|
|
||||||
async function downloadMEGA(url, savepath) {
|
|
||||||
// The URL is masked
|
|
||||||
const browser = await prepareBrowser();
|
|
||||||
const page = await preparePage(browser);
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
await page.goto(url);
|
|
||||||
await page.waitForSelector("a.host_link");
|
|
||||||
|
|
||||||
// Obtain the link for the unmasked page and click it
|
|
||||||
const link = await page.$("a.host_link");
|
|
||||||
await link.click();
|
|
||||||
await page.goto(url, {
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
}); // Go to the game page and wait until it loads
|
|
||||||
|
|
||||||
// Obtain the URL after the redirect
|
|
||||||
const downloadURL = page.url();
|
|
||||||
|
|
||||||
// Close browser and page
|
|
||||||
await page.close();
|
|
||||||
await browser.close();
|
|
||||||
|
|
||||||
const stream = fs.createWriteStream(savepath);
|
|
||||||
const file = File.fromURL(downloadURL);
|
|
||||||
file.download().pipe(stream);
|
|
||||||
return fs.existsSync(savepath);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function downloadNOPY(url, savepath) {
|
|
||||||
// Prepare browser
|
|
||||||
const browser = await prepareBrowser();
|
|
||||||
const page = await preparePage(browser);
|
|
||||||
await page.goto(url);
|
|
||||||
await page.waitForSelector("#download");
|
|
||||||
|
|
||||||
// Set the save path
|
|
||||||
await page._client.send("Page.setDownloadBehavior", {
|
|
||||||
behavior: "allow",
|
|
||||||
downloadPath: path.basename(path.dirname(savepath)), // It's a directory
|
|
||||||
});
|
|
||||||
|
|
||||||
// Obtain the download button and click it
|
|
||||||
const downloadButton = await page.$("#download");
|
|
||||||
await downloadButton.click();
|
|
||||||
|
|
||||||
// Await for all the connections to close
|
|
||||||
await page.waitForNavigation({
|
|
||||||
waitUntil: "networkidle0",
|
|
||||||
timeout: 0, // Disable timeout
|
|
||||||
});
|
|
||||||
|
|
||||||
// Close browser and page
|
|
||||||
await page.close();
|
|
||||||
await browser.close();
|
|
||||||
|
|
||||||
return fs.existsSync(savepath);
|
|
||||||
}
|
|
||||||
|
|
@ -1,119 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
class GameInfo {
|
|
||||||
constructor() {
|
|
||||||
//#region Properties
|
|
||||||
/**
|
|
||||||
* Game name
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.name = null;
|
|
||||||
/**
|
|
||||||
* Game author
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.author = null;
|
|
||||||
/**
|
|
||||||
* URL to the game's official conversation on the F95Zone portal
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.f95url = null;
|
|
||||||
/**
|
|
||||||
* Game description
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.overview = null;
|
|
||||||
/**
|
|
||||||
* List of tags associated with the game
|
|
||||||
* @type String[]
|
|
||||||
*/
|
|
||||||
this.tags = [];
|
|
||||||
/**
|
|
||||||
* Graphics engine used for game development
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.engine = null;
|
|
||||||
/**
|
|
||||||
* Progress of the game
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.status = null;
|
|
||||||
/**
|
|
||||||
* Game description image URL
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.previewSource = null;
|
|
||||||
/**
|
|
||||||
* Game version
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.version = null;
|
|
||||||
/**
|
|
||||||
* Last time the game underwent updates
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.lastUpdate = null;
|
|
||||||
/**
|
|
||||||
* Last time the local copy of the game was run
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.lastPlayed = null;
|
|
||||||
/**
|
|
||||||
* Specifies if the game is original or a mod
|
|
||||||
* @type Boolean
|
|
||||||
*/
|
|
||||||
this.isMod = false;
|
|
||||||
/**
|
|
||||||
* Changelog for the last version.
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.changelog = null;
|
|
||||||
/**
|
|
||||||
* Directory containing the local copy of the game
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.gameDir = null;
|
|
||||||
/**
|
|
||||||
* Information on game file download links,
|
|
||||||
* including information on hosting platforms
|
|
||||||
* and operating system supported by the specific link
|
|
||||||
* @type GameDownload[]
|
|
||||||
*/
|
|
||||||
this.downloadInfo = [];
|
|
||||||
//#endregion Properties
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Converts the object to a dictionary used for JSON serialization
|
|
||||||
*/
|
|
||||||
/* istanbul ignore next */
|
|
||||||
toJSON() {
|
|
||||||
return {
|
|
||||||
name: this.name,
|
|
||||||
author: this.author,
|
|
||||||
f95url: this.f95url,
|
|
||||||
overview: this.overview,
|
|
||||||
engine: this.engine,
|
|
||||||
status: this.status,
|
|
||||||
previewSource: this.previewSource,
|
|
||||||
version: this.version,
|
|
||||||
lastUpdate: this.lastUpdate,
|
|
||||||
lastPlayed: this.lastPlayed,
|
|
||||||
isMod: this.isMod,
|
|
||||||
changelog: this.changelog,
|
|
||||||
gameDir: this.gameDir,
|
|
||||||
downloadInfo: this.downloadInfo,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Return a new GameInfo from a JSON string
|
|
||||||
* @param {String} json JSON string used to create the new object
|
|
||||||
* @returns {GameInfo}
|
|
||||||
*/
|
|
||||||
/* istanbul ignore next */
|
|
||||||
static fromJSON(json) {
|
|
||||||
return Object.assign(new GameInfo(), json);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
module.exports = GameInfo;
|
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Object obtained in response to an attempt to login to the portal.
|
|
||||||
*/
|
|
||||||
class LoginResult {
|
|
||||||
constructor(success, message) {
|
|
||||||
/**
|
|
||||||
* Result of the login operation
|
|
||||||
* @type Boolean
|
|
||||||
*/
|
|
||||||
this.success = success;
|
|
||||||
/**
|
|
||||||
* Login response message
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.message = message;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
module.exports = LoginResult;
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Class containing the data of the user currently connected to the F95Zone platform.
|
|
||||||
*/
|
|
||||||
class UserData {
|
|
||||||
constructor() {
|
|
||||||
/**
|
|
||||||
* User username.
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.username = "";
|
|
||||||
/**
|
|
||||||
* Path to the user's profile picture.
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
this.avatarSrc = null;
|
|
||||||
/**
|
|
||||||
* List of followed thread URLs.
|
|
||||||
* @type URL[]
|
|
||||||
*/
|
|
||||||
this.watchedThreads = [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = UserData;
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
module.exports = Object.freeze({
|
|
||||||
AVATAR_INFO: "span.avatar",
|
|
||||||
AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]",
|
|
||||||
ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span",
|
|
||||||
FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]",
|
|
||||||
GAME_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]",
|
|
||||||
GAME_TAGS: "a.tagItem",
|
|
||||||
GAME_TITLE: "h1.p-title-value",
|
|
||||||
GAME_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]",
|
|
||||||
LOGIN_BUTTON: "button.button--icon--login",
|
|
||||||
LOGIN_MESSAGE_ERROR:
|
|
||||||
"div.blockMessage.blockMessage--error.blockMessage--iconic",
|
|
||||||
ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]",
|
|
||||||
PASSWORD_INPUT: "input[name=\"password\"]",
|
|
||||||
SEARCH_BUTTON: "form.block > * button.button--icon--search",
|
|
||||||
SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]",
|
|
||||||
SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]",
|
|
||||||
STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span",
|
|
||||||
THREAD_POSTS:
|
|
||||||
"article.message-body:first-child > div.bbWrapper:first-of-type",
|
|
||||||
THREAD_TITLE: "h3.contentRow-title",
|
|
||||||
TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]",
|
|
||||||
UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]",
|
|
||||||
USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText",
|
|
||||||
USERNAME_INPUT: "input[name=\"login\"]",
|
|
||||||
WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
|
|
||||||
WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next",
|
|
||||||
WATCHED_THREAD_URLS: "a[href^=\"/threads/\"][data-tp-primary]",
|
|
||||||
DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]",
|
|
||||||
SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main",
|
|
||||||
SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)",
|
|
||||||
THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type",
|
|
||||||
});
|
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
module.exports = Object.freeze({
|
|
||||||
F95_BASE_URL: "https://f95zone.to",
|
|
||||||
F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
|
|
||||||
F95_LATEST_UPDATES: "https://f95zone.to/latest",
|
|
||||||
F95_LOGIN_URL: "https://f95zone.to/login",
|
|
||||||
F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
|
|
||||||
});
|
|
||||||
|
|
@ -1,467 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
// Public modules from npm
|
|
||||||
const HTMLParser = require("node-html-parser");
|
|
||||||
const puppeteer = require("puppeteer"); // skipcq: JS-0128
|
|
||||||
|
|
||||||
// Modules from file
|
|
||||||
const shared = require("./shared.js");
|
|
||||||
const selectorK = require("./constants/css-selector.js");
|
|
||||||
const { preparePage } = require("./puppeteer-helper.js");
|
|
||||||
const GameDownload = require("./classes/game-download.js");
|
|
||||||
const GameInfo = require("./classes/game-info.js");
|
|
||||||
const urlHelper = require("./url-helper.js");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Get information from the game's main page.
|
|
||||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
|
||||||
* @param {String} url URL (String) of the game/mod to extract data from
|
|
||||||
* @return {Promise<GameInfo>} Complete information about the game you are
|
|
||||||
* looking for
|
|
||||||
*/
|
|
||||||
module.exports.getGameInfo = async function (browser, url) {
|
|
||||||
shared.logger.info("Obtaining game info");
|
|
||||||
|
|
||||||
// Verify the correctness of the URL
|
|
||||||
const exists = await urlHelper.urlExists(url);
|
|
||||||
if (!exists) throw new URIError(`${url} is not a valid URL`);
|
|
||||||
if (!urlHelper.isF95URL(url))
|
|
||||||
throw new Error(`${url} is not a valid F95Zone URL`);
|
|
||||||
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
await page.goto(url, {
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
}); // Go to the game page and wait until it loads
|
|
||||||
|
|
||||||
// It asynchronously searches for the elements and
|
|
||||||
// then waits at the end to compile the object to be returned
|
|
||||||
let info = new GameInfo();
|
|
||||||
const title = getGameTitle(page);
|
|
||||||
const author = getGameAuthor(page);
|
|
||||||
const tags = getGameTags(page);
|
|
||||||
const redirectUrl = urlHelper.getUrlRedirect(url);
|
|
||||||
info = await parsePrefixes(page, info); // Fill status/engines/isMod
|
|
||||||
const structuredText = await getMainPostStructuredText(page);
|
|
||||||
const overview = getOverview(structuredText, info.isMod);
|
|
||||||
const parsedInfos = parseConversationPage(structuredText);
|
|
||||||
const previewSource = getGamePreviewSource(page);
|
|
||||||
const changelog = getLastChangelog(page);
|
|
||||||
|
|
||||||
// Fill in the GameInfo element with the information obtained
|
|
||||||
info.name = await title;
|
|
||||||
info.author = await author;
|
|
||||||
info.tags = await tags;
|
|
||||||
info.f95url = await redirectUrl;
|
|
||||||
info.overview = overview;
|
|
||||||
info.lastUpdate = info.isMod
|
|
||||||
? parsedInfos.UPDATED
|
|
||||||
: parsedInfos.THREAD_UPDATED;
|
|
||||||
info.previewSource = await previewSource;
|
|
||||||
info.changelog = await changelog;
|
|
||||||
info.version = await exports.getGameVersionFromTitle(browser, info);
|
|
||||||
|
|
||||||
//let downloadData = getGameDownloadLink(page);
|
|
||||||
//info.downloadInfo = await downloadData;
|
|
||||||
/* Downloading games without going directly to
|
|
||||||
* the platform appears to be prohibited by
|
|
||||||
* the guidelines. It is therefore useless to
|
|
||||||
* keep the links for downloading the games. */
|
|
||||||
|
|
||||||
await page.close(); // Close the page
|
|
||||||
shared.logger.info("Founded data for " + info.name);
|
|
||||||
return info;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Obtain the game version without parsing again all the data of the game.
|
|
||||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
|
||||||
* @param {GameInfo} info Information about the game
|
|
||||||
* @returns {Promise<String>} Online version of the game
|
|
||||||
*/
|
|
||||||
module.exports.getGameVersionFromTitle = async function (browser, info) {
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
await page.goto(info.f95url, {
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
}); // Go to the game page and wait until it loads
|
|
||||||
|
|
||||||
// Get the title
|
|
||||||
const titleHTML = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(selector) => document.querySelector(selector).innerHTML,
|
|
||||||
selectorK.GAME_TITLE
|
|
||||||
);
|
|
||||||
const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText;
|
|
||||||
|
|
||||||
// The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR]
|
|
||||||
const startIndex = title.indexOf("[") + 1;
|
|
||||||
const endIndex = title.indexOf("]", startIndex);
|
|
||||||
let version = title.substring(startIndex, endIndex).trim().toUpperCase();
|
|
||||||
if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence
|
|
||||||
await page.close();
|
|
||||||
return cleanFSString(version);
|
|
||||||
};
|
|
||||||
|
|
||||||
//#region Private methods
|
|
||||||
/**
|
|
||||||
* Clean a string from invalid File System chars.
|
|
||||||
* @param {String} s
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
function cleanFSString(s) {
|
|
||||||
const rx = /[/\\?%*:|"<>]/g;
|
|
||||||
return s.replace(rx, "");
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Get the game description from its web page.
|
|
||||||
* Different processing depending on whether the game is a mod or not.
|
|
||||||
* @param {String} text Structured text extracted from the game's web page
|
|
||||||
* @param {Boolean} isMod Specify if it is a game or a mod
|
|
||||||
* @returns {Promise<String>} Game description
|
|
||||||
*/
|
|
||||||
function getOverview(text, isMod) {
|
|
||||||
// Get overview (different parsing for game and mod)
|
|
||||||
let overviewEndIndex;
|
|
||||||
if (isMod) overviewEndIndex = text.indexOf("Updated");
|
|
||||||
else overviewEndIndex = text.indexOf("Thread Updated");
|
|
||||||
return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Extrapolate the page structure by removing the element tags
|
|
||||||
* and leaving only the text and its spacing.
|
|
||||||
* @param {puppeteer.Page} page Page containing the text
|
|
||||||
* @returns {Promise<String>} Structured text
|
|
||||||
*/
|
|
||||||
async function getMainPostStructuredText(page) {
|
|
||||||
// Gets the first post, where are listed all the game's informations
|
|
||||||
const post = (await page.$$(selectorK.THREAD_POSTS))[0];
|
|
||||||
|
|
||||||
// The info are plain text so we need to parse the HTML code
|
|
||||||
const bodyHTML = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(mainPost) => mainPost.innerHTML,
|
|
||||||
post
|
|
||||||
);
|
|
||||||
return HTMLParser.parse(bodyHTML).structuredText;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Extrapolates and cleans the author from the page passed by parameter.
|
|
||||||
* @param {puppeteer.Page} page Page containing the author to be extrapolated
|
|
||||||
* @returns {Promise<String>} Game author
|
|
||||||
*/
|
|
||||||
async function getGameAuthor(page) {
|
|
||||||
// Get the game/mod name (without square brackets)
|
|
||||||
const titleHTML = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(selector) => document.querySelector(selector).innerHTML,
|
|
||||||
selectorK.GAME_TITLE
|
|
||||||
);
|
|
||||||
const structuredTitle = HTMLParser.parse(titleHTML);
|
|
||||||
|
|
||||||
// The last element **shoud be** the title without prefixes (engines, status, other...)
|
|
||||||
const gameTitle = structuredTitle.childNodes.pop().rawText;
|
|
||||||
|
|
||||||
// The last square brackets contain the author
|
|
||||||
const startTitleIndex = gameTitle.lastIndexOf("[") + 1;
|
|
||||||
return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Process the post text to get all the useful
|
|
||||||
* information in the format *DESCRIPTOR : VALUE*.
|
|
||||||
* @param {String} text Structured text of the post
|
|
||||||
* @returns {Object} Dictionary of information
|
|
||||||
*/
|
|
||||||
function parseConversationPage(text) {
|
|
||||||
const dataPairs = {};
|
|
||||||
|
|
||||||
// The information searched in the game post are one per line
|
|
||||||
const splittedText = text.split("\n");
|
|
||||||
for (const line of splittedText) {
|
|
||||||
if (!line.includes(":")) continue;
|
|
||||||
|
|
||||||
// Create pair key/value
|
|
||||||
const splitted = line.split(":");
|
|
||||||
const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch
|
|
||||||
const value = splitted[1].trim();
|
|
||||||
|
|
||||||
// Add pair to the dict if valid
|
|
||||||
if (value !== "") dataPairs[key] = value;
|
|
||||||
}
|
|
||||||
|
|
||||||
return dataPairs;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Gets the URL of the image used as a preview for the game in the conversation.
|
|
||||||
* @param {puppeteer.Page} page Page containing the URL to be extrapolated
|
|
||||||
* @returns {Promise<String>} URL (String) of the image or null if failed to get it
|
|
||||||
*/
|
|
||||||
async function getGamePreviewSource(page) {
|
|
||||||
// Wait for the selector or return an empty value
|
|
||||||
try {
|
|
||||||
await page.waitForSelector(selectorK.GAME_IMAGES);
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const src = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(selector) => {
|
|
||||||
// Get the firs image available
|
|
||||||
const img = document.querySelector(selector);
|
|
||||||
|
|
||||||
if (img) return img.getAttribute("src");
|
|
||||||
else return null;
|
|
||||||
},
|
|
||||||
selectorK.GAME_IMAGES
|
|
||||||
);
|
|
||||||
|
|
||||||
// Check if the URL is valid
|
|
||||||
return urlHelper.isStringAValidURL(src) ? src : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Extrapolates and cleans the title from the page passed by parameter.
|
|
||||||
* @param {puppeteer.Page} page Page containing the title to be extrapolated
|
|
||||||
* @returns {Promise<String>} Game title
|
|
||||||
*/
|
|
||||||
async function getGameTitle(page) {
|
|
||||||
// Get the game/mod name (without square brackets)
|
|
||||||
const titleHTML = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(selector) => document.querySelector(selector).innerHTML,
|
|
||||||
selectorK.GAME_TITLE
|
|
||||||
);
|
|
||||||
const structuredTitle = HTMLParser.parse(titleHTML);
|
|
||||||
|
|
||||||
// The last element **shoud be** the title without prefixes (engines, status, other...)
|
|
||||||
const gameTitle = structuredTitle.childNodes.pop().rawText;
|
|
||||||
const endTitleIndex = gameTitle.indexOf("[");
|
|
||||||
return gameTitle.substring(0, endTitleIndex).trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Get the alphabetically sorted list of tags associated with the game.
|
|
||||||
* @param {puppeteer.Page} page Page containing the tags to be extrapolated
|
|
||||||
* @returns {Promise<String[]>} List of uppercase tags
|
|
||||||
*/
|
|
||||||
async function getGameTags(page) {
|
|
||||||
const tags = [];
|
|
||||||
|
|
||||||
// Get the game tags
|
|
||||||
for (const handle of await page.$$(selectorK.GAME_TAGS)) {
|
|
||||||
const tag = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(element) => element.innerText,
|
|
||||||
handle
|
|
||||||
);
|
|
||||||
tags.push(tag.toUpperCase());
|
|
||||||
}
|
|
||||||
return tags.sort();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Process the game title prefixes to extract information such as game status,
|
|
||||||
* graphics engine used, and whether it is a mod or original game.
|
|
||||||
* @param {puppeteer.Page} page Page containing the prefixes to be extrapolated
|
|
||||||
* @param {GameInfo} info Object to assign the identified information to
|
|
||||||
* @returns {Promise<GameInfo>} GameInfo object passed in to which the identified information has been added
|
|
||||||
*/
|
|
||||||
async function parsePrefixes(page, info) {
|
|
||||||
// The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete'
|
|
||||||
info.status = "ONGOING";
|
|
||||||
for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) {
|
|
||||||
const value = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(element) => element.innerText,
|
|
||||||
handle
|
|
||||||
);
|
|
||||||
|
|
||||||
// Clean the prefix
|
|
||||||
const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim();
|
|
||||||
|
|
||||||
// Getting infos...
|
|
||||||
if (shared.statuses.includes(prefix)) info.status = prefix;
|
|
||||||
else if (shared.engines.includes(prefix)) info.engine = prefix;
|
|
||||||
// This is not a game but a mod
|
|
||||||
else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true;
|
|
||||||
}
|
|
||||||
return info;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Get the last changelog available for the game.
|
|
||||||
* @param {puppeteer.Page} page Page containing the changelog
|
|
||||||
* @returns {Promise<String>} Changelog for the last version or a empty string if no changelog is found
|
|
||||||
*/
|
|
||||||
async function getLastChangelog(page) {
|
|
||||||
// Gets the first post, where are listed all the game's informations
|
|
||||||
const post = (await page.$$(selectorK.THREAD_POSTS))[0];
|
|
||||||
|
|
||||||
const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG);
|
|
||||||
if (!spoiler) return "";
|
|
||||||
|
|
||||||
const changelogHTML = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(e) => e.innerText,
|
|
||||||
spoiler
|
|
||||||
);
|
|
||||||
let parsedText = HTMLParser.parse(changelogHTML).structuredText;
|
|
||||||
|
|
||||||
// Clean the text
|
|
||||||
if (parsedText.startsWith("Spoiler"))
|
|
||||||
parsedText = parsedText.replace("Spoiler", "");
|
|
||||||
if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", "");
|
|
||||||
return parsedText.trim();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Get game download links for different platforms.
|
|
||||||
* @param {puppeteer.Page} page Page containing the links to be extrapolated
|
|
||||||
* @returns {Promise<GameDownload[]>} List of objects used for game download
|
|
||||||
* @deprecated
|
|
||||||
*/
|
|
||||||
/* istanbul ignore next */
|
|
||||||
// skipcq: JS-0128
|
|
||||||
async function getGameDownloadLink(page) {
|
|
||||||
// Most used hosting platforms
|
|
||||||
const hostingPlatforms = [
|
|
||||||
"MEGA",
|
|
||||||
"NOPY",
|
|
||||||
"FILESUPLOAD",
|
|
||||||
"MIXDROP",
|
|
||||||
"UPLOADHAVEN",
|
|
||||||
"PIXELDRAIN",
|
|
||||||
"FILESFM",
|
|
||||||
];
|
|
||||||
|
|
||||||
// Supported OS platforms
|
|
||||||
const platformOS = ["WIN", "LINUX", "MAC", "ALL"];
|
|
||||||
|
|
||||||
// Gets the <span> which contains the download links
|
|
||||||
const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER);
|
|
||||||
if (temp.length === 0) return [];
|
|
||||||
|
|
||||||
// Look for the container that contains the links
|
|
||||||
// It is necessary because the same css selector
|
|
||||||
// also identifies other elements on the page
|
|
||||||
let container = null;
|
|
||||||
for (const candidate of temp) {
|
|
||||||
if (container !== null) break;
|
|
||||||
const upperText = (
|
|
||||||
await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(e) => e.innerText,
|
|
||||||
candidate
|
|
||||||
)
|
|
||||||
).toUpperCase();
|
|
||||||
|
|
||||||
// Search if the container contains the name of a hosting platform
|
|
||||||
for (const p of hostingPlatforms) {
|
|
||||||
if (upperText.includes(p)) {
|
|
||||||
container = candidate;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (container === null) return [];
|
|
||||||
|
|
||||||
// Extract the HTML text from the container
|
|
||||||
const searchText = (
|
|
||||||
await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(e) => e.innerHTML,
|
|
||||||
container
|
|
||||||
)
|
|
||||||
).toLowerCase();
|
|
||||||
|
|
||||||
// Parse the download links
|
|
||||||
const downloadData = [];
|
|
||||||
for (const platform of platformOS) {
|
|
||||||
const data = extractGameHostingData(platform, searchText);
|
|
||||||
downloadData.push(...data);
|
|
||||||
}
|
|
||||||
return downloadData;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* From the HTML text it extracts the game download links for the specified operating system.
|
|
||||||
* @param {String} platform Name of the operating system to look for a compatible link to.
|
|
||||||
* It can only be *WIN/LINUX/MAC/ALL*
|
|
||||||
* @param {String} text HTML string to extract links from
|
|
||||||
* @returns {GameDownload[]} List of game download links for the selected platform
|
|
||||||
* @deprecated
|
|
||||||
*/
|
|
||||||
/* istanbul ignore next */
|
|
||||||
function extractGameHostingData(platform, text) {
|
|
||||||
const PLATFORM_BOLD_OPEN = "<b>";
|
|
||||||
const CONTAINER_SPAN_CLOSE = "</span>";
|
|
||||||
const LINK_OPEN = "<a";
|
|
||||||
const LINK_CLOSE = "</a>";
|
|
||||||
const HREF_START = "href='";
|
|
||||||
const HREF_END = "'";
|
|
||||||
const TAG_CLOSE = ">";
|
|
||||||
|
|
||||||
// Identify the individual platforms
|
|
||||||
let startIndex = text.indexOf(platform.toLowerCase());
|
|
||||||
if (startIndex === -1) return [];
|
|
||||||
else startIndex += platform.length;
|
|
||||||
|
|
||||||
// Find the <b>platform</b>
|
|
||||||
let endIndex =
|
|
||||||
text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length;
|
|
||||||
|
|
||||||
// Find the end of the container
|
|
||||||
if (endIndex === -1)
|
|
||||||
endIndex =
|
|
||||||
text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) +
|
|
||||||
CONTAINER_SPAN_CLOSE.length;
|
|
||||||
|
|
||||||
text = text.substring(startIndex, endIndex);
|
|
||||||
|
|
||||||
const downloadData = [];
|
|
||||||
const linkTags = text.split(LINK_OPEN);
|
|
||||||
for (const tag of linkTags) {
|
|
||||||
// Ignore non-link string
|
|
||||||
if (!tag.includes(HREF_START)) continue;
|
|
||||||
|
|
||||||
// Find the hosting platform name
|
|
||||||
startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length;
|
|
||||||
endIndex = tag.indexOf(LINK_CLOSE, startIndex);
|
|
||||||
const hosting = tag.substring(startIndex, endIndex);
|
|
||||||
|
|
||||||
// Find the 'href' attribute
|
|
||||||
startIndex = tag.indexOf(HREF_START) + HREF_START.length;
|
|
||||||
endIndex = tag.indexOf(HREF_END, startIndex);
|
|
||||||
const link = tag.substring(startIndex, endIndex);
|
|
||||||
|
|
||||||
if (urlHelper.isStringAValidURL(link)) {
|
|
||||||
const gd = new GameDownload();
|
|
||||||
gd.hosting = hosting.toUpperCase();
|
|
||||||
gd.link = link;
|
|
||||||
gd.supportedOS = platform.toUpperCase();
|
|
||||||
|
|
||||||
downloadData.push(gd);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return downloadData;
|
|
||||||
}
|
|
||||||
|
|
||||||
//#endregion Private methods
|
|
||||||
|
|
@ -1,132 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
// Public modules from npm
|
|
||||||
const puppeteer = require("puppeteer"); // skipcq: JS-0128
|
|
||||||
|
|
||||||
// Modules from file
|
|
||||||
const shared = require("./shared.js");
|
|
||||||
const urlK = require("./constants/url.js");
|
|
||||||
const selectorK = require("./constants/css-selector.js");
|
|
||||||
const { preparePage } = require("./puppeteer-helper.js");
|
|
||||||
const { isF95URL } = require("./url-helper.js");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Search the F95Zone portal to find possible conversations regarding the game you are looking for.
|
|
||||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
|
||||||
* @param {String} gamename Name of the game to search for
|
|
||||||
* @returns {Promise<String[]>} List of URL of possible games obtained from the preliminary research on the F95 portal
|
|
||||||
*/
|
|
||||||
module.exports.getSearchGameResults = async function (browser, gamename) {
|
|
||||||
shared.logger.info(`Searching ${gamename} on F95Zone`);
|
|
||||||
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
|
||||||
await page.goto(urlK.F95_SEARCH_URL, {
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
}); // Go to the search form and wait for it
|
|
||||||
|
|
||||||
// Explicitly wait for the required items to load
|
|
||||||
await Promise.all([
|
|
||||||
page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX),
|
|
||||||
page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX),
|
|
||||||
page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION),
|
|
||||||
page.waitForSelector(selectorK.SEARCH_BUTTON),
|
|
||||||
]);
|
|
||||||
|
|
||||||
await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire
|
|
||||||
await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles
|
|
||||||
await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod
|
|
||||||
await Promise.all([
|
|
||||||
page.click(selectorK.SEARCH_BUTTON), // Execute search
|
|
||||||
page.waitForNavigation({
|
|
||||||
waitUntil: shared.WAIT_STATEMENT,
|
|
||||||
}), // Wait for page to load
|
|
||||||
]);
|
|
||||||
|
|
||||||
// Select all conversation titles
|
|
||||||
const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY);
|
|
||||||
|
|
||||||
// For each element found extract the info about the conversation
|
|
||||||
shared.logger.info("Extracting info from conversations");
|
|
||||||
const results = [];
|
|
||||||
for (const element of resultsThread) {
|
|
||||||
const gameUrl = await getOnlyGameThreads(page, element);
|
|
||||||
if (gameUrl !== null) results.push(gameUrl);
|
|
||||||
}
|
|
||||||
shared.logger.info(`Find ${results.length} conversations`);
|
|
||||||
await page.close(); // Close the page
|
|
||||||
|
|
||||||
return results;
|
|
||||||
};
|
|
||||||
|
|
||||||
//#region Private methods
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Return the link of a conversation if it is a game or a mod.
|
|
||||||
* @param {puppeteer.Page} page Page containing the conversation to be analyzed
|
|
||||||
* @param {puppeteer.ElementHandle} divHandle Element of the conversation to be analyzed
|
|
||||||
* @return {Promise<String>} URL of the game/mod or null if the URL is not of a game
|
|
||||||
*/
|
|
||||||
async function getOnlyGameThreads(page, divHandle) {
|
|
||||||
// Obtain the elements containing the basic information
|
|
||||||
const titleHandle = await divHandle.$(selectorK.THREAD_TITLE);
|
|
||||||
const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP);
|
|
||||||
|
|
||||||
// Get the forum where the thread was posted
|
|
||||||
const forum = await getMembershipForum(page, forumHandle);
|
|
||||||
if (forum !== "GAMES" && forum !== "MODS") return null;
|
|
||||||
|
|
||||||
// Get the URL of the thread from the title
|
|
||||||
return await getThreadURL(page, titleHandle);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Obtain the membership forum of the thread passed throught 'handle'.
|
|
||||||
* @param {puppeteer.Page} page Page containing the conversation to be analyzed
|
|
||||||
* @param {puppeteer.ElementHandle} handle Handle containing the forum membership
|
|
||||||
* @returns {Promise<String>} Uppercase membership category
|
|
||||||
*/
|
|
||||||
async function getMembershipForum(page, handle) {
|
|
||||||
// The link can be something like:
|
|
||||||
// + /forums/request.NUMBER/
|
|
||||||
// + /forums/game-recommendations-identification.NUMBER/
|
|
||||||
// + /forums/games.NUMBER/ <-- We need this
|
|
||||||
|
|
||||||
let link = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(e) => e.getAttribute("href"),
|
|
||||||
handle
|
|
||||||
);
|
|
||||||
|
|
||||||
// Parse link
|
|
||||||
link = link.replace("/forums/", "");
|
|
||||||
const endIndex = link.indexOf(".");
|
|
||||||
const forum = link.substring(0, endIndex);
|
|
||||||
|
|
||||||
return forum.toUpperCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @private
|
|
||||||
* Obtain the URL of the thread passed through 'handle'.
|
|
||||||
* @param {puppeteer.Page} page Page containing the conversation to be analyzed
|
|
||||||
* @param {puppeteer.ElementHandle} handle Handle containing the thread title
|
|
||||||
* @returns {Promise<String>} URL of the thread
|
|
||||||
*/
|
|
||||||
async function getThreadURL(page, handle) {
|
|
||||||
const relativeURLThread = await page.evaluate(
|
|
||||||
/* istanbul ignore next */
|
|
||||||
(e) => e.querySelector("a").href,
|
|
||||||
handle
|
|
||||||
);
|
|
||||||
|
|
||||||
// Some game already have a full URL...
|
|
||||||
if (isF95URL(relativeURLThread)) return relativeURLThread;
|
|
||||||
|
|
||||||
// ... else compose the URL and return
|
|
||||||
const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString();
|
|
||||||
return urlThread;
|
|
||||||
}
|
|
||||||
//#endregion Private methods
|
|
||||||
|
|
@ -1,60 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
// Public modules from npm
|
|
||||||
const puppeteer = require("puppeteer");
|
|
||||||
|
|
||||||
// Modules from file
|
|
||||||
const shared = require("./shared.js");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Create a Chromium instance used to navigate with Puppeteer.
|
|
||||||
* By default the browser is headless.
|
|
||||||
* @returns {Promise<puppeteer.Browser>} Created browser
|
|
||||||
*/
|
|
||||||
module.exports.prepareBrowser = async function () {
|
|
||||||
// Create a headless browser
|
|
||||||
let browser = null;
|
|
||||||
if (shared.chromiumLocalPath) {
|
|
||||||
browser = await puppeteer.launch({
|
|
||||||
executablePath: shared.chromiumLocalPath,
|
|
||||||
headless: !shared.debug, // Use GUI when debug = true
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
browser = await puppeteer.launch({
|
|
||||||
headless: !shared.debug, // Use GUI when debug = true
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
return browser;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Prepare a page used to navigate the browser.
|
|
||||||
* The page is set up to reject image download requests. The user agent is also changed.
|
|
||||||
* @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created
|
|
||||||
* @returns {Promise<puppeteer.Page>} New page
|
|
||||||
*/
|
|
||||||
module.exports.preparePage = async function (browser) {
|
|
||||||
// Create new page in the browser argument
|
|
||||||
const page = await browser.newPage();
|
|
||||||
|
|
||||||
// Block image download
|
|
||||||
await page.setRequestInterception(true);
|
|
||||||
page.on("request", (request) => {
|
|
||||||
if (request.resourceType() === "image") request.abort();
|
|
||||||
else if (request.resourceType === "font") request.abort();
|
|
||||||
// else if (request.resourceType() == 'stylesheet') request.abort();
|
|
||||||
// else if(request.resourceType == 'media') request.abort();
|
|
||||||
else request.continue();
|
|
||||||
});
|
|
||||||
|
|
||||||
// Set custom user-agent
|
|
||||||
const userAgent =
|
|
||||||
"Mozilla/5.0 (X11; Linux x86_64)" +
|
|
||||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
|
|
||||||
await page.setUserAgent(userAgent);
|
|
||||||
|
|
||||||
return page;
|
|
||||||
};
|
|
||||||
|
|
@ -1,173 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
// Core modules
|
|
||||||
const { join } = require("path");
|
|
||||||
|
|
||||||
const log4js = require("log4js");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Class containing variables shared between modules.
|
|
||||||
*/
|
|
||||||
class Shared {
|
|
||||||
//#region Properties
|
|
||||||
/**
|
|
||||||
* Shows log messages and other useful functions for module debugging.
|
|
||||||
* @type Boolean
|
|
||||||
*/
|
|
||||||
static #_debug = false;
|
|
||||||
/**
|
|
||||||
* Indicates whether a user is logged in to the F95Zone platform or not.
|
|
||||||
* @type Boolean
|
|
||||||
*/
|
|
||||||
static #_isLogged = false;
|
|
||||||
/**
|
|
||||||
* List of cookies obtained from the F95Zone platform.
|
|
||||||
* @type Object[]
|
|
||||||
*/
|
|
||||||
static #_cookies = null;
|
|
||||||
/**
|
|
||||||
* List of possible game engines used for development.
|
|
||||||
* @type String[]
|
|
||||||
*/
|
|
||||||
static #_engines = null;
|
|
||||||
/**
|
|
||||||
* List of possible development statuses that a game can assume.
|
|
||||||
* @type String[]
|
|
||||||
*/
|
|
||||||
static #_statuses = null;
|
|
||||||
/**
|
|
||||||
* Wait instruction for the browser created by puppeteer.
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
static WAIT_STATEMENT = "domcontentloaded";
|
|
||||||
/**
|
|
||||||
* Path to the directory to save the cache generated by the API.
|
|
||||||
* @type String
|
|
||||||
*/
|
|
||||||
static #_cacheDir = "./f95cache";
|
|
||||||
/**
|
|
||||||
* If true, it opens a new browser for each request to
|
|
||||||
* the F95Zone platform, otherwise it reuses the same.
|
|
||||||
* @type Boolean
|
|
||||||
*/
|
|
||||||
static #_isolation = false;
|
|
||||||
/**
|
|
||||||
* Logger object used to write to both file and console.
|
|
||||||
* @type log4js.Logger
|
|
||||||
*/
|
|
||||||
static #_logger = log4js.getLogger();
|
|
||||||
//#endregion Properties
|
|
||||||
|
|
||||||
//#region Getters
|
|
||||||
/**
|
|
||||||
* Shows log messages and other useful functions for module debugging.
|
|
||||||
* @returns {Boolean}
|
|
||||||
*/
|
|
||||||
static get debug() {
|
|
||||||
return this.#_debug;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Indicates whether a user is logged in to the F95Zone platform or not.
|
|
||||||
* @returns {Boolean}
|
|
||||||
*/
|
|
||||||
static get isLogged() {
|
|
||||||
return this.#_isLogged;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* List of cookies obtained from the F95Zone platform.
|
|
||||||
* @returns {Object[]}
|
|
||||||
*/
|
|
||||||
static get cookies() {
|
|
||||||
return this.#_cookies;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* List of possible game engines used for development.
|
|
||||||
* @returns {String[]}
|
|
||||||
*/
|
|
||||||
static get engines() {
|
|
||||||
return this.#_engines;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* List of possible development states that a game can assume.
|
|
||||||
* @returns {String[]}
|
|
||||||
*/
|
|
||||||
static get statuses() {
|
|
||||||
return this.#_statuses;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Directory to save the API cache.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
static get cacheDir() {
|
|
||||||
return this.#_cacheDir;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Path to the F95 platform cache.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
static get cookiesCachePath() {
|
|
||||||
return join(this.#_cacheDir, "cookies.json");
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Path to the game engine cache.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
static get enginesCachePath() {
|
|
||||||
return join(this.#_cacheDir, "engines.json");
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Path to the cache of possible game states.
|
|
||||||
* @returns {String}
|
|
||||||
*/
|
|
||||||
static get statusesCachePath() {
|
|
||||||
return join(this.#_cacheDir, "statuses.json");
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* If true, it opens a new browser for each request
|
|
||||||
* to the F95Zone platform, otherwise it reuses the same.
|
|
||||||
* @returns {Boolean}
|
|
||||||
*/
|
|
||||||
static get isolation() {
|
|
||||||
return this.#_isolation;
|
|
||||||
}
|
|
||||||
/**
|
|
||||||
* Logger object used to write to both file and console.
|
|
||||||
* @returns {log4js.Logger}
|
|
||||||
*/
|
|
||||||
static get logger() {
|
|
||||||
return this.#_logger;
|
|
||||||
}
|
|
||||||
//#endregion Getters
|
|
||||||
|
|
||||||
//#region Setters
|
|
||||||
static set cookies(val) {
|
|
||||||
this.#_cookies = val;
|
|
||||||
}
|
|
||||||
|
|
||||||
static set engines(val) {
|
|
||||||
this.#_engines = val;
|
|
||||||
}
|
|
||||||
|
|
||||||
static set statuses(val) {
|
|
||||||
this.#_statuses = val;
|
|
||||||
}
|
|
||||||
|
|
||||||
static set cacheDir(val) {
|
|
||||||
this.#_cacheDir = val;
|
|
||||||
}
|
|
||||||
|
|
||||||
static set debug(val) {
|
|
||||||
this.#_debug = val;
|
|
||||||
}
|
|
||||||
|
|
||||||
static set isLogged(val) {
|
|
||||||
this.#_isLogged = val;
|
|
||||||
}
|
|
||||||
|
|
||||||
static set isolation(val) {
|
|
||||||
this.#_isolation = val;
|
|
||||||
}
|
|
||||||
//#endregion Setters
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = Shared;
|
|
||||||
|
|
@ -1,72 +0,0 @@
|
||||||
"use strict";
|
|
||||||
|
|
||||||
// Public modules from npm
|
|
||||||
const ky = require("ky-universal").create({
|
|
||||||
throwHttpErrors: false,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Modules from file
|
|
||||||
const { F95_BASE_URL } = require("./constants/url.js");
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Check if the url belongs to the domain of the F95 platform.
|
|
||||||
* @param {String} url URL to check
|
|
||||||
* @returns {Boolean} true if the url belongs to the domain, false otherwise
|
|
||||||
*/
|
|
||||||
module.exports.isF95URL = function (url) {
|
|
||||||
if (url.toString().startsWith(F95_BASE_URL)) return true;
|
|
||||||
else return false;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Checks if the string passed by parameter has a properly formatted and valid path to a URL.
|
|
||||||
* @param {String} url String to check for correctness
|
|
||||||
* @returns {Boolean} true if the string is a valid URL, false otherwise
|
|
||||||
*/
|
|
||||||
module.exports.isStringAValidURL = function (url) {
|
|
||||||
try {
|
|
||||||
new URL(url); // skipcq: JS-0078
|
|
||||||
return true;
|
|
||||||
} catch (err) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Check if a particular URL is valid and reachable on the web.
|
|
||||||
* @param {String} url URL to check
|
|
||||||
* @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false
|
|
||||||
* @returns {Promise<Boolean>} true if the URL exists, false otherwise
|
|
||||||
*/
|
|
||||||
module.exports.urlExists = async function (url, checkRedirect) {
|
|
||||||
if (!exports.isStringAValidURL(url)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
const response = await ky.head(url);
|
|
||||||
let valid = response !== undefined && !/4\d\d/.test(response.status);
|
|
||||||
|
|
||||||
if (!valid) return false;
|
|
||||||
|
|
||||||
if (checkRedirect) {
|
|
||||||
const redirectUrl = await exports.getUrlRedirect(url);
|
|
||||||
if (redirectUrl === url) valid = true;
|
|
||||||
else valid = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return valid;
|
|
||||||
};
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @protected
|
|
||||||
* Check if the URL has a redirect to another page.
|
|
||||||
* @param {String} url URL to check for redirect
|
|
||||||
* @returns {Promise<String>} Redirect URL or the passed URL
|
|
||||||
*/
|
|
||||||
module.exports.getUrlRedirect = async function (url) {
|
|
||||||
const response = await ky.head(url);
|
|
||||||
return response.url;
|
|
||||||
};
|
|
||||||
Loading…
Reference in New Issue