diff --git a/app/index.js b/app/index.js index cb26ad8..a8d62cf 100644 --- a/app/index.js +++ b/app/index.js @@ -1,24 +1,18 @@ "use strict"; -// Core modules -const fs = require("fs"); - // Modules from file -const shared = require("../app/scripts/shared.js"); -const urlK = require("../app/scripts/constants/url.js"); -const selectorK = require("../app/scripts/constants/css-selector.js"); -const urlHelper = require("../app/scripts/url-helper.js"); -const scraper = require("../app/scripts/game-scraper.js"); -const { - prepareBrowser, - preparePage, -} = require("../app/scripts/puppeteer-helper.js"); -const searcher = require("../app/scripts/game-searcher.js"); +const shared = require("./scripts/shared.js"); +const f95url = require("./scripts/constants/url.js"); +const f95selector = require("./scripts/constants/css-selector.js"); +const networkHelper = require("./scripts/network-helper.js"); +const scraper = require("./scripts/scraper.js"); +const searcher = require("./scripts/searcher.js"); // Classes from file -const GameInfo = require("../app/scripts/classes/game-info.js"); -const LoginResult = require("../app/scripts/classes/login-result.js"); -const UserData = require("../app/scripts/classes/user-data.js"); +const Credentials = require("./scripts/classes/credentials.js"); +const GameInfo = require("./scripts/classes/game-info.js"); +const LoginResult = require("./scripts/classes/login-result.js"); +const UserData = require("./scripts/classes/user-data.js"); //#region Export classes module.exports.GameInfo = GameInfo; @@ -35,7 +29,7 @@ module.exports.debug = function (value) { shared.debug = value; // Configure logger - shared.logger.level = value ? "debug" : "warn"; + shared.logger.level = value ? "trace" : "warn"; }; /** * @public @@ -45,46 +39,9 @@ module.exports.debug = function (value) { module.exports.isLogged = function () { return shared.isLogged; }; -/** - * @public - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {String} - */ -module.exports.setIsolation = function (value) { - shared.isolation = value; -}; -/** - * @public - * Path to the cache directory - * @returns {String} - */ -module.exports.getCacheDir = function () { - return shared.cacheDir; -}; -/** - * @public - * Set path to the cache directory - * @returns {String} - */ -module.exports.setCacheDir = function (value) { - shared.cacheDir = value; - - // Create directory if it doesn't exist - if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); -}; -/** - * @public - * Set local chromium path. - * @returns {String} - */ -module.exports.setChromiumPath = function (value) { - shared.chromiumLocalPath = value; -}; //#endregion Export properties //#region Global variables -var _browser = null; const USER_NOT_LOGGED = "User not authenticated, unable to continue"; //#endregion @@ -99,91 +56,24 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue"; */ module.exports.login = async function (username, password) { if (shared.isLogged) { - shared.logger.info("Already logged in"); - const result = new LoginResult(true, "Already logged in"); - return result; + shared.logger.info(`${username} already authenticated`); + return new LoginResult(true, `${username} already authenticated`); } - // If cookies are loaded, use them to authenticate - shared.cookies = loadCookies(); - if (shared.cookies !== null) { - shared.logger.info("Valid session, no need to re-authenticate"); - shared.isLogged = true; - const result = new LoginResult(true, "Logged with cookies"); - return result; - } + shared.logger.trace("Fetching token..."); + const creds = new Credentials(username, password); + await creds.fetchToken(); - // Else, log in throught browser - shared.logger.info( - "No saved sessions or expired session, login on the platform" - ); - - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const result = await loginF95(browser, username, password); + shared.logger.trace(`Authentication for ${username}`); + const result = await networkHelper.autenticate(creds); shared.isLogged = result.success; - if (result.success) { - // Reload cookies - shared.cookies = loadCookies(); - shared.logger.info("User logged in through the platform"); - } else { - shared.logger.warn("Error during authentication: " + result.message); - } - if (shared.isolation) await browser.close(); + if (result.success) shared.logger.info("User logged in through the platform"); + else shared.logger.warn(`Error during authentication: ${result.message}`); + return result; }; -/** - * @public - * This method loads the main data from the F95 portal - * used to provide game information. You **must** be logged - * in to the portal before calling this method. - * @returns {Promise} Result of the operation - */ -module.exports.loadF95BaseData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } - shared.logger.info("Loading base data..."); - - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - - // Go to latest update page and wait for it to load - await page.goto(urlK.F95_LATEST_UPDATES, { - waitUntil: shared.WAIT_STATEMENT, - }); - - // Obtain engines (disk/online) - await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); - shared.engines = await loadValuesFromLatestPage( - page, - shared.enginesCachePath, - selectorK.ENGINE_ID_SELECTOR, - "engines" - ); - - // Obtain statuses (disk/online) - await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); - shared.statuses = await loadValuesFromLatestPage( - page, - shared.statusesCachePath, - selectorK.STATUS_ID_SELECTOR, - "statuses" - ); - - await page.close(); - if (shared.isolation) await browser.close(); - shared.logger.info("Base data loaded"); - return true; -}; /** * @public * Chek if exists a new version of the game. @@ -191,68 +81,54 @@ module.exports.loadF95BaseData = async function () { * @param {GameInfo} info Information about the game to get the version for * @returns {Promise} true if an update is available, false otherwise */ -module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged || !shared.cookies) { +module.exports.checkIfGameHasUpdate = async function (info) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return false; } // F95 change URL at every game update, // so if the URL is different an update is available - const exists = await urlHelper.urlExists(info.f95url, true); + const exists = await networkHelper.urlExists(info.url, true); if (!exists) return true; // Parse version from title - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); - - if (shared.isolation) await browser.close(); - + const onlineVersion = await scraper.getGameInfo(info.url).version; + + // Compare the versions return onlineVersion.toUpperCase() !== info.version.toUpperCase(); }; + /** * @public * Starting from the name, it gets all the information about the game you are looking for. * You **must** be logged in to the portal before calling this method. * @param {String} name Name of the game searched - * @param {Boolean} includeMods Indicates whether to also take mods into account when searching + * @param {Boolean} mod Indicate if you are looking for mods or games * @returns {Promise} List of information obtained where each item corresponds to * an identified game (in the case of homonymy of titles) */ -module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged || !shared.cookies) { +module.exports.getGameData = async function (name, mod) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return null; } - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const urlList = await searcher.getSearchGameResults(browser, name); + // Gets the search results of the game/mod being searched for + let urls = []; + if(mod) urls = await searcher.searchMod(name); + else urls = await searcher.searchGame(name); // Process previous partial results - const promiseList = []; - for (const url of urlList) { - // Start looking for information - promiseList.push(scraper.getGameInfo(browser, url)); + const results = []; + for (const url of urls) { + // Start looking for information + const info = scraper.getGameInfo(url); + results.push(info); } - - // Filter for mods - const result = []; - for (const info of await Promise.all(promiseList)) { - // Ignore empty results - if (!info) continue; - // Skip mods if not required - if (info.isMod && !includeMods) continue; - // Else save data - result.push(info); - } - - if (shared.isolation) await browser.close(); - return result; + return results; }; + /** * @public * Starting from the url, it gets all the information about the game you are looking for. @@ -261,27 +137,20 @@ module.exports.getGameData = async function (name, includeMods) { * @returns {Promise} Information about the game. If no game was found, null is returned */ module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged || !shared.cookies) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return null; } - // Check URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); - if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); - - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - + // Check URL validity + const exists = await networkHelper.urlExists(url); + if (!exists) throw new URIError(`${url} is not a valid URL`); + if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`); + // Get game data - const result = await scraper.getGameInfo(browser, url); - - if (shared.isolation) await browser.close(); - return result; + return await scraper.getGameInfo(url); }; + /** * @public * Gets the data of the currently logged in user. @@ -289,297 +158,76 @@ module.exports.getGameDataFromURL = async function (url) { * @returns {Promise} Data of the user currently logged in */ module.exports.getUserData = async function () { - if (!shared.isLogged || !shared.cookies) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return null; } - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_BASE_URL); // Go to base page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_ELEMENT), - page.waitForSelector(selectorK.AVATAR_PIC), - ]); - - const threads = getUserWatchedGameThreads(browser); + const threads = await getUserWatchedGameThreads(null); const username = await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector).innerText, - selectorK.USERNAME_ELEMENT + f95selector.USERNAME_ELEMENT ); const avatarSrc = await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector).getAttribute("src"), - selectorK.AVATAR_PIC + f95selector.AVATAR_PIC ); const ud = new UserData(); ud.username = username; - ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = await threads; - - await page.close(); - if (shared.isolation) await browser.close(); + ud.avatarSrc = networkHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; + ud.watchedThreads = threads; return ud; }; -/** - * @public - * Logout from the current user and gracefully close shared browser. - * You **must** be logged in to the portal before calling this method. - */ -module.exports.logout = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return; - } - - // Logout - shared.isLogged = false; - - // Gracefully close shared browser - if (!shared.isolation && _browser !== null) { - await _browser.close(); - _browser = null; - } -}; //#endregion //#region Private methods -//#region Cookies functions -/** - * @private - * Loads and verifies the expiration of previously stored cookies from disk - * if they exist, otherwise it returns null. - * @return {object[]} List of dictionaries or null if cookies don't exist - */ -function loadCookies() { - // Check the existence of the cookie file - if (fs.existsSync(shared.cookiesCachePath)) { - // Read cookies - const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); - const cookies = JSON.parse(cookiesJSON); - - // Check if the cookies have expired - for (const cookie of cookies) { - if (isCookieExpired(cookie)) return null; - } - - // Cookies loaded and verified - return cookies; - } else return null; -} -/** - * @private - * Check the validity of a cookie. - * @param {object} cookie Cookies to verify the validity. It's a dictionary - * @returns {Boolean} true if the cookie has expired, false otherwise - */ -function isCookieExpired(cookie) { - // Local variables - let expiredCookies = false; - - // Ignore cookies that never expire - const expirationUnixTimestamp = cookie.expire; - - if (expirationUnixTimestamp !== "-1") { - // Convert UNIX epoch timestamp to normal Date - const expirationDate = new Date(expirationUnixTimestamp * 1000); - - if (expirationDate < Date.now()) { - shared.logger.warn( - "Cookie " + cookie.name + " expired, you need to re-authenticate" - ); - expiredCookies = true; - } - } - - return expiredCookies; -} -//#endregion Cookies functions - -//#region Latest Updates page parserer -/** - * @private - * If present, it reads the file containing the searched values (engines or states) - * from the disk, otherwise it connects to the F95 portal (at the page - * https://f95zone.to/latest) and downloads them. - * @param {puppeteer.Page} page Page used to locate the required elements - * @param {String} path Path to disk of the JSON file containing the data to read / write - * @param {String} selector CSS selector of the required elements - * @param {String} elementRequested Required element (engines or states) used to detail log messages - * @returns {Promise} List of required values in uppercase - */ -async function loadValuesFromLatestPage( - page, - path, - selector, - elementRequested -) { - // If the values already exist they are loaded from disk without having to connect to F95 - shared.logger.info("Load " + elementRequested + " from disk..."); - if (fs.existsSync(path)) { - const valueJSON = fs.readFileSync(path); - return JSON.parse(valueJSON); - } - - // Otherwise, connect and download the data from the portal - shared.logger.info("No " + elementRequested + " cached, downloading..."); - const values = await getValuesFromLatestPage( - page, - selector, - "Getting " + elementRequested + " from page" - ); - fs.writeFileSync(path, JSON.stringify(values)); - return values; -} -/** - * @private - * Gets all the textual values of the elements present - * in the F95 portal page and identified by the selector - * passed by parameter - * @param {puppeteer.Page} page Page used to locate items specified by the selector - * @param {String} selector CSS selector - * @param {String} logMessage Log message indicating which items the selector is requesting - * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector - */ -async function getValuesFromLatestPage(page, selector, logMessage) { - shared.logger.info(logMessage); - - const result = []; - const elements = await page.$$(selector); - - for (const element of elements) { - const text = await element.evaluate( - /* istanbul ignore next */ (e) => e.innerText - ); - - // Save as upper text for better match if used in query - result.push(text.toUpperCase()); - } - return result; -} -//#endregion - //#region User -/** - * @private - * Log in to the F95Zone portal and, if successful, save the cookies. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} username Username to use during login - * @param {String} password Password to use during login - * @returns {Promise} Result of the operation - */ -async function loginF95(browser, username, password) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_LOGIN_URL); // Go to login page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_INPUT), - page.waitForSelector(selectorK.PASSWORD_INPUT), - page.waitForSelector(selectorK.LOGIN_BUTTON), - ]); - - await page.type(selectorK.USERNAME_INPUT, username); // Insert username - await page.type(selectorK.PASSWORD_INPUT, password); // Insert password - await Promise.all([ - page.click(selectorK.LOGIN_BUTTON), // Click on the login button - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Prepare result - let message = ""; - - // Check if the user is logged in - const success = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector) !== null, - selectorK.AVATAR_INFO - ); - - const errorMessageExists = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector) !== null, - selectorK.LOGIN_MESSAGE_ERROR - ); - - // Save cookies to avoid re-auth - if (success) { - const c = await page.cookies(); - fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); - message = "Authentication successful"; - } else if (errorMessageExists) { - const errorMessage = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.LOGIN_MESSAGE_ERROR - ); - - if (errorMessage === "Incorrect password. Please try again.") { - message = "Incorrect password"; - } else if ( - errorMessage === - "The requested user '" + username + "' could not be found." - ) { - // The escaped quotes are important! - message = "Incorrect username"; - } else message = errorMessage; - } else message = "Unknown error"; - - await page.close(); // Close the page - return new LoginResult(success, message); -} /** * @private * Gets the list of URLs of threads the user follows. * @param {puppeteer.Browser} browser Browser object used for navigation * @returns {Promise} URL list */ -async function getUserWatchedGameThreads(browser) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page +async function getUserWatchedGameThreads() { + const page = null; + await page.goto(f95url.F95_WATCHED_THREADS); // Go to the thread page // Explicitly wait for the required items to load - await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); + await page.waitForSelector(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON); // Show the popup await Promise.all([ - page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), + page.click(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON), + page.waitForSelector(f95selector.UNREAD_THREAD_CHECKBOX), + page.waitForSelector(f95selector.ONLY_GAMES_THREAD_OPTION), + page.waitForSelector(f95selector.FILTER_THREADS_BUTTON), ]); // Set the filters await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector).removeAttribute("checked"), - selectorK.UNREAD_THREAD_CHECKBOX + f95selector.UNREAD_THREAD_CHECKBOX ); // Also read the threads already read // Filter the threads - await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); - await page.click(selectorK.FILTER_THREADS_BUTTON); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + await page.click(f95selector.ONLY_GAMES_THREAD_OPTION); + await page.click(f95selector.FILTER_THREADS_BUTTON); + await page.waitForSelector(f95selector.WATCHED_THREAD_URLS); // Get the threads urls const urls = []; let nextPageExists = false; do { // Get all the URLs - for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { + for (const handle of await page.$$(f95selector.WATCHED_THREAD_URLS)) { const src = await page.evaluate( /* istanbul ignore next */ (element) => element.href, handle @@ -591,13 +239,13 @@ async function getUserWatchedGameThreads(browser) { nextPageExists = await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector), - selectorK.WATCHED_THREAD_NEXT_PAGE + f95selector.WATCHED_THREAD_NEXT_PAGE ); // Click to next page if (nextPageExists) { - await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + await page.click(f95selector.WATCHED_THREAD_NEXT_PAGE); + await page.waitForSelector(f95selector.WATCHED_THREAD_URLS); } } while (nextPageExists); diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 15434a7..19b609b 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -59,12 +59,6 @@ module.exports.autenticate = async function (credentials) { shared.logger.info(`Authenticating with user ${credentials.username}`); if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); - // If the user is already logged, return - if(shared.isLogged) { - shared.logger.warn(`${credentials.username} already authenticated`); - return new LoginResult(true, "Already authenticated"); - } - // Secure the URL const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL); @@ -91,7 +85,6 @@ module.exports.autenticate = async function (credentials) { const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, ""); // Return the result of the authentication - shared.isLogged = errorMessage === ""; if (errorMessage === "") return new LoginResult(true, "Authentication successful"); else return new LoginResult(false, errorMessage); } catch (e) { diff --git a/app/scripts/user-scraper.js b/app/scripts/user-scraper.js new file mode 100644 index 0000000..9a390c3 --- /dev/null +++ b/app/scripts/user-scraper.js @@ -0,0 +1 @@ +"use strict"; \ No newline at end of file diff --git a/legacy/index.js b/legacy/index.js deleted file mode 100644 index cb26ad8..0000000 --- a/legacy/index.js +++ /dev/null @@ -1,609 +0,0 @@ -"use strict"; - -// Core modules -const fs = require("fs"); - -// Modules from file -const shared = require("../app/scripts/shared.js"); -const urlK = require("../app/scripts/constants/url.js"); -const selectorK = require("../app/scripts/constants/css-selector.js"); -const urlHelper = require("../app/scripts/url-helper.js"); -const scraper = require("../app/scripts/game-scraper.js"); -const { - prepareBrowser, - preparePage, -} = require("../app/scripts/puppeteer-helper.js"); -const searcher = require("../app/scripts/game-searcher.js"); - -// Classes from file -const GameInfo = require("../app/scripts/classes/game-info.js"); -const LoginResult = require("../app/scripts/classes/login-result.js"); -const UserData = require("../app/scripts/classes/user-data.js"); - -//#region Export classes -module.exports.GameInfo = GameInfo; -module.exports.LoginResult = LoginResult; -module.exports.UserData = UserData; -//#endregion Export classes - -//#region Export properties -/** - * Shows log messages and other useful functions for module debugging. - * @param {Boolean} value - */ -module.exports.debug = function (value) { - shared.debug = value; - - // Configure logger - shared.logger.level = value ? "debug" : "warn"; -}; -/** - * @public - * Indicates whether a user is logged in to the F95Zone platform or not. - * @returns {String} - */ -module.exports.isLogged = function () { - return shared.isLogged; -}; -/** - * @public - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {String} - */ -module.exports.setIsolation = function (value) { - shared.isolation = value; -}; -/** - * @public - * Path to the cache directory - * @returns {String} - */ -module.exports.getCacheDir = function () { - return shared.cacheDir; -}; -/** - * @public - * Set path to the cache directory - * @returns {String} - */ -module.exports.setCacheDir = function (value) { - shared.cacheDir = value; - - // Create directory if it doesn't exist - if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); -}; -/** - * @public - * Set local chromium path. - * @returns {String} - */ -module.exports.setChromiumPath = function (value) { - shared.chromiumLocalPath = value; -}; -//#endregion Export properties - -//#region Global variables -var _browser = null; -const USER_NOT_LOGGED = "User not authenticated, unable to continue"; -//#endregion - -//#region Export methods -/** - * @public - * Log in to the F95Zone platform. - * This **must** be the first operation performed before accessing any other script functions. - * @param {String} username Username used for login - * @param {String} password Password used for login - * @returns {Promise} Result of the operation - */ -module.exports.login = async function (username, password) { - if (shared.isLogged) { - shared.logger.info("Already logged in"); - const result = new LoginResult(true, "Already logged in"); - return result; - } - - // If cookies are loaded, use them to authenticate - shared.cookies = loadCookies(); - if (shared.cookies !== null) { - shared.logger.info("Valid session, no need to re-authenticate"); - shared.isLogged = true; - const result = new LoginResult(true, "Logged with cookies"); - return result; - } - - // Else, log in throught browser - shared.logger.info( - "No saved sessions or expired session, login on the platform" - ); - - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const result = await loginF95(browser, username, password); - shared.isLogged = result.success; - - if (result.success) { - // Reload cookies - shared.cookies = loadCookies(); - shared.logger.info("User logged in through the platform"); - } else { - shared.logger.warn("Error during authentication: " + result.message); - } - if (shared.isolation) await browser.close(); - return result; -}; -/** - * @public - * This method loads the main data from the F95 portal - * used to provide game information. You **must** be logged - * in to the portal before calling this method. - * @returns {Promise} Result of the operation - */ -module.exports.loadF95BaseData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } - - shared.logger.info("Loading base data..."); - - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - - // Go to latest update page and wait for it to load - await page.goto(urlK.F95_LATEST_UPDATES, { - waitUntil: shared.WAIT_STATEMENT, - }); - - // Obtain engines (disk/online) - await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); - shared.engines = await loadValuesFromLatestPage( - page, - shared.enginesCachePath, - selectorK.ENGINE_ID_SELECTOR, - "engines" - ); - - // Obtain statuses (disk/online) - await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); - shared.statuses = await loadValuesFromLatestPage( - page, - shared.statusesCachePath, - selectorK.STATUS_ID_SELECTOR, - "statuses" - ); - - await page.close(); - if (shared.isolation) await browser.close(); - shared.logger.info("Base data loaded"); - return true; -}; -/** - * @public - * Chek if exists a new version of the game. - * You **must** be logged in to the portal before calling this method. - * @param {GameInfo} info Information about the game to get the version for - * @returns {Promise} true if an update is available, false otherwise - */ -module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } - - // F95 change URL at every game update, - // so if the URL is different an update is available - const exists = await urlHelper.urlExists(info.f95url, true); - if (!exists) return true; - - // Parse version from title - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); - - if (shared.isolation) await browser.close(); - - return onlineVersion.toUpperCase() !== info.version.toUpperCase(); -}; -/** - * @public - * Starting from the name, it gets all the information about the game you are looking for. - * You **must** be logged in to the portal before calling this method. - * @param {String} name Name of the game searched - * @param {Boolean} includeMods Indicates whether to also take mods into account when searching - * @returns {Promise} List of information obtained where each item corresponds to - * an identified game (in the case of homonymy of titles) - */ -module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } - - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const urlList = await searcher.getSearchGameResults(browser, name); - - // Process previous partial results - const promiseList = []; - for (const url of urlList) { - // Start looking for information - promiseList.push(scraper.getGameInfo(browser, url)); - } - - // Filter for mods - const result = []; - for (const info of await Promise.all(promiseList)) { - // Ignore empty results - if (!info) continue; - // Skip mods if not required - if (info.isMod && !includeMods) continue; - // Else save data - result.push(info); - } - - if (shared.isolation) await browser.close(); - return result; -}; -/** - * @public - * Starting from the url, it gets all the information about the game you are looking for. - * You **must** be logged in to the portal before calling this method. - * @param {String} url URL of the game to obtain information of - * @returns {Promise} Information about the game. If no game was found, null is returned - */ -module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } - - // Check URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); - if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); - - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - // Get game data - const result = await scraper.getGameInfo(browser, url); - - if (shared.isolation) await browser.close(); - return result; -}; -/** - * @public - * Gets the data of the currently logged in user. - * You **must** be logged in to the portal before calling this method. - * @returns {Promise} Data of the user currently logged in - */ -module.exports.getUserData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } - - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_BASE_URL); // Go to base page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_ELEMENT), - page.waitForSelector(selectorK.AVATAR_PIC), - ]); - - const threads = getUserWatchedGameThreads(browser); - - const username = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.USERNAME_ELEMENT - ); - - const avatarSrc = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).getAttribute("src"), - selectorK.AVATAR_PIC - ); - - const ud = new UserData(); - ud.username = username; - ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = await threads; - - await page.close(); - if (shared.isolation) await browser.close(); - - return ud; -}; -/** - * @public - * Logout from the current user and gracefully close shared browser. - * You **must** be logged in to the portal before calling this method. - */ -module.exports.logout = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return; - } - - // Logout - shared.isLogged = false; - - // Gracefully close shared browser - if (!shared.isolation && _browser !== null) { - await _browser.close(); - _browser = null; - } -}; -//#endregion - -//#region Private methods - -//#region Cookies functions -/** - * @private - * Loads and verifies the expiration of previously stored cookies from disk - * if they exist, otherwise it returns null. - * @return {object[]} List of dictionaries or null if cookies don't exist - */ -function loadCookies() { - // Check the existence of the cookie file - if (fs.existsSync(shared.cookiesCachePath)) { - // Read cookies - const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); - const cookies = JSON.parse(cookiesJSON); - - // Check if the cookies have expired - for (const cookie of cookies) { - if (isCookieExpired(cookie)) return null; - } - - // Cookies loaded and verified - return cookies; - } else return null; -} -/** - * @private - * Check the validity of a cookie. - * @param {object} cookie Cookies to verify the validity. It's a dictionary - * @returns {Boolean} true if the cookie has expired, false otherwise - */ -function isCookieExpired(cookie) { - // Local variables - let expiredCookies = false; - - // Ignore cookies that never expire - const expirationUnixTimestamp = cookie.expire; - - if (expirationUnixTimestamp !== "-1") { - // Convert UNIX epoch timestamp to normal Date - const expirationDate = new Date(expirationUnixTimestamp * 1000); - - if (expirationDate < Date.now()) { - shared.logger.warn( - "Cookie " + cookie.name + " expired, you need to re-authenticate" - ); - expiredCookies = true; - } - } - - return expiredCookies; -} -//#endregion Cookies functions - -//#region Latest Updates page parserer -/** - * @private - * If present, it reads the file containing the searched values (engines or states) - * from the disk, otherwise it connects to the F95 portal (at the page - * https://f95zone.to/latest) and downloads them. - * @param {puppeteer.Page} page Page used to locate the required elements - * @param {String} path Path to disk of the JSON file containing the data to read / write - * @param {String} selector CSS selector of the required elements - * @param {String} elementRequested Required element (engines or states) used to detail log messages - * @returns {Promise} List of required values in uppercase - */ -async function loadValuesFromLatestPage( - page, - path, - selector, - elementRequested -) { - // If the values already exist they are loaded from disk without having to connect to F95 - shared.logger.info("Load " + elementRequested + " from disk..."); - if (fs.existsSync(path)) { - const valueJSON = fs.readFileSync(path); - return JSON.parse(valueJSON); - } - - // Otherwise, connect and download the data from the portal - shared.logger.info("No " + elementRequested + " cached, downloading..."); - const values = await getValuesFromLatestPage( - page, - selector, - "Getting " + elementRequested + " from page" - ); - fs.writeFileSync(path, JSON.stringify(values)); - return values; -} -/** - * @private - * Gets all the textual values of the elements present - * in the F95 portal page and identified by the selector - * passed by parameter - * @param {puppeteer.Page} page Page used to locate items specified by the selector - * @param {String} selector CSS selector - * @param {String} logMessage Log message indicating which items the selector is requesting - * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector - */ -async function getValuesFromLatestPage(page, selector, logMessage) { - shared.logger.info(logMessage); - - const result = []; - const elements = await page.$$(selector); - - for (const element of elements) { - const text = await element.evaluate( - /* istanbul ignore next */ (e) => e.innerText - ); - - // Save as upper text for better match if used in query - result.push(text.toUpperCase()); - } - return result; -} -//#endregion - -//#region User -/** - * @private - * Log in to the F95Zone portal and, if successful, save the cookies. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} username Username to use during login - * @param {String} password Password to use during login - * @returns {Promise} Result of the operation - */ -async function loginF95(browser, username, password) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_LOGIN_URL); // Go to login page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_INPUT), - page.waitForSelector(selectorK.PASSWORD_INPUT), - page.waitForSelector(selectorK.LOGIN_BUTTON), - ]); - - await page.type(selectorK.USERNAME_INPUT, username); // Insert username - await page.type(selectorK.PASSWORD_INPUT, password); // Insert password - await Promise.all([ - page.click(selectorK.LOGIN_BUTTON), // Click on the login button - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Prepare result - let message = ""; - - // Check if the user is logged in - const success = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector) !== null, - selectorK.AVATAR_INFO - ); - - const errorMessageExists = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector) !== null, - selectorK.LOGIN_MESSAGE_ERROR - ); - - // Save cookies to avoid re-auth - if (success) { - const c = await page.cookies(); - fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); - message = "Authentication successful"; - } else if (errorMessageExists) { - const errorMessage = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.LOGIN_MESSAGE_ERROR - ); - - if (errorMessage === "Incorrect password. Please try again.") { - message = "Incorrect password"; - } else if ( - errorMessage === - "The requested user '" + username + "' could not be found." - ) { - // The escaped quotes are important! - message = "Incorrect username"; - } else message = errorMessage; - } else message = "Unknown error"; - - await page.close(); // Close the page - return new LoginResult(success, message); -} -/** - * @private - * Gets the list of URLs of threads the user follows. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @returns {Promise} URL list - */ -async function getUserWatchedGameThreads(browser) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page - - // Explicitly wait for the required items to load - await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); - - // Show the popup - await Promise.all([ - page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), - ]); - - // Set the filters - await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).removeAttribute("checked"), - selectorK.UNREAD_THREAD_CHECKBOX - ); // Also read the threads already read - - // Filter the threads - await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); - await page.click(selectorK.FILTER_THREADS_BUTTON); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - - // Get the threads urls - const urls = []; - let nextPageExists = false; - do { - // Get all the URLs - for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { - const src = await page.evaluate( - /* istanbul ignore next */ (element) => element.href, - handle - ); - // If 'unread' is left, it will redirect to the last unread post - const url = src.replace("/unread", ""); - urls.push(url); - } - - nextPageExists = await page.evaluate( - /* istanbul ignore next */ (selector) => document.querySelector(selector), - selectorK.WATCHED_THREAD_NEXT_PAGE - ); - - // Click to next page - if (nextPageExists) { - await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - } - } while (nextPageExists); - - await page.close(); - return urls; -} -//#endregion User - -//#endregion Private methods diff --git a/legacy/scripts/classes/game-download.js b/legacy/scripts/classes/game-download.js deleted file mode 100644 index e1240fb..0000000 --- a/legacy/scripts/classes/game-download.js +++ /dev/null @@ -1,111 +0,0 @@ -/* istanbul ignore file */ - -"use strict"; - -// Core modules -const fs = require("fs"); -const path = require("path"); - -// Public modules from npm -// const { File } = require('megajs'); - -// Modules from file -const { prepareBrowser, preparePage } = require("../puppeteer-helper.js"); -const shared = require("../shared.js"); - -class GameDownload { - constructor() { - /** - * @public - * Platform that hosts game files - * @type String - */ - this.hosting = ""; - /** - * @public - * Link to game files - * @type String - */ - this.link = null; - /** - * @public - * Operating systems supported by the game version indicated in this class. - * Can be *WINDOWS/LINUX/MACOS* - * @type String[] - */ - this.supportedOS = []; - } - - /** - * @public - * Download the game data in the indicated path. - * Supported hosting platforms: MEGA, NOPY - * @param {String} path Save path - * @return {Promise} Result of the operation - */ - async download(path) { - if (this.link.includes("mega.nz")) - return await downloadMEGA(this.link, path); - else if (this.link.includes("nopy.to")) - return await downloadNOPY(this.link, path); - } -} -module.exports = GameDownload; - -async function downloadMEGA(url, savepath) { - // The URL is masked - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url); - await page.waitForSelector("a.host_link"); - - // Obtain the link for the unmasked page and click it - const link = await page.$("a.host_link"); - await link.click(); - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // Obtain the URL after the redirect - const downloadURL = page.url(); - - // Close browser and page - await page.close(); - await browser.close(); - - const stream = fs.createWriteStream(savepath); - const file = File.fromURL(downloadURL); - file.download().pipe(stream); - return fs.existsSync(savepath); -} - -async function downloadNOPY(url, savepath) { - // Prepare browser - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.goto(url); - await page.waitForSelector("#download"); - - // Set the save path - await page._client.send("Page.setDownloadBehavior", { - behavior: "allow", - downloadPath: path.basename(path.dirname(savepath)), // It's a directory - }); - - // Obtain the download button and click it - const downloadButton = await page.$("#download"); - await downloadButton.click(); - - // Await for all the connections to close - await page.waitForNavigation({ - waitUntil: "networkidle0", - timeout: 0, // Disable timeout - }); - - // Close browser and page - await page.close(); - await browser.close(); - - return fs.existsSync(savepath); -} diff --git a/legacy/scripts/classes/game-info.js b/legacy/scripts/classes/game-info.js deleted file mode 100644 index c9fcac5..0000000 --- a/legacy/scripts/classes/game-info.js +++ /dev/null @@ -1,119 +0,0 @@ -"use strict"; - -class GameInfo { - constructor() { - //#region Properties - /** - * Game name - * @type String - */ - this.name = null; - /** - * Game author - * @type String - */ - this.author = null; - /** - * URL to the game's official conversation on the F95Zone portal - * @type String - */ - this.f95url = null; - /** - * Game description - * @type String - */ - this.overview = null; - /** - * List of tags associated with the game - * @type String[] - */ - this.tags = []; - /** - * Graphics engine used for game development - * @type String - */ - this.engine = null; - /** - * Progress of the game - * @type String - */ - this.status = null; - /** - * Game description image URL - * @type String - */ - this.previewSource = null; - /** - * Game version - * @type String - */ - this.version = null; - /** - * Last time the game underwent updates - * @type String - */ - this.lastUpdate = null; - /** - * Last time the local copy of the game was run - * @type String - */ - this.lastPlayed = null; - /** - * Specifies if the game is original or a mod - * @type Boolean - */ - this.isMod = false; - /** - * Changelog for the last version. - * @type String - */ - this.changelog = null; - /** - * Directory containing the local copy of the game - * @type String - */ - this.gameDir = null; - /** - * Information on game file download links, - * including information on hosting platforms - * and operating system supported by the specific link - * @type GameDownload[] - */ - this.downloadInfo = []; - //#endregion Properties - } - - /** - * Converts the object to a dictionary used for JSON serialization - */ - /* istanbul ignore next */ - toJSON() { - return { - name: this.name, - author: this.author, - f95url: this.f95url, - overview: this.overview, - engine: this.engine, - status: this.status, - previewSource: this.previewSource, - version: this.version, - lastUpdate: this.lastUpdate, - lastPlayed: this.lastPlayed, - isMod: this.isMod, - changelog: this.changelog, - gameDir: this.gameDir, - downloadInfo: this.downloadInfo, - }; - } - - /** - * Return a new GameInfo from a JSON string - * @param {String} json JSON string used to create the new object - * @returns {GameInfo} - */ - /* istanbul ignore next */ - static fromJSON(json) { - return Object.assign(new GameInfo(), json); - } -} -module.exports = GameInfo; diff --git a/legacy/scripts/classes/login-result.js b/legacy/scripts/classes/login-result.js deleted file mode 100644 index 51fd9bc..0000000 --- a/legacy/scripts/classes/login-result.js +++ /dev/null @@ -1,20 +0,0 @@ -"use strict"; - -/** - * Object obtained in response to an attempt to login to the portal. - */ -class LoginResult { - constructor(success, message) { - /** - * Result of the login operation - * @type Boolean - */ - this.success = success; - /** - * Login response message - * @type String - */ - this.message = message; - } -} -module.exports = LoginResult; diff --git a/legacy/scripts/classes/user-data.js b/legacy/scripts/classes/user-data.js deleted file mode 100644 index 7edb904..0000000 --- a/legacy/scripts/classes/user-data.js +++ /dev/null @@ -1,26 +0,0 @@ -"use strict"; - -/** - * Class containing the data of the user currently connected to the F95Zone platform. - */ -class UserData { - constructor() { - /** - * User username. - * @type String - */ - this.username = ""; - /** - * Path to the user's profile picture. - * @type String - */ - this.avatarSrc = null; - /** - * List of followed thread URLs. - * @type URL[] - */ - this.watchedThreads = []; - } -} - -module.exports = UserData; diff --git a/legacy/scripts/constants/css-selector.js b/legacy/scripts/constants/css-selector.js deleted file mode 100644 index bf0f60b..0000000 --- a/legacy/scripts/constants/css-selector.js +++ /dev/null @@ -1,33 +0,0 @@ -module.exports = Object.freeze({ - AVATAR_INFO: "span.avatar", - AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", - ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", - FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", - GAME_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", - GAME_TAGS: "a.tagItem", - GAME_TITLE: "h1.p-title-value", - GAME_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", - LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: - "div.blockMessage.blockMessage--error.blockMessage--iconic", - ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", - PASSWORD_INPUT: "input[name=\"password\"]", - SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", - SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", - STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", - THREAD_POSTS: - "article.message-body:first-child > div.bbWrapper:first-of-type", - THREAD_TITLE: "h3.contentRow-title", - TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", - UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", - USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", - USERNAME_INPUT: "input[name=\"login\"]", - WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", - WATCHED_THREAD_URLS: "a[href^=\"/threads/\"][data-tp-primary]", - DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", - SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", - SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", - THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", -}); diff --git a/legacy/scripts/constants/url.js b/legacy/scripts/constants/url.js deleted file mode 100644 index fac63ba..0000000 --- a/legacy/scripts/constants/url.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = Object.freeze({ - F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search/?type=post", - F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_LOGIN_URL: "https://f95zone.to/login", - F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", -}); diff --git a/legacy/scripts/game-scraper.js b/legacy/scripts/game-scraper.js deleted file mode 100644 index 7b20e47..0000000 --- a/legacy/scripts/game-scraper.js +++ /dev/null @@ -1,467 +0,0 @@ -"use strict"; - -// Public modules from npm -const HTMLParser = require("node-html-parser"); -const puppeteer = require("puppeteer"); // skipcq: JS-0128 - -// Modules from file -const shared = require("./shared.js"); -const selectorK = require("./constants/css-selector.js"); -const { preparePage } = require("./puppeteer-helper.js"); -const GameDownload = require("./classes/game-download.js"); -const GameInfo = require("./classes/game-info.js"); -const urlHelper = require("./url-helper.js"); - -/** - * @protected - * Get information from the game's main page. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} url URL (String) of the game/mod to extract data from - * @return {Promise} Complete information about the game you are - * looking for - */ -module.exports.getGameInfo = async function (browser, url) { - shared.logger.info("Obtaining game info"); - - // Verify the correctness of the URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(`${url} is not a valid URL`); - if (!urlHelper.isF95URL(url)) - throw new Error(`${url} is not a valid F95Zone URL`); - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // It asynchronously searches for the elements and - // then waits at the end to compile the object to be returned - let info = new GameInfo(); - const title = getGameTitle(page); - const author = getGameAuthor(page); - const tags = getGameTags(page); - const redirectUrl = urlHelper.getUrlRedirect(url); - info = await parsePrefixes(page, info); // Fill status/engines/isMod - const structuredText = await getMainPostStructuredText(page); - const overview = getOverview(structuredText, info.isMod); - const parsedInfos = parseConversationPage(structuredText); - const previewSource = getGamePreviewSource(page); - const changelog = getLastChangelog(page); - - // Fill in the GameInfo element with the information obtained - info.name = await title; - info.author = await author; - info.tags = await tags; - info.f95url = await redirectUrl; - info.overview = overview; - info.lastUpdate = info.isMod - ? parsedInfos.UPDATED - : parsedInfos.THREAD_UPDATED; - info.previewSource = await previewSource; - info.changelog = await changelog; - info.version = await exports.getGameVersionFromTitle(browser, info); - - //let downloadData = getGameDownloadLink(page); - //info.downloadInfo = await downloadData; - /* Downloading games without going directly to - * the platform appears to be prohibited by - * the guidelines. It is therefore useless to - * keep the links for downloading the games. */ - - await page.close(); // Close the page - shared.logger.info("Founded data for " + info.name); - return info; -}; - -/** - * Obtain the game version without parsing again all the data of the game. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {GameInfo} info Information about the game - * @returns {Promise} Online version of the game - */ -module.exports.getGameVersionFromTitle = async function (browser, info) { - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(info.f95url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // Get the title - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; - - // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] - const startIndex = title.indexOf("[") + 1; - const endIndex = title.indexOf("]", startIndex); - let version = title.substring(startIndex, endIndex).trim().toUpperCase(); - if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence - await page.close(); - return cleanFSString(version); -}; - -//#region Private methods -/** - * Clean a string from invalid File System chars. - * @param {String} s - * @returns {String} - */ -function cleanFSString(s) { - const rx = /[/\\?%*:|"<>]/g; - return s.replace(rx, ""); -} - -/** - * @private - * Get the game description from its web page. - * Different processing depending on whether the game is a mod or not. - * @param {String} text Structured text extracted from the game's web page - * @param {Boolean} isMod Specify if it is a game or a mod - * @returns {Promise} Game description - */ -function getOverview(text, isMod) { - // Get overview (different parsing for game and mod) - let overviewEndIndex; - if (isMod) overviewEndIndex = text.indexOf("Updated"); - else overviewEndIndex = text.indexOf("Thread Updated"); - return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); -} - -/** - * @private - * Extrapolate the page structure by removing the element tags - * and leaving only the text and its spacing. - * @param {puppeteer.Page} page Page containing the text - * @returns {Promise} Structured text - */ -async function getMainPostStructuredText(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - - // The info are plain text so we need to parse the HTML code - const bodyHTML = await page.evaluate( - /* istanbul ignore next */ - (mainPost) => mainPost.innerHTML, - post - ); - return HTMLParser.parse(bodyHTML).structuredText; -} - -/** - * @private - * Extrapolates and cleans the author from the page passed by parameter. - * @param {puppeteer.Page} page Page containing the author to be extrapolated - * @returns {Promise} Game author - */ -async function getGameAuthor(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); - - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - - // The last square brackets contain the author - const startTitleIndex = gameTitle.lastIndexOf("[") + 1; - return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); -} - -/** - * @private - * Process the post text to get all the useful - * information in the format *DESCRIPTOR : VALUE*. - * @param {String} text Structured text of the post - * @returns {Object} Dictionary of information - */ -function parseConversationPage(text) { - const dataPairs = {}; - - // The information searched in the game post are one per line - const splittedText = text.split("\n"); - for (const line of splittedText) { - if (!line.includes(":")) continue; - - // Create pair key/value - const splitted = line.split(":"); - const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch - const value = splitted[1].trim(); - - // Add pair to the dict if valid - if (value !== "") dataPairs[key] = value; - } - - return dataPairs; -} - -/** - * @private - * Gets the URL of the image used as a preview for the game in the conversation. - * @param {puppeteer.Page} page Page containing the URL to be extrapolated - * @returns {Promise} URL (String) of the image or null if failed to get it - */ -async function getGamePreviewSource(page) { - // Wait for the selector or return an empty value - try { - await page.waitForSelector(selectorK.GAME_IMAGES); - } catch { - return null; - } - - const src = await page.evaluate( - /* istanbul ignore next */ - (selector) => { - // Get the firs image available - const img = document.querySelector(selector); - - if (img) return img.getAttribute("src"); - else return null; - }, - selectorK.GAME_IMAGES - ); - - // Check if the URL is valid - return urlHelper.isStringAValidURL(src) ? src : null; -} - -/** - * @private - * Extrapolates and cleans the title from the page passed by parameter. - * @param {puppeteer.Page} page Page containing the title to be extrapolated - * @returns {Promise} Game title - */ -async function getGameTitle(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); - - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - const endTitleIndex = gameTitle.indexOf("["); - return gameTitle.substring(0, endTitleIndex).trim(); -} - -/** - * @private - * Get the alphabetically sorted list of tags associated with the game. - * @param {puppeteer.Page} page Page containing the tags to be extrapolated - * @returns {Promise} List of uppercase tags - */ -async function getGameTags(page) { - const tags = []; - - // Get the game tags - for (const handle of await page.$$(selectorK.GAME_TAGS)) { - const tag = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - tags.push(tag.toUpperCase()); - } - return tags.sort(); -} - -/** - * @private - * Process the game title prefixes to extract information such as game status, - * graphics engine used, and whether it is a mod or original game. - * @param {puppeteer.Page} page Page containing the prefixes to be extrapolated - * @param {GameInfo} info Object to assign the identified information to - * @returns {Promise} GameInfo object passed in to which the identified information has been added - */ -async function parsePrefixes(page, info) { - // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' - info.status = "ONGOING"; - for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { - const value = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - - // Clean the prefix - const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); - - // Getting infos... - if (shared.statuses.includes(prefix)) info.status = prefix; - else if (shared.engines.includes(prefix)) info.engine = prefix; - // This is not a game but a mod - else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; - } - return info; -} - -/** - * @private - * Get the last changelog available for the game. - * @param {puppeteer.Page} page Page containing the changelog - * @returns {Promise} Changelog for the last version or a empty string if no changelog is found - */ -async function getLastChangelog(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - - const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG); - if (!spoiler) return ""; - - const changelogHTML = await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - spoiler - ); - let parsedText = HTMLParser.parse(changelogHTML).structuredText; - - // Clean the text - if (parsedText.startsWith("Spoiler")) - parsedText = parsedText.replace("Spoiler", ""); - if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", ""); - return parsedText.trim(); -} - -/** - * @private - * Get game download links for different platforms. - * @param {puppeteer.Page} page Page containing the links to be extrapolated - * @returns {Promise} List of objects used for game download - * @deprecated - */ -/* istanbul ignore next */ -// skipcq: JS-0128 -async function getGameDownloadLink(page) { - // Most used hosting platforms - const hostingPlatforms = [ - "MEGA", - "NOPY", - "FILESUPLOAD", - "MIXDROP", - "UPLOADHAVEN", - "PIXELDRAIN", - "FILESFM", - ]; - - // Supported OS platforms - const platformOS = ["WIN", "LINUX", "MAC", "ALL"]; - - // Gets the which contains the download links - const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER); - if (temp.length === 0) return []; - - // Look for the container that contains the links - // It is necessary because the same css selector - // also identifies other elements on the page - let container = null; - for (const candidate of temp) { - if (container !== null) break; - const upperText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - candidate - ) - ).toUpperCase(); - - // Search if the container contains the name of a hosting platform - for (const p of hostingPlatforms) { - if (upperText.includes(p)) { - container = candidate; - break; - } - } - } - if (container === null) return []; - - // Extract the HTML text from the container - const searchText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerHTML, - container - ) - ).toLowerCase(); - - // Parse the download links - const downloadData = []; - for (const platform of platformOS) { - const data = extractGameHostingData(platform, searchText); - downloadData.push(...data); - } - return downloadData; -} - -/** - * @private - * From the HTML text it extracts the game download links for the specified operating system. - * @param {String} platform Name of the operating system to look for a compatible link to. - * It can only be *WIN/LINUX/MAC/ALL* - * @param {String} text HTML string to extract links from - * @returns {GameDownload[]} List of game download links for the selected platform - * @deprecated - */ -/* istanbul ignore next */ -function extractGameHostingData(platform, text) { - const PLATFORM_BOLD_OPEN = ""; - const CONTAINER_SPAN_CLOSE = ""; - const LINK_OPEN = "platform - let endIndex = - text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length; - - // Find the end of the container - if (endIndex === -1) - endIndex = - text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) + - CONTAINER_SPAN_CLOSE.length; - - text = text.substring(startIndex, endIndex); - - const downloadData = []; - const linkTags = text.split(LINK_OPEN); - for (const tag of linkTags) { - // Ignore non-link string - if (!tag.includes(HREF_START)) continue; - - // Find the hosting platform name - startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length; - endIndex = tag.indexOf(LINK_CLOSE, startIndex); - const hosting = tag.substring(startIndex, endIndex); - - // Find the 'href' attribute - startIndex = tag.indexOf(HREF_START) + HREF_START.length; - endIndex = tag.indexOf(HREF_END, startIndex); - const link = tag.substring(startIndex, endIndex); - - if (urlHelper.isStringAValidURL(link)) { - const gd = new GameDownload(); - gd.hosting = hosting.toUpperCase(); - gd.link = link; - gd.supportedOS = platform.toUpperCase(); - - downloadData.push(gd); - } - } - return downloadData; -} - -//#endregion Private methods diff --git a/legacy/scripts/game-searcher.js b/legacy/scripts/game-searcher.js deleted file mode 100644 index 497edfe..0000000 --- a/legacy/scripts/game-searcher.js +++ /dev/null @@ -1,132 +0,0 @@ -"use strict"; - -// Public modules from npm -const puppeteer = require("puppeteer"); // skipcq: JS-0128 - -// Modules from file -const shared = require("./shared.js"); -const urlK = require("./constants/url.js"); -const selectorK = require("./constants/css-selector.js"); -const { preparePage } = require("./puppeteer-helper.js"); -const { isF95URL } = require("./url-helper.js"); - -/** - * @protected - * Search the F95Zone portal to find possible conversations regarding the game you are looking for. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} gamename Name of the game to search for - * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal - */ -module.exports.getSearchGameResults = async function (browser, gamename) { - shared.logger.info(`Searching ${gamename} on F95Zone`); - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_SEARCH_URL, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the search form and wait for it - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), - page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), - page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), - page.waitForSelector(selectorK.SEARCH_BUTTON), - ]); - - await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire - await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles - await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod - await Promise.all([ - page.click(selectorK.SEARCH_BUTTON), // Execute search - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Select all conversation titles - const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY); - - // For each element found extract the info about the conversation - shared.logger.info("Extracting info from conversations"); - const results = []; - for (const element of resultsThread) { - const gameUrl = await getOnlyGameThreads(page, element); - if (gameUrl !== null) results.push(gameUrl); - } - shared.logger.info(`Find ${results.length} conversations`); - await page.close(); // Close the page - - return results; -}; - -//#region Private methods -/** - * @private - * Return the link of a conversation if it is a game or a mod. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} divHandle Element of the conversation to be analyzed - * @return {Promise} URL of the game/mod or null if the URL is not of a game - */ -async function getOnlyGameThreads(page, divHandle) { - // Obtain the elements containing the basic information - const titleHandle = await divHandle.$(selectorK.THREAD_TITLE); - const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP); - - // Get the forum where the thread was posted - const forum = await getMembershipForum(page, forumHandle); - if (forum !== "GAMES" && forum !== "MODS") return null; - - // Get the URL of the thread from the title - return await getThreadURL(page, titleHandle); -} - -/** - * @private - * Obtain the membership forum of the thread passed throught 'handle'. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} handle Handle containing the forum membership - * @returns {Promise} Uppercase membership category - */ -async function getMembershipForum(page, handle) { - // The link can be something like: - // + /forums/request.NUMBER/ - // + /forums/game-recommendations-identification.NUMBER/ - // + /forums/games.NUMBER/ <-- We need this - - let link = await page.evaluate( - /* istanbul ignore next */ - (e) => e.getAttribute("href"), - handle - ); - - // Parse link - link = link.replace("/forums/", ""); - const endIndex = link.indexOf("."); - const forum = link.substring(0, endIndex); - - return forum.toUpperCase(); -} - -/** - * @private - * Obtain the URL of the thread passed through 'handle'. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} handle Handle containing the thread title - * @returns {Promise} URL of the thread - */ -async function getThreadURL(page, handle) { - const relativeURLThread = await page.evaluate( - /* istanbul ignore next */ - (e) => e.querySelector("a").href, - handle - ); - - // Some game already have a full URL... - if (isF95URL(relativeURLThread)) return relativeURLThread; - - // ... else compose the URL and return - const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString(); - return urlThread; -} -//#endregion Private methods diff --git a/legacy/scripts/puppeteer-helper.js b/legacy/scripts/puppeteer-helper.js deleted file mode 100644 index 942da84..0000000 --- a/legacy/scripts/puppeteer-helper.js +++ /dev/null @@ -1,60 +0,0 @@ -"use strict"; - -// Public modules from npm -const puppeteer = require("puppeteer"); - -// Modules from file -const shared = require("./shared.js"); - -/** - * @protected - * Create a Chromium instance used to navigate with Puppeteer. - * By default the browser is headless. - * @returns {Promise} Created browser - */ -module.exports.prepareBrowser = async function () { - // Create a headless browser - let browser = null; - if (shared.chromiumLocalPath) { - browser = await puppeteer.launch({ - executablePath: shared.chromiumLocalPath, - headless: !shared.debug, // Use GUI when debug = true - }); - } else { - browser = await puppeteer.launch({ - headless: !shared.debug, // Use GUI when debug = true - }); - } - - return browser; -}; - -/** - * @protected - * Prepare a page used to navigate the browser. - * The page is set up to reject image download requests. The user agent is also changed. - * @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created - * @returns {Promise} New page - */ -module.exports.preparePage = async function (browser) { - // Create new page in the browser argument - const page = await browser.newPage(); - - // Block image download - await page.setRequestInterception(true); - page.on("request", (request) => { - if (request.resourceType() === "image") request.abort(); - else if (request.resourceType === "font") request.abort(); - // else if (request.resourceType() == 'stylesheet') request.abort(); - // else if(request.resourceType == 'media') request.abort(); - else request.continue(); - }); - - // Set custom user-agent - const userAgent = - "Mozilla/5.0 (X11; Linux x86_64)" + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; - await page.setUserAgent(userAgent); - - return page; -}; diff --git a/legacy/scripts/shared.js b/legacy/scripts/shared.js deleted file mode 100644 index 85a5ba4..0000000 --- a/legacy/scripts/shared.js +++ /dev/null @@ -1,173 +0,0 @@ -"use strict"; - -// Core modules -const { join } = require("path"); - -const log4js = require("log4js"); - -/** - * Class containing variables shared between modules. - */ -class Shared { - //#region Properties - /** - * Shows log messages and other useful functions for module debugging. - * @type Boolean - */ - static #_debug = false; - /** - * Indicates whether a user is logged in to the F95Zone platform or not. - * @type Boolean - */ - static #_isLogged = false; - /** - * List of cookies obtained from the F95Zone platform. - * @type Object[] - */ - static #_cookies = null; - /** - * List of possible game engines used for development. - * @type String[] - */ - static #_engines = null; - /** - * List of possible development statuses that a game can assume. - * @type String[] - */ - static #_statuses = null; - /** - * Wait instruction for the browser created by puppeteer. - * @type String - */ - static WAIT_STATEMENT = "domcontentloaded"; - /** - * Path to the directory to save the cache generated by the API. - * @type String - */ - static #_cacheDir = "./f95cache"; - /** - * If true, it opens a new browser for each request to - * the F95Zone platform, otherwise it reuses the same. - * @type Boolean - */ - static #_isolation = false; - /** - * Logger object used to write to both file and console. - * @type log4js.Logger - */ - static #_logger = log4js.getLogger(); - //#endregion Properties - - //#region Getters - /** - * Shows log messages and other useful functions for module debugging. - * @returns {Boolean} - */ - static get debug() { - return this.#_debug; - } - /** - * Indicates whether a user is logged in to the F95Zone platform or not. - * @returns {Boolean} - */ - static get isLogged() { - return this.#_isLogged; - } - /** - * List of cookies obtained from the F95Zone platform. - * @returns {Object[]} - */ - static get cookies() { - return this.#_cookies; - } - /** - * List of possible game engines used for development. - * @returns {String[]} - */ - static get engines() { - return this.#_engines; - } - /** - * List of possible development states that a game can assume. - * @returns {String[]} - */ - static get statuses() { - return this.#_statuses; - } - /** - * Directory to save the API cache. - * @returns {String} - */ - static get cacheDir() { - return this.#_cacheDir; - } - /** - * Path to the F95 platform cache. - * @returns {String} - */ - static get cookiesCachePath() { - return join(this.#_cacheDir, "cookies.json"); - } - /** - * Path to the game engine cache. - * @returns {String} - */ - static get enginesCachePath() { - return join(this.#_cacheDir, "engines.json"); - } - /** - * Path to the cache of possible game states. - * @returns {String} - */ - static get statusesCachePath() { - return join(this.#_cacheDir, "statuses.json"); - } - /** - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {Boolean} - */ - static get isolation() { - return this.#_isolation; - } - /** - * Logger object used to write to both file and console. - * @returns {log4js.Logger} - */ - static get logger() { - return this.#_logger; - } - //#endregion Getters - - //#region Setters - static set cookies(val) { - this.#_cookies = val; - } - - static set engines(val) { - this.#_engines = val; - } - - static set statuses(val) { - this.#_statuses = val; - } - - static set cacheDir(val) { - this.#_cacheDir = val; - } - - static set debug(val) { - this.#_debug = val; - } - - static set isLogged(val) { - this.#_isLogged = val; - } - - static set isolation(val) { - this.#_isolation = val; - } - //#endregion Setters -} - -module.exports = Shared; diff --git a/legacy/scripts/url-helper.js b/legacy/scripts/url-helper.js deleted file mode 100644 index aae60a0..0000000 --- a/legacy/scripts/url-helper.js +++ /dev/null @@ -1,72 +0,0 @@ -"use strict"; - -// Public modules from npm -const ky = require("ky-universal").create({ - throwHttpErrors: false, -}); - -// Modules from file -const { F95_BASE_URL } = require("./constants/url.js"); - -/** - * @protected - * Check if the url belongs to the domain of the F95 platform. - * @param {String} url URL to check - * @returns {Boolean} true if the url belongs to the domain, false otherwise - */ -module.exports.isF95URL = function (url) { - if (url.toString().startsWith(F95_BASE_URL)) return true; - else return false; -}; - -/** - * @protected - * Checks if the string passed by parameter has a properly formatted and valid path to a URL. - * @param {String} url String to check for correctness - * @returns {Boolean} true if the string is a valid URL, false otherwise - */ -module.exports.isStringAValidURL = function (url) { - try { - new URL(url); // skipcq: JS-0078 - return true; - } catch (err) { - return false; - } -}; - -/** - * @protected - * Check if a particular URL is valid and reachable on the web. - * @param {String} url URL to check - * @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false - * @returns {Promise} true if the URL exists, false otherwise - */ -module.exports.urlExists = async function (url, checkRedirect) { - if (!exports.isStringAValidURL(url)) { - return false; - } - - const response = await ky.head(url); - let valid = response !== undefined && !/4\d\d/.test(response.status); - - if (!valid) return false; - - if (checkRedirect) { - const redirectUrl = await exports.getUrlRedirect(url); - if (redirectUrl === url) valid = true; - else valid = false; - } - - return valid; -}; - -/** - * @protected - * Check if the URL has a redirect to another page. - * @param {String} url URL to check for redirect - * @returns {Promise} Redirect URL or the passed URL - */ -module.exports.getUrlRedirect = async function (url) { - const response = await ky.head(url); - return response.url; -};