diff --git a/app/index.js b/app/index.js index d03987b..1c99110 100644 --- a/app/index.js +++ b/app/index.js @@ -10,8 +10,8 @@ const selectorK = require("./scripts/constants/css-selector.js"); const urlHelper = require("./scripts/url-helper.js"); const scraper = require("./scripts/game-scraper.js"); const { - prepareBrowser, - preparePage, + prepareBrowser, + preparePage, } = require("./scripts/puppeteer-helper.js"); const searcher = require("./scripts/game-searcher.js"); @@ -32,10 +32,10 @@ module.exports.UserData = UserData; * @param {Boolean} value */ module.exports.debug = function (value) { - shared.debug = value; + shared.debug = value; - // Configure logger - shared.logger.level = value ? "debug" : "warn"; + // Configure logger + shared.logger.level = value ? "debug" : "warn"; }; /** * @public @@ -43,7 +43,7 @@ module.exports.debug = function (value) { * @returns {String} */ module.exports.isLogged = function () { - return shared.isLogged; + return shared.isLogged; }; /** * @public @@ -52,7 +52,7 @@ module.exports.isLogged = function () { * @returns {String} */ module.exports.setIsolation = function (value) { - shared.isolation = value; + shared.isolation = value; }; /** * @public @@ -60,7 +60,7 @@ module.exports.setIsolation = function (value) { * @returns {String} */ module.exports.getCacheDir = function () { - return shared.cacheDir; + return shared.cacheDir; }; /** * @public @@ -68,10 +68,10 @@ module.exports.getCacheDir = function () { * @returns {String} */ module.exports.setCacheDir = function (value) { - shared.cacheDir = value; + shared.cacheDir = value; - // Create directory if it doesn't exist - if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); + // Create directory if it doesn't exist + if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); }; /** * @public @@ -79,7 +79,7 @@ module.exports.setCacheDir = function (value) { * @returns {String} */ module.exports.setChromiumPath = function (value) { - shared.chromiumLocalPath = value; + shared.chromiumLocalPath = value; }; //#endregion Export properties @@ -98,41 +98,41 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue"; * @returns {Promise} Result of the operation */ module.exports.login = async function (username, password) { - if (shared.isLogged) { - shared.logger.info("Already logged in"); - const result = new LoginResult(true, "Already logged in"); - return result; - } + if (shared.isLogged) { + shared.logger.info("Already logged in"); + const result = new LoginResult(true, "Already logged in"); + return result; + } - // If cookies are loaded, use them to authenticate - shared.cookies = loadCookies(); - if (shared.cookies !== null) { - shared.logger.info("Valid session, no need to re-authenticate"); - shared.isLogged = true; - const result = new LoginResult(true, "Logged with cookies"); - return result; - } - - // Else, log in throught browser - shared.logger.info( - "No saved sessions or expired session, login on the platform" - ); - - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const result = await loginF95(browser, username, password); - shared.isLogged = result.success; - - if (result.success) { - // Reload cookies + // If cookies are loaded, use them to authenticate shared.cookies = loadCookies(); - shared.logger.info("User logged in through the platform"); - } else { - shared.logger.warn("Error during authentication: " + result.message); - } - if (shared.isolation) await browser.close(); - return result; + if (shared.cookies !== null) { + shared.logger.info("Valid session, no need to re-authenticate"); + shared.isLogged = true; + const result = new LoginResult(true, "Logged with cookies"); + return result; + } + + // Else, log in throught browser + shared.logger.info( + "No saved sessions or expired session, login on the platform" + ); + + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + + const result = await loginF95(browser, username, password); + shared.isLogged = result.success; + + if (result.success) { + // Reload cookies + shared.cookies = loadCookies(); + shared.logger.info("User logged in through the platform"); + } else { + shared.logger.warn("Error during authentication: " + result.message); + } + if (shared.isolation) await browser.close(); + return result; }; /** * @public @@ -142,47 +142,47 @@ module.exports.login = async function (username, password) { * @returns {Promise} Result of the operation */ module.exports.loadF95BaseData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return false; + } - shared.logger.info("Loading base data..."); + shared.logger.info("Loading base data..."); - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; + // Prepare a new web page + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login - // Go to latest update page and wait for it to load - await page.goto(urlK.F95_LATEST_UPDATES, { - waitUntil: shared.WAIT_STATEMENT, - }); + // Go to latest update page and wait for it to load + await page.goto(urlK.F95_LATEST_UPDATES, { + waitUntil: shared.WAIT_STATEMENT, + }); - // Obtain engines (disk/online) - await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); - shared.engines = await loadValuesFromLatestPage( - page, - shared.enginesCachePath, - selectorK.ENGINE_ID_SELECTOR, - "engines" - ); + // Obtain engines (disk/online) + await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); + shared.engines = await loadValuesFromLatestPage( + page, + shared.enginesCachePath, + selectorK.ENGINE_ID_SELECTOR, + "engines" + ); - // Obtain statuses (disk/online) - await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); - shared.statuses = await loadValuesFromLatestPage( - page, - shared.statusesCachePath, - selectorK.STATUS_ID_SELECTOR, - "statuses" - ); + // Obtain statuses (disk/online) + await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); + shared.statuses = await loadValuesFromLatestPage( + page, + shared.statusesCachePath, + selectorK.STATUS_ID_SELECTOR, + "statuses" + ); - await page.close(); - if (shared.isolation) await browser.close(); - shared.logger.info("Base data loaded"); - return true; + await page.close(); + if (shared.isolation) await browser.close(); + shared.logger.info("Base data loaded"); + return true; }; /** * @public @@ -192,25 +192,25 @@ module.exports.loadF95BaseData = async function () { * @returns {Promise} true if an update is available, false otherwise */ module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return false; + } - // F95 change URL at every game update, - // so if the URL is different an update is available - const exists = await urlHelper.urlExists(info.f95url, true); - if (!exists) return true; + // F95 change URL at every game update, + // so if the URL is different an update is available + const exists = await urlHelper.urlExists(info.f95url, true); + if (!exists) return true; - // Parse version from title - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; + // Parse version from title + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; - const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); + const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); - if (shared.isolation) await browser.close(); + if (shared.isolation) await browser.close(); - return onlineVersion.toUpperCase() !== info.version.toUpperCase(); + return onlineVersion.toUpperCase() !== info.version.toUpperCase(); }; /** * @public @@ -222,36 +222,36 @@ module.exports.chekIfGameHasUpdate = async function (info) { * an identified game (in the case of homonymy of titles) */ module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const urlList = await searcher.getSearchGameResults(browser, name); + // Gets the search results of the game being searched for + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + const urlList = await searcher.getSearchGameResults(browser, name); - // Process previous partial results - const promiseList = []; - for (const url of urlList) { + // Process previous partial results + const promiseList = []; + for (const url of urlList) { // Start looking for information - promiseList.push(scraper.getGameInfo(browser, url)); - } + promiseList.push(scraper.getGameInfo(browser, url)); + } - // Filter for mods - const result = []; - for (const info of await Promise.all(promiseList)) { + // Filter for mods + const result = []; + for (const info of await Promise.all(promiseList)) { // Ignore empty results - if (!info) continue; - // Skip mods if not required - if (info.isMod && !includeMods) continue; - // Else save data - result.push(info); - } + if (!info) continue; + // Skip mods if not required + if (info.isMod && !includeMods) continue; + // Else save data + result.push(info); + } - if (shared.isolation) await browser.close(); - return result; + if (shared.isolation) await browser.close(); + return result; }; /** * @public @@ -261,26 +261,26 @@ module.exports.getGameData = async function (name, includeMods) { * @returns {Promise} Information about the game. If no game was found, null is returned */ module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Check URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); - if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); + // Check URL + const exists = await urlHelper.urlExists(url); + if (!exists) throw new URIError(url + " is not a valid URL"); + if (!urlHelper.isF95URL(url)) + throw new Error(url + " is not a valid F95Zone URL"); - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; + // Gets the search results of the game being searched for + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; - // Get game data - const result = await scraper.getGameInfo(browser, url); + // Get game data + const result = await scraper.getGameInfo(browser, url); - if (shared.isolation) await browser.close(); - return result; + if (shared.isolation) await browser.close(); + return result; }; /** * @public @@ -289,47 +289,47 @@ module.exports.getGameDataFromURL = async function (url) { * @returns {Promise} Data of the user currently logged in */ module.exports.getUserData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_BASE_URL); // Go to base page + // Prepare a new web page + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(urlK.F95_BASE_URL); // Go to base page - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_ELEMENT), - page.waitForSelector(selectorK.AVATAR_PIC), - ]); + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.USERNAME_ELEMENT), + page.waitForSelector(selectorK.AVATAR_PIC), + ]); - const threads = getUserWatchedGameThreads(browser); + const threads = getUserWatchedGameThreads(browser); - const username = await page.evaluate( + const username = await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.USERNAME_ELEMENT - ); + document.querySelector(selector).innerText, + selectorK.USERNAME_ELEMENT + ); - const avatarSrc = await page.evaluate( + const avatarSrc = await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector).getAttribute("src"), - selectorK.AVATAR_PIC - ); + document.querySelector(selector).getAttribute("src"), + selectorK.AVATAR_PIC + ); - const ud = new UserData(); - ud.username = username; - ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = await threads; + const ud = new UserData(); + ud.username = username; + ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; + ud.watchedThreads = await threads; - await page.close(); - if (shared.isolation) await browser.close(); + await page.close(); + if (shared.isolation) await browser.close(); - return ud; + return ud; }; /** * @public @@ -337,19 +337,19 @@ module.exports.getUserData = async function () { * You **must** be logged in to the portal before calling this method. */ module.exports.logout = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return; + } - // Logout - shared.isLogged = false; + // Logout + shared.isLogged = false; - // Gracefully close shared browser - if (!shared.isolation && _browser !== null) { - await _browser.close(); - _browser = null; - } + // Gracefully close shared browser + if (!shared.isolation && _browser !== null) { + await _browser.close(); + _browser = null; + } }; //#endregion @@ -363,20 +363,20 @@ module.exports.logout = async function () { * @return {object[]} List of dictionaries or null if cookies don't exist */ function loadCookies() { - // Check the existence of the cookie file - if (fs.existsSync(shared.cookiesCachePath)) { + // Check the existence of the cookie file + if (fs.existsSync(shared.cookiesCachePath)) { // Read cookies - const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); - const cookies = JSON.parse(cookiesJSON); + const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); + const cookies = JSON.parse(cookiesJSON); - // Check if the cookies have expired - for (const cookie of cookies) { - if (isCookieExpired(cookie)) return null; - } + // Check if the cookies have expired + for (const cookie of cookies) { + if (isCookieExpired(cookie)) return null; + } - // Cookies loaded and verified - return cookies; - } else return null; + // Cookies loaded and verified + return cookies; + } else return null; } /** * @private @@ -385,25 +385,25 @@ function loadCookies() { * @returns {Boolean} true if the cookie has expired, false otherwise */ function isCookieExpired(cookie) { - // Local variables - let expiredCookies = false; + // Local variables + let expiredCookies = false; - // Ignore cookies that never expire - const expirationUnixTimestamp = cookie.expire; + // Ignore cookies that never expire + const expirationUnixTimestamp = cookie.expire; - if (expirationUnixTimestamp !== "-1") { + if (expirationUnixTimestamp !== "-1") { // Convert UNIX epoch timestamp to normal Date - const expirationDate = new Date(expirationUnixTimestamp * 1000); + const expirationDate = new Date(expirationUnixTimestamp * 1000); - if (expirationDate < Date.now()) { - shared.logger.warn( - "Cookie " + cookie.name + " expired, you need to re-authenticate" - ); - expiredCookies = true; + if (expirationDate < Date.now()) { + shared.logger.warn( + "Cookie " + cookie.name + " expired, you need to re-authenticate" + ); + expiredCookies = true; + } } - } - return expiredCookies; + return expiredCookies; } //#endregion Cookies functions @@ -420,27 +420,27 @@ function isCookieExpired(cookie) { * @returns {Promise} List of required values in uppercase */ async function loadValuesFromLatestPage( - page, - path, - selector, - elementRequested -) { - // If the values already exist they are loaded from disk without having to connect to F95 - shared.logger.info("Load " + elementRequested + " from disk..."); - if (fs.existsSync(path)) { - const valueJSON = fs.readFileSync(path); - return JSON.parse(valueJSON); - } - - // Otherwise, connect and download the data from the portal - shared.logger.info("No " + elementRequested + " cached, downloading..."); - const values = await getValuesFromLatestPage( page, + path, selector, - "Getting " + elementRequested + " from page" - ); - fs.writeFileSync(path, JSON.stringify(values)); - return values; + elementRequested +) { + // If the values already exist they are loaded from disk without having to connect to F95 + shared.logger.info("Load " + elementRequested + " from disk..."); + if (fs.existsSync(path)) { + const valueJSON = fs.readFileSync(path); + return JSON.parse(valueJSON); + } + + // Otherwise, connect and download the data from the portal + shared.logger.info("No " + elementRequested + " cached, downloading..."); + const values = await getValuesFromLatestPage( + page, + selector, + "Getting " + elementRequested + " from page" + ); + fs.writeFileSync(path, JSON.stringify(values)); + return values; } /** * @private @@ -453,20 +453,20 @@ async function loadValuesFromLatestPage( * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector */ async function getValuesFromLatestPage(page, selector, logMessage) { - shared.logger.info(logMessage); + shared.logger.info(logMessage); - const result = []; - const elements = await page.$$(selector); + const result = []; + const elements = await page.$$(selector); - for (const element of elements) { - const text = await element.evaluate( - /* istanbul ignore next */ (e) => e.innerText - ); + for (const element of elements) { + const text = await element.evaluate( + /* istanbul ignore next */ (e) => e.innerText + ); - // Save as upper text for better match if used in query - result.push(text.toUpperCase()); - } - return result; + // Save as upper text for better match if used in query + result.push(text.toUpperCase()); + } + return result; } //#endregion @@ -480,66 +480,66 @@ async function getValuesFromLatestPage(page, selector, logMessage) { * @returns {Promise} Result of the operation */ async function loginF95(browser, username, password) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_LOGIN_URL); // Go to login page + const page = await preparePage(browser); // Set new isolated page + await page.goto(urlK.F95_LOGIN_URL); // Go to login page - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_INPUT), - page.waitForSelector(selectorK.PASSWORD_INPUT), - page.waitForSelector(selectorK.LOGIN_BUTTON), - ]); + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.USERNAME_INPUT), + page.waitForSelector(selectorK.PASSWORD_INPUT), + page.waitForSelector(selectorK.LOGIN_BUTTON), + ]); - await page.type(selectorK.USERNAME_INPUT, username); // Insert username - await page.type(selectorK.PASSWORD_INPUT, password); // Insert password - await Promise.all([ - page.click(selectorK.LOGIN_BUTTON), // Click on the login button - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); + await page.type(selectorK.USERNAME_INPUT, username); // Insert username + await page.type(selectorK.PASSWORD_INPUT, password); // Insert password + await Promise.all([ + page.click(selectorK.LOGIN_BUTTON), // Click on the login button + page.waitForNavigation({ + waitUntil: shared.WAIT_STATEMENT, + }), // Wait for page to load + ]); - // Prepare result - let message = ""; + // Prepare result + let message = ""; - // Check if the user is logged in - const success = await page.evaluate( + // Check if the user is logged in + const success = await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector) !== null, - selectorK.AVATAR_INFO - ); - - const errorMessageExists = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector) !== null, - selectorK.LOGIN_MESSAGE_ERROR - ); - - // Save cookies to avoid re-auth - if (success) { - const c = await page.cookies(); - fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); - message = "Authentication successful"; - } else if (errorMessageExists) { - const errorMessage = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.LOGIN_MESSAGE_ERROR + document.querySelector(selector) !== null, + selectorK.AVATAR_INFO ); - if (errorMessage === "Incorrect password. Please try again.") { - message = "Incorrect password"; - } else if ( - errorMessage === - "The requested user '" + username + "' could not be found." - ) { - // The escaped quotes are important! - message = "Incorrect username"; - } else message = errorMessage; - } else message = "Unknown error"; + const errorMessageExists = await page.evaluate( + /* istanbul ignore next */ + (selector) => document.querySelector(selector) !== null, + selectorK.LOGIN_MESSAGE_ERROR + ); - await page.close(); // Close the page - return new LoginResult(success, message); + // Save cookies to avoid re-auth + if (success) { + const c = await page.cookies(); + fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); + message = "Authentication successful"; + } else if (errorMessageExists) { + const errorMessage = await page.evaluate( + /* istanbul ignore next */ (selector) => + document.querySelector(selector).innerText, + selectorK.LOGIN_MESSAGE_ERROR + ); + + if (errorMessage === "Incorrect password. Please try again.") { + message = "Incorrect password"; + } else if ( + errorMessage === + "The requested user '" + username + "' could not be found." + ) { + // The escaped quotes are important! + message = "Incorrect username"; + } else message = errorMessage; + } else message = "Unknown error"; + + await page.close(); // Close the page + return new LoginResult(success, message); } /** * @private @@ -548,61 +548,61 @@ async function loginF95(browser, username, password) { * @returns {Promise} URL list */ async function getUserWatchedGameThreads(browser) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page + const page = await preparePage(browser); // Set new isolated page + await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page - // Explicitly wait for the required items to load - await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); + // Explicitly wait for the required items to load + await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); - // Show the popup - await Promise.all([ - page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), - ]); + // Show the popup + await Promise.all([ + page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), + page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), + page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), + page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), + ]); - // Set the filters - await page.evaluate( + // Set the filters + await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector).removeAttribute("checked"), - selectorK.UNREAD_THREAD_CHECKBOX - ); // Also read the threads already read + document.querySelector(selector).removeAttribute("checked"), + selectorK.UNREAD_THREAD_CHECKBOX + ); // Also read the threads already read - // Filter the threads - await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); - await page.click(selectorK.FILTER_THREADS_BUTTON); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + // Filter the threads + await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); + await page.click(selectorK.FILTER_THREADS_BUTTON); + await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - // Get the threads urls - const urls = []; - let nextPageExists = false; - do { + // Get the threads urls + const urls = []; + let nextPageExists = false; + do { // Get all the URLs - for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { - const src = await page.evaluate( - /* istanbul ignore next */ (element) => element.href, - handle - ); - // If 'unread' is left, it will redirect to the last unread post - const url = src.replace("/unread", ""); - urls.push(url); - } + for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { + const src = await page.evaluate( + /* istanbul ignore next */ (element) => element.href, + handle + ); + // If 'unread' is left, it will redirect to the last unread post + const url = src.replace("/unread", ""); + urls.push(url); + } - nextPageExists = await page.evaluate( - /* istanbul ignore next */ (selector) => document.querySelector(selector), - selectorK.WATCHED_THREAD_NEXT_PAGE - ); + nextPageExists = await page.evaluate( + /* istanbul ignore next */ (selector) => document.querySelector(selector), + selectorK.WATCHED_THREAD_NEXT_PAGE + ); - // Click to next page - if (nextPageExists) { - await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - } - } while (nextPageExists); + // Click to next page + if (nextPageExists) { + await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); + await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + } + } while (nextPageExists); - await page.close(); - return urls; + await page.close(); + return urls; } //#endregion User diff --git a/app/scripts/classes/game-download.js b/app/scripts/classes/game-download.js index 4cc5868..e1240fb 100644 --- a/app/scripts/classes/game-download.js +++ b/app/scripts/classes/game-download.js @@ -4,6 +4,7 @@ // Core modules const fs = require("fs"); +const path = require("path"); // Public modules from npm // const { File } = require('megajs'); @@ -13,98 +14,98 @@ const { prepareBrowser, preparePage } = require("../puppeteer-helper.js"); const shared = require("../shared.js"); class GameDownload { - constructor() { + constructor() { /** * @public * Platform that hosts game files * @type String */ - this.hosting = ""; - /** + this.hosting = ""; + /** * @public * Link to game files * @type String */ - this.link = null; - /** + this.link = null; + /** * @public * Operating systems supported by the game version indicated in this class. * Can be *WINDOWS/LINUX/MACOS* * @type String[] */ - this.supportedOS = []; - } + this.supportedOS = []; + } - /** + /** * @public * Download the game data in the indicated path. * Supported hosting platforms: MEGA, NOPY * @param {String} path Save path * @return {Promise} Result of the operation */ - async download(path) { - if (this.link.includes("mega.nz")) - return await downloadMEGA(this.link, path); - else if (this.link.includes("nopy.to")) - return await downloadNOPY(this.link, path); - } + async download(path) { + if (this.link.includes("mega.nz")) + return await downloadMEGA(this.link, path); + else if (this.link.includes("nopy.to")) + return await downloadNOPY(this.link, path); + } } module.exports = GameDownload; async function downloadMEGA(url, savepath) { - // The URL is masked - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url); - await page.waitForSelector("a.host_link"); + // The URL is masked + const browser = await prepareBrowser(); + const page = await preparePage(browser); + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(url); + await page.waitForSelector("a.host_link"); - // Obtain the link for the unmasked page and click it - const link = await page.$("a.host_link"); - await link.click(); - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads + // Obtain the link for the unmasked page and click it + const link = await page.$("a.host_link"); + await link.click(); + await page.goto(url, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the game page and wait until it loads - // Obtain the URL after the redirect - const downloadURL = page.url(); + // Obtain the URL after the redirect + const downloadURL = page.url(); - // Close browser and page - await page.close(); - await browser.close(); + // Close browser and page + await page.close(); + await browser.close(); - const stream = fs.createWriteStream(savepath); - const file = File.fromURL(downloadURL); - file.download().pipe(stream); - return fs.existsSync(savepath); + const stream = fs.createWriteStream(savepath); + const file = File.fromURL(downloadURL); + file.download().pipe(stream); + return fs.existsSync(savepath); } async function downloadNOPY(url, savepath) { - // Prepare browser - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.goto(url); - await page.waitForSelector("#download"); + // Prepare browser + const browser = await prepareBrowser(); + const page = await preparePage(browser); + await page.goto(url); + await page.waitForSelector("#download"); - // Set the save path - await page._client.send("Page.setDownloadBehavior", { - behavior: "allow", - downloadPath: path.basename(path.dirname(savepath)), // It's a directory - }); + // Set the save path + await page._client.send("Page.setDownloadBehavior", { + behavior: "allow", + downloadPath: path.basename(path.dirname(savepath)), // It's a directory + }); - // Obtain the download button and click it - const downloadButton = await page.$("#download"); - await downloadButton.click(); + // Obtain the download button and click it + const downloadButton = await page.$("#download"); + await downloadButton.click(); - // Await for all the connections to close - await page.waitForNavigation({ - waitUntil: "networkidle0", - timeout: 0, // Disable timeout - }); + // Await for all the connections to close + await page.waitForNavigation({ + waitUntil: "networkidle0", + timeout: 0, // Disable timeout + }); - // Close browser and page - await page.close(); - await browser.close(); + // Close browser and page + await page.close(); + await browser.close(); - return fs.existsSync(savepath); + return fs.existsSync(savepath); } diff --git a/app/scripts/classes/game-info.js b/app/scripts/classes/game-info.js index 2b26de7..c9fcac5 100644 --- a/app/scripts/classes/game-info.js +++ b/app/scripts/classes/game-info.js @@ -1,119 +1,119 @@ "use strict"; class GameInfo { - constructor() { + constructor() { //#region Properties /** * Game name * @type String */ - this.name = null; - /** + this.name = null; + /** * Game author * @type String */ - this.author = null; - /** + this.author = null; + /** * URL to the game's official conversation on the F95Zone portal * @type String */ - this.f95url = null; - /** + this.f95url = null; + /** * Game description * @type String */ - this.overview = null; - /** + this.overview = null; + /** * List of tags associated with the game * @type String[] */ - this.tags = []; - /** + this.tags = []; + /** * Graphics engine used for game development * @type String */ - this.engine = null; - /** + this.engine = null; + /** * Progress of the game * @type String */ - this.status = null; - /** + this.status = null; + /** * Game description image URL * @type String */ - this.previewSource = null; - /** + this.previewSource = null; + /** * Game version * @type String */ - this.version = null; - /** + this.version = null; + /** * Last time the game underwent updates * @type String */ - this.lastUpdate = null; - /** + this.lastUpdate = null; + /** * Last time the local copy of the game was run * @type String */ - this.lastPlayed = null; - /** + this.lastPlayed = null; + /** * Specifies if the game is original or a mod * @type Boolean */ - this.isMod = false; - /** + this.isMod = false; + /** * Changelog for the last version. * @type String */ - this.changelog = null; - /** + this.changelog = null; + /** * Directory containing the local copy of the game * @type String */ - this.gameDir = null; - /** + this.gameDir = null; + /** * Information on game file download links, * including information on hosting platforms * and operating system supported by the specific link * @type GameDownload[] */ - this.downloadInfo = []; + this.downloadInfo = []; //#endregion Properties - } + } - /** + /** * Converts the object to a dictionary used for JSON serialization */ - /* istanbul ignore next */ - toJSON() { - return { - name: this.name, - author: this.author, - f95url: this.f95url, - overview: this.overview, - engine: this.engine, - status: this.status, - previewSource: this.previewSource, - version: this.version, - lastUpdate: this.lastUpdate, - lastPlayed: this.lastPlayed, - isMod: this.isMod, - changelog: this.changelog, - gameDir: this.gameDir, - downloadInfo: this.downloadInfo, - }; - } + /* istanbul ignore next */ + toJSON() { + return { + name: this.name, + author: this.author, + f95url: this.f95url, + overview: this.overview, + engine: this.engine, + status: this.status, + previewSource: this.previewSource, + version: this.version, + lastUpdate: this.lastUpdate, + lastPlayed: this.lastPlayed, + isMod: this.isMod, + changelog: this.changelog, + gameDir: this.gameDir, + downloadInfo: this.downloadInfo, + }; + } - /** + /** * Return a new GameInfo from a JSON string * @param {String} json JSON string used to create the new object * @returns {GameInfo} */ - /* istanbul ignore next */ - static fromJSON(json) { - return Object.assign(new GameInfo(), json); - } + /* istanbul ignore next */ + static fromJSON(json) { + return Object.assign(new GameInfo(), json); + } } module.exports = GameInfo; diff --git a/app/scripts/classes/login-result.js b/app/scripts/classes/login-result.js index 5eb9646..51fd9bc 100644 --- a/app/scripts/classes/login-result.js +++ b/app/scripts/classes/login-result.js @@ -4,17 +4,17 @@ * Object obtained in response to an attempt to login to the portal. */ class LoginResult { - constructor(success, message) { - /** - * Result of the login operation - * @type Boolean - */ - this.success = success; - /** - * Login response message - * @type String - */ - this.message = message; - } + constructor(success, message) { + /** + * Result of the login operation + * @type Boolean + */ + this.success = success; + /** + * Login response message + * @type String + */ + this.message = message; + } } module.exports = LoginResult; diff --git a/app/scripts/classes/user-data.js b/app/scripts/classes/user-data.js index d6fead0..7edb904 100644 --- a/app/scripts/classes/user-data.js +++ b/app/scripts/classes/user-data.js @@ -4,23 +4,23 @@ * Class containing the data of the user currently connected to the F95Zone platform. */ class UserData { - constructor() { + constructor() { /** * User username. * @type String */ - this.username = ""; - /** + this.username = ""; + /** * Path to the user's profile picture. * @type String */ - this.avatarSrc = null; - /** + this.avatarSrc = null; + /** * List of followed thread URLs. * @type URL[] */ - this.watchedThreads = []; - } + this.watchedThreads = []; + } } module.exports = UserData; diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index 6ef347e..bf0f60b 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -1,33 +1,33 @@ module.exports = Object.freeze({ - AVATAR_INFO: "span.avatar", - AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]', - ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span', - FILTER_THREADS_BUTTON: 'button[class="button--primary button"]', - GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]', - GAME_TAGS: "a.tagItem", - GAME_TITLE: "h1.p-title-value", - GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]', - LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: + AVATAR_INFO: "span.avatar", + AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", + ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", + FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", + GAME_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", + GAME_TAGS: "a.tagItem", + GAME_TITLE: "h1.p-title-value", + GAME_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", + LOGIN_BUTTON: "button.button--icon--login", + LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", - ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]', - PASSWORD_INPUT: 'input[name="password"]', - SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: 'input[name="keywords"][type="search"]', - SEARCH_ONLY_GAMES_OPTION: 'select[name="c[nodes][]"] > option[value="1"]', - STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span', - THREAD_POSTS: + ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", + PASSWORD_INPUT: "input[name=\"password\"]", + SEARCH_BUTTON: "form.block > * button.button--icon--search", + SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", + SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", + STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", + THREAD_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", - THREAD_TITLE: "h3.contentRow-title", - TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]', - UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]', - USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText', - USERNAME_INPUT: 'input[name="login"]', - WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", - WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]', - DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]', - SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", - SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", - THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", + THREAD_TITLE: "h3.contentRow-title", + TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", + UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", + USERNAME_INPUT: "input[name=\"login\"]", + WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", + WATCHED_THREAD_URLS: "a[href^=\"/threads/\"][data-tp-primary]", + DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", + SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", + SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", + THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", }); diff --git a/app/scripts/constants/url.js b/app/scripts/constants/url.js index 87a8847..fac63ba 100644 --- a/app/scripts/constants/url.js +++ b/app/scripts/constants/url.js @@ -1,7 +1,7 @@ module.exports = Object.freeze({ - F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search/?type=post", - F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_LOGIN_URL: "https://f95zone.to/login", - F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", + F95_BASE_URL: "https://f95zone.to", + F95_SEARCH_URL: "https://f95zone.to/search/?type=post", + F95_LATEST_UPDATES: "https://f95zone.to/latest", + F95_LOGIN_URL: "https://f95zone.to/login", + F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", }); diff --git a/app/scripts/game-scraper.js b/app/scripts/game-scraper.js index dd52d52..7b20e47 100644 --- a/app/scripts/game-scraper.js +++ b/app/scripts/game-scraper.js @@ -21,57 +21,57 @@ const urlHelper = require("./url-helper.js"); * looking for */ module.exports.getGameInfo = async function (browser, url) { - shared.logger.info("Obtaining game info"); + shared.logger.info("Obtaining game info"); - // Verify the correctness of the URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(`${url} is not a valid URL`); - if (!urlHelper.isF95URL(url)) - throw new Error(`${url} is not a valid F95Zone URL`); + // Verify the correctness of the URL + const exists = await urlHelper.urlExists(url); + if (!exists) throw new URIError(`${url} is not a valid URL`); + if (!urlHelper.isF95URL(url)) + throw new Error(`${url} is not a valid F95Zone URL`); - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(url, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the game page and wait until it loads - // It asynchronously searches for the elements and - // then waits at the end to compile the object to be returned - let info = new GameInfo(); - const title = getGameTitle(page); - const author = getGameAuthor(page); - const tags = getGameTags(page); - const redirectUrl = urlHelper.getUrlRedirect(url); - info = await parsePrefixes(page, info); // Fill status/engines/isMod - const structuredText = await getMainPostStructuredText(page); - const overview = getOverview(structuredText, info.isMod); - const parsedInfos = parseConversationPage(structuredText); - const previewSource = getGamePreviewSource(page); - const changelog = getLastChangelog(page); + // It asynchronously searches for the elements and + // then waits at the end to compile the object to be returned + let info = new GameInfo(); + const title = getGameTitle(page); + const author = getGameAuthor(page); + const tags = getGameTags(page); + const redirectUrl = urlHelper.getUrlRedirect(url); + info = await parsePrefixes(page, info); // Fill status/engines/isMod + const structuredText = await getMainPostStructuredText(page); + const overview = getOverview(structuredText, info.isMod); + const parsedInfos = parseConversationPage(structuredText); + const previewSource = getGamePreviewSource(page); + const changelog = getLastChangelog(page); - // Fill in the GameInfo element with the information obtained - info.name = await title; - info.author = await author; - info.tags = await tags; - info.f95url = await redirectUrl; - info.overview = overview; - info.lastUpdate = info.isMod - ? parsedInfos.UPDATED - : parsedInfos.THREAD_UPDATED; - info.previewSource = await previewSource; - info.changelog = await changelog; - info.version = await exports.getGameVersionFromTitle(browser, info); + // Fill in the GameInfo element with the information obtained + info.name = await title; + info.author = await author; + info.tags = await tags; + info.f95url = await redirectUrl; + info.overview = overview; + info.lastUpdate = info.isMod + ? parsedInfos.UPDATED + : parsedInfos.THREAD_UPDATED; + info.previewSource = await previewSource; + info.changelog = await changelog; + info.version = await exports.getGameVersionFromTitle(browser, info); - //let downloadData = getGameDownloadLink(page); - //info.downloadInfo = await downloadData; - /* Downloading games without going directly to + //let downloadData = getGameDownloadLink(page); + //info.downloadInfo = await downloadData; + /* Downloading games without going directly to * the platform appears to be prohibited by * the guidelines. It is therefore useless to * keep the links for downloading the games. */ - await page.close(); // Close the page - shared.logger.info("Founded data for " + info.name); - return info; + await page.close(); // Close the page + shared.logger.info("Founded data for " + info.name); + return info; }; /** @@ -81,27 +81,27 @@ module.exports.getGameInfo = async function (browser, url) { * @returns {Promise} Online version of the game */ module.exports.getGameVersionFromTitle = async function (browser, info) { - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(info.f95url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(info.f95url, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the game page and wait until it loads - // Get the title - const titleHTML = await page.evaluate( + // Get the title + const titleHTML = await page.evaluate( /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; + (selector) => document.querySelector(selector).innerHTML, + selectorK.GAME_TITLE + ); + const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; - // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] - const startIndex = title.indexOf("[") + 1; - const endIndex = title.indexOf("]", startIndex); - let version = title.substring(startIndex, endIndex).trim().toUpperCase(); - if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence - await page.close(); - return cleanFSString(version); + // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] + const startIndex = title.indexOf("[") + 1; + const endIndex = title.indexOf("]", startIndex); + let version = title.substring(startIndex, endIndex).trim().toUpperCase(); + if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence + await page.close(); + return cleanFSString(version); }; //#region Private methods @@ -111,8 +111,8 @@ module.exports.getGameVersionFromTitle = async function (browser, info) { * @returns {String} */ function cleanFSString(s) { - const rx = /[/\\?%*:|"<>]/g; - return s.replace(rx, ""); + const rx = /[/\\?%*:|"<>]/g; + return s.replace(rx, ""); } /** @@ -124,11 +124,11 @@ function cleanFSString(s) { * @returns {Promise} Game description */ function getOverview(text, isMod) { - // Get overview (different parsing for game and mod) - let overviewEndIndex; - if (isMod) overviewEndIndex = text.indexOf("Updated"); - else overviewEndIndex = text.indexOf("Thread Updated"); - return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); + // Get overview (different parsing for game and mod) + let overviewEndIndex; + if (isMod) overviewEndIndex = text.indexOf("Updated"); + else overviewEndIndex = text.indexOf("Thread Updated"); + return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); } /** @@ -139,16 +139,16 @@ function getOverview(text, isMod) { * @returns {Promise} Structured text */ async function getMainPostStructuredText(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; + // Gets the first post, where are listed all the game's informations + const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - // The info are plain text so we need to parse the HTML code - const bodyHTML = await page.evaluate( + // The info are plain text so we need to parse the HTML code + const bodyHTML = await page.evaluate( /* istanbul ignore next */ - (mainPost) => mainPost.innerHTML, - post - ); - return HTMLParser.parse(bodyHTML).structuredText; + (mainPost) => mainPost.innerHTML, + post + ); + return HTMLParser.parse(bodyHTML).structuredText; } /** @@ -158,20 +158,20 @@ async function getMainPostStructuredText(page) { * @returns {Promise} Game author */ async function getGameAuthor(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( + // Get the game/mod name (without square brackets) + const titleHTML = await page.evaluate( /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); + (selector) => document.querySelector(selector).innerHTML, + selectorK.GAME_TITLE + ); + const structuredTitle = HTMLParser.parse(titleHTML); - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; + // The last element **shoud be** the title without prefixes (engines, status, other...) + const gameTitle = structuredTitle.childNodes.pop().rawText; - // The last square brackets contain the author - const startTitleIndex = gameTitle.lastIndexOf("[") + 1; - return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); + // The last square brackets contain the author + const startTitleIndex = gameTitle.lastIndexOf("[") + 1; + return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); } /** @@ -182,23 +182,23 @@ async function getGameAuthor(page) { * @returns {Object} Dictionary of information */ function parseConversationPage(text) { - const dataPairs = {}; + const dataPairs = {}; - // The information searched in the game post are one per line - const splittedText = text.split("\n"); - for (const line of splittedText) { - if (!line.includes(":")) continue; + // The information searched in the game post are one per line + const splittedText = text.split("\n"); + for (const line of splittedText) { + if (!line.includes(":")) continue; - // Create pair key/value - const splitted = line.split(":"); - const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch - const value = splitted[1].trim(); + // Create pair key/value + const splitted = line.split(":"); + const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch + const value = splitted[1].trim(); - // Add pair to the dict if valid - if (value !== "") dataPairs[key] = value; - } + // Add pair to the dict if valid + if (value !== "") dataPairs[key] = value; + } - return dataPairs; + return dataPairs; } /** @@ -208,27 +208,27 @@ function parseConversationPage(text) { * @returns {Promise} URL (String) of the image or null if failed to get it */ async function getGamePreviewSource(page) { - // Wait for the selector or return an empty value - try { - await page.waitForSelector(selectorK.GAME_IMAGES); - } catch { - return null; - } + // Wait for the selector or return an empty value + try { + await page.waitForSelector(selectorK.GAME_IMAGES); + } catch { + return null; + } - const src = await page.evaluate( + const src = await page.evaluate( /* istanbul ignore next */ - (selector) => { - // Get the firs image available - const img = document.querySelector(selector); + (selector) => { + // Get the firs image available + const img = document.querySelector(selector); - if (img) return img.getAttribute("src"); - else return null; - }, - selectorK.GAME_IMAGES - ); + if (img) return img.getAttribute("src"); + else return null; + }, + selectorK.GAME_IMAGES + ); - // Check if the URL is valid - return urlHelper.isStringAValidURL(src) ? src : null; + // Check if the URL is valid + return urlHelper.isStringAValidURL(src) ? src : null; } /** @@ -238,18 +238,18 @@ async function getGamePreviewSource(page) { * @returns {Promise} Game title */ async function getGameTitle(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( + // Get the game/mod name (without square brackets) + const titleHTML = await page.evaluate( /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); + (selector) => document.querySelector(selector).innerHTML, + selectorK.GAME_TITLE + ); + const structuredTitle = HTMLParser.parse(titleHTML); - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - const endTitleIndex = gameTitle.indexOf("["); - return gameTitle.substring(0, endTitleIndex).trim(); + // The last element **shoud be** the title without prefixes (engines, status, other...) + const gameTitle = structuredTitle.childNodes.pop().rawText; + const endTitleIndex = gameTitle.indexOf("["); + return gameTitle.substring(0, endTitleIndex).trim(); } /** @@ -259,18 +259,18 @@ async function getGameTitle(page) { * @returns {Promise} List of uppercase tags */ async function getGameTags(page) { - const tags = []; + const tags = []; - // Get the game tags - for (const handle of await page.$$(selectorK.GAME_TAGS)) { - const tag = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - tags.push(tag.toUpperCase()); - } - return tags.sort(); + // Get the game tags + for (const handle of await page.$$(selectorK.GAME_TAGS)) { + const tag = await page.evaluate( + /* istanbul ignore next */ + (element) => element.innerText, + handle + ); + tags.push(tag.toUpperCase()); + } + return tags.sort(); } /** @@ -282,25 +282,25 @@ async function getGameTags(page) { * @returns {Promise} GameInfo object passed in to which the identified information has been added */ async function parsePrefixes(page, info) { - // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' - info.status = "ONGOING"; - for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { - const value = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); + // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' + info.status = "ONGOING"; + for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { + const value = await page.evaluate( + /* istanbul ignore next */ + (element) => element.innerText, + handle + ); - // Clean the prefix - const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); + // Clean the prefix + const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); - // Getting infos... - if (shared.statuses.includes(prefix)) info.status = prefix; - else if (shared.engines.includes(prefix)) info.engine = prefix; - // This is not a game but a mod - else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; - } - return info; + // Getting infos... + if (shared.statuses.includes(prefix)) info.status = prefix; + else if (shared.engines.includes(prefix)) info.engine = prefix; + // This is not a game but a mod + else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; + } + return info; } /** @@ -310,24 +310,24 @@ async function parsePrefixes(page, info) { * @returns {Promise} Changelog for the last version or a empty string if no changelog is found */ async function getLastChangelog(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; + // Gets the first post, where are listed all the game's informations + const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG); - if (!spoiler) return ""; + const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG); + if (!spoiler) return ""; - const changelogHTML = await page.evaluate( + const changelogHTML = await page.evaluate( /* istanbul ignore next */ - (e) => e.innerText, - spoiler - ); - let parsedText = HTMLParser.parse(changelogHTML).structuredText; + (e) => e.innerText, + spoiler + ); + let parsedText = HTMLParser.parse(changelogHTML).structuredText; - // Clean the text - if (parsedText.startsWith("Spoiler")) - parsedText = parsedText.replace("Spoiler", ""); - if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", ""); - return parsedText.trim(); + // Clean the text + if (parsedText.startsWith("Spoiler")) + parsedText = parsedText.replace("Spoiler", ""); + if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", ""); + return parsedText.trim(); } /** @@ -340,64 +340,64 @@ async function getLastChangelog(page) { /* istanbul ignore next */ // skipcq: JS-0128 async function getGameDownloadLink(page) { - // Most used hosting platforms - const hostingPlatforms = [ - "MEGA", - "NOPY", - "FILESUPLOAD", - "MIXDROP", - "UPLOADHAVEN", - "PIXELDRAIN", - "FILESFM", - ]; + // Most used hosting platforms + const hostingPlatforms = [ + "MEGA", + "NOPY", + "FILESUPLOAD", + "MIXDROP", + "UPLOADHAVEN", + "PIXELDRAIN", + "FILESFM", + ]; - // Supported OS platforms - const platformOS = ["WIN", "LINUX", "MAC", "ALL"]; + // Supported OS platforms + const platformOS = ["WIN", "LINUX", "MAC", "ALL"]; - // Gets the which contains the download links - const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER); - if (temp.length === 0) return []; + // Gets the which contains the download links + const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER); + if (temp.length === 0) return []; - // Look for the container that contains the links - // It is necessary because the same css selector - // also identifies other elements on the page - let container = null; - for (const candidate of temp) { - if (container !== null) break; - const upperText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - candidate - ) - ).toUpperCase(); + // Look for the container that contains the links + // It is necessary because the same css selector + // also identifies other elements on the page + let container = null; + for (const candidate of temp) { + if (container !== null) break; + const upperText = ( + await page.evaluate( + /* istanbul ignore next */ + (e) => e.innerText, + candidate + ) + ).toUpperCase(); - // Search if the container contains the name of a hosting platform - for (const p of hostingPlatforms) { - if (upperText.includes(p)) { - container = candidate; - break; - } + // Search if the container contains the name of a hosting platform + for (const p of hostingPlatforms) { + if (upperText.includes(p)) { + container = candidate; + break; + } + } } - } - if (container === null) return []; + if (container === null) return []; - // Extract the HTML text from the container - const searchText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerHTML, - container - ) - ).toLowerCase(); + // Extract the HTML text from the container + const searchText = ( + await page.evaluate( + /* istanbul ignore next */ + (e) => e.innerHTML, + container + ) + ).toLowerCase(); - // Parse the download links - const downloadData = []; - for (const platform of platformOS) { - const data = extractGameHostingData(platform, searchText); - downloadData.push(...data); - } - return downloadData; + // Parse the download links + const downloadData = []; + for (const platform of platformOS) { + const data = extractGameHostingData(platform, searchText); + downloadData.push(...data); + } + return downloadData; } /** @@ -411,57 +411,57 @@ async function getGameDownloadLink(page) { */ /* istanbul ignore next */ function extractGameHostingData(platform, text) { - const PLATFORM_BOLD_OPEN = ""; - const CONTAINER_SPAN_CLOSE = ""; - const LINK_OPEN = "platform - let endIndex = + // Find the platform + let endIndex = text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length; - // Find the end of the container - if (endIndex === -1) - endIndex = + // Find the end of the container + if (endIndex === -1) + endIndex = text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) + CONTAINER_SPAN_CLOSE.length; - text = text.substring(startIndex, endIndex); + text = text.substring(startIndex, endIndex); - const downloadData = []; - const linkTags = text.split(LINK_OPEN); - for (const tag of linkTags) { + const downloadData = []; + const linkTags = text.split(LINK_OPEN); + for (const tag of linkTags) { // Ignore non-link string - if (!tag.includes(HREF_START)) continue; + if (!tag.includes(HREF_START)) continue; - // Find the hosting platform name - startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length; - endIndex = tag.indexOf(LINK_CLOSE, startIndex); - const hosting = tag.substring(startIndex, endIndex); + // Find the hosting platform name + startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length; + endIndex = tag.indexOf(LINK_CLOSE, startIndex); + const hosting = tag.substring(startIndex, endIndex); - // Find the 'href' attribute - startIndex = tag.indexOf(HREF_START) + HREF_START.length; - endIndex = tag.indexOf(HREF_END, startIndex); - const link = tag.substring(startIndex, endIndex); + // Find the 'href' attribute + startIndex = tag.indexOf(HREF_START) + HREF_START.length; + endIndex = tag.indexOf(HREF_END, startIndex); + const link = tag.substring(startIndex, endIndex); - if (urlHelper.isStringAValidURL(link)) { - const gd = new GameDownload(); - gd.hosting = hosting.toUpperCase(); - gd.link = link; - gd.supportedOS = platform.toUpperCase(); + if (urlHelper.isStringAValidURL(link)) { + const gd = new GameDownload(); + gd.hosting = hosting.toUpperCase(); + gd.link = link; + gd.supportedOS = platform.toUpperCase(); - downloadData.push(gd); + downloadData.push(gd); + } } - } - return downloadData; + return downloadData; } //#endregion Private methods diff --git a/app/scripts/game-searcher.js b/app/scripts/game-searcher.js index 592b490..497edfe 100644 --- a/app/scripts/game-searcher.js +++ b/app/scripts/game-searcher.js @@ -18,46 +18,46 @@ const { isF95URL } = require("./url-helper.js"); * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal */ module.exports.getSearchGameResults = async function (browser, gamename) { - shared.logger.info(`Searching ${gamename} on F95Zone`); + shared.logger.info(`Searching ${gamename} on F95Zone`); - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_SEARCH_URL, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the search form and wait for it + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(urlK.F95_SEARCH_URL, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the search form and wait for it - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), - page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), - page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), - page.waitForSelector(selectorK.SEARCH_BUTTON), - ]); + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), + page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), + page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), + page.waitForSelector(selectorK.SEARCH_BUTTON), + ]); - await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire - await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles - await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod - await Promise.all([ - page.click(selectorK.SEARCH_BUTTON), // Execute search - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); + await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire + await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles + await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod + await Promise.all([ + page.click(selectorK.SEARCH_BUTTON), // Execute search + page.waitForNavigation({ + waitUntil: shared.WAIT_STATEMENT, + }), // Wait for page to load + ]); - // Select all conversation titles - const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY); + // Select all conversation titles + const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY); - // For each element found extract the info about the conversation - shared.logger.info("Extracting info from conversations"); - const results = []; - for (const element of resultsThread) { - const gameUrl = await getOnlyGameThreads(page, element); - if (gameUrl !== null) results.push(gameUrl); - } - shared.logger.info(`Find ${results.length} conversations`); - await page.close(); // Close the page + // For each element found extract the info about the conversation + shared.logger.info("Extracting info from conversations"); + const results = []; + for (const element of resultsThread) { + const gameUrl = await getOnlyGameThreads(page, element); + if (gameUrl !== null) results.push(gameUrl); + } + shared.logger.info(`Find ${results.length} conversations`); + await page.close(); // Close the page - return results; + return results; }; //#region Private methods @@ -69,16 +69,16 @@ module.exports.getSearchGameResults = async function (browser, gamename) { * @return {Promise} URL of the game/mod or null if the URL is not of a game */ async function getOnlyGameThreads(page, divHandle) { - // Obtain the elements containing the basic information - const titleHandle = await divHandle.$(selectorK.THREAD_TITLE); - const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP); + // Obtain the elements containing the basic information + const titleHandle = await divHandle.$(selectorK.THREAD_TITLE); + const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP); - // Get the forum where the thread was posted - const forum = await getMembershipForum(page, forumHandle); - if (forum !== "GAMES" && forum !== "MODS") return null; + // Get the forum where the thread was posted + const forum = await getMembershipForum(page, forumHandle); + if (forum !== "GAMES" && forum !== "MODS") return null; - // Get the URL of the thread from the title - return await getThreadURL(page, titleHandle); + // Get the URL of the thread from the title + return await getThreadURL(page, titleHandle); } /** @@ -89,23 +89,23 @@ async function getOnlyGameThreads(page, divHandle) { * @returns {Promise} Uppercase membership category */ async function getMembershipForum(page, handle) { - // The link can be something like: - // + /forums/request.NUMBER/ - // + /forums/game-recommendations-identification.NUMBER/ - // + /forums/games.NUMBER/ <-- We need this + // The link can be something like: + // + /forums/request.NUMBER/ + // + /forums/game-recommendations-identification.NUMBER/ + // + /forums/games.NUMBER/ <-- We need this - let link = await page.evaluate( + let link = await page.evaluate( /* istanbul ignore next */ - (e) => e.getAttribute("href"), - handle - ); + (e) => e.getAttribute("href"), + handle + ); - // Parse link - link = link.replace("/forums/", ""); - const endIndex = link.indexOf("."); - const forum = link.substring(0, endIndex); + // Parse link + link = link.replace("/forums/", ""); + const endIndex = link.indexOf("."); + const forum = link.substring(0, endIndex); - return forum.toUpperCase(); + return forum.toUpperCase(); } /** @@ -116,17 +116,17 @@ async function getMembershipForum(page, handle) { * @returns {Promise} URL of the thread */ async function getThreadURL(page, handle) { - const relativeURLThread = await page.evaluate( + const relativeURLThread = await page.evaluate( /* istanbul ignore next */ - (e) => e.querySelector("a").href, - handle - ); + (e) => e.querySelector("a").href, + handle + ); - // Some game already have a full URL... - if (isF95URL(relativeURLThread)) return relativeURLThread; + // Some game already have a full URL... + if (isF95URL(relativeURLThread)) return relativeURLThread; - // ... else compose the URL and return - const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString(); - return urlThread; + // ... else compose the URL and return + const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString(); + return urlThread; } //#endregion Private methods diff --git a/app/scripts/puppeteer-helper.js b/app/scripts/puppeteer-helper.js index 18a40a4..942da84 100644 --- a/app/scripts/puppeteer-helper.js +++ b/app/scripts/puppeteer-helper.js @@ -13,20 +13,20 @@ const shared = require("./shared.js"); * @returns {Promise} Created browser */ module.exports.prepareBrowser = async function () { - // Create a headless browser - let browser = null; - if (shared.chromiumLocalPath) { - browser = await puppeteer.launch({ - executablePath: shared.chromiumLocalPath, - headless: !shared.debug, // Use GUI when debug = true - }); - } else { - browser = await puppeteer.launch({ - headless: !shared.debug, // Use GUI when debug = true - }); - } + // Create a headless browser + let browser = null; + if (shared.chromiumLocalPath) { + browser = await puppeteer.launch({ + executablePath: shared.chromiumLocalPath, + headless: !shared.debug, // Use GUI when debug = true + }); + } else { + browser = await puppeteer.launch({ + headless: !shared.debug, // Use GUI when debug = true + }); + } - return browser; + return browser; }; /** @@ -37,24 +37,24 @@ module.exports.prepareBrowser = async function () { * @returns {Promise} New page */ module.exports.preparePage = async function (browser) { - // Create new page in the browser argument - const page = await browser.newPage(); + // Create new page in the browser argument + const page = await browser.newPage(); - // Block image download - await page.setRequestInterception(true); - page.on("request", (request) => { - if (request.resourceType() === "image") request.abort(); - else if (request.resourceType === "font") request.abort(); - // else if (request.resourceType() == 'stylesheet') request.abort(); - // else if(request.resourceType == 'media') request.abort(); - else request.continue(); - }); + // Block image download + await page.setRequestInterception(true); + page.on("request", (request) => { + if (request.resourceType() === "image") request.abort(); + else if (request.resourceType === "font") request.abort(); + // else if (request.resourceType() == 'stylesheet') request.abort(); + // else if(request.resourceType == 'media') request.abort(); + else request.continue(); + }); - // Set custom user-agent - const userAgent = + // Set custom user-agent + const userAgent = "Mozilla/5.0 (X11; Linux x86_64)" + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; - await page.setUserAgent(userAgent); + await page.setUserAgent(userAgent); - return page; + return page; }; diff --git a/app/scripts/shared.js b/app/scripts/shared.js index ead66fc..85a5ba4 100644 --- a/app/scripts/shared.js +++ b/app/scripts/shared.js @@ -9,7 +9,7 @@ const log4js = require("log4js"); * Class containing variables shared between modules. */ class Shared { - //#region Properties + //#region Properties /** * Shows log messages and other useful functions for module debugging. * @type Boolean @@ -64,63 +64,63 @@ class Shared { * @returns {Boolean} */ static get debug() { - return this.#_debug; + return this.#_debug; } /** * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {Boolean} */ static get isLogged() { - return this.#_isLogged; + return this.#_isLogged; } /** * List of cookies obtained from the F95Zone platform. * @returns {Object[]} */ static get cookies() { - return this.#_cookies; + return this.#_cookies; } /** * List of possible game engines used for development. * @returns {String[]} */ static get engines() { - return this.#_engines; + return this.#_engines; } /** * List of possible development states that a game can assume. * @returns {String[]} */ static get statuses() { - return this.#_statuses; + return this.#_statuses; } /** * Directory to save the API cache. * @returns {String} */ static get cacheDir() { - return this.#_cacheDir; + return this.#_cacheDir; } /** * Path to the F95 platform cache. * @returns {String} */ static get cookiesCachePath() { - return join(this.#_cacheDir, "cookies.json"); + return join(this.#_cacheDir, "cookies.json"); } /** * Path to the game engine cache. * @returns {String} */ static get enginesCachePath() { - return join(this.#_cacheDir, "engines.json"); + return join(this.#_cacheDir, "engines.json"); } /** * Path to the cache of possible game states. * @returns {String} */ static get statusesCachePath() { - return join(this.#_cacheDir, "statuses.json"); + return join(this.#_cacheDir, "statuses.json"); } /** * If true, it opens a new browser for each request @@ -128,44 +128,44 @@ class Shared { * @returns {Boolean} */ static get isolation() { - return this.#_isolation; + return this.#_isolation; } /** * Logger object used to write to both file and console. * @returns {log4js.Logger} */ static get logger() { - return this.#_logger; + return this.#_logger; } //#endregion Getters //#region Setters static set cookies(val) { - this.#_cookies = val; + this.#_cookies = val; } static set engines(val) { - this.#_engines = val; + this.#_engines = val; } static set statuses(val) { - this.#_statuses = val; + this.#_statuses = val; } static set cacheDir(val) { - this.#_cacheDir = val; + this.#_cacheDir = val; } static set debug(val) { - this.#_debug = val; + this.#_debug = val; } static set isLogged(val) { - this.#_isLogged = val; + this.#_isLogged = val; } static set isolation(val) { - this.#_isolation = val; + this.#_isolation = val; } //#endregion Setters } diff --git a/app/scripts/url-helper.js b/app/scripts/url-helper.js index 221a56c..aae60a0 100644 --- a/app/scripts/url-helper.js +++ b/app/scripts/url-helper.js @@ -2,7 +2,7 @@ // Public modules from npm const ky = require("ky-universal").create({ - throwHttpErrors: false, + throwHttpErrors: false, }); // Modules from file @@ -15,8 +15,8 @@ const { F95_BASE_URL } = require("./constants/url.js"); * @returns {Boolean} true if the url belongs to the domain, false otherwise */ module.exports.isF95URL = function (url) { - if (url.toString().startsWith(F95_BASE_URL)) return true; - else return false; + if (url.toString().startsWith(F95_BASE_URL)) return true; + else return false; }; /** @@ -26,12 +26,12 @@ module.exports.isF95URL = function (url) { * @returns {Boolean} true if the string is a valid URL, false otherwise */ module.exports.isStringAValidURL = function (url) { - try { - new URL(url); // skipcq: JS-0078 - return true; - } catch (err) { - return false; - } + try { + new URL(url); // skipcq: JS-0078 + return true; + } catch (err) { + return false; + } }; /** @@ -42,22 +42,22 @@ module.exports.isStringAValidURL = function (url) { * @returns {Promise} true if the URL exists, false otherwise */ module.exports.urlExists = async function (url, checkRedirect) { - if (!exports.isStringAValidURL(url)) { - return false; - } + if (!exports.isStringAValidURL(url)) { + return false; + } - const response = await ky.head(url); - let valid = response !== undefined && !/4\d\d/.test(response.status); + const response = await ky.head(url); + let valid = response !== undefined && !/4\d\d/.test(response.status); - if (!valid) return false; + if (!valid) return false; - if (checkRedirect) { - const redirectUrl = await exports.getUrlRedirect(url); - if (redirectUrl === url) valid = true; - else valid = false; - } + if (checkRedirect) { + const redirectUrl = await exports.getUrlRedirect(url); + if (redirectUrl === url) valid = true; + else valid = false; + } - return valid; + return valid; }; /** @@ -67,6 +67,6 @@ module.exports.urlExists = async function (url, checkRedirect) { * @returns {Promise} Redirect URL or the passed URL */ module.exports.getUrlRedirect = async function (url) { - const response = await ky.head(url); - return response.url; + const response = await ky.head(url); + return response.url; }; diff --git a/test/index-test.js b/test/index-test.js index 4d89e73..7b69d4c 100644 --- a/test/index-test.js +++ b/test/index-test.js @@ -27,264 +27,264 @@ const FAKE_PASSWORD = "fake_password"; //F95API.debug(false); function randomSleep() { - const random = Math.floor(Math.random() * 500) + 50; - sleep.msleep(500 + random); + const random = Math.floor(Math.random() * 500) + 50; + sleep.msleep(500 + random); } describe("Login without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout - before("Set isolation", function () { - F95API.setIsolation(true); - }); + before("Set isolation", function () { + F95API.setIsolation(true); + }); - beforeEach("Remove all cookies", function () { + beforeEach("Remove all cookies", function () { // Runs before each test in this block - if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up + if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up - let testOrder = 0; + let testOrder = 0; - it("Test with valid credentials", async function () { + it("Test with valid credentials", async function () { // Gain exclusive use of the cookies - while (testOrder !== 0) randomSleep(); + while (testOrder !== 0) randomSleep(); - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Authentication successful"); + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(result.message).equal("Authentication successful"); - testOrder = 1; - }); - it("Test with invalid username", async function () { + testOrder = 1; + }); + it("Test with invalid username", async function () { // Gain exclusive use of the cookies - while (testOrder !== 1) randomSleep(); + while (testOrder !== 1) randomSleep(); - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect username"); - testOrder = 2; - }); - it("Test with invalid password", async function () { + testOrder = 2; + }); + it("Test with invalid password", async function () { // Gain exclusive use of the cookies - while (testOrder !== 2) randomSleep(); + while (testOrder !== 2) randomSleep(); - const result = await F95API.login(USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect password"); + const result = await F95API.login(USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect password"); - testOrder = 3; - }); - it("Test with invalid credentials", async function () { + testOrder = 3; + }); + it("Test with invalid credentials", async function () { // Gain exclusive use of the cookies - while (testOrder !== 3) randomSleep(); + while (testOrder !== 3) randomSleep(); - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); // It should first check the username + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect username"); // It should first check the username - testOrder = 4; - }); + testOrder = 4; + }); }); describe("Login with cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout - before("Log in to create cookies then logout", async function () { + before("Log in to create cookies then logout", async function () { // Runs once before the first test in this block - if (!fs.existsSync(COOKIES_SAVE_PATH)) - await F95API.login(USERNAME, PASSWORD); // Download cookies - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up + if (!fs.existsSync(COOKIES_SAVE_PATH)) + await F95API.login(USERNAME, PASSWORD); // Download cookies + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up - it("Test with valid credentials", async function () { - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Logged with cookies"); - }); + it("Test with valid credentials", async function () { + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(result.message).equal("Logged with cookies"); + }); }); describe("Load base data without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout - before("Delete cache if exists", function () { + before("Delete cache if exists", function () { // Runs once before the first test in this block - if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); - if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); - }); - //#endregion Set-up + if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); + if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); + }); + //#endregion Set-up - it("With login", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("With login", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - const result = await F95API.loadF95BaseData(); + const result = await F95API.loadF95BaseData(); - const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); - const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); + const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); + const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); - expect(result).to.be.true; - expect(enginesCacheExists).to.be.true; - expect(statusesCacheExists).to.be.true; - }); + expect(result).to.be.true; + expect(enginesCacheExists).to.be.true; + expect(statusesCacheExists).to.be.true; + }); - it("Without login", async function () { - if (F95API.isLogged()) F95API.logout(); - const result = await F95API.loadF95BaseData(); - expect(result).to.be.false; - }); + it("Without login", async function () { + if (F95API.isLogged()) F95API.logout(); + const result = await F95API.loadF95BaseData(); + expect(result).to.be.false; + }); }); describe("Search game data", function () { - //#region Set-up - this.timeout(60000); // All tests in this suite get 60 seconds before timeout + //#region Set-up + this.timeout(60000); // All tests in this suite get 60 seconds before timeout - beforeEach("Prepare API", function () { + beforeEach("Prepare API", function () { // Runs once before the first test in this block - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up - let testGame = null; + let testGame = null; - it("Search game when logged", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("Search game when logged", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; + const loadResult = await F95API.loadF95BaseData(); + expect(loadResult).to.be.true; - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const gamesList = await F95API.getGameData("Kingdom of Deception", false); - expect(gamesList.length, "Should find only the game").to.equal(1); - const result = gamesList[0]; - const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ + const gamesList = await F95API.getGameData("Kingdom of Deception", false); + expect(gamesList.length, "Should find only the game").to.equal(1); + const result = gamesList[0]; + const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; - // Test only the main information - expect(result.name).to.equal("Kingdom of Deception"); - expect(result.author).to.equal("Hreinn Games"); - expect(result.isMod, "Should be false").to.be.false; - expect(result.engine).to.equal("REN'PY"); - expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? - testGame = Object.assign({}, result); - }); - it("Search game when not logged", async function () { - const result = await F95API.getGameData("Kingdom of Deception", false); - expect(result, "Without being logged should return null").to.be.null; - }); - it("Test game serialization", function () { - const json = JSON.stringify(testGame); - const parsedGameInfo = JSON.parse(json); - const result = _.isEqual(parsedGameInfo, testGame); - expect(result).to.be.true; - }); + // Test only the main information + expect(result.name).to.equal("Kingdom of Deception"); + expect(result.author).to.equal("Hreinn Games"); + expect(result.isMod, "Should be false").to.be.false; + expect(result.engine).to.equal("REN'PY"); + expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? + testGame = Object.assign({}, result); + }); + it("Search game when not logged", async function () { + const result = await F95API.getGameData("Kingdom of Deception", false); + expect(result, "Without being logged should return null").to.be.null; + }); + it("Test game serialization", function () { + const json = JSON.stringify(testGame); + const parsedGameInfo = JSON.parse(json); + const result = _.isEqual(parsedGameInfo, testGame); + expect(result).to.be.true; + }); }); describe("Load user data", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up - it("Retrieve when logged", async function () { + it("Retrieve when logged", async function () { // Login - await F95API.login(USERNAME, PASSWORD); + await F95API.login(USERNAME, PASSWORD); - // Then retrieve user data - const data = await F95API.getUserData(); + // Then retrieve user data + const data = await F95API.getUserData(); - expect(data).to.exist; - expect(data.username).to.equal(USERNAME); - }); - it("Retrieve when not logged", async function () { + expect(data).to.exist; + expect(data.username).to.equal(USERNAME); + }); + it("Retrieve when not logged", async function () { // Logout - if (F95API.isLogged()) F95API.logout(); + if (F95API.isLogged()) F95API.logout(); - // Try to retrieve user data - const data = await F95API.getUserData(); + // Try to retrieve user data + const data = await F95API.getUserData(); - expect(data).to.be.null; - }); + expect(data).to.be.null; + }); }); describe("Check game update", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up - it("Get online game and verify that no update exists", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("Get online game and verify that no update exists", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; + const loadResult = await F95API.loadF95BaseData(); + expect(loadResult).to.be.true; - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ + const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.false; - }); + const update = await F95API.chekIfGameHasUpdate(result); + expect(update).to.be.false; + }); - it("Verify that update exists from old URL", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("Verify that update exists from old URL", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ - const url = + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ + const url = "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; - const result = await F95API.getGameDataFromURL(url); - result.version = "0.9600"; + const result = await F95API.getGameDataFromURL(url); + result.version = "0.9600"; - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.true; - }); + const update = await F95API.chekIfGameHasUpdate(result); + expect(update).to.be.true; + }); }); describe("Test url-helper", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up - it("Check if URL exists", async function () { + it("Check if URL exists", async function () { // Check generic URLs... - let exists = await urlHelper.urlExists("https://www.google.com/"); - expect(exists, "Complete valid URL").to.be.true; + let exists = await urlHelper.urlExists("https://www.google.com/"); + expect(exists, "Complete valid URL").to.be.true; - exists = await urlHelper.urlExists("www.google.com"); - expect(exists, "URl without protocol prefix").to.be.false; + exists = await urlHelper.urlExists("www.google.com"); + expect(exists, "URl without protocol prefix").to.be.false; - exists = await urlHelper.urlExists("https://www.google/"); - expect(exists, "URL without third level domain").to.be.false; + exists = await urlHelper.urlExists("https://www.google/"); + expect(exists, "URL without third level domain").to.be.false; - // Now check for more specific URLs (with redirect)... - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(exists, "URL with redirect without check").to.be.true; + // Now check for more specific URLs (with redirect)... + exists = await urlHelper.urlExists( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" + ); + expect(exists, "URL with redirect without check").to.be.true; - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", - true - ); - expect(exists, "URL with redirect with check").to.be.false; - }); + exists = await urlHelper.urlExists( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", + true + ); + expect(exists, "URL with redirect with check").to.be.false; + }); - it("Check if URL belong to the platform", async function () { - let belong = urlHelper.isF95URL( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(belong).to.be.true; + it("Check if URL belong to the platform", async function () { + let belong = urlHelper.isF95URL( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" + ); + expect(belong).to.be.true; - belong = urlHelper.isF95URL("https://www.google/"); - expect(belong).to.be.false; - }); + belong = urlHelper.isF95URL("https://www.google/"); + expect(belong).to.be.false; + }); }); diff --git a/test/user-test.js b/test/user-test.js index c6f6b29..9819d66 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -4,12 +4,12 @@ F95API.debug(true); main(); async function main() { - const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); + const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); - if (loginResult.success) { - await F95API.loadF95BaseData(); - const gameData = await F95API.getGameData("a struggle with sin", false); - console.log(gameData); - } - F95API.logout(); + if (loginResult.success) { + await F95API.loadF95BaseData(); + const gameData = await F95API.getGameData("a struggle with sin", false); + console.log(gameData); + } + F95API.logout(); }