diff --git a/.eslintrc.json b/.eslintrc.json index c439502..10f4dca 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,20 +1,38 @@ { - "env": { - "browser": true, - "commonjs": true, - "es2021": true, - "node": true - }, - "extends": "eslint:recommended", - "parser": "./node_modules/babel-eslint", - "parserOptions": { - "ecmaVersion": 12 - }, - "rules": { - "indent": ["error", 4], - "linebreak-style": ["error", "windows"], - "quotes": ["error", "double"], - "semi": ["error", "always"], - "no-unused-vars": ["error", "after-used"] - } + "env": { + "browser": true, + "commonjs": true, + "es2021": true, + "node": true, + "mocha": true + }, + "extends": "eslint:recommended", + "parser": "babel-eslint", + "parserOptions": { + "ecmaVersion": 12 + }, + "rules": { + "indent": [ + "error", + 4 + ], + "linebreak-style": [ + "error", + "windows" + ], + "quotes": [ + "error", + "double" + ], + "semi": [ + "error", + "always" + ], + "no-unused-vars": [ + "error", + { + "args": "after-used" + } + ] + } } diff --git a/README.md b/README.md index 56ff3af..565a308 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,75 @@ Unofficial Node JS module for scraping F95Zone platform +These APIs have been developed to support this application and allow you to obtain data on games and mods on the platform [F95zone](https://f95zone.to/) (**NSFW**) + +A simple usage example can be found in [app/example.js](https://github.com/MillenniumEarl/F95API/blob/master/app/example.js) + +**Attention**: Two-factor authentication is not supported + +# Data scraping +Games/mods can be obtained by name or URL + +```javascript +// The name is case insensitive +let listOfFoundGames = await F95API.getGameData("your game name"); +let listOfFoundMods = await F95API.getGameData("your mod name", true); + +let specificGame = await F95API.getGameDataFromURL("the URL of your game"); +``` + +While user data (after authenticating) with + +```javascript +let authResult = await F95API.login(username, password); + +let loggedUserData = await F95API.getUserData(); +``` + +# Classes +## Games and mods +Information about games and mods is stored in a GameInfo object with the following fields: + +``` +name: The game name +author: The game developer +url: The URL that leads to the game thread on F95Zone +overview: Description of the game +language: Main language of the game +supportedOS: List of supported OS (Windows/Linux/Mac/Android...) +censored: Are the NSFW parts censored? +engine: Game engine (Unity, Ren'Py, RPGM...) +status: Completed/Abandoned/Ongoing/Onhold +tags: List of tags +previewSrc: Source URL of the game description image +version: Version of the game +lastUpdate: Date of the last update (it's a Date object) +isMod: Is it a game or a mod? +changelog: Latest changelog available +``` + +The serialization in JSON format of this object is possible through `JSON.stringfy()` while the deserialization must happen through the static method `GameInfo.fromJSON()`. + +## User data +User data (after authentication) can be stored in a UserData object, consisting of the following fields: + +``` +username: Name of the logged in user +avatarSrc: Source URL of the user's profile picture +watchedThreads: List of URLs of threads followed by the user +``` + +## Login results +The outcome of the authentication process is represented by the LoginResult object: + +``` +success: Was the authentication successful?; +message: Possible error message (unrecognized user, wrong password ...) or authentication successful message +``` + +# Logging +To log the behavior of the application [log4js](https://github.com/log4js-node/log4js-node) is used with a default level of "warn". This option can be changed with the `loggerLevel` property. + # Guidelines for errors - If you can, return a meaningful value diff --git a/app/example.js b/app/example.js new file mode 100644 index 0000000..48eac22 --- /dev/null +++ b/app/example.js @@ -0,0 +1,56 @@ +/* +to use this example, create an .env file +in the project root with the following values: + +F95_USERNAME = YOUR_USERNAME +F95_PASSWORD = YOUR_PASSWORD +*/ + +"use strict"; + +// Public modules from npm +const dotenv = require("dotenv"); + +// Modules from file +const F95API = require("./index.js"); + +// Configure the .env reader +dotenv.config(); + +main(); + +async function main() { + // Local variables + const gameList = [ + "kingdom of deception", + "perverted education", + "corrupted kingdoms", + "summertime saga", + "brothel king" + ]; + + // Log in the platform + console.log("Authenticating..."); + const result = await F95API.login(process.env.F95_USERNAME, process.env.F95_PASSWORD); + console.log(`Authentication result: ${result.message}`); + + // Get user data + console.log("Fetching user data..."); + const userdata = await F95API.getUserData(); + console.log(`${userdata.username} follows ${userdata.watchedThreads.length} threads`); + + for(const gamename of gameList) { + console.log(`Searching '${gamename}'...`); + const found = await F95API.getGameData(gamename, false); + + // If no game is found + if (found.length === 0) { + console.log(`No data found for '${gamename}'`); + continue; + } + + // Extract first game + const gamedata = found[0]; + console.log(`Found ${gamedata.name} (${gamedata.version}) by ${gamedata.author}`); + } +} diff --git a/app/index.js b/app/index.js index d03987b..1d59db9 100644 --- a/app/index.js +++ b/app/index.js @@ -1,21 +1,14 @@ "use strict"; -// Core modules -const fs = require("fs"); - // Modules from file const shared = require("./scripts/shared.js"); -const urlK = require("./scripts/constants/url.js"); -const selectorK = require("./scripts/constants/css-selector.js"); -const urlHelper = require("./scripts/url-helper.js"); -const scraper = require("./scripts/game-scraper.js"); -const { - prepareBrowser, - preparePage, -} = require("./scripts/puppeteer-helper.js"); -const searcher = require("./scripts/game-searcher.js"); +const networkHelper = require("./scripts/network-helper.js"); +const scraper = require("./scripts/scraper.js"); +const searcher = require("./scripts/searcher.js"); +const uScraper = require("./scripts/user-scraper.js"); // Classes from file +const Credentials = require("./scripts/classes/credentials.js"); const GameInfo = require("./scripts/classes/game-info.js"); const LoginResult = require("./scripts/classes/login-result.js"); const UserData = require("./scripts/classes/user-data.js"); @@ -28,63 +21,24 @@ module.exports.UserData = UserData; //#region Export properties /** - * Shows log messages and other useful functions for module debugging. - * @param {Boolean} value + * @public + * Set the logger level for module debugging. */ -module.exports.debug = function (value) { - shared.debug = value; - - // Configure logger - shared.logger.level = value ? "debug" : "warn"; -}; +/* istambul ignore next */ +module.exports.loggerLevel = shared.logger.level; +exports.loggerLevel = "warn"; // By default log only the warn messages /** * @public * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {String} */ -module.exports.isLogged = function () { - return shared.isLogged; -}; -/** - * @public - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {String} - */ -module.exports.setIsolation = function (value) { - shared.isolation = value; -}; -/** - * @public - * Path to the cache directory - * @returns {String} - */ -module.exports.getCacheDir = function () { - return shared.cacheDir; -}; -/** - * @public - * Set path to the cache directory - * @returns {String} - */ -module.exports.setCacheDir = function (value) { - shared.cacheDir = value; - - // Create directory if it doesn't exist - if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); -}; -/** - * @public - * Set local chromium path. - * @returns {String} - */ -module.exports.setChromiumPath = function (value) { - shared.chromiumLocalPath = value; +/* istambul ignore next */ +module.exports.isLogged = function isLogged() { + return shared.isLogged; }; //#endregion Export properties //#region Global variables -var _browser = null; const USER_NOT_LOGGED = "User not authenticated, unable to continue"; //#endregion @@ -98,92 +52,25 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue"; * @returns {Promise} Result of the operation */ module.exports.login = async function (username, password) { - if (shared.isLogged) { - shared.logger.info("Already logged in"); - const result = new LoginResult(true, "Already logged in"); + if (shared.isLogged) { + shared.logger.info(`${username} already authenticated`); + return new LoginResult(true, `${username} already authenticated`); + } + + shared.logger.trace("Fetching token..."); + const creds = new Credentials(username, password); + await creds.fetchToken(); + + shared.logger.trace(`Authentication for ${username}`); + const result = await networkHelper.authenticate(creds); + shared.isLogged = result.success; + + if (result.success) shared.logger.info("User logged in through the platform"); + else shared.logger.warn(`Error during authentication: ${result.message}`); + return result; - } - - // If cookies are loaded, use them to authenticate - shared.cookies = loadCookies(); - if (shared.cookies !== null) { - shared.logger.info("Valid session, no need to re-authenticate"); - shared.isLogged = true; - const result = new LoginResult(true, "Logged with cookies"); - return result; - } - - // Else, log in throught browser - shared.logger.info( - "No saved sessions or expired session, login on the platform" - ); - - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const result = await loginF95(browser, username, password); - shared.isLogged = result.success; - - if (result.success) { - // Reload cookies - shared.cookies = loadCookies(); - shared.logger.info("User logged in through the platform"); - } else { - shared.logger.warn("Error during authentication: " + result.message); - } - if (shared.isolation) await browser.close(); - return result; }; -/** - * @public - * This method loads the main data from the F95 portal - * used to provide game information. You **must** be logged - * in to the portal before calling this method. - * @returns {Promise} Result of the operation - */ -module.exports.loadF95BaseData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } - shared.logger.info("Loading base data..."); - - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - - // Go to latest update page and wait for it to load - await page.goto(urlK.F95_LATEST_UPDATES, { - waitUntil: shared.WAIT_STATEMENT, - }); - - // Obtain engines (disk/online) - await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); - shared.engines = await loadValuesFromLatestPage( - page, - shared.enginesCachePath, - selectorK.ENGINE_ID_SELECTOR, - "engines" - ); - - // Obtain statuses (disk/online) - await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); - shared.statuses = await loadValuesFromLatestPage( - page, - shared.statusesCachePath, - selectorK.STATUS_ID_SELECTOR, - "statuses" - ); - - await page.close(); - if (shared.isolation) await browser.close(); - shared.logger.info("Base data loaded"); - return true; -}; /** * @public * Chek if exists a new version of the game. @@ -191,68 +78,54 @@ module.exports.loadF95BaseData = async function () { * @param {GameInfo} info Information about the game to get the version for * @returns {Promise} true if an update is available, false otherwise */ -module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } +module.exports.checkIfGameHasUpdate = async function (info) { + if (!shared.isLogged) { + shared.logger.warn(USER_NOT_LOGGED); + return false; + } - // F95 change URL at every game update, - // so if the URL is different an update is available - const exists = await urlHelper.urlExists(info.f95url, true); - if (!exists) return true; + // F95 change URL at every game update, + // so if the URL is different an update is available + const exists = await networkHelper.urlExists(info.url, true); + if (!exists) return true; - // Parse version from title - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); - - if (shared.isolation) await browser.close(); - - return onlineVersion.toUpperCase() !== info.version.toUpperCase(); + // Parse version from title + const onlineVersion = await scraper.getGameInfo(info.url).version; + + // Compare the versions + return onlineVersion.toUpperCase() !== info.version.toUpperCase(); }; + /** * @public * Starting from the name, it gets all the information about the game you are looking for. * You **must** be logged in to the portal before calling this method. * @param {String} name Name of the game searched - * @param {Boolean} includeMods Indicates whether to also take mods into account when searching + * @param {Boolean} mod Indicate if you are looking for mods or games * @returns {Promise} List of information obtained where each item corresponds to * an identified game (in the case of homonymy of titles) */ -module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } +module.exports.getGameData = async function (name, mod) { + if (!shared.isLogged) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const urlList = await searcher.getSearchGameResults(browser, name); + // Gets the search results of the game/mod being searched for + let urls = []; + if(mod) urls = await searcher.searchMod(name); + else urls = await searcher.searchGame(name); - // Process previous partial results - const promiseList = []; - for (const url of urlList) { - // Start looking for information - promiseList.push(scraper.getGameInfo(browser, url)); - } - - // Filter for mods - const result = []; - for (const info of await Promise.all(promiseList)) { - // Ignore empty results - if (!info) continue; - // Skip mods if not required - if (info.isMod && !includeMods) continue; - // Else save data - result.push(info); - } - - if (shared.isolation) await browser.close(); - return result; + // Process previous partial results + const results = []; + for (const url of urls) { + // Start looking for information + const info = await scraper.getGameInfo(url); + results.push(info); + } + return results; }; + /** * @public * Starting from the url, it gets all the information about the game you are looking for. @@ -261,27 +134,20 @@ module.exports.getGameData = async function (name, includeMods) { * @returns {Promise} Information about the game. If no game was found, null is returned */ module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Check URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); - if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); - - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - // Get game data - const result = await scraper.getGameInfo(browser, url); - - if (shared.isolation) await browser.close(); - return result; + // Check URL validity + const exists = await networkHelper.urlExists(url); + if (!exists) throw new URIError(`${url} is not a valid URL`); + if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`); + + // Get game data + return await scraper.getGameInfo(url); }; + /** * @public * Gets the data of the currently logged in user. @@ -289,321 +155,11 @@ module.exports.getGameDataFromURL = async function (url) { * @returns {Promise} Data of the user currently logged in */ module.exports.getUserData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_BASE_URL); // Go to base page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_ELEMENT), - page.waitForSelector(selectorK.AVATAR_PIC), - ]); - - const threads = getUserWatchedGameThreads(browser); - - const username = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.USERNAME_ELEMENT - ); - - const avatarSrc = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).getAttribute("src"), - selectorK.AVATAR_PIC - ); - - const ud = new UserData(); - ud.username = username; - ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = await threads; - - await page.close(); - if (shared.isolation) await browser.close(); - - return ud; -}; -/** - * @public - * Logout from the current user and gracefully close shared browser. - * You **must** be logged in to the portal before calling this method. - */ -module.exports.logout = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return; - } - - // Logout - shared.isLogged = false; - - // Gracefully close shared browser - if (!shared.isolation && _browser !== null) { - await _browser.close(); - _browser = null; - } + return await uScraper.getUserData(); }; //#endregion - -//#region Private methods - -//#region Cookies functions -/** - * @private - * Loads and verifies the expiration of previously stored cookies from disk - * if they exist, otherwise it returns null. - * @return {object[]} List of dictionaries or null if cookies don't exist - */ -function loadCookies() { - // Check the existence of the cookie file - if (fs.existsSync(shared.cookiesCachePath)) { - // Read cookies - const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); - const cookies = JSON.parse(cookiesJSON); - - // Check if the cookies have expired - for (const cookie of cookies) { - if (isCookieExpired(cookie)) return null; - } - - // Cookies loaded and verified - return cookies; - } else return null; -} -/** - * @private - * Check the validity of a cookie. - * @param {object} cookie Cookies to verify the validity. It's a dictionary - * @returns {Boolean} true if the cookie has expired, false otherwise - */ -function isCookieExpired(cookie) { - // Local variables - let expiredCookies = false; - - // Ignore cookies that never expire - const expirationUnixTimestamp = cookie.expire; - - if (expirationUnixTimestamp !== "-1") { - // Convert UNIX epoch timestamp to normal Date - const expirationDate = new Date(expirationUnixTimestamp * 1000); - - if (expirationDate < Date.now()) { - shared.logger.warn( - "Cookie " + cookie.name + " expired, you need to re-authenticate" - ); - expiredCookies = true; - } - } - - return expiredCookies; -} -//#endregion Cookies functions - -//#region Latest Updates page parserer -/** - * @private - * If present, it reads the file containing the searched values (engines or states) - * from the disk, otherwise it connects to the F95 portal (at the page - * https://f95zone.to/latest) and downloads them. - * @param {puppeteer.Page} page Page used to locate the required elements - * @param {String} path Path to disk of the JSON file containing the data to read / write - * @param {String} selector CSS selector of the required elements - * @param {String} elementRequested Required element (engines or states) used to detail log messages - * @returns {Promise} List of required values in uppercase - */ -async function loadValuesFromLatestPage( - page, - path, - selector, - elementRequested -) { - // If the values already exist they are loaded from disk without having to connect to F95 - shared.logger.info("Load " + elementRequested + " from disk..."); - if (fs.existsSync(path)) { - const valueJSON = fs.readFileSync(path); - return JSON.parse(valueJSON); - } - - // Otherwise, connect and download the data from the portal - shared.logger.info("No " + elementRequested + " cached, downloading..."); - const values = await getValuesFromLatestPage( - page, - selector, - "Getting " + elementRequested + " from page" - ); - fs.writeFileSync(path, JSON.stringify(values)); - return values; -} -/** - * @private - * Gets all the textual values of the elements present - * in the F95 portal page and identified by the selector - * passed by parameter - * @param {puppeteer.Page} page Page used to locate items specified by the selector - * @param {String} selector CSS selector - * @param {String} logMessage Log message indicating which items the selector is requesting - * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector - */ -async function getValuesFromLatestPage(page, selector, logMessage) { - shared.logger.info(logMessage); - - const result = []; - const elements = await page.$$(selector); - - for (const element of elements) { - const text = await element.evaluate( - /* istanbul ignore next */ (e) => e.innerText - ); - - // Save as upper text for better match if used in query - result.push(text.toUpperCase()); - } - return result; -} -//#endregion - -//#region User -/** - * @private - * Log in to the F95Zone portal and, if successful, save the cookies. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} username Username to use during login - * @param {String} password Password to use during login - * @returns {Promise} Result of the operation - */ -async function loginF95(browser, username, password) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_LOGIN_URL); // Go to login page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_INPUT), - page.waitForSelector(selectorK.PASSWORD_INPUT), - page.waitForSelector(selectorK.LOGIN_BUTTON), - ]); - - await page.type(selectorK.USERNAME_INPUT, username); // Insert username - await page.type(selectorK.PASSWORD_INPUT, password); // Insert password - await Promise.all([ - page.click(selectorK.LOGIN_BUTTON), // Click on the login button - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Prepare result - let message = ""; - - // Check if the user is logged in - const success = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector) !== null, - selectorK.AVATAR_INFO - ); - - const errorMessageExists = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector) !== null, - selectorK.LOGIN_MESSAGE_ERROR - ); - - // Save cookies to avoid re-auth - if (success) { - const c = await page.cookies(); - fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); - message = "Authentication successful"; - } else if (errorMessageExists) { - const errorMessage = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.LOGIN_MESSAGE_ERROR - ); - - if (errorMessage === "Incorrect password. Please try again.") { - message = "Incorrect password"; - } else if ( - errorMessage === - "The requested user '" + username + "' could not be found." - ) { - // The escaped quotes are important! - message = "Incorrect username"; - } else message = errorMessage; - } else message = "Unknown error"; - - await page.close(); // Close the page - return new LoginResult(success, message); -} -/** - * @private - * Gets the list of URLs of threads the user follows. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @returns {Promise} URL list - */ -async function getUserWatchedGameThreads(browser) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page - - // Explicitly wait for the required items to load - await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); - - // Show the popup - await Promise.all([ - page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), - ]); - - // Set the filters - await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).removeAttribute("checked"), - selectorK.UNREAD_THREAD_CHECKBOX - ); // Also read the threads already read - - // Filter the threads - await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); - await page.click(selectorK.FILTER_THREADS_BUTTON); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - - // Get the threads urls - const urls = []; - let nextPageExists = false; - do { - // Get all the URLs - for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { - const src = await page.evaluate( - /* istanbul ignore next */ (element) => element.href, - handle - ); - // If 'unread' is left, it will redirect to the last unread post - const url = src.replace("/unread", ""); - urls.push(url); - } - - nextPageExists = await page.evaluate( - /* istanbul ignore next */ (selector) => document.querySelector(selector), - selectorK.WATCHED_THREAD_NEXT_PAGE - ); - - // Click to next page - if (nextPageExists) { - await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - } - } while (nextPageExists); - - await page.close(); - return urls; -} -//#endregion User - -//#endregion Private methods diff --git a/app/scripts/classes/credentials.js b/app/scripts/classes/credentials.js new file mode 100644 index 0000000..3041f03 --- /dev/null +++ b/app/scripts/classes/credentials.js @@ -0,0 +1,18 @@ +"use strict"; + +// Modules from file +const { getF95Token } = require("../network-helper.js"); + +class Credentials { + constructor(username, password) { + this.username = username; + this.password = password; + this.token = null; + } + + async fetchToken() { + this.token = await getF95Token(); + } +} + +module.exports = Credentials; \ No newline at end of file diff --git a/app/scripts/classes/game-download.js b/app/scripts/classes/game-download.js deleted file mode 100644 index 4cc5868..0000000 --- a/app/scripts/classes/game-download.js +++ /dev/null @@ -1,110 +0,0 @@ -/* istanbul ignore file */ - -"use strict"; - -// Core modules -const fs = require("fs"); - -// Public modules from npm -// const { File } = require('megajs'); - -// Modules from file -const { prepareBrowser, preparePage } = require("../puppeteer-helper.js"); -const shared = require("../shared.js"); - -class GameDownload { - constructor() { - /** - * @public - * Platform that hosts game files - * @type String - */ - this.hosting = ""; - /** - * @public - * Link to game files - * @type String - */ - this.link = null; - /** - * @public - * Operating systems supported by the game version indicated in this class. - * Can be *WINDOWS/LINUX/MACOS* - * @type String[] - */ - this.supportedOS = []; - } - - /** - * @public - * Download the game data in the indicated path. - * Supported hosting platforms: MEGA, NOPY - * @param {String} path Save path - * @return {Promise} Result of the operation - */ - async download(path) { - if (this.link.includes("mega.nz")) - return await downloadMEGA(this.link, path); - else if (this.link.includes("nopy.to")) - return await downloadNOPY(this.link, path); - } -} -module.exports = GameDownload; - -async function downloadMEGA(url, savepath) { - // The URL is masked - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url); - await page.waitForSelector("a.host_link"); - - // Obtain the link for the unmasked page and click it - const link = await page.$("a.host_link"); - await link.click(); - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // Obtain the URL after the redirect - const downloadURL = page.url(); - - // Close browser and page - await page.close(); - await browser.close(); - - const stream = fs.createWriteStream(savepath); - const file = File.fromURL(downloadURL); - file.download().pipe(stream); - return fs.existsSync(savepath); -} - -async function downloadNOPY(url, savepath) { - // Prepare browser - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.goto(url); - await page.waitForSelector("#download"); - - // Set the save path - await page._client.send("Page.setDownloadBehavior", { - behavior: "allow", - downloadPath: path.basename(path.dirname(savepath)), // It's a directory - }); - - // Obtain the download button and click it - const downloadButton = await page.$("#download"); - await downloadButton.click(); - - // Await for all the connections to close - await page.waitForNavigation({ - waitUntil: "networkidle0", - timeout: 0, // Disable timeout - }); - - // Close browser and page - await page.close(); - await browser.close(); - - return fs.existsSync(savepath); -} diff --git a/app/scripts/classes/game-info.js b/app/scripts/classes/game-info.js index 2b26de7..84a0aac 100644 --- a/app/scripts/classes/game-info.js +++ b/app/scripts/classes/game-info.js @@ -1,119 +1,135 @@ "use strict"; class GameInfo { - constructor() { - //#region Properties - /** - * Game name - * @type String - */ - this.name = null; - /** - * Game author - * @type String - */ - this.author = null; - /** - * URL to the game's official conversation on the F95Zone portal - * @type String - */ - this.f95url = null; - /** - * Game description - * @type String - */ - this.overview = null; - /** - * List of tags associated with the game - * @type String[] - */ - this.tags = []; - /** - * Graphics engine used for game development - * @type String - */ - this.engine = null; - /** - * Progress of the game - * @type String - */ - this.status = null; - /** - * Game description image URL - * @type String - */ - this.previewSource = null; - /** - * Game version - * @type String - */ - this.version = null; - /** - * Last time the game underwent updates - * @type String - */ - this.lastUpdate = null; - /** - * Last time the local copy of the game was run - * @type String - */ - this.lastPlayed = null; - /** - * Specifies if the game is original or a mod - * @type Boolean - */ - this.isMod = false; - /** - * Changelog for the last version. - * @type String - */ - this.changelog = null; - /** - * Directory containing the local copy of the game - * @type String - */ - this.gameDir = null; - /** - * Information on game file download links, - * including information on hosting platforms - * and operating system supported by the specific link - * @type GameDownload[] - */ - this.downloadInfo = []; - //#endregion Properties - } + constructor() { + //#region Properties + /** + * Game name + * @type String + */ + this.name = null; + /** + * Game author + * @type String + */ + this.author = null; + /** + * URL to the game's official conversation on the F95Zone portal + * @type String + */ + this.url = null; + /** + * Game description + * @type String + */ + this.overview = null; + /** + * Game language. + * @type String + */ + this.language = null; + /** + * List of supported OS. + * @type + */ + this.supportedOS = []; + /** + * Specify whether the game has censorship + * measures regarding NSFW scenes. + * @type Boolean + */ + this.censored = null; + /** + * List of tags associated with the game + * @type String[] + */ + this.tags = []; + /** + * Graphics engine used for game development + * @type String + */ + this.engine = null; + /** + * Progress of the game + * @type String + */ + this.status = null; + /** + * Game description image URL + * @type String + */ + this.previewSrc = null; + /** + * Game version + * @type String + */ + this.version = null; + /** + * Last time the game underwent updates + * @type String + */ + this.lastUpdate = null; + /** + * Last time the local copy of the game was run + * @type String + */ + this.lastPlayed = null; + /** + * Specifies if the game is original or a mod + * @type Boolean + */ + this.isMod = false; + /** + * Changelog for the last version. + * @type String + */ + this.changelog = null; + /** + * Directory containing the local copy of the game + * @type String + */ + this.gameDir = null; + //#endregion Properties + } - /** - * Converts the object to a dictionary used for JSON serialization - */ - /* istanbul ignore next */ - toJSON() { - return { - name: this.name, - author: this.author, - f95url: this.f95url, - overview: this.overview, - engine: this.engine, - status: this.status, - previewSource: this.previewSource, - version: this.version, - lastUpdate: this.lastUpdate, - lastPlayed: this.lastPlayed, - isMod: this.isMod, - changelog: this.changelog, - gameDir: this.gameDir, - downloadInfo: this.downloadInfo, - }; - } + /** + * Converts the object to a dictionary used for JSON serialization. + */ + /* istanbul ignore next */ + toJSON() { + return { + name: this.name, + author: this.author, + url: this.url, + overview: this.overview, + language: this.language, + supportedOS: this.supportedOS, + censored: this.censored, + engine: this.engine, + status: this.status, + tags: this.tags, + previewSrc: this.previewSrc, + version: this.version, + lastUpdate: this.lastUpdate, + lastPlayed: this.lastPlayed, + isMod: this.isMod, + changelog: this.changelog, + gameDir: this.gameDir, + }; + } - /** - * Return a new GameInfo from a JSON string - * @param {String} json JSON string used to create the new object - * @returns {GameInfo} - */ - /* istanbul ignore next */ - static fromJSON(json) { - return Object.assign(new GameInfo(), json); - } + /** + * Return a new GameInfo from a JSON string. + * @param {String} json JSON string used to create the new object + * @returns {GameInfo} + */ + static fromJSON(json) { + // Convert string + const temp = Object.assign(new GameInfo(), JSON.parse(json)); + + // JSON cannot transform a string to a date implicitly + temp.lastUpdate = new Date(temp.lastUpdate); + return temp; + } } module.exports = GameInfo; diff --git a/app/scripts/classes/login-result.js b/app/scripts/classes/login-result.js index 5eb9646..51fd9bc 100644 --- a/app/scripts/classes/login-result.js +++ b/app/scripts/classes/login-result.js @@ -4,17 +4,17 @@ * Object obtained in response to an attempt to login to the portal. */ class LoginResult { - constructor(success, message) { - /** - * Result of the login operation - * @type Boolean - */ - this.success = success; - /** - * Login response message - * @type String - */ - this.message = message; - } + constructor(success, message) { + /** + * Result of the login operation + * @type Boolean + */ + this.success = success; + /** + * Login response message + * @type String + */ + this.message = message; + } } module.exports = LoginResult; diff --git a/app/scripts/classes/user-data.js b/app/scripts/classes/user-data.js index d6fead0..bdc2457 100644 --- a/app/scripts/classes/user-data.js +++ b/app/scripts/classes/user-data.js @@ -4,23 +4,23 @@ * Class containing the data of the user currently connected to the F95Zone platform. */ class UserData { - constructor() { - /** - * User username. - * @type String - */ - this.username = ""; - /** - * Path to the user's profile picture. - * @type String - */ - this.avatarSrc = null; - /** - * List of followed thread URLs. - * @type URL[] - */ - this.watchedThreads = []; - } + constructor() { + /** + * User name. + * @type String + */ + this.username = ""; + /** + * Path to the user's profile picture. + * @type String + */ + this.avatarSrc = null; + /** + * List of followed thread URLs. + * @type String[] + */ + this.watchedThreads = []; + } } module.exports = UserData; diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index 6ef347e..e985b2d 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -1,33 +1,23 @@ module.exports = Object.freeze({ - AVATAR_INFO: "span.avatar", - AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]', - ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span', - FILTER_THREADS_BUTTON: 'button[class="button--primary button"]', - GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]', - GAME_TAGS: "a.tagItem", - GAME_TITLE: "h1.p-title-value", - GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]', - LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: - "div.blockMessage.blockMessage--error.blockMessage--iconic", - ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]', - PASSWORD_INPUT: 'input[name="password"]', - SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: 'input[name="keywords"][type="search"]', - SEARCH_ONLY_GAMES_OPTION: 'select[name="c[nodes][]"] > option[value="1"]', - STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span', - THREAD_POSTS: - "article.message-body:first-child > div.bbWrapper:first-of-type", - THREAD_TITLE: "h3.contentRow-title", - TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]', - UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]', - USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText', - USERNAME_INPUT: 'input[name="login"]', - WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", - WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]', - DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]', - SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", - SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", - THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", + BD_ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", + BD_STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", + + GT_IMAGES: "img:not([title])[data-src^=\"https://attachments.f95zone.to\"][data-url=\"\"]", + GT_TAGS: "a.tagItem", + GT_TITLE: "h1.p-title-value", + GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", + GT_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", + GT_JSONLD: "script[type=\"application/ld+json\"]", + WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WT_NEXT_PAGE: "a.pageNav-jump--next", + WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]", + WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", + GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", + GS_RESULT_BODY: "div.contentRow-main", + GS_MEMBERSHIP: "li > a:not(.username)", + GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]", + UD_USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", + UD_AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", + LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", }); diff --git a/app/scripts/constants/url.js b/app/scripts/constants/url.js index 87a8847..6931f82 100644 --- a/app/scripts/constants/url.js +++ b/app/scripts/constants/url.js @@ -1,7 +1,7 @@ module.exports = Object.freeze({ - F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search/?type=post", - F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_LOGIN_URL: "https://f95zone.to/login", - F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", + F95_BASE_URL: "https://f95zone.to", + F95_SEARCH_URL: "https://f95zone.to/search/?type=post", + F95_LATEST_UPDATES: "https://f95zone.to/latest", + F95_LOGIN_URL: "https://f95zone.to/login/login", + F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", }); diff --git a/app/scripts/game-scraper.js b/app/scripts/game-scraper.js deleted file mode 100644 index dd52d52..0000000 --- a/app/scripts/game-scraper.js +++ /dev/null @@ -1,467 +0,0 @@ -"use strict"; - -// Public modules from npm -const HTMLParser = require("node-html-parser"); -const puppeteer = require("puppeteer"); // skipcq: JS-0128 - -// Modules from file -const shared = require("./shared.js"); -const selectorK = require("./constants/css-selector.js"); -const { preparePage } = require("./puppeteer-helper.js"); -const GameDownload = require("./classes/game-download.js"); -const GameInfo = require("./classes/game-info.js"); -const urlHelper = require("./url-helper.js"); - -/** - * @protected - * Get information from the game's main page. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} url URL (String) of the game/mod to extract data from - * @return {Promise} Complete information about the game you are - * looking for - */ -module.exports.getGameInfo = async function (browser, url) { - shared.logger.info("Obtaining game info"); - - // Verify the correctness of the URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(`${url} is not a valid URL`); - if (!urlHelper.isF95URL(url)) - throw new Error(`${url} is not a valid F95Zone URL`); - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // It asynchronously searches for the elements and - // then waits at the end to compile the object to be returned - let info = new GameInfo(); - const title = getGameTitle(page); - const author = getGameAuthor(page); - const tags = getGameTags(page); - const redirectUrl = urlHelper.getUrlRedirect(url); - info = await parsePrefixes(page, info); // Fill status/engines/isMod - const structuredText = await getMainPostStructuredText(page); - const overview = getOverview(structuredText, info.isMod); - const parsedInfos = parseConversationPage(structuredText); - const previewSource = getGamePreviewSource(page); - const changelog = getLastChangelog(page); - - // Fill in the GameInfo element with the information obtained - info.name = await title; - info.author = await author; - info.tags = await tags; - info.f95url = await redirectUrl; - info.overview = overview; - info.lastUpdate = info.isMod - ? parsedInfos.UPDATED - : parsedInfos.THREAD_UPDATED; - info.previewSource = await previewSource; - info.changelog = await changelog; - info.version = await exports.getGameVersionFromTitle(browser, info); - - //let downloadData = getGameDownloadLink(page); - //info.downloadInfo = await downloadData; - /* Downloading games without going directly to - * the platform appears to be prohibited by - * the guidelines. It is therefore useless to - * keep the links for downloading the games. */ - - await page.close(); // Close the page - shared.logger.info("Founded data for " + info.name); - return info; -}; - -/** - * Obtain the game version without parsing again all the data of the game. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {GameInfo} info Information about the game - * @returns {Promise} Online version of the game - */ -module.exports.getGameVersionFromTitle = async function (browser, info) { - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(info.f95url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // Get the title - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; - - // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] - const startIndex = title.indexOf("[") + 1; - const endIndex = title.indexOf("]", startIndex); - let version = title.substring(startIndex, endIndex).trim().toUpperCase(); - if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence - await page.close(); - return cleanFSString(version); -}; - -//#region Private methods -/** - * Clean a string from invalid File System chars. - * @param {String} s - * @returns {String} - */ -function cleanFSString(s) { - const rx = /[/\\?%*:|"<>]/g; - return s.replace(rx, ""); -} - -/** - * @private - * Get the game description from its web page. - * Different processing depending on whether the game is a mod or not. - * @param {String} text Structured text extracted from the game's web page - * @param {Boolean} isMod Specify if it is a game or a mod - * @returns {Promise} Game description - */ -function getOverview(text, isMod) { - // Get overview (different parsing for game and mod) - let overviewEndIndex; - if (isMod) overviewEndIndex = text.indexOf("Updated"); - else overviewEndIndex = text.indexOf("Thread Updated"); - return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); -} - -/** - * @private - * Extrapolate the page structure by removing the element tags - * and leaving only the text and its spacing. - * @param {puppeteer.Page} page Page containing the text - * @returns {Promise} Structured text - */ -async function getMainPostStructuredText(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - - // The info are plain text so we need to parse the HTML code - const bodyHTML = await page.evaluate( - /* istanbul ignore next */ - (mainPost) => mainPost.innerHTML, - post - ); - return HTMLParser.parse(bodyHTML).structuredText; -} - -/** - * @private - * Extrapolates and cleans the author from the page passed by parameter. - * @param {puppeteer.Page} page Page containing the author to be extrapolated - * @returns {Promise} Game author - */ -async function getGameAuthor(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); - - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - - // The last square brackets contain the author - const startTitleIndex = gameTitle.lastIndexOf("[") + 1; - return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); -} - -/** - * @private - * Process the post text to get all the useful - * information in the format *DESCRIPTOR : VALUE*. - * @param {String} text Structured text of the post - * @returns {Object} Dictionary of information - */ -function parseConversationPage(text) { - const dataPairs = {}; - - // The information searched in the game post are one per line - const splittedText = text.split("\n"); - for (const line of splittedText) { - if (!line.includes(":")) continue; - - // Create pair key/value - const splitted = line.split(":"); - const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch - const value = splitted[1].trim(); - - // Add pair to the dict if valid - if (value !== "") dataPairs[key] = value; - } - - return dataPairs; -} - -/** - * @private - * Gets the URL of the image used as a preview for the game in the conversation. - * @param {puppeteer.Page} page Page containing the URL to be extrapolated - * @returns {Promise} URL (String) of the image or null if failed to get it - */ -async function getGamePreviewSource(page) { - // Wait for the selector or return an empty value - try { - await page.waitForSelector(selectorK.GAME_IMAGES); - } catch { - return null; - } - - const src = await page.evaluate( - /* istanbul ignore next */ - (selector) => { - // Get the firs image available - const img = document.querySelector(selector); - - if (img) return img.getAttribute("src"); - else return null; - }, - selectorK.GAME_IMAGES - ); - - // Check if the URL is valid - return urlHelper.isStringAValidURL(src) ? src : null; -} - -/** - * @private - * Extrapolates and cleans the title from the page passed by parameter. - * @param {puppeteer.Page} page Page containing the title to be extrapolated - * @returns {Promise} Game title - */ -async function getGameTitle(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); - - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - const endTitleIndex = gameTitle.indexOf("["); - return gameTitle.substring(0, endTitleIndex).trim(); -} - -/** - * @private - * Get the alphabetically sorted list of tags associated with the game. - * @param {puppeteer.Page} page Page containing the tags to be extrapolated - * @returns {Promise} List of uppercase tags - */ -async function getGameTags(page) { - const tags = []; - - // Get the game tags - for (const handle of await page.$$(selectorK.GAME_TAGS)) { - const tag = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - tags.push(tag.toUpperCase()); - } - return tags.sort(); -} - -/** - * @private - * Process the game title prefixes to extract information such as game status, - * graphics engine used, and whether it is a mod or original game. - * @param {puppeteer.Page} page Page containing the prefixes to be extrapolated - * @param {GameInfo} info Object to assign the identified information to - * @returns {Promise} GameInfo object passed in to which the identified information has been added - */ -async function parsePrefixes(page, info) { - // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' - info.status = "ONGOING"; - for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { - const value = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - - // Clean the prefix - const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); - - // Getting infos... - if (shared.statuses.includes(prefix)) info.status = prefix; - else if (shared.engines.includes(prefix)) info.engine = prefix; - // This is not a game but a mod - else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; - } - return info; -} - -/** - * @private - * Get the last changelog available for the game. - * @param {puppeteer.Page} page Page containing the changelog - * @returns {Promise} Changelog for the last version or a empty string if no changelog is found - */ -async function getLastChangelog(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - - const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG); - if (!spoiler) return ""; - - const changelogHTML = await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - spoiler - ); - let parsedText = HTMLParser.parse(changelogHTML).structuredText; - - // Clean the text - if (parsedText.startsWith("Spoiler")) - parsedText = parsedText.replace("Spoiler", ""); - if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", ""); - return parsedText.trim(); -} - -/** - * @private - * Get game download links for different platforms. - * @param {puppeteer.Page} page Page containing the links to be extrapolated - * @returns {Promise} List of objects used for game download - * @deprecated - */ -/* istanbul ignore next */ -// skipcq: JS-0128 -async function getGameDownloadLink(page) { - // Most used hosting platforms - const hostingPlatforms = [ - "MEGA", - "NOPY", - "FILESUPLOAD", - "MIXDROP", - "UPLOADHAVEN", - "PIXELDRAIN", - "FILESFM", - ]; - - // Supported OS platforms - const platformOS = ["WIN", "LINUX", "MAC", "ALL"]; - - // Gets the which contains the download links - const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER); - if (temp.length === 0) return []; - - // Look for the container that contains the links - // It is necessary because the same css selector - // also identifies other elements on the page - let container = null; - for (const candidate of temp) { - if (container !== null) break; - const upperText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - candidate - ) - ).toUpperCase(); - - // Search if the container contains the name of a hosting platform - for (const p of hostingPlatforms) { - if (upperText.includes(p)) { - container = candidate; - break; - } - } - } - if (container === null) return []; - - // Extract the HTML text from the container - const searchText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerHTML, - container - ) - ).toLowerCase(); - - // Parse the download links - const downloadData = []; - for (const platform of platformOS) { - const data = extractGameHostingData(platform, searchText); - downloadData.push(...data); - } - return downloadData; -} - -/** - * @private - * From the HTML text it extracts the game download links for the specified operating system. - * @param {String} platform Name of the operating system to look for a compatible link to. - * It can only be *WIN/LINUX/MAC/ALL* - * @param {String} text HTML string to extract links from - * @returns {GameDownload[]} List of game download links for the selected platform - * @deprecated - */ -/* istanbul ignore next */ -function extractGameHostingData(platform, text) { - const PLATFORM_BOLD_OPEN = ""; - const CONTAINER_SPAN_CLOSE = ""; - const LINK_OPEN = "platform - let endIndex = - text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length; - - // Find the end of the container - if (endIndex === -1) - endIndex = - text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) + - CONTAINER_SPAN_CLOSE.length; - - text = text.substring(startIndex, endIndex); - - const downloadData = []; - const linkTags = text.split(LINK_OPEN); - for (const tag of linkTags) { - // Ignore non-link string - if (!tag.includes(HREF_START)) continue; - - // Find the hosting platform name - startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length; - endIndex = tag.indexOf(LINK_CLOSE, startIndex); - const hosting = tag.substring(startIndex, endIndex); - - // Find the 'href' attribute - startIndex = tag.indexOf(HREF_START) + HREF_START.length; - endIndex = tag.indexOf(HREF_END, startIndex); - const link = tag.substring(startIndex, endIndex); - - if (urlHelper.isStringAValidURL(link)) { - const gd = new GameDownload(); - gd.hosting = hosting.toUpperCase(); - gd.link = link; - gd.supportedOS = platform.toUpperCase(); - - downloadData.push(gd); - } - } - return downloadData; -} - -//#endregion Private methods diff --git a/app/scripts/game-searcher.js b/app/scripts/game-searcher.js deleted file mode 100644 index 592b490..0000000 --- a/app/scripts/game-searcher.js +++ /dev/null @@ -1,132 +0,0 @@ -"use strict"; - -// Public modules from npm -const puppeteer = require("puppeteer"); // skipcq: JS-0128 - -// Modules from file -const shared = require("./shared.js"); -const urlK = require("./constants/url.js"); -const selectorK = require("./constants/css-selector.js"); -const { preparePage } = require("./puppeteer-helper.js"); -const { isF95URL } = require("./url-helper.js"); - -/** - * @protected - * Search the F95Zone portal to find possible conversations regarding the game you are looking for. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} gamename Name of the game to search for - * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal - */ -module.exports.getSearchGameResults = async function (browser, gamename) { - shared.logger.info(`Searching ${gamename} on F95Zone`); - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_SEARCH_URL, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the search form and wait for it - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), - page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), - page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), - page.waitForSelector(selectorK.SEARCH_BUTTON), - ]); - - await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire - await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles - await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod - await Promise.all([ - page.click(selectorK.SEARCH_BUTTON), // Execute search - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Select all conversation titles - const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY); - - // For each element found extract the info about the conversation - shared.logger.info("Extracting info from conversations"); - const results = []; - for (const element of resultsThread) { - const gameUrl = await getOnlyGameThreads(page, element); - if (gameUrl !== null) results.push(gameUrl); - } - shared.logger.info(`Find ${results.length} conversations`); - await page.close(); // Close the page - - return results; -}; - -//#region Private methods -/** - * @private - * Return the link of a conversation if it is a game or a mod. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} divHandle Element of the conversation to be analyzed - * @return {Promise} URL of the game/mod or null if the URL is not of a game - */ -async function getOnlyGameThreads(page, divHandle) { - // Obtain the elements containing the basic information - const titleHandle = await divHandle.$(selectorK.THREAD_TITLE); - const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP); - - // Get the forum where the thread was posted - const forum = await getMembershipForum(page, forumHandle); - if (forum !== "GAMES" && forum !== "MODS") return null; - - // Get the URL of the thread from the title - return await getThreadURL(page, titleHandle); -} - -/** - * @private - * Obtain the membership forum of the thread passed throught 'handle'. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} handle Handle containing the forum membership - * @returns {Promise} Uppercase membership category - */ -async function getMembershipForum(page, handle) { - // The link can be something like: - // + /forums/request.NUMBER/ - // + /forums/game-recommendations-identification.NUMBER/ - // + /forums/games.NUMBER/ <-- We need this - - let link = await page.evaluate( - /* istanbul ignore next */ - (e) => e.getAttribute("href"), - handle - ); - - // Parse link - link = link.replace("/forums/", ""); - const endIndex = link.indexOf("."); - const forum = link.substring(0, endIndex); - - return forum.toUpperCase(); -} - -/** - * @private - * Obtain the URL of the thread passed through 'handle'. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} handle Handle containing the thread title - * @returns {Promise} URL of the thread - */ -async function getThreadURL(page, handle) { - const relativeURLThread = await page.evaluate( - /* istanbul ignore next */ - (e) => e.querySelector("a").href, - handle - ); - - // Some game already have a full URL... - if (isF95URL(relativeURLThread)) return relativeURLThread; - - // ... else compose the URL and return - const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString(); - return urlThread; -} -//#endregion Private methods diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js new file mode 100644 index 0000000..7e3ae4a --- /dev/null +++ b/app/scripts/network-helper.js @@ -0,0 +1,243 @@ +"use strict"; + +// Public modules from npm +const axios = require("axios").default; +const ky = require("ky-universal").create({ + throwHttpErrors: false, +}); +const cheerio = require("cheerio"); +const axiosCookieJarSupport = require("axios-cookiejar-support").default; +const tough = require("tough-cookie"); + +// Modules from file +const shared = require("./shared.js"); +const f95url = require("./constants/url.js"); +const f95selector = require("./constants/css-selector.js"); +const LoginResult = require("./classes/login-result.js"); + +// Global variables +const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15"; +axiosCookieJarSupport(axios); +const cookieJar = new tough.CookieJar(); + +const commonConfig = { + headers: { + "User-Agent": userAgent, + "Connection": "keep-alive" + }, + withCredentials: true, + jar: cookieJar // Used to store the token in the PC +}; + +/** + * @protected + * Gets the HTML code of a page. + * @param {String} url URL to fetch + * @returns {Promise} HTML code or `null` if an error arise + */ +module.exports.fetchHTML = async function (url) { + // Fetch the response of the platform + const response = await exports.fetchGETResponse(url); + if (!response) { + shared.logger.warn(`Unable to fetch HTML for ${url}`); + return null; + } + return response.data; +}; + +/** + * @protected + * It authenticates to the platform using the credentials + * and token obtained previously. Save cookies on your + * device after authentication. + * @param {Credentials} credentials Platform access credentials + * @param {Boolea} force Specifies whether the request should be forced, ignoring any saved cookies + * @returns {Promise} Result of the operation + */ +module.exports.authenticate = async function (credentials, force) { + shared.logger.info(`Authenticating with user ${credentials.username}`); + if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); + + // Secure the URL + const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL); + + // Prepare the parameters to send to the platform to authenticate + const params = new URLSearchParams(); + params.append("login", credentials.username); + params.append("url", ""); + params.append("password", credentials.password); + params.append("password_confirm", ""); + params.append("additional_security", ""); + params.append("remember", "1"); + params.append("_xfRedirect", "https://f95zone.to/"); + params.append("website_code", ""); + params.append("_xfToken", credentials.token); + + try { + // Try to log-in + let config = Object.assign({}, commonConfig); + if (force) delete config.jar; + const response = await axios.post(secureURL, params, config); + + // Parse the response HTML + const $ = cheerio.load(response.data); + + // Get the error message (if any) and remove the new line chars + const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, ""); + + // Return the result of the authentication + if (errorMessage === "") return new LoginResult(true, "Authentication successful"); + else return new LoginResult(false, errorMessage); + } catch (e) { + shared.logger.error(`Error ${e.message} occurred while authenticating to ${secureURL}`); + return new LoginResult(false, `Error ${e.message} while authenticating`); + } +}; + +/** + * Obtain the token used to authenticate the user to the platform. + * @returns {Promise} Token or `null` if an error arise + */ +module.exports.getF95Token = async function() { + // Fetch the response of the platform + const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL); + if (!response) { + shared.logger.warn("Unable to get the token for the session"); + return null; + } + + // The response is a HTML page, we need to find the with name "_xfToken" + const $ = cheerio.load(response.data); + const token = $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value"); + return token; +}; + +/** + * @protected + * Gets the basic data used for game data processing + * (such as graphics engines and progress statuses) + * @deprecated + */ +/* istanbul ignore next */ +module.exports.fetchPlatformData = async function() { + // Fetch the response of the platform + const response = await exports.fetchGETResponse(f95url.F95_LATEST_UPDATES); + if (!response) { + shared.logger.warn("Unable to get the token for the session"); + return; + } + + // The response is a HTML page, we need to find + // the base data, used when scraping the games + const $ = cheerio.load(response.data); + + // Extract the elements + const engineElements = $("body").find(f95selector.BD_ENGINE_ID_SELECTOR); + const statusesElements = $("body").find(f95selector.BD_STATUS_ID_SELECTOR); + + // Extract the raw text + engineElements.each(function extractEngineNames(idx, el) { + const engine = cheerio.load(el).text().trim(); + shared.engines.push(engine); + }); + + statusesElements.each(function extractEngineNames(idx, el) { + const status = cheerio.load(el).text().trim(); + shared.statuses.push(status); + }); +}; + +//#region Utility methods +/** + * @protected + * Performs a GET request to a specific URL and returns the response. + * If the request generates an error (for example 400) `null` is returned. + * @param {String} url + */ +module.exports.fetchGETResponse = async function(url) { + // Secure the URL + const secureURL = exports.enforceHttpsUrl(url); + + try { + // Fetch and return the response + return await axios.get(secureURL, commonConfig); + } catch (e) { + shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`); + return null; + } +}; + +/** + * @protected + * Enforces the scheme of the URL is https and returns the new URL. + * @param {String} url + * @returns {String} Secure URL or `null` if the argument is not a string + */ +module.exports.enforceHttpsUrl = function (url) { + return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; +}; + +/** + * @protected + * Check if the url belongs to the domain of the F95 platform. + * @param {String} url URL to check + * @returns {Boolean} true if the url belongs to the domain, false otherwise + */ +module.exports.isF95URL = function (url) { + if (url.toString().startsWith(f95url.F95_BASE_URL)) return true; + else return false; +}; + +/** + * @protected + * Checks if the string passed by parameter has a + * properly formatted and valid path to a URL (HTTP/HTTPS). + * @param {String} url String to check for correctness + * @returns {Boolean} true if the string is a valid URL, false otherwise + */ +module.exports.isStringAValidURL = function (url) { + // Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc) + const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/; + const regex = new RegExp(expression); + if (url.match(regex)) return true; + else return false; +}; + +/** + * @protected + * Check if a particular URL is valid and reachable on the web. + * @param {String} url URL to check + * @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false + * @returns {Promise} true if the URL exists, false otherwise + */ +module.exports.urlExists = async function (url, checkRedirect) { + if (!exports.isStringAValidURL(url)) { + return false; + } + + const response = await ky.head(url); + let valid = response !== undefined && !/4\d\d/.test(response.status); + + if (!valid) return false; + + if (checkRedirect) { + const redirectUrl = await exports.getUrlRedirect(url); + if (redirectUrl === url) valid = true; + else valid = false; + } + + return valid; +}; + +/** + * @protected + * Check if the URL has a redirect to another page. + * @param {String} url URL to check for redirect + * @returns {Promise} Redirect URL or the passed URL + */ +module.exports.getUrlRedirect = async function (url) { + const response = await ky.head(url); + return response.url; +}; +//#endregion Utility methods \ No newline at end of file diff --git a/app/scripts/puppeteer-helper.js b/app/scripts/puppeteer-helper.js deleted file mode 100644 index 18a40a4..0000000 --- a/app/scripts/puppeteer-helper.js +++ /dev/null @@ -1,60 +0,0 @@ -"use strict"; - -// Public modules from npm -const puppeteer = require("puppeteer"); - -// Modules from file -const shared = require("./shared.js"); - -/** - * @protected - * Create a Chromium instance used to navigate with Puppeteer. - * By default the browser is headless. - * @returns {Promise} Created browser - */ -module.exports.prepareBrowser = async function () { - // Create a headless browser - let browser = null; - if (shared.chromiumLocalPath) { - browser = await puppeteer.launch({ - executablePath: shared.chromiumLocalPath, - headless: !shared.debug, // Use GUI when debug = true - }); - } else { - browser = await puppeteer.launch({ - headless: !shared.debug, // Use GUI when debug = true - }); - } - - return browser; -}; - -/** - * @protected - * Prepare a page used to navigate the browser. - * The page is set up to reject image download requests. The user agent is also changed. - * @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created - * @returns {Promise} New page - */ -module.exports.preparePage = async function (browser) { - // Create new page in the browser argument - const page = await browser.newPage(); - - // Block image download - await page.setRequestInterception(true); - page.on("request", (request) => { - if (request.resourceType() === "image") request.abort(); - else if (request.resourceType === "font") request.abort(); - // else if (request.resourceType() == 'stylesheet') request.abort(); - // else if(request.resourceType == 'media') request.abort(); - else request.continue(); - }); - - // Set custom user-agent - const userAgent = - "Mozilla/5.0 (X11; Linux x86_64)" + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; - await page.setUserAgent(userAgent); - - return page; -}; diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js new file mode 100644 index 0000000..129c90c --- /dev/null +++ b/app/scripts/scraper.js @@ -0,0 +1,371 @@ +"use strict"; + +// Public modules from npm +const cheerio = require("cheerio"); + +// Modules from file +const { fetchHTML, getUrlRedirect } = require("./network-helper.js"); +const shared = require("./shared.js"); +const GameInfo = require("./classes/game-info.js"); +const f95Selector = require("./constants/css-selector.js"); + +/** + * @protected + * Get information from the game's main page. + * @param {String} url URL of the game/mod to extract data from + * @return {Promise} Complete information about the game you are + * looking for + */ +module.exports.getGameInfo = async function (url) { + shared.logger.info("Obtaining game info"); + + // Fetch HTML and prepare Cheerio + const html = await fetchHTML(url); + const $ = cheerio.load(html); + const body = $("body"); + const mainPost = $(f95Selector.GS_POSTS).first(); + + // Extract data + const titleData = extractInfoFromTitle(body); + const tags = extractTags(body); + const prefixesData = parseGamePrefixes(body); + const src = extractPreviewSource(body); + const changelog = extractChangelog(mainPost); + const structuredData = extractStructuredData(body); + const parsedInfos = parseMainPostText(structuredData["description"]); + const overview = getOverview(structuredData["description"], prefixesData.mod); + + // Obtain the updated URL + const redirectUrl = await getUrlRedirect(url); + + // Fill in the GameInfo element with the information obtained + const info = new GameInfo(); + info.name = titleData.name; + info.author = titleData.author; + info.isMod = prefixesData.mod; + info.engine = prefixesData.engine; + info.status = prefixesData.status; + info.tags = tags; + info.url = redirectUrl; + info.language = parsedInfos.Language; + info.overview = overview; + info.supportedOS = parsedInfos.SupportedOS; + info.censored = parsedInfos.Censored; + info.lastUpdate = parsedInfos.LastUpdate; + info.previewSrc = src; + info.changelog = changelog; + info.version = titleData.version; + + shared.logger.info(`Founded data for ${info.name}`); + return info; +}; + +//#region Private methods +/** + * @private + * Parse the game prefixes obtaining the engine used, + * the advancement status and if the game is actually a game or a mod. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {Object.} Dictionary of values with keys `engine`, `status`, `mod` + */ +function parseGamePrefixes(body) { + shared.logger.trace("Parsing prefixes..."); + + // Local variables + let mod = false, + engine = null, + status = null; + + // Obtain the title prefixes + const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES); + + prefixeElements.each(function parseGamePrefix(idx, el) { + // Obtain the prefix text + let prefix = cheerio.load(el).text().trim(); + + // Remove the square brackets + prefix = prefix.replace("[", "").replace("]", ""); + + // Check what the prefix indicates + if (isEngine(prefix)) engine = prefix; + else if (isStatus(prefix)) status = prefix; + else if (isMod(prefix)) mod = true; + }); + + // If the status is not set, then the game in in development (Ongoing) + if (!status) status = "Ongoing"; + + return { + engine, + status, + mod + }; +} + +/** + * @private + * Extracts all the possible informations from the title. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {Object.} Dictionary of values with keys `name`, `author`, `version` + */ +function extractInfoFromTitle(body) { + shared.logger.trace("Extracting information from title..."); + const title = body + .find(f95Selector.GT_TITLE) + .text() + .trim(); + + // From the title we can extract: Name, author and version + // [PREFIXES] TITLE [VERSION] [AUTHOR] + const matches = title.match(/\[(.*?)\]/g); + + // Get the title name + let name = title; + matches.forEach(function replaceElementsInTitle(e) { + name = name.replace(e, ""); + }); + name = name.trim(); + + // The regex [[\]]+ remove the square brackets + + // The version is the penultimate element. + // If the matches are less than 2, than the title + // is malformes and only the author is fetched + // (usually the author is always present) + let version = null; + if (matches.length >= 2) version = matches[matches.length - 2].replace(/[[\]]+/g, "").trim(); + else shared.logger.trace(`Malformed title: ${title}`); + + // Last element + const author = matches[matches.length - 1].replace(/[[\]]+/g, "").trim(); + + return { + name, + version, + author, + }; +} + +/** + * @private + * Gets the tags used to classify the game. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String[]} List of tags + */ +function extractTags(body) { + shared.logger.trace("Extracting tags..."); + + // Get the game tags + const tagResults = body.find(f95Selector.GT_TAGS); + return tagResults.map(function parseGameTags(idx, el) { + return cheerio.load(el).text().trim(); + }).get(); +} + +/** + * @private + * Gets the URL of the image used as a preview. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String} URL of the image + */ +function extractPreviewSource(body) { + shared.logger.trace("Extracting image preview source..."); + const image = body.find(f95Selector.GT_IMAGES); + + // The "src" attribute is rendered only in a second moment, + // we need the "static" src value saved in the attribute "data-src" + const source = image ? image.attr("data-src") : null; + return source; +} + +/** + * @private + * Gets the changelog of the latest version. + * @param {cheerio.Cheerio} mainPost main post selector + * @returns {String} Changelog of the last version or `null` if no changelog is fetched + */ +function extractChangelog(mainPost) { + shared.logger.trace("Extracting last changelog..."); + + // Obtain changelog + let changelog = mainPost.find(f95Selector.GT_LAST_CHANGELOG).text().trim(); + + // Clean changelog + changelog = changelog.replace("Spoiler", ""); + changelog = changelog.replace(/\n+/g, "\n"); + + // Return changelog + return changelog ? changelog : null; +} + +/** + * @private + * Process the main post text to get all the useful + * information in the format *DESCRIPTOR : VALUE*. + * Gets "standard" values such as: `Language`, `SupportedOS`, `Censored`, and `LastUpdate`. + * All non-canonical values are instead grouped together as a dictionary with the key `Various`. + * @param {String} text Structured text of the post + * @returns {Object.} Dictionary of information + */ +function parseMainPostText(text) { + shared.logger.trace("Parsing main post raw text..."); + + const data = {}; + + // The information searched in the game post are one per line + const splittedText = text.split("\n"); + for (const line of splittedText) { + if (!line.includes(":")) continue; + + // Create pair key/value + const splitted = line.split(":"); + const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch + const value = splitted[1].trim(); + + // Add pair to the dict if valid + if (value !== "") data[key] = value; + } + + // Parse the standard pairs + const parsedDict = {}; + + // Check if the game is censored + if (data.CENSORED) { + const censored = data.CENSORED.toUpperCase() === "NO" ? false : true; + parsedDict["Censored"] = censored; + delete data.CENSORED; + } + + // Last update of the main post + if (data.UPDATED) { + parsedDict["LastUpdate"] = new Date(data.UPDATED); + delete data.UPDATED; + } + else if (data.THREAD_UPDATED) { + parsedDict["LastUpdate"] = new Date(data.THREAD_UPDATED); + delete data.THREAD_UPDATED; + } + + // Parse the supported OS + if (data.OS) { + const listOS = []; + + // Usually the string is something like "Windows, Linux, Mac" + const splitted = data.OS.split(","); + splitted.forEach(function (os) { + listOS.push(os.trim()); + }); + + parsedDict["SupportedOS"] = listOS; + delete data.OS; + } + + // Rename the key for the language + if (data.LANGUAGE) { + parsedDict["Language"] = data.LANGUAGE; + delete data.LANGUAGE; + } + + // What remains is added to a sub dictionary + parsedDict["Various"] = data; + + return parsedDict; +} + +/** + * @private + * Extracts and processes the JSON-LD values found at the bottom of the page. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {Object.} JSON-LD or `null` if no valid JSON is found + */ +function extractStructuredData(body) { + shared.logger.trace("Extracting JSON-LD data..."); + const structuredDataElements = body.find(f95Selector.GT_JSONLD); + const json = structuredDataElements.map(function parseScriptTag(idx, el) { + // Get the element HTML + const html = cheerio.load(el).html().trim(); + + // Obtain the JSON-LD + const data = html + .replace("", ""); + + // Convert the string to an object + const json = JSON.parse(data); + + // Return only the data of the game + if (json["@type"] === "Book") return json; + }).get(); + return json[0] ? json[0] : null; +} + +/** + * @private + * Get the game description from its web page. + * Different processing depending on whether the game is a mod or not. + * @param {String} text Structured text extracted from the game's web page + * @param {Boolean} mod Specify if it is a game or a mod + * @returns {String} Game description + */ +function getOverview(text, mod) { + shared.logger.trace("Extracting game overview..."); + + // Get overview (different parsing for game and mod) + const overviewEndIndex = mod ? text.indexOf("Updated") : text.indexOf("Thread Updated"); + return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); +} + +/** + * @private + * Check if the prefix is a game's engine. + * @param {String} prefix Prefix to check + * @return {Boolean} + */ +function isEngine(prefix) { + const engines = toUpperCaseArray(shared.engines); + return engines.includes(prefix.toUpperCase()); +} + +/** + * @private + * Check if the prefix is a game's status. + * @param {String} prefix Prefix to check + * @return {Boolean} + */ +function isStatus(prefix) { + const statuses = toUpperCaseArray(shared.statuses); + return statuses.includes(prefix.toUpperCase()); +} + +/** + * @private + * Check if the prefix indicates a mod. + * @param {String} prefix Prefix to check + * @return {Boolean} + */ +function isMod(prefix) { + const modPrefixes = ["MOD", "CHEAT MOD"]; + return modPrefixes.includes(prefix.toUpperCase()); +} + +/** + * @private + * Makes an array of strings uppercase. + * @param {String[]} a + * @returns {String[]} + */ +function toUpperCaseArray(a) { + // If the array is empty, return + if(a.length === 0) return []; + + /** + * Makes a string uppercase. + * @param {String} s + * @returns {String} + */ + function toUpper(s) { + return s.toUpperCase(); + } + return a.map(toUpper); +} +//#endregion Private methods \ No newline at end of file diff --git a/app/scripts/searcher.js b/app/scripts/searcher.js new file mode 100644 index 0000000..f310669 --- /dev/null +++ b/app/scripts/searcher.js @@ -0,0 +1,95 @@ +"use strict"; + +// Public modules from npm +const cheerio = require("cheerio"); + +// Modules from file +const { fetchHTML } = require("./network-helper.js"); +const shared = require("./shared.js"); +const f95Selector = require("./constants/css-selector.js"); +const { F95_BASE_URL } = require("./constants/url.js"); + +//#region Public methods +/** + * @protected + * Search for a game on F95Zone and return a list of URLs, one for each search result. + * @param {String} name Game name + * @returns {Promise} URLs of results + */ +module.exports.searchGame = async function (name) { + shared.logger.info(`Searching games with name ${name}`); + + // Replace the whitespaces with + + const searchName = encodeURIComponent(name.toUpperCase()); + + // Prepare the URL (only title, search in the "Games" section, order by relevance) + const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; + + // Fetch and parse the result URLs + return await fetchResultURLs(url); +}; + +/** + * @protected + * Search for a mod on F95Zone and return a list of URLs, one for each search result. + * @param {String} name Mod name + * @returns {Promise} URLs of results + */ +module.exports.searchMod = async function (name) { + shared.logger.info(`Searching mods with name ${name}`); + + // Replace the whitespaces with + + const searchName = encodeURIComponent(name.toUpperCase()); + + // Prepare the URL (only title, search in the "Mods" section, order by relevance) + const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; + + // Fetch and parse the result URLs + return await fetchResultURLs(url); +}; +//#endregion Public methods + +//#region Private methods +/** + * @private + * Gets the URLs of the threads resulting from the F95Zone search. + * @param {String} url Search URL + * @return {Promise} List of URLs + */ +async function fetchResultURLs(url) { + shared.logger.trace(`Fetching ${url}...`); + + // Fetch HTML and prepare Cheerio + const html = await fetchHTML(url); + const $ = cheerio.load(html); + + // Here we get all the DIV that are the body of the various query results + const results = $("body").find(f95Selector.GS_RESULT_BODY); + + // Than we extract the URLs + const urls = results.map((idx, el) => { + const elementSelector = $(el); + return extractLinkFromResult(elementSelector); + }).get(); + + return urls; +} + +/** + * @private + * Look for the URL to the thread referenced by the item. + * @param {cheerio.Cheerio} selector Element to search + * @returns {String} URL to thread + */ +function extractLinkFromResult(selector) { + shared.logger.trace("Extracting thread link from result..."); + + const partialLink = selector + .find(f95Selector.GS_RESULT_THREAD_TITLE) + .attr("href") + .trim(); + + // Compose and return the URL + return new URL(partialLink, F95_BASE_URL).toString(); +} +//#endregion Private methods diff --git a/app/scripts/shared.js b/app/scripts/shared.js index ead66fc..cbefd62 100644 --- a/app/scripts/shared.js +++ b/app/scripts/shared.js @@ -1,173 +1,80 @@ +/* istanbul ignore file */ "use strict"; -// Core modules -const { join } = require("path"); - +// Public modules from npm const log4js = require("log4js"); /** * Class containing variables shared between modules. */ class Shared { - //#region Properties - /** - * Shows log messages and other useful functions for module debugging. - * @type Boolean - */ - static #_debug = false; - /** - * Indicates whether a user is logged in to the F95Zone platform or not. - * @type Boolean - */ - static #_isLogged = false; - /** - * List of cookies obtained from the F95Zone platform. - * @type Object[] - */ - static #_cookies = null; - /** - * List of possible game engines used for development. - * @type String[] - */ - static #_engines = null; - /** - * List of possible development statuses that a game can assume. - * @type String[] - */ - static #_statuses = null; - /** - * Wait instruction for the browser created by puppeteer. - * @type String - */ - static WAIT_STATEMENT = "domcontentloaded"; - /** - * Path to the directory to save the cache generated by the API. - * @type String - */ - static #_cacheDir = "./f95cache"; - /** - * If true, it opens a new browser for each request to - * the F95Zone platform, otherwise it reuses the same. - * @type Boolean - */ - static #_isolation = false; - /** - * Logger object used to write to both file and console. - * @type log4js.Logger - */ - static #_logger = log4js.getLogger(); - //#endregion Properties + //#region Properties + /** + * Indicates whether a user is logged in to the F95Zone platform or not. + * @type Boolean + */ + static #_isLogged = false; + /** + * List of possible game engines used for development. + * @type String[] + */ + static #_engines = ["ADRIFT", "Flash", "HTML", "Java", "Others", "QSP", "RAGS", "RPGM", "Ren'Py", "Tads", "Unity", "Unreal Engine", "WebGL", "Wolf RPG"]; + /** + * List of possible development statuses that a game can assume. + * @type String[] + */ + static #_statuses = ["Completed", "Onhold", "Abandoned"]; + /** + * Logger object used to write to both file and console. + * @type log4js.Logger + */ + static #_logger = log4js.getLogger(); + //#endregion Properties - //#region Getters - /** - * Shows log messages and other useful functions for module debugging. - * @returns {Boolean} - */ - static get debug() { - return this.#_debug; - } - /** + //#region Getters + /** * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {Boolean} */ - static get isLogged() { - return this.#_isLogged; - } - /** - * List of cookies obtained from the F95Zone platform. - * @returns {Object[]} - */ - static get cookies() { - return this.#_cookies; - } - /** + static get isLogged() { + return this.#_isLogged; + } + /** * List of possible game engines used for development. * @returns {String[]} */ - static get engines() { - return this.#_engines; - } - /** + static get engines() { + return this.#_engines; + } + /** * List of possible development states that a game can assume. * @returns {String[]} */ - static get statuses() { - return this.#_statuses; - } - /** - * Directory to save the API cache. - * @returns {String} - */ - static get cacheDir() { - return this.#_cacheDir; - } - /** - * Path to the F95 platform cache. - * @returns {String} - */ - static get cookiesCachePath() { - return join(this.#_cacheDir, "cookies.json"); - } - /** - * Path to the game engine cache. - * @returns {String} - */ - static get enginesCachePath() { - return join(this.#_cacheDir, "engines.json"); - } - /** - * Path to the cache of possible game states. - * @returns {String} - */ - static get statusesCachePath() { - return join(this.#_cacheDir, "statuses.json"); - } - /** - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {Boolean} - */ - static get isolation() { - return this.#_isolation; - } - /** + static get statuses() { + return this.#_statuses; + } + /** * Logger object used to write to both file and console. * @returns {log4js.Logger} */ - static get logger() { - return this.#_logger; - } - //#endregion Getters + static get logger() { + return this.#_logger; + } + //#endregion Getters - //#region Setters - static set cookies(val) { - this.#_cookies = val; - } + //#region Setters + static set engines(val) { + this.#_engines = val; + } - static set engines(val) { - this.#_engines = val; - } + static set statuses(val) { + this.#_statuses = val; + } - static set statuses(val) { - this.#_statuses = val; - } - - static set cacheDir(val) { - this.#_cacheDir = val; - } - - static set debug(val) { - this.#_debug = val; - } - - static set isLogged(val) { - this.#_isLogged = val; - } - - static set isolation(val) { - this.#_isolation = val; - } - //#endregion Setters + static set isLogged(val) { + this.#_isLogged = val; + } + //#endregion Setters } module.exports = Shared; diff --git a/app/scripts/url-helper.js b/app/scripts/url-helper.js deleted file mode 100644 index 221a56c..0000000 --- a/app/scripts/url-helper.js +++ /dev/null @@ -1,72 +0,0 @@ -"use strict"; - -// Public modules from npm -const ky = require("ky-universal").create({ - throwHttpErrors: false, -}); - -// Modules from file -const { F95_BASE_URL } = require("./constants/url.js"); - -/** - * @protected - * Check if the url belongs to the domain of the F95 platform. - * @param {String} url URL to check - * @returns {Boolean} true if the url belongs to the domain, false otherwise - */ -module.exports.isF95URL = function (url) { - if (url.toString().startsWith(F95_BASE_URL)) return true; - else return false; -}; - -/** - * @protected - * Checks if the string passed by parameter has a properly formatted and valid path to a URL. - * @param {String} url String to check for correctness - * @returns {Boolean} true if the string is a valid URL, false otherwise - */ -module.exports.isStringAValidURL = function (url) { - try { - new URL(url); // skipcq: JS-0078 - return true; - } catch (err) { - return false; - } -}; - -/** - * @protected - * Check if a particular URL is valid and reachable on the web. - * @param {String} url URL to check - * @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false - * @returns {Promise} true if the URL exists, false otherwise - */ -module.exports.urlExists = async function (url, checkRedirect) { - if (!exports.isStringAValidURL(url)) { - return false; - } - - const response = await ky.head(url); - let valid = response !== undefined && !/4\d\d/.test(response.status); - - if (!valid) return false; - - if (checkRedirect) { - const redirectUrl = await exports.getUrlRedirect(url); - if (redirectUrl === url) valid = true; - else valid = false; - } - - return valid; -}; - -/** - * @protected - * Check if the URL has a redirect to another page. - * @param {String} url URL to check for redirect - * @returns {Promise} Redirect URL or the passed URL - */ -module.exports.getUrlRedirect = async function (url) { - const response = await ky.head(url); - return response.url; -}; diff --git a/app/scripts/user-scraper.js b/app/scripts/user-scraper.js new file mode 100644 index 0000000..c9c96e3 --- /dev/null +++ b/app/scripts/user-scraper.js @@ -0,0 +1,123 @@ +"use strict"; + +// Public modules from npm +const cheerio = require("cheerio"); + +// Modules from file +const networkHelper = require("./network-helper.js"); +const f95Selector = require("./constants/css-selector.js"); +const f95url = require("./constants/url.js"); +const UserData = require("./classes/user-data.js"); + +/** + * @protected + * Gets user data, such as username, url of watched threads, and profile picture url. + * @return {Promise} User data + */ +module.exports.getUserData = async function() { + // Fetch data + const data = await fetchUsernameAndAvatar(); + const urls = await fetchWatchedThreadURLs(); + + // Create object + const ud = new UserData(); + ud.username = data.username; + ud.avatarSrc = data.source; + ud.watchedThreads = urls; + + return ud; +}; + +//#region Private methods +/** + * @private + * It connects to the page and extracts the name + * of the currently logged in user and the URL + * of their profile picture. + * @return {Promise>} + */ +async function fetchUsernameAndAvatar() { + // Fetch page + const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL); + + // Load HTML response + const $ = cheerio.load(html); + const body = $("body"); + + // Fetch username + const username = body.find(f95Selector.UD_USERNAME_ELEMENT).first().text().trim(); + + // Fetch user avatar image source + const source = body.find(f95Selector.UD_AVATAR_PIC).first().attr("src"); + + return { + username, + source + }; +} + +/** + * @private + * Gets the list of URLs of threads watched by the user. + * @returns {Promise} List of URLs + */ +async function fetchWatchedThreadURLs() { + // Local variables + let currentURL = f95url.F95_WATCHED_THREADS; + const wathcedThreadURLs = []; + + do { + // Fetch page + const html = await networkHelper.fetchHTML(currentURL); + + // Load HTML response + const $ = cheerio.load(html); + const body = $("body"); + + // Find the URLs + const urls = fetchPageURLs(body); + wathcedThreadURLs.push(...urls); + + // Find the next page (if any) + currentURL = fetchNextPageURL(body); + } + while (currentURL); + + return wathcedThreadURLs; +} + +/** + * @private + * Gets the URLs of the watched threads on the page. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String[]} + */ +function fetchPageURLs(body) { + const elements = body.find(f95Selector.WT_URLS); + + return elements.map(function extractURLs(idx, e) { + // Obtain the link (replace "unread" only for the unread threads) + const partialLink = e.attribs.href.replace("unread", ""); + + // Compose and return the URL + return new URL(partialLink, f95url.F95_BASE_URL).toString(); + }).get(); +} + +/** + * @private + * Gets the URL of the next page containing the watched threads + * or `null` if that page does not exist. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String} + */ +function fetchNextPageURL(body) { + const element = body.find(f95Selector.WT_NEXT_PAGE).first(); + + // No element found + if(element.length === 0) return null; + + // Compose and return the URL + return new URL(element.attr("href"), f95url.F95_BASE_URL).toString(); +} +//#endregion Private methods \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 1d49d83..fc06d26 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "f95api", - "version": "1.3.5", + "version": "1.5.0", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -285,17 +285,7 @@ "@types/node": { "version": "14.11.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", - "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==", - "optional": true - }, - "@types/yauzl": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", - "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", - "optional": true, - "requires": { - "@types/node": "*" - } + "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==" }, "abort-controller": { "version": "3.0.0", @@ -317,11 +307,6 @@ "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", "dev": true }, - "agent-base": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", - "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" - }, "aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -423,6 +408,23 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "axios": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz", + "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, + "axios-cookiejar-support": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz", + "integrity": "sha512-IZJxnAJ99XxiLqNeMOqrPbfR7fRyIfaoSLdPUf4AMQEGkH8URs0ghJK/xtqBsD+KsSr3pKl4DEQjCn834pHMig==", + "requires": { + "is-redirect": "^1.0.0", + "pify": "^5.0.0" + } + }, "babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -435,17 +437,21 @@ "@babel/types": "^7.7.0", "eslint-visitor-keys": "^1.0.0", "resolve": "^1.12.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } } }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true }, "binary-extensions": { "version": "2.1.0", @@ -453,20 +459,16 @@ "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", "dev": true }, - "bl": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", - "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", - "requires": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -487,20 +489,6 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, - "buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" - }, "caching-transform": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", @@ -556,6 +544,19 @@ "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "chokidar": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz", @@ -572,11 +573,6 @@ "readdirp": "~3.4.0" } }, - "chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" - }, "clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -618,7 +614,8 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true }, "convert-source-map": { "version": "1.7.0", @@ -640,6 +637,22 @@ "which": "^2.0.1" } }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==" + }, "data-uri-to-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", @@ -697,11 +710,6 @@ "object-keys": "^1.0.12" } }, - "devtools-protocol": { - "version": "0.0.799653", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz", - "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg==" - }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -717,6 +725,37 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dotenv": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", @@ -729,14 +768,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { - "once": "^1.4.0" - } - }, "enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -746,6 +777,11 @@ "ansi-colors": "^4.1.1" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, "es-abstract": { "version": "1.17.6", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", @@ -888,12 +924,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "eslint-visitor-keys": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", - "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", - "dev": true - }, "globals": { "version": "12.4.0", "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", @@ -949,12 +979,20 @@ "dev": true, "requires": { "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } } }, "eslint-visitor-keys": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", - "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", "dev": true }, "espree": { @@ -966,6 +1004,14 @@ "acorn": "^7.4.0", "acorn-jsx": "^5.2.0", "eslint-visitor-keys": "^1.3.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } } }, "esprima": { @@ -1025,17 +1071,6 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, - "extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "requires": { - "@types/yauzl": "^2.9.1", - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - } - }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1054,14 +1089,6 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, - "fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", - "requires": { - "pend": "~1.2.0" - } - }, "fetch-blob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.1.tgz", @@ -1100,6 +1127,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1141,6 +1169,11 @@ "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" + }, "foreground-child": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", @@ -1157,11 +1190,6 @@ "integrity": "sha512-Xu2Qh8yqYuDhQGOhD5iJGninErSfI9A3FrriD3tjUgV5VbJFeH8vfgZ9HnC6jWN80QDVNQK5vmxRAmEAp7Mevw==", "dev": true }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -1175,7 +1203,8 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true }, "fsevents": { "version": "2.1.3", @@ -1220,18 +1249,11 @@ "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true }, - "get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "requires": { - "pump": "^3.0.0" - } - }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1301,7 +1323,8 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true }, "html-escaper": { "version": "2.0.2", @@ -1309,19 +1332,42 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, - "https-proxy-agent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", - "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", "requires": { - "agent-base": "5", - "debug": "4" + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" } }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "dependencies": { + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + } + } }, "ignore": { "version": "4.0.6", @@ -1363,6 +1409,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1445,6 +1492,11 @@ "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", "dev": true }, + "is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=" + }, "is-regex": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", @@ -1699,6 +1751,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, "requires": { "p-locate": "^4.1.0" } @@ -1706,8 +1759,7 @@ "lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", - "dev": true + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, "lodash.flattendeep": { "version": "4.4.0", @@ -1801,6 +1853,7 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1820,11 +1873,6 @@ "minimist": "^1.2.5" } }, - "mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" - }, "mocha": { "version": "8.1.3", "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.1.3.tgz", @@ -2065,10 +2113,10 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, "natural-compare": { @@ -2086,14 +2134,6 @@ "fetch-blob": "^2.1.1" } }, - "node-html-parser": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-1.3.1.tgz", - "integrity": "sha512-AwYVI6GyEKj9NGoyMfSx4j5l7Axf7obQgLWGxtasLjED6RggTTQoq5ZRzjwSUfgSZ+Mv8Nzbi3pID0gFGqNUsA==", - "requires": { - "he": "1.2.0" - } - }, "node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -2109,6 +2149,14 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, "nyc": { "version": "15.1.0", "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", @@ -2172,6 +2220,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "requires": { "wrappy": "1" } @@ -2194,6 +2243,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -2202,6 +2252,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, "requires": { "p-limit": "^2.2.0" } @@ -2218,7 +2269,8 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true }, "package-hash": { "version": "4.0.0", @@ -2241,15 +2293,25 @@ "callsites": "^3.0.0" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true }, "path-key": { "version": "3.1.1", @@ -2269,21 +2331,22 @@ "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", "dev": true }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" - }, "picomatch": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", "dev": true }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" + }, "pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, "requires": { "find-up": "^4.0.0" } @@ -2306,7 +2369,8 @@ "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true }, "promise.allsettled": { "version": "1.0.2", @@ -2321,43 +2385,15 @@ "iterate-value": "^1.0.0" } }, - "proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, - "pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } + "psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true - }, - "puppeteer": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz", - "integrity": "sha512-YTM1RaBeYrj6n7IlRXRYLqJHF+GM7tasbvrNFx6w1S16G76NrPq7oYFKLDO+BQsXNtS8kW2GxWCXjIMPvfDyaQ==", - "requires": { - "debug": "^4.1.0", - "devtools-protocol": "0.0.799653", - "extract-zip": "^2.0.0", - "https-proxy-agent": "^4.0.0", - "pkg-dir": "^4.2.0", - "progress": "^2.0.1", - "proxy-from-env": "^1.0.0", - "rimraf": "^3.0.2", - "tar-fs": "^2.0.0", - "unbzip2-stream": "^1.3.3", - "ws": "^7.2.3" - } + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "randombytes": { "version": "2.1.0", @@ -2438,6 +2474,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -2490,13 +2527,23 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, - "sleep": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/sleep/-/sleep-6.3.0.tgz", - "integrity": "sha512-+WgYl951qdUlb1iS97UvQ01pkauoBK9ML9I/CMPg41v0Ze4EyMlTgFTDDo32iYj98IYqxIjDMRd+L71lawFfpQ==", + "slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", "dev": true, "requires": { - "nan": "^2.14.1" + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + } } }, "slice-ansi": { @@ -2689,29 +2736,6 @@ } } }, - "tar-fs": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", - "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", - "requires": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.0.0" - } - }, - "tar-stream": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz", - "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==", - "requires": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - } - }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -2729,11 +2753,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -2749,6 +2768,16 @@ "is-number": "^7.0.0" } }, + "tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.1.2" + } + }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2779,15 +2808,6 @@ "is-typedarray": "^1.0.0" } }, - "unbzip2-stream": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", - "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", - "requires": { - "buffer": "^5.2.1", - "through": "^2.3.8" - } - }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -2814,9 +2834,9 @@ "dev": true }, "v8-compile-cache": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz", - "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==", + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz", + "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==", "dev": true }, "which": { @@ -2929,7 +2949,17 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true + }, + "write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "requires": { + "mkdirp": "^0.5.1" + } }, "write": { "version": "1.0.3", @@ -2952,11 +2982,6 @@ "typedarray-to-buffer": "^3.1.5" } }, - "ws": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz", - "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==" - }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", @@ -3129,15 +3154,6 @@ } } } - }, - "yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", - "requires": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/package.json b/package.json index d78bd34..621ee58 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "main": "./app/index.js", "name": "f95api", - "version": "1.3.5", + "version": "1.5.0", "author": { "name": "Millennium Earl" }, @@ -19,7 +19,10 @@ "scraping", "login", "game", - "games" + "games", + "data", + "userdata", + "user data" ], "scripts": { "unit-test-mocha": "nyc --reporter=text mocha './test/index-test.js'", @@ -31,11 +34,13 @@ "node": ">=10.0" }, "dependencies": { + "axios": "^0.21.0", + "axios-cookiejar-support": "^1.0.1", + "cheerio": "^1.0.0-rc.3", "ky": "^0.24.0", "ky-universal": "^0.8.2", "log4js": "^6.3.0", - "node-html-parser": "^1.3.1", - "puppeteer": "^5.3.1" + "tough-cookie": "^4.0.0" }, "devDependencies": { "babel-eslint": "^10.1.0", @@ -44,7 +49,6 @@ "eslint": "^7.12.1", "lodash": "^4.17.20", "mocha": "^8.1.3", - "nyc": "^15.1.0", - "sleep": "^6.3.0" + "nyc": "^15.1.0" } } diff --git a/test/index-test.js b/test/index-test.js index 4d89e73..069c9bc 100644 --- a/test/index-test.js +++ b/test/index-test.js @@ -1,290 +1,36 @@ "use strict"; -// Core modules -const fs = require("fs"); +// Test suite +const api = require("./suites/api-test.js").suite; +const credentials = require("./suites/credentials-test.js").suite; +const network = require("./suites/network-helper-test.js").suite; +const scraper = require("./suites/scraper-test.js").suite; +const searcher = require("./suites/searcher-test.js").suite; +const uScraper = require("./suites/user-scraper-test.js").suite; -// Public modules from npm -const _ = require("lodash"); -const expect = require("chai").expect; -const sleep = require("sleep"); -const dotenv = require("dotenv"); +describe("Test basic function", function testBasic() { + //#region Set-up + this.timeout(15000); // All tests in this suite get 15 seconds before timeout + //#endregion Set-up -// Modules from file -const urlHelper = require("../app/scripts/url-helper.js"); -const F95API = require("../app/index.js"); - -// Configure the .env reader -dotenv.config(); - -const COOKIES_SAVE_PATH = "./f95cache/cookies.json"; -const ENGINES_SAVE_PATH = "./f95cache/engines.json"; -const STATUSES_SAVE_PATH = "./f95cache/statuses.json"; -const USERNAME = process.env.F95_USERNAME; -const PASSWORD = process.env.F95_PASSWORD; -const FAKE_USERNAME = "FakeUsername091276"; -const FAKE_PASSWORD = "fake_password"; - -//F95API.debug(false); - -function randomSleep() { - const random = Math.floor(Math.random() * 500) + 50; - sleep.msleep(500 + random); -} - -describe("Login without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - - before("Set isolation", function () { - F95API.setIsolation(true); - }); - - beforeEach("Remove all cookies", function () { - // Runs before each test in this block - if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up - - let testOrder = 0; - - it("Test with valid credentials", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 0) randomSleep(); - - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Authentication successful"); - - testOrder = 1; - }); - it("Test with invalid username", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 1) randomSleep(); - - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); - - testOrder = 2; - }); - it("Test with invalid password", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 2) randomSleep(); - - const result = await F95API.login(USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect password"); - - testOrder = 3; - }); - it("Test with invalid credentials", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 3) randomSleep(); - - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); // It should first check the username - - testOrder = 4; - }); + describe("Test credentials class", credentials.bind(this)); + describe("Test network helper", network.bind(this)); }); -describe("Login with cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - - before("Log in to create cookies then logout", async function () { - // Runs once before the first test in this block - if (!fs.existsSync(COOKIES_SAVE_PATH)) - await F95API.login(USERNAME, PASSWORD); // Download cookies - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up - - it("Test with valid credentials", async function () { - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Logged with cookies"); - }); +describe("Test F95 modules", function testF95Modules() { + //#region Set-up + this.timeout(15000); // All tests in this suite get 15 seconds before timeout + //#endregion Set-up + + describe("Test scraper methods", scraper.bind(this)); + describe("Test searcher methods", searcher.bind(this)); + describe("Test user scraper methods", uScraper.bind(this)); }); -describe("Load base data without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout +describe("Test complete API", function testAPI() { + //#region Set-up + this.timeout(15000); // All tests in this suite get 15 seconds before timeout + //#endregion Set-up - before("Delete cache if exists", function () { - // Runs once before the first test in this block - if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); - if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); - }); - //#endregion Set-up - - it("With login", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const result = await F95API.loadF95BaseData(); - - const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); - const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); - - expect(result).to.be.true; - expect(enginesCacheExists).to.be.true; - expect(statusesCacheExists).to.be.true; - }); - - it("Without login", async function () { - if (F95API.isLogged()) F95API.logout(); - const result = await F95API.loadF95BaseData(); - expect(result).to.be.false; - }); -}); - -describe("Search game data", function () { - //#region Set-up - this.timeout(60000); // All tests in this suite get 60 seconds before timeout - - beforeEach("Prepare API", function () { - // Runs once before the first test in this block - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up - - let testGame = null; - - it("Search game when logged", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const gamesList = await F95API.getGameData("Kingdom of Deception", false); - expect(gamesList.length, "Should find only the game").to.equal(1); - const result = gamesList[0]; - const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; - - // Test only the main information - expect(result.name).to.equal("Kingdom of Deception"); - expect(result.author).to.equal("Hreinn Games"); - expect(result.isMod, "Should be false").to.be.false; - expect(result.engine).to.equal("REN'PY"); - expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? - testGame = Object.assign({}, result); - }); - it("Search game when not logged", async function () { - const result = await F95API.getGameData("Kingdom of Deception", false); - expect(result, "Without being logged should return null").to.be.null; - }); - it("Test game serialization", function () { - const json = JSON.stringify(testGame); - const parsedGameInfo = JSON.parse(json); - const result = _.isEqual(parsedGameInfo, testGame); - expect(result).to.be.true; - }); -}); - -describe("Load user data", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Retrieve when logged", async function () { - // Login - await F95API.login(USERNAME, PASSWORD); - - // Then retrieve user data - const data = await F95API.getUserData(); - - expect(data).to.exist; - expect(data.username).to.equal(USERNAME); - }); - it("Retrieve when not logged", async function () { - // Logout - if (F95API.isLogged()) F95API.logout(); - - // Try to retrieve user data - const data = await F95API.getUserData(); - - expect(data).to.be.null; - }); -}); - -describe("Check game update", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Get online game and verify that no update exists", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; - - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.false; - }); - - it("Verify that update exists from old URL", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ - const url = - "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; - const result = await F95API.getGameDataFromURL(url); - result.version = "0.9600"; - - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.true; - }); -}); - -describe("Test url-helper", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Check if URL exists", async function () { - // Check generic URLs... - let exists = await urlHelper.urlExists("https://www.google.com/"); - expect(exists, "Complete valid URL").to.be.true; - - exists = await urlHelper.urlExists("www.google.com"); - expect(exists, "URl without protocol prefix").to.be.false; - - exists = await urlHelper.urlExists("https://www.google/"); - expect(exists, "URL without third level domain").to.be.false; - - // Now check for more specific URLs (with redirect)... - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(exists, "URL with redirect without check").to.be.true; - - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", - true - ); - expect(exists, "URL with redirect with check").to.be.false; - }); - - it("Check if URL belong to the platform", async function () { - let belong = urlHelper.isF95URL( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(belong).to.be.true; - - belong = urlHelper.isF95URL("https://www.google/"); - expect(belong).to.be.false; - }); -}); + describe("Test API", api.bind(this)); +}); \ No newline at end of file diff --git a/test/suites/api-test.js b/test/suites/api-test.js new file mode 100644 index 0000000..07e6016 --- /dev/null +++ b/test/suites/api-test.js @@ -0,0 +1,64 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); +const { + isEqual +} = require("lodash"); + +// Modules from file +const F95API = require("../../app/index.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; + +module.exports.suite = function suite() { + // Global suite variables + const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"; + + it("Test login", async function testLogin() { + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(F95API.isLogged()).to.be.true; + }); + + it("Test user data fetching", async function testUserDataFetch() { + const userdata = await F95API.getUserData(); + expect(userdata.username).to.be.equal(USERNAME); + }); + + it("Test game update checking", async function testGameUpdateCheck() { + // We force the creation of a GameInfo object, + // knowing that the checkIfGameHasUpdate() function + // only needs the game URL + const info = new F95API.GameInfo(); + + // The gameURL identifies a game for which we know there is an update + info.url = gameURL; + + // Check for updates + const update = await F95API.checkIfGameHasUpdate(info); + expect(update).to.be.true; + }); + + it("Test game data fetching", async function testGameDataFetch() { + // Search a game by name + const gameList = await F95API.getGameData("perverted education", false); + + // We know that there is only one game with the selected name + expect(gameList.length).to.be.equal(1, `There should be only one game, not ${gameList.length}`); + const game = gameList[0]; + + // Than we fetch a game from URL + const gameFromURL = await F95API.getGameDataFromURL(game.url); + + // The two games must be equal + const equal = isEqual(game, gameFromURL); + expect(equal).to.be.true; + }); +}; \ No newline at end of file diff --git a/test/suites/credentials-test.js b/test/suites/credentials-test.js new file mode 100644 index 0000000..3bf60e2 --- /dev/null +++ b/test/suites/credentials-test.js @@ -0,0 +1,52 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); + +module.exports.suite = function suite() { + it("Check token formatting", async function testValidToken() { + // Token example: + // 1604309951,0338213c00fcbd894fd9415e6ba08403 + // 1604309986,ebdb75502337699381f0f55c86353555 + // 1604310008,2d50d55808e5ec3a157ec01953da9d26 + + // Fetch token (is a GET request, we don't need the credentials) + const cred = new Credentials(null, null); + await cred.fetchToken(); + + // Parse token for assert + const splitted = cred.token.split(","); + const unique = splitted[0]; + const hash = splitted[1]; + expect(splitted.length).to.be.equal(2, "The token consists of two parts"); + + // Check type of parts + expect(isNumeric(unique)).to.be.true; + expect(isNumeric(hash)).to.be.false; + + // The second part is most probably the MD5 hash of something + expect(hash.length).to.be.equal(32, "Hash should have 32 hex chars"); + }); +}; + +//#region Private methods +/** + * @private + * Check if a string is a number + * @param {String} str + * @author Dan, Ben Aston + * @see https://preview.tinyurl.com/y46jqwkt + */ +function isNumeric(str) { + // We only process strings! + if (typeof str != "string") return false; + + // Use type coercion to parse the _entirety_ of the string + // (`parseFloat` alone does not do this) and ensure strings + // of whitespace fail + return !isNaN(str) && !isNaN(parseFloat(str)); +} +//#endregion \ No newline at end of file diff --git a/test/suites/network-helper-test.js b/test/suites/network-helper-test.js new file mode 100644 index 0000000..a650c99 --- /dev/null +++ b/test/suites/network-helper-test.js @@ -0,0 +1,107 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); +const networkHelper = require("../../app/scripts/network-helper.js"); +const { + F95_SEARCH_URL +} = require("../../app/scripts/constants/url.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; +const FAKE_USERNAME = "Fake_Username091276"; +const FAKE_PASSWORD = "fake_password"; + +module.exports.suite = function suite() { + // Global suite variables + const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"; + + it("Check if URL exists", async function checkURLExistence() { + // Check generic URLs... + let exists = await networkHelper.urlExists("https://www.google.com/"); + expect(exists, "Complete valid URL").to.be.true; + + exists = await networkHelper.urlExists("www.google.com"); + expect(exists, "URl without protocol prefix").to.be.false; + + exists = await networkHelper.urlExists("https://www.google/"); + expect(exists, "URL without third level domain").to.be.false; + + // Now check for more specific URLs (with redirect)... + exists = await networkHelper.urlExists(gameURL); + expect(exists, "URL with redirect without check").to.be.true; + + exists = await networkHelper.urlExists(gameURL, true); + expect(exists, "URL with redirect with check").to.be.false; + }); + + it("Check if URL belong to the platform", function checkIfURLIsF95() { + let belong = networkHelper.isF95URL(gameURL); + expect(belong).to.be.true; + + belong = networkHelper.isF95URL("https://www.google/"); + expect(belong).to.be.false; + }); + + it("Enforce secure URLs", function testSecureURLEnforcement() { + // This URL is already secure, should remain the same + let enforced = networkHelper.enforceHttpsUrl(gameURL); + expect(enforced).to.be.equal(gameURL, "The game URL is already secure"); + + // This URL is not secure + enforced = networkHelper.enforceHttpsUrl("http://www.google.com"); + expect(enforced).to.be.equal("https://www.google.com", "The URL was without SSL/TLS (HTTPs)"); + + // Finally, we check when we pass a invalid URL + enforced = networkHelper.enforceHttpsUrl("http://invalidurl"); + expect(enforced).to.be.null; + }); + + it("Check URL redirect", async function checkURLRedirect() { + // gameURL is an old URL it has been verified that it generates a redirect + const redirectURL = await networkHelper.getUrlRedirect(gameURL); + expect(redirectURL).to.not.be.equal(gameURL, "The original URL has redirect"); + + // If we recheck the new URL, we find that no redirect happens + const secondRedirectURL = await networkHelper.getUrlRedirect(redirectURL); + expect(secondRedirectURL).to.be.equal(redirectURL, "The URL has no redirect"); + }); + + it("Check response to GET request", async function testGETResponse() { + // We should be able to fetch a game page + let response = await networkHelper.fetchGETResponse(gameURL); + expect(response.status).to.be.equal(200, "The operation must be successful"); + + // We should NOT be able to fetch the search page (we must be logged) + response = await networkHelper.fetchGETResponse(F95_SEARCH_URL); + expect(response).to.be.null; + }); + + it("Test for authentication to platform", async function testAuthentication() { + // Try to authenticate with valid credentials + const creds = new Credentials(USERNAME, PASSWORD); + await creds.fetchToken(); + const validResult = await networkHelper.authenticate(creds); + expect(validResult.success).to.be.true; + + // Now we use fake credentials + const fakeCreds = new Credentials(FAKE_USERNAME, FAKE_PASSWORD); + await fakeCreds.fetchToken(); + const invalidResult = await networkHelper.authenticate(fakeCreds, true); + expect(invalidResult.success).to.be.false; + }); + + it("Test fetching HTML", async function testFetchHTML() { + // This should return the HTML code of the page + const html = await networkHelper.fetchHTML(gameURL); + expect(html.startsWith("")).to.be.true; + }); +}; \ No newline at end of file diff --git a/test/suites/scraper-test.js b/test/suites/scraper-test.js new file mode 100644 index 0000000..a68666b --- /dev/null +++ b/test/suites/scraper-test.js @@ -0,0 +1,42 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const { isEqual } = require("lodash"); +const GameInfo = require("../../app/scripts/classes/game-info.js"); + +// Modules from file +const scraper = require("../../app/scripts/scraper.js"); + +module.exports.suite = function suite() { + // Global suite variables + const gameURL = "https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/"; + const previewSrc = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; + + it("Search game", async function () { + // This test depend on the data on F95Zone at gameURL + const result = await scraper.getGameInfo(gameURL); + + // Test only the main information + expect(result.name).to.equal("Kingdom of Deception"); + expect(result.author).to.equal("Hreinn Games"); + expect(result.isMod, "Should be false").to.be.false; + expect(result.engine).to.equal("Ren'Py"); + expect(result.previewSrc).to.equal(previewSrc, "Preview not equals"); + }); + + it("Test game serialization", async function testGameSerialization() { + // This test depend on the data on F95Zone at gameURL + const testGame = await scraper.getGameInfo(gameURL); + + // Serialize... + const json = JSON.stringify(testGame); + + // Deserialize... + const parsedGameInfo = GameInfo.fromJSON(json); + + // Compare with lodash + const result = isEqual(parsedGameInfo, testGame); + expect(result).to.be.true; + }); +}; diff --git a/test/suites/searcher-test.js b/test/suites/searcher-test.js new file mode 100644 index 0000000..6b8aea4 --- /dev/null +++ b/test/suites/searcher-test.js @@ -0,0 +1,64 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); +const searcher = require("../../app/scripts/searcher.js"); +const { + authenticate +} = require("../../app/scripts/network-helper.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; + +module.exports.suite = function suite() { + // TODO: + // This method should delete the store F95Zone cookies, + // but what if the other tests require them? + + // it("Search game when not logged", async function searchGameWhenNotLogged() { + // // Search for a game that we know has only one result + // // but without logging in first + // const urls = await searcher.searchGame("kingdom of deception"); + // expect(urls.lenght).to.be.equal(0, "There should not be any URL"); + // }); + + it("Search game", async function searchGame() { + // Authenticate + const result = await auth(); + expect(result.success, "Authentication should be successful").to.be.true; + + // Search for a game that we know has only one result + const urls = await searcher.searchGame("kingdom of deception"); + expect(urls.length).to.be.equal(1, `There should be only one game result instead of ${urls.length}`); + }); + + it("Search mod", async function searchMod() { + // Authenticate + const result = await auth(); + expect(result.success, "Authentication should be successful").to.be.true; + + // Search for a mod that we know has only one result + const urls = await searcher.searchMod("kingdom of deception jdmod"); + expect(urls.length).to.be.equal(1, `There should be only one mod result instead of ${urls.length}`); + }); +}; + +//#region Private methods +/** + * @private + * Simple wrapper for authentication. + */ +async function auth() { + const creds = new Credentials(USERNAME, PASSWORD); + await creds.fetchToken(); + return await authenticate(creds); +} +//#endregion Private methods \ No newline at end of file diff --git a/test/suites/user-scraper-test.js b/test/suites/user-scraper-test.js new file mode 100644 index 0000000..e3abe0e --- /dev/null +++ b/test/suites/user-scraper-test.js @@ -0,0 +1,54 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); +const uScraper = require("../../app/scripts/user-scraper.js"); +const { + authenticate +} = require("../../app/scripts/network-helper.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; + +module.exports.suite = function suite() { + // TODO: + // This method should delete the store F95Zone cookies, + // but what if the other tests require them? + + // it("Fetch data when not logged", async function fetchUserDataWhenLogged() { + // const data = await uScraper.getUserData(); + // expect(data.username).to.be.equal(""); + // expect(data.avatarSrc).to.be.equal(""); + // expect(data.watchedThreads.length).to.be.equal(0); + // }); + + it("Fetch data when logged", async function fetchUserDataWhenNotLogged() { + // Authenticate + const result = await auth(); + expect(result.success, "Authentication should be successful").to.be.true; + + // We test only for the username, the other test data depends on the user logged + const data = await uScraper.getUserData(); + expect(data.username).to.be.equal(USERNAME); + }); +}; + +//#region Private methods +/** + * @private + * Simple wrapper for authentication. + */ +async function auth() { + const creds = new Credentials(USERNAME, PASSWORD); + await creds.fetchToken(); + return await authenticate(creds); +} +//#endregion Private methods \ No newline at end of file diff --git a/test/user-test.js b/test/user-test.js deleted file mode 100644 index c6f6b29..0000000 --- a/test/user-test.js +++ /dev/null @@ -1,15 +0,0 @@ -const F95API = require("../app/index.js"); - -F95API.debug(true); -main(); - -async function main() { - const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); - - if (loginResult.success) { - await F95API.loadF95BaseData(); - const gameData = await F95API.getGameData("a struggle with sin", false); - console.log(gameData); - } - F95API.logout(); -}