Merge pull request #44 from MillenniumEarl/Use-plain-HTML-scraper

Switch to plain HTML scraper

.eslintrc.json

@@ -1,20 +1,38 @@
 {
-  "env": {
+    "env": {
-    "browser": true,
+        "browser": true,
-    "commonjs": true,
+        "commonjs": true,
-    "es2021": true,
+        "es2021": true,
-    "node": true
+        "node": true,
-  },
+        "mocha": true
-  "extends": "eslint:recommended",
+    },
-  "parser": "./node_modules/babel-eslint",
+    "extends": "eslint:recommended",
-  "parserOptions": {
+    "parser": "babel-eslint",
-    "ecmaVersion": 12
+    "parserOptions": {
-  },
+        "ecmaVersion": 12
-  "rules": {
+    },
-    "indent": ["error", 4],
+    "rules": {
-    "linebreak-style": ["error", "windows"],
+        "indent": [
-    "quotes": ["error", "double"],
+            "error",
-    "semi": ["error", "always"],
+            4
-    "no-unused-vars": ["error", "after-used"]
+        ],
-  }
+        "linebreak-style": [
+            "error",
+            "windows"
+        ],
+        "quotes": [
+            "error",
+            "double"
+        ],
+        "semi": [
+            "error",
+            "always"
+        ],
+        "no-unused-vars": [
+            "error",
+            {
+                "args": "after-used"
+            }
+        ]
+    }
 }

README.md

@@ -8,6 +8,75 @@
 
 Unofficial Node JS module for scraping F95Zone platform
 
+These APIs have been developed to support this application and allow you to obtain data on games and mods on the platform [F95zone](https://f95zone.to/) (**NSFW**)
+
+A simple usage example can be found in [app/example.js](https://github.com/MillenniumEarl/F95API/blob/master/app/example.js)
+
+**Attention**: Two-factor authentication is not supported
+
+# Data scraping
+Games/mods can be obtained by name or URL
+
+```javascript
+// The name is case insensitive
+let listOfFoundGames = await F95API.getGameData("your game name");
+let listOfFoundMods = await F95API.getGameData("your mod name", true);
+
+let specificGame = await F95API.getGameDataFromURL("the URL of your game");
+```
+
+While user data (after authenticating) with 
+
+```javascript
+let authResult = await F95API.login(username, password);
+
+let loggedUserData = await F95API.getUserData();
+```
+
+# Classes
+## Games and mods
+Information about games and mods is stored in a GameInfo object with the following fields:
+
+```
+name: The game name
+author: The game developer
+url: The URL that leads to the game thread on F95Zone
+overview: Description of the game
+language: Main language of the game
+supportedOS: List of supported OS (Windows/Linux/Mac/Android...)
+censored: Are the NSFW parts censored?
+engine: Game engine (Unity, Ren'Py, RPGM...)
+status: Completed/Abandoned/Ongoing/Onhold
+tags: List of tags
+previewSrc: Source URL of the game description image
+version: Version of the game
+lastUpdate: Date of the last update (it's a Date object)
+isMod: Is it a game or a mod?
+changelog: Latest changelog available
+```
+
+The serialization in JSON format of this object is possible through `JSON.stringfy()` while the deserialization must happen through the static method `GameInfo.fromJSON()`.
+
+## User data
+User data (after authentication) can be stored in a UserData object, consisting of the following fields:
+
+```
+username: Name of the logged in user
+avatarSrc: Source URL of the user's profile picture
+watchedThreads: List of URLs of threads followed by the user
+```
+
+## Login results
+The outcome of the authentication process is represented by the LoginResult object:
+
+```
+success: Was the authentication successful?;
+message: Possible error message (unrecognized user, wrong password ...) or authentication successful message
+```
+
+# Logging
+To log the behavior of the application [log4js](https://github.com/log4js-node/log4js-node) is used with a default level of "warn". This option can be changed with the `loggerLevel` property.
+
 # Guidelines for errors
 
 - If you can, return a meaningful value

app/example.js

@@ -0,0 +1,56 @@
+/* 
+to use this example, create an .env file 
+in the project root with the following values:
+
+F95_USERNAME = YOUR_USERNAME
+F95_PASSWORD = YOUR_PASSWORD
+*/
+
+"use strict";
+
+// Public modules from npm
+const dotenv = require("dotenv");
+
+// Modules from file
+const F95API = require("./index.js");
+
+// Configure the .env reader
+dotenv.config();
+
+main();
+
+async function main() {
+    // Local variables
+    const gameList = [
+        "kingdom of deception",
+        "perverted education",
+        "corrupted kingdoms",
+        "summertime saga",
+        "brothel king"
+    ];
+
+    // Log in the platform
+    console.log("Authenticating...");
+    const result = await F95API.login(process.env.F95_USERNAME, process.env.F95_PASSWORD);
+    console.log(`Authentication result: ${result.message}`);
+
+    // Get user data
+    console.log("Fetching user data...");
+    const userdata = await F95API.getUserData();
+    console.log(`${userdata.username} follows ${userdata.watchedThreads.length} threads`);
+
+    for(const gamename of gameList) {
+        console.log(`Searching '${gamename}'...`);
+        const found = await F95API.getGameData(gamename, false);
+
+        // If no game is found
+        if (found.length === 0) {
+            console.log(`No data found for '${gamename}'`);
+            continue;
+        }
+
+        // Extract first game
+        const gamedata = found[0];
+        console.log(`Found ${gamedata.name} (${gamedata.version}) by ${gamedata.author}`);
+    }
+}

app/index.js

@@ -1,21 +1,14 @@
 "use strict";
 
-// Core modules
-const fs = require("fs");
-
 // Modules from file
 const shared = require("./scripts/shared.js");
-const urlK = require("./scripts/constants/url.js");
+const networkHelper = require("./scripts/network-helper.js");
-const selectorK = require("./scripts/constants/css-selector.js");
+const scraper = require("./scripts/scraper.js");
-const urlHelper = require("./scripts/url-helper.js");
+const searcher = require("./scripts/searcher.js");
-const scraper = require("./scripts/game-scraper.js");
+const uScraper = require("./scripts/user-scraper.js");
-const {
-  prepareBrowser,
-  preparePage,
-} = require("./scripts/puppeteer-helper.js");
-const searcher = require("./scripts/game-searcher.js");
 
 // Classes from file
+const Credentials = require("./scripts/classes/credentials.js");
 const GameInfo = require("./scripts/classes/game-info.js");
 const LoginResult = require("./scripts/classes/login-result.js");
 const UserData = require("./scripts/classes/user-data.js");
@@ -28,63 +21,24 @@ module.exports.UserData = UserData;
 
 //#region Export properties
 /**
- * Shows log messages and other useful functions for module debugging.
+ * @public
- * @param {Boolean} value
+ * Set the logger level for module debugging.
  */
-module.exports.debug = function (value) {
+/* istambul ignore next */
-  shared.debug = value;
+module.exports.loggerLevel = shared.logger.level;
-
+exports.loggerLevel = "warn"; // By default log only the warn messages
-  // Configure logger
-  shared.logger.level = value ? "debug" : "warn";
-};
 /**
  * @public
  * Indicates whether a user is logged in to the F95Zone platform or not.
  * @returns {String}
  */
-module.exports.isLogged = function () {
+/* istambul ignore next */
-  return shared.isLogged;
+module.exports.isLogged = function isLogged() {
-};
+    return shared.isLogged;
-/**
- * @public
- * If true, it opens a new browser for each request
- * to the F95Zone platform, otherwise it reuses the same.
- * @returns {String}
- */
-module.exports.setIsolation = function (value) {
-  shared.isolation = value;
-};
-/**
- * @public
- * Path to the cache directory
- * @returns {String}
- */
-module.exports.getCacheDir = function () {
-  return shared.cacheDir;
-};
-/**
- * @public
- * Set path to the cache directory
- * @returns {String}
- */
-module.exports.setCacheDir = function (value) {
-  shared.cacheDir = value;
-
-  // Create directory if it doesn't exist
-  if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
-};
-/**
- * @public
- * Set local chromium path.
- * @returns {String}
- */
-module.exports.setChromiumPath = function (value) {
-  shared.chromiumLocalPath = value;
 };
 //#endregion Export properties
 
 //#region Global variables
-var _browser = null;
 const USER_NOT_LOGGED = "User not authenticated, unable to continue";
 //#endregion
 
@@ -98,92 +52,25 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue";
  * @returns {Promise<LoginResult>} Result of the operation
  */
 module.exports.login = async function (username, password) {
-  if (shared.isLogged) {
+    if (shared.isLogged) {
-    shared.logger.info("Already logged in");
+        shared.logger.info(`${username} already authenticated`);
-    const result = new LoginResult(true, "Already logged in");
+        return new LoginResult(true, `${username} already authenticated`);
+    }
+
+    shared.logger.trace("Fetching token...");
+    const creds = new Credentials(username, password);
+    await creds.fetchToken();
+
+    shared.logger.trace(`Authentication for ${username}`);
+    const result = await networkHelper.authenticate(creds);
+    shared.isLogged = result.success;
+
+    if (result.success) shared.logger.info("User logged in through the platform");
+    else shared.logger.warn(`Error during authentication: ${result.message}`);
+
     return result;
-  }
-
-  // If cookies are loaded, use them to authenticate
-  shared.cookies = loadCookies();
-  if (shared.cookies !== null) {
-    shared.logger.info("Valid session, no need to re-authenticate");
-    shared.isLogged = true;
-    const result = new LoginResult(true, "Logged with cookies");
-    return result;
-  }
-
-  // Else, log in throught browser
-  shared.logger.info(
-    "No saved sessions or expired session, login on the platform"
-  );
-
-  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  const browser = shared.isolation ? await prepareBrowser() : _browser;
-
-  const result = await loginF95(browser, username, password);
-  shared.isLogged = result.success;
-
-  if (result.success) {
-    // Reload cookies
-    shared.cookies = loadCookies();
-    shared.logger.info("User logged in through the platform");
-  } else {
-    shared.logger.warn("Error during authentication: " + result.message);
-  }
-  if (shared.isolation) await browser.close();
-  return result;
 };
-/**
- * @public
- * This method loads the main data from the F95 portal
- * used to provide game information. You **must** be logged
- * in to the portal before calling this method.
- * @returns {Promise<Boolean>} Result of the operation
- */
-module.exports.loadF95BaseData = async function () {
-  if (!shared.isLogged || !shared.cookies) {
-    shared.logger.warn(USER_NOT_LOGGED);
-    return false;
-  }
 
-  shared.logger.info("Loading base data...");
-
-  // Prepare a new web page
-  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  const browser = shared.isolation ? await prepareBrowser() : _browser;
-
-  const page = await preparePage(browser); // Set new isolated page
-  await page.setCookie(...shared.cookies); // Set cookies to avoid login
-
-  // Go to latest update page and wait for it to load
-  await page.goto(urlK.F95_LATEST_UPDATES, {
-    waitUntil: shared.WAIT_STATEMENT,
-  });
-
-  // Obtain engines (disk/online)
-  await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR);
-  shared.engines = await loadValuesFromLatestPage(
-    page,
-    shared.enginesCachePath,
-    selectorK.ENGINE_ID_SELECTOR,
-    "engines"
-  );
-
-  // Obtain statuses (disk/online)
-  await page.waitForSelector(selectorK.STATUS_ID_SELECTOR);
-  shared.statuses = await loadValuesFromLatestPage(
-    page,
-    shared.statusesCachePath,
-    selectorK.STATUS_ID_SELECTOR,
-    "statuses"
-  );
-
-  await page.close();
-  if (shared.isolation) await browser.close();
-  shared.logger.info("Base data loaded");
-  return true;
-};
 /**
  * @public
  * Chek if exists a new version of the game.
@@ -191,68 +78,54 @@ module.exports.loadF95BaseData = async function () {
  * @param {GameInfo} info Information about the game to get the version for
  * @returns {Promise<Boolean>} true if an update is available, false otherwise
  */
-module.exports.chekIfGameHasUpdate = async function (info) {
+module.exports.checkIfGameHasUpdate = async function (info) {
-  if (!shared.isLogged || !shared.cookies) {
+    if (!shared.isLogged) {
-    shared.logger.warn(USER_NOT_LOGGED);
+        shared.logger.warn(USER_NOT_LOGGED);
-    return false;
+        return false;
-  }
+    }
 
-  // F95 change URL at every game update,
+    // F95 change URL at every game update,
-  // so if the URL is different an update is available
+    // so if the URL is different an update is available
-  const exists = await urlHelper.urlExists(info.f95url, true);
+    const exists = await networkHelper.urlExists(info.url, true);
-  if (!exists) return true;
+    if (!exists) return true;
 
-  // Parse version from title
+    // Parse version from title
-  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
+    const onlineVersion = await scraper.getGameInfo(info.url).version;
-  const browser = shared.isolation ? await prepareBrowser() : _browser;
     
-  const onlineVersion = await scraper.getGameVersionFromTitle(browser, info);
+    // Compare the versions
-
+    return onlineVersion.toUpperCase() !== info.version.toUpperCase();
-  if (shared.isolation) await browser.close();
-
-  return onlineVersion.toUpperCase() !== info.version.toUpperCase();
 };
+
 /**
  * @public
  * Starting from the name, it gets all the information about the game you are looking for.
  * You **must** be logged in to the portal before calling this method.
  * @param {String} name Name of the game searched
- * @param {Boolean} includeMods Indicates whether to also take mods into account when searching
+ * @param {Boolean} mod Indicate if you are looking for mods or games
  * @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
  * an identified game (in the case of homonymy of titles)
  */
-module.exports.getGameData = async function (name, includeMods) {
+module.exports.getGameData = async function (name, mod) {
-  if (!shared.isLogged || !shared.cookies) {
+    if (!shared.isLogged) {
-    shared.logger.warn(USER_NOT_LOGGED);
+        shared.logger.warn(USER_NOT_LOGGED);
-    return null;
+        return null;
-  }
+    }
 
-  // Gets the search results of the game being searched for
+    // Gets the search results of the game/mod being searched for
-  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
+    let urls = [];
-  const browser = shared.isolation ? await prepareBrowser() : _browser;
+    if(mod) urls = await searcher.searchMod(name);
-  const urlList = await searcher.getSearchGameResults(browser, name);
+    else urls = await searcher.searchGame(name);
 
-  // Process previous partial results
+    // Process previous partial results
-  const promiseList = [];
+    const results = [];
-  for (const url of urlList) {
+    for (const url of urls) {
-    // Start looking for information
+        // Start looking for information
-    promiseList.push(scraper.getGameInfo(browser, url));
+        const info = await scraper.getGameInfo(url);
-  }
+        results.push(info);
-
+    }
-  // Filter for mods
+    return results;
-  const result = [];
-  for (const info of await Promise.all(promiseList)) {
-    // Ignore empty results
-    if (!info) continue;
-    // Skip mods if not required
-    if (info.isMod && !includeMods) continue;
-    // Else save data
-    result.push(info);
-  }
-
-  if (shared.isolation) await browser.close();
-  return result;
 };
+
 /**
  * @public
  * Starting from the url, it gets all the information about the game you are looking for.
@@ -261,27 +134,20 @@ module.exports.getGameData = async function (name, includeMods) {
  * @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
  */
 module.exports.getGameDataFromURL = async function (url) {
-  if (!shared.isLogged || !shared.cookies) {
+    if (!shared.isLogged) {
-    shared.logger.warn(USER_NOT_LOGGED);
+        shared.logger.warn(USER_NOT_LOGGED);
-    return null;
+        return null;
-  }
+    }
 
-  // Check URL
+    // Check URL validity
-  const exists = await urlHelper.urlExists(url);
+    const exists = await networkHelper.urlExists(url);
-  if (!exists) throw new URIError(url + " is not a valid URL");
+    if (!exists) throw new URIError(`${url} is not a valid URL`);
-  if (!urlHelper.isF95URL(url))
+    if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
-    throw new Error(url + " is not a valid F95Zone URL");
     
-  // Gets the search results of the game being searched for
+    // Get game data
-  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
+    return await scraper.getGameInfo(url);
-  const browser = shared.isolation ? await prepareBrowser() : _browser;
-
-  // Get game data
-  const result = await scraper.getGameInfo(browser, url);
-
-  if (shared.isolation) await browser.close();
-  return result;
 };
+
 /**
  * @public
  * Gets the data of the currently logged in user.
@@ -289,321 +155,11 @@ module.exports.getGameDataFromURL = async function (url) {
  * @returns {Promise<UserData>} Data of the user currently logged in
  */
 module.exports.getUserData = async function () {
-  if (!shared.isLogged || !shared.cookies) {
+    if (!shared.isLogged) {
-    shared.logger.warn(USER_NOT_LOGGED);
+        shared.logger.warn(USER_NOT_LOGGED);
-    return null;
+        return null;
-  }
+    }
 
-  // Prepare a new web page
+    return await uScraper.getUserData();
-  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  const browser = shared.isolation ? await prepareBrowser() : _browser;
-  const page = await preparePage(browser); // Set new isolated page
-  await page.setCookie(...shared.cookies); // Set cookies to avoid login
-  await page.goto(urlK.F95_BASE_URL); // Go to base page
-
-  // Explicitly wait for the required items to load
-  await Promise.all([
-    page.waitForSelector(selectorK.USERNAME_ELEMENT),
-    page.waitForSelector(selectorK.AVATAR_PIC),
-  ]);
-
-  const threads = getUserWatchedGameThreads(browser);
-
-  const username = await page.evaluate(
-    /* istanbul ignore next */ (selector) =>
-      document.querySelector(selector).innerText,
-    selectorK.USERNAME_ELEMENT
-  );
-
-  const avatarSrc = await page.evaluate(
-    /* istanbul ignore next */ (selector) =>
-      document.querySelector(selector).getAttribute("src"),
-    selectorK.AVATAR_PIC
-  );
-
-  const ud = new UserData();
-  ud.username = username;
-  ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null;
-  ud.watchedThreads = await threads;
-
-  await page.close();
-  if (shared.isolation) await browser.close();
-
-  return ud;
-};
-/**
- * @public
- * Logout from the current user and gracefully close shared browser.
- * You **must** be logged in to the portal before calling this method.
- */
-module.exports.logout = async function () {
-  if (!shared.isLogged || !shared.cookies) {
-    shared.logger.warn(USER_NOT_LOGGED);
-    return;
-  }
-
-  // Logout
-  shared.isLogged = false;
-
-  // Gracefully close shared browser
-  if (!shared.isolation && _browser !== null) {
-    await _browser.close();
-    _browser = null;
-  }
 };
 //#endregion
-
-//#region Private methods
-
-//#region Cookies functions
-/**
- * @private
- * Loads and verifies the expiration of previously stored cookies from disk
- * if they exist, otherwise it returns null.
- * @return {object[]} List of dictionaries or null if cookies don't exist
- */
-function loadCookies() {
-  // Check the existence of the cookie file
-  if (fs.existsSync(shared.cookiesCachePath)) {
-    // Read cookies
-    const cookiesJSON = fs.readFileSync(shared.cookiesCachePath);
-    const cookies = JSON.parse(cookiesJSON);
-
-    // Check if the cookies have expired
-    for (const cookie of cookies) {
-      if (isCookieExpired(cookie)) return null;
-    }
-
-    // Cookies loaded and verified
-    return cookies;
-  } else return null;
-}
-/**
- * @private
- * Check the validity of a cookie.
- * @param {object} cookie Cookies to verify the validity. It's a dictionary
- * @returns {Boolean} true if the cookie has expired, false otherwise
- */
-function isCookieExpired(cookie) {
-  // Local variables
-  let expiredCookies = false;
-
-  // Ignore cookies that never expire
-  const expirationUnixTimestamp = cookie.expire;
-
-  if (expirationUnixTimestamp !== "-1") {
-    // Convert UNIX epoch timestamp to normal Date
-    const expirationDate = new Date(expirationUnixTimestamp * 1000);
-
-    if (expirationDate < Date.now()) {
-      shared.logger.warn(
-        "Cookie " + cookie.name + " expired, you need to re-authenticate"
-      );
-      expiredCookies = true;
-    }
-  }
-
-  return expiredCookies;
-}
-//#endregion Cookies functions
-
-//#region Latest Updates page parserer
-/**
- * @private
- * If present, it reads the file containing the searched values (engines or states)
- * from the disk, otherwise it connects to the F95 portal (at the page
- * https://f95zone.to/latest) and downloads them.
- * @param {puppeteer.Page} page Page used to locate the required elements
- * @param {String} path Path to disk of the JSON file containing the data to read / write
- * @param {String} selector CSS selector of the required elements
- * @param {String} elementRequested Required element (engines or states) used to detail log messages
- * @returns {Promise<String[]>} List of required values in uppercase
- */
-async function loadValuesFromLatestPage(
-  page,
-  path,
-  selector,
-  elementRequested
-) {
-  // If the values already exist they are loaded from disk without having to connect to F95
-  shared.logger.info("Load " + elementRequested + " from disk...");
-  if (fs.existsSync(path)) {
-    const valueJSON = fs.readFileSync(path);
-    return JSON.parse(valueJSON);
-  }
-
-  // Otherwise, connect and download the data from the portal
-  shared.logger.info("No " + elementRequested + " cached, downloading...");
-  const values = await getValuesFromLatestPage(
-    page,
-    selector,
-    "Getting " + elementRequested + " from page"
-  );
-  fs.writeFileSync(path, JSON.stringify(values));
-  return values;
-}
-/**
- * @private
- * Gets all the textual values of the elements present
- * in the F95 portal page and identified by the selector
- * passed by parameter
- * @param {puppeteer.Page} page Page used to locate items specified by the selector
- * @param {String} selector CSS selector
- * @param {String} logMessage Log message indicating which items the selector is requesting
- * @return {Promise<String[]>} List of uppercase strings indicating the textual values of the elements identified by the selector
- */
-async function getValuesFromLatestPage(page, selector, logMessage) {
-  shared.logger.info(logMessage);
-
-  const result = [];
-  const elements = await page.$$(selector);
-
-  for (const element of elements) {
-    const text = await element.evaluate(
-      /* istanbul ignore next */ (e) => e.innerText
-    );
-
-    // Save as upper text for better match if used in query
-    result.push(text.toUpperCase());
-  }
-  return result;
-}
-//#endregion
-
-//#region User
-/**
- * @private
- * Log in to the F95Zone portal and, if successful, save the cookies.
- * @param {puppeteer.Browser} browser Browser object used for navigation
- * @param {String} username Username to use during login
- * @param {String} password Password to use during login
- * @returns {Promise<LoginResult>} Result of the operation
- */
-async function loginF95(browser, username, password) {
-  const page = await preparePage(browser); // Set new isolated page
-  await page.goto(urlK.F95_LOGIN_URL); // Go to login page
-
-  // Explicitly wait for the required items to load
-  await Promise.all([
-    page.waitForSelector(selectorK.USERNAME_INPUT),
-    page.waitForSelector(selectorK.PASSWORD_INPUT),
-    page.waitForSelector(selectorK.LOGIN_BUTTON),
-  ]);
-
-  await page.type(selectorK.USERNAME_INPUT, username); // Insert username
-  await page.type(selectorK.PASSWORD_INPUT, password); // Insert password
-  await Promise.all([
-    page.click(selectorK.LOGIN_BUTTON), // Click on the login button
-    page.waitForNavigation({
-      waitUntil: shared.WAIT_STATEMENT,
-    }), // Wait for page to load
-  ]);
-
-  // Prepare result
-  let message = "";
-
-  // Check if the user is logged in
-  const success = await page.evaluate(
-    /* istanbul ignore next */ (selector) =>
-      document.querySelector(selector) !== null,
-    selectorK.AVATAR_INFO
-  );
-
-  const errorMessageExists = await page.evaluate(
-    /* istanbul ignore next */
-    (selector) => document.querySelector(selector) !== null,
-    selectorK.LOGIN_MESSAGE_ERROR
-  );
-
-  // Save cookies to avoid re-auth
-  if (success) {
-    const c = await page.cookies();
-    fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
-    message = "Authentication successful";
-  } else if (errorMessageExists) {
-    const errorMessage = await page.evaluate(
-      /* istanbul ignore next */ (selector) =>
-        document.querySelector(selector).innerText,
-      selectorK.LOGIN_MESSAGE_ERROR
-    );
-
-    if (errorMessage === "Incorrect password. Please try again.") {
-      message = "Incorrect password";
-    } else if (
-      errorMessage ===
-      "The requested user '" + username + "' could not be found."
-    ) {
-      // The escaped quotes are important!
-      message = "Incorrect username";
-    } else message = errorMessage;
-  } else message = "Unknown error";
-
-  await page.close(); // Close the page
-  return new LoginResult(success, message);
-}
-/**
- * @private
- * Gets the list of URLs of threads the user follows.
- * @param {puppeteer.Browser} browser Browser object used for navigation
- * @returns {Promise<String[]>} URL list
- */
-async function getUserWatchedGameThreads(browser) {
-  const page = await preparePage(browser); // Set new isolated page
-  await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page
-
-  // Explicitly wait for the required items to load
-  await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON);
-
-  // Show the popup
-  await Promise.all([
-    page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON),
-    page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX),
-    page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION),
-    page.waitForSelector(selectorK.FILTER_THREADS_BUTTON),
-  ]);
-
-  // Set the filters
-  await page.evaluate(
-    /* istanbul ignore next */ (selector) =>
-      document.querySelector(selector).removeAttribute("checked"),
-    selectorK.UNREAD_THREAD_CHECKBOX
-  ); // Also read the threads already read
-
-  // Filter the threads
-  await page.click(selectorK.ONLY_GAMES_THREAD_OPTION);
-  await page.click(selectorK.FILTER_THREADS_BUTTON);
-  await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
-
-  // Get the threads urls
-  const urls = [];
-  let nextPageExists = false;
-  do {
-    // Get all the URLs
-    for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) {
-      const src = await page.evaluate(
-        /* istanbul ignore next */ (element) => element.href,
-        handle
-      );
-      // If 'unread' is left, it will redirect to the last unread post
-      const url = src.replace("/unread", "");
-      urls.push(url);
-    }
-
-    nextPageExists = await page.evaluate(
-      /* istanbul ignore next */ (selector) => document.querySelector(selector),
-      selectorK.WATCHED_THREAD_NEXT_PAGE
-    );
-
-    // Click to next page
-    if (nextPageExists) {
-      await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE);
-      await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
-    }
-  } while (nextPageExists);
-
-  await page.close();
-  return urls;
-}
-//#endregion User
-
-//#endregion Private methods

app/scripts/classes/credentials.js

@@ -0,0 +1,18 @@
+"use strict";
+
+// Modules from file
+const { getF95Token } = require("../network-helper.js");
+
+class Credentials {
+    constructor(username, password) {
+        this.username = username;
+        this.password = password;
+        this.token = null;
+    }
+
+    async fetchToken() {
+        this.token = await getF95Token();
+    }
+}
+
+module.exports = Credentials;

app/scripts/classes/game-download.js

@@ -1,110 +0,0 @@
-/* istanbul ignore file */
-
-"use strict";
-
-// Core modules
-const fs = require("fs");
-
-// Public modules from npm
-// const { File } = require('megajs');
-
-// Modules from file
-const { prepareBrowser, preparePage } = require("../puppeteer-helper.js");
-const shared = require("../shared.js");
-
-class GameDownload {
-  constructor() {
-    /**
-     * @public
-     * Platform that hosts game files
-     * @type String
-     */
-    this.hosting = "";
-    /**
-     * @public
-     * Link to game files
-     * @type String
-     */
-    this.link = null;
-    /**
-     * @public
-     * Operating systems supported by the game version indicated in this class.
-     * Can be *WINDOWS/LINUX/MACOS*
-     * @type String[]
-     */
-    this.supportedOS = [];
-  }
-
-  /**
-   * @public
-   * Download the game data in the indicated path.
-   * Supported hosting platforms: MEGA, NOPY
-   * @param {String} path Save path
-   * @return {Promise<Boolean>} Result of the operation
-   */
-  async download(path) {
-    if (this.link.includes("mega.nz"))
-      return await downloadMEGA(this.link, path);
-    else if (this.link.includes("nopy.to"))
-      return await downloadNOPY(this.link, path);
-  }
-}
-module.exports = GameDownload;
-
-async function downloadMEGA(url, savepath) {
-  // The URL is masked
-  const browser = await prepareBrowser();
-  const page = await preparePage(browser);
-  await page.setCookie(...shared.cookies); // Set cookies to avoid login
-  await page.goto(url);
-  await page.waitForSelector("a.host_link");
-
-  // Obtain the link for the unmasked page and click it
-  const link = await page.$("a.host_link");
-  await link.click();
-  await page.goto(url, {
-    waitUntil: shared.WAIT_STATEMENT,
-  }); // Go to the game page and wait until it loads
-
-  // Obtain the URL after the redirect
-  const downloadURL = page.url();
-
-  // Close browser and page
-  await page.close();
-  await browser.close();
-
-  const stream = fs.createWriteStream(savepath);
-  const file = File.fromURL(downloadURL);
-  file.download().pipe(stream);
-  return fs.existsSync(savepath);
-}
-
-async function downloadNOPY(url, savepath) {
-  // Prepare browser
-  const browser = await prepareBrowser();
-  const page = await preparePage(browser);
-  await page.goto(url);
-  await page.waitForSelector("#download");
-
-  // Set the save path
-  await page._client.send("Page.setDownloadBehavior", {
-    behavior: "allow",
-    downloadPath: path.basename(path.dirname(savepath)), // It's a directory
-  });
-
-  // Obtain the download button and click it
-  const downloadButton = await page.$("#download");
-  await downloadButton.click();
-
-  // Await for all the connections to close
-  await page.waitForNavigation({
-    waitUntil: "networkidle0",
-    timeout: 0, // Disable timeout
-  });
-
-  // Close browser and page
-  await page.close();
-  await browser.close();
-
-  return fs.existsSync(savepath);
-}

app/scripts/classes/game-info.js

@@ -1,119 +1,135 @@
 "use strict";
 
 class GameInfo {
-  constructor() {
+    constructor() {
-    //#region Properties
+        //#region Properties
-    /**
+        /**
-     * Game name
+       * Game name
-     * @type String
+       * @type String
-     */
+       */
-    this.name = null;
+        this.name = null;
-    /**
+        /**
-     * Game author
+        * Game author
-     * @type String
+        * @type String
-     */
+        */
-    this.author = null;
+        this.author = null;
-    /**
+        /**
-     * URL to the game's official conversation on the F95Zone portal
+       * URL to the game's official conversation on the F95Zone portal
-     * @type String
+       * @type String
-     */
+       */
-    this.f95url = null;
+        this.url = null;
-    /**
+        /**
-     * Game description
+       * Game description
-     * @type String
+       * @type String
-     */
+       */
-    this.overview = null;
+        this.overview = null;
-    /**
+        /**
-     * List of tags associated with the game
+         * Game language.
-     * @type String[]
+         * @type String
-     */
+         */
-    this.tags = [];
+        this.language = null;
-    /**
+        /**
-     * Graphics engine used for game development
+         * List of supported OS.
-     * @type String
+         * @type
-     */
+         */
-    this.engine = null;
+        this.supportedOS = [];
-    /**
+        /**
-     * Progress of the game
+         * Specify whether the game has censorship 
-     * @type String
+         * measures regarding NSFW scenes.
-     */
+         * @type Boolean
-    this.status = null;
+         */
-    /**
+        this.censored = null;
-     * Game description image URL
+        /**
-     * @type String
+        * List of tags associated with the game
-     */
+        * @type String[]
-    this.previewSource = null;
+        */
-    /**
+        this.tags = [];
-     * Game version
+        /**
-     * @type String
+       * Graphics engine used for game development
-     */
+       * @type String
-    this.version = null;
+       */
-    /**
+        this.engine = null;
-     * Last time the game underwent updates
+        /**
-     * @type String
+       * Progress of the game
-     */
+       * @type String
-    this.lastUpdate = null;
+       */
-    /**
+        this.status = null;
-     * Last time the local copy of the game was run
+        /**
-     * @type String
+       * Game description image URL
-     */
+       * @type String
-    this.lastPlayed = null;
+       */
-    /**
+        this.previewSrc = null;
-     * Specifies if the game is original or a mod
+        /**
-     * @type Boolean
+       * Game version
-     */
+       * @type String
-    this.isMod = false;
+       */
-    /**
+        this.version = null;
-     * Changelog for the last version.
+        /**
-     * @type String
+       * Last time the game underwent updates
-     */
+       * @type String
-    this.changelog = null;
+       */
-    /**
+        this.lastUpdate = null;
-     * Directory containing the local copy of the game
+        /**
-     * @type String
+       * Last time the local copy of the game was run
-     */
+       * @type String
-    this.gameDir = null;
+       */
-    /**
+        this.lastPlayed = null;
-     * Information on game file download links,
+        /**
-     * including information on hosting platforms
+       * Specifies if the game is original or a mod
-     * and operating system supported by the specific link
+       * @type Boolean
-     * @type GameDownload[]
+       */
-     */
+        this.isMod = false;
-    this.downloadInfo = [];
+        /**
-    //#endregion Properties
+       * Changelog for the last version.
-  }
+       * @type String
+       */
+        this.changelog = null;
+        /**
+       * Directory containing the local copy of the game
+       * @type String
+       */
+        this.gameDir = null;
+        //#endregion Properties
+    }
 
-  /**
+    /**
-   * Converts the object to a dictionary used for JSON serialization
+    * Converts the object to a dictionary used for JSON serialization.
-   */
+    */
-  /* istanbul ignore next */
+    /* istanbul ignore next */
-  toJSON() {
+    toJSON() {
-    return {
+        return {
-      name: this.name,
+            name: this.name,
-      author: this.author,
+            author: this.author,
-      f95url: this.f95url,
+            url: this.url,
-      overview: this.overview,
+            overview: this.overview,
-      engine: this.engine,
+            language: this.language,
-      status: this.status,
+            supportedOS: this.supportedOS,
-      previewSource: this.previewSource,
+            censored: this.censored,
-      version: this.version,
+            engine: this.engine,
-      lastUpdate: this.lastUpdate,
+            status: this.status,
-      lastPlayed: this.lastPlayed,
+            tags: this.tags,
-      isMod: this.isMod,
+            previewSrc: this.previewSrc,
-      changelog: this.changelog,
+            version: this.version,
-      gameDir: this.gameDir,
+            lastUpdate: this.lastUpdate,
-      downloadInfo: this.downloadInfo,
+            lastPlayed: this.lastPlayed,
-    };
+            isMod: this.isMod,
-  }
+            changelog: this.changelog,
+            gameDir: this.gameDir,
+        };
+    }
 
-  /**
+    /**
-   * Return a new GameInfo from a JSON string
+    * Return a new GameInfo from a JSON string.
-   * @param {String} json JSON string used to create the new object
+    * @param {String} json JSON string used to create the new object
-   * @returns {GameInfo}
+    * @returns {GameInfo}
-   */
+    */
-  /* istanbul ignore next */
+    static fromJSON(json) {
-  static fromJSON(json) {
+        // Convert string
-    return Object.assign(new GameInfo(), json);
+        const temp = Object.assign(new GameInfo(), JSON.parse(json));
-  }
+        
+        // JSON cannot transform a string to a date implicitly
+        temp.lastUpdate = new Date(temp.lastUpdate);
+        return temp;
+    }
 }
 module.exports = GameInfo;

app/scripts/classes/login-result.js

@@ -4,17 +4,17 @@
  * Object obtained in response to an attempt to login to the portal.
  */
 class LoginResult {
-  constructor(success, message) {
+    constructor(success, message) {
-    /**
+        /**
-     * Result of the login operation
+        * Result of the login operation
-     * @type Boolean
+        * @type Boolean
-     */
+        */
-    this.success = success;
+        this.success = success;
-    /**
+        /**
-     * Login response message
+        * Login response message
-     * @type String
+        * @type String
-     */
+        */
-    this.message = message;
+        this.message = message;
-  }
+    }
 }
 module.exports = LoginResult;

app/scripts/classes/user-data.js

@@ -4,23 +4,23 @@
  * Class containing the data of the user currently connected to the F95Zone platform.
  */
 class UserData {
-  constructor() {
+    constructor() {
-    /**
+        /**
-     * User username.
+        * User name.
-     * @type String
+        * @type String
-     */
+        */
-    this.username = "";
+        this.username = "";
-    /**
+        /**
-     * Path to the user's profile picture.
+        * Path to the user's profile picture.
-     * @type String
+        * @type String
-     */
+        */
-    this.avatarSrc = null;
+        this.avatarSrc = null;
-    /**
+        /**
-     * List of followed thread URLs.
+        * List of followed thread URLs.
-     * @type URL[]
+        * @type String[]
-     */
+        */
-    this.watchedThreads = [];
+        this.watchedThreads = [];
-  }
+    }
 }
 
 module.exports = UserData;

app/scripts/constants/css-selector.js

@@ -1,33 +1,23 @@
 module.exports = Object.freeze({
-  AVATAR_INFO: "span.avatar",
+    BD_ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span",
-  AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]',
+    BD_STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span",
-  ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span',
+    
-  FILTER_THREADS_BUTTON: 'button[class="button--primary button"]',
+    GT_IMAGES: "img:not([title])[data-src^=\"https://attachments.f95zone.to\"][data-url=\"\"]",
-  GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]',
+    GT_TAGS: "a.tagItem",
-  GAME_TAGS: "a.tagItem",
+    GT_TITLE: "h1.p-title-value",
-  GAME_TITLE: "h1.p-title-value",
+    GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]",
-  GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]',
+    GT_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type",
-  LOGIN_BUTTON: "button.button--icon--login",
+    GT_JSONLD: "script[type=\"application/ld+json\"]",
-  LOGIN_MESSAGE_ERROR:
+    WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
-    "div.blockMessage.blockMessage--error.blockMessage--iconic",
+    WT_NEXT_PAGE: "a.pageNav-jump--next",
-  ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]',
+    WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]",
-  PASSWORD_INPUT: 'input[name="password"]',
+    WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]",
-  SEARCH_BUTTON: "form.block > * button.button--icon--search",
+    GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type",
-  SEARCH_FORM_TEXTBOX: 'input[name="keywords"][type="search"]',
+    GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a",
-  SEARCH_ONLY_GAMES_OPTION: 'select[name="c[nodes][]"] > option[value="1"]',
+    GS_RESULT_BODY: "div.contentRow-main",
-  STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span',
+    GS_MEMBERSHIP: "li > a:not(.username)",
-  THREAD_POSTS:
+    GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]",
-    "article.message-body:first-child > div.bbWrapper:first-of-type",
+    UD_USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText",
-  THREAD_TITLE: "h3.contentRow-title",
+    UD_AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]",
-  TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]',
+    LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic",
-  UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]',
-  USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText',
-  USERNAME_INPUT: 'input[name="login"]',
-  WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
-  WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next",
-  WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]',
-  DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]',
-  SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main",
-  SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)",
-  THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type",
 });

app/scripts/constants/url.js

@@ -1,7 +1,7 @@
 module.exports = Object.freeze({
-  F95_BASE_URL: "https://f95zone.to",
+    F95_BASE_URL: "https://f95zone.to",
-  F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
+    F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
-  F95_LATEST_UPDATES: "https://f95zone.to/latest",
+    F95_LATEST_UPDATES: "https://f95zone.to/latest",
-  F95_LOGIN_URL: "https://f95zone.to/login",
+    F95_LOGIN_URL: "https://f95zone.to/login/login",
-  F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
+    F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
 });

app/scripts/game-scraper.js

@@ -1,467 +0,0 @@
-"use strict";
-
-// Public modules from npm
-const HTMLParser = require("node-html-parser");
-const puppeteer = require("puppeteer"); // skipcq: JS-0128
-
-// Modules from file
-const shared = require("./shared.js");
-const selectorK = require("./constants/css-selector.js");
-const { preparePage } = require("./puppeteer-helper.js");
-const GameDownload = require("./classes/game-download.js");
-const GameInfo = require("./classes/game-info.js");
-const urlHelper = require("./url-helper.js");
-
-/**
- * @protected
- * Get information from the game's main page.
- * @param {puppeteer.Browser} browser Browser object used for navigation
- * @param {String} url URL (String) of the game/mod to extract data from
- * @return {Promise<GameInfo>} Complete information about the game you are
- * looking for
- */
-module.exports.getGameInfo = async function (browser, url) {
-  shared.logger.info("Obtaining game info");
-
-  // Verify the correctness of the URL
-  const exists = await urlHelper.urlExists(url);
-  if (!exists) throw new URIError(`${url} is not a valid URL`);
-  if (!urlHelper.isF95URL(url))
-    throw new Error(`${url} is not a valid F95Zone URL`);
-
-  const page = await preparePage(browser); // Set new isolated page
-  await page.setCookie(...shared.cookies); // Set cookies to avoid login
-  await page.goto(url, {
-    waitUntil: shared.WAIT_STATEMENT,
-  }); // Go to the game page and wait until it loads
-
-  // It asynchronously searches for the elements and
-  // then waits at the end to compile the object to be returned
-  let info = new GameInfo();
-  const title = getGameTitle(page);
-  const author = getGameAuthor(page);
-  const tags = getGameTags(page);
-  const redirectUrl = urlHelper.getUrlRedirect(url);
-  info = await parsePrefixes(page, info); // Fill status/engines/isMod
-  const structuredText = await getMainPostStructuredText(page);
-  const overview = getOverview(structuredText, info.isMod);
-  const parsedInfos = parseConversationPage(structuredText);
-  const previewSource = getGamePreviewSource(page);
-  const changelog = getLastChangelog(page);
-
-  // Fill in the GameInfo element with the information obtained
-  info.name = await title;
-  info.author = await author;
-  info.tags = await tags;
-  info.f95url = await redirectUrl;
-  info.overview = overview;
-  info.lastUpdate = info.isMod
-    ? parsedInfos.UPDATED
-    : parsedInfos.THREAD_UPDATED;
-  info.previewSource = await previewSource;
-  info.changelog = await changelog;
-  info.version = await exports.getGameVersionFromTitle(browser, info);
-
-  //let downloadData = getGameDownloadLink(page);
-  //info.downloadInfo = await downloadData;
-  /* Downloading games without going directly to
-   * the platform appears to be prohibited by
-   * the guidelines. It is therefore useless to
-   * keep the links for downloading the games. */
-
-  await page.close(); // Close the page
-  shared.logger.info("Founded data for " + info.name);
-  return info;
-};
-
-/**
- * Obtain the game version without parsing again all the data of the game.
- * @param {puppeteer.Browser} browser Browser object used for navigation
- * @param {GameInfo} info Information about the game
- * @returns {Promise<String>} Online version of the game
- */
-module.exports.getGameVersionFromTitle = async function (browser, info) {
-  const page = await preparePage(browser); // Set new isolated page
-  await page.setCookie(...shared.cookies); // Set cookies to avoid login
-  await page.goto(info.f95url, {
-    waitUntil: shared.WAIT_STATEMENT,
-  }); // Go to the game page and wait until it loads
-
-  // Get the title
-  const titleHTML = await page.evaluate(
-    /* istanbul ignore next */
-    (selector) => document.querySelector(selector).innerHTML,
-    selectorK.GAME_TITLE
-  );
-  const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText;
-
-  // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR]
-  const startIndex = title.indexOf("[") + 1;
-  const endIndex = title.indexOf("]", startIndex);
-  let version = title.substring(startIndex, endIndex).trim().toUpperCase();
-  if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence
-  await page.close();
-  return cleanFSString(version);
-};
-
-//#region Private methods
-/**
- * Clean a string from invalid File System chars.
- * @param {String} s
- * @returns {String}
- */
-function cleanFSString(s) {
-  const rx = /[/\\?%*:|"<>]/g;
-  return s.replace(rx, "");
-}
-
-/**
- * @private
- * Get the game description from its web page.
- * Different processing depending on whether the game is a mod or not.
- * @param {String} text Structured text extracted from the game's web page
- * @param {Boolean} isMod Specify if it is a game or a mod
- * @returns {Promise<String>} Game description
- */
-function getOverview(text, isMod) {
-  // Get overview (different parsing for game and mod)
-  let overviewEndIndex;
-  if (isMod) overviewEndIndex = text.indexOf("Updated");
-  else overviewEndIndex = text.indexOf("Thread Updated");
-  return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim();
-}
-
-/**
- * @private
- * Extrapolate the page structure by removing the element tags
- * and leaving only the text and its spacing.
- * @param {puppeteer.Page} page Page containing the text
- * @returns {Promise<String>} Structured text
- */
-async function getMainPostStructuredText(page) {
-  // Gets the first post, where are listed all the game's informations
-  const post = (await page.$$(selectorK.THREAD_POSTS))[0];
-
-  // The info are plain text so we need to parse the HTML code
-  const bodyHTML = await page.evaluate(
-    /* istanbul ignore next */
-    (mainPost) => mainPost.innerHTML,
-    post
-  );
-  return HTMLParser.parse(bodyHTML).structuredText;
-}
-
-/**
- * @private
- * Extrapolates and cleans the author from the page passed by parameter.
- * @param {puppeteer.Page} page Page containing the author to be extrapolated
- * @returns {Promise<String>} Game author
- */
-async function getGameAuthor(page) {
-  // Get the game/mod name (without square brackets)
-  const titleHTML = await page.evaluate(
-    /* istanbul ignore next */
-    (selector) => document.querySelector(selector).innerHTML,
-    selectorK.GAME_TITLE
-  );
-  const structuredTitle = HTMLParser.parse(titleHTML);
-
-  // The last element **shoud be** the title without prefixes (engines, status, other...)
-  const gameTitle = structuredTitle.childNodes.pop().rawText;
-
-  // The last square brackets contain the author
-  const startTitleIndex = gameTitle.lastIndexOf("[") + 1;
-  return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim();
-}
-
-/**
- * @private
- * Process the post text to get all the useful
- * information in the format *DESCRIPTOR : VALUE*.
- * @param {String} text Structured text of the post
- * @returns {Object} Dictionary of information
- */
-function parseConversationPage(text) {
-  const dataPairs = {};
-
-  // The information searched in the game post are one per line
-  const splittedText = text.split("\n");
-  for (const line of splittedText) {
-    if (!line.includes(":")) continue;
-
-    // Create pair key/value
-    const splitted = line.split(":");
-    const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch
-    const value = splitted[1].trim();
-
-    // Add pair to the dict if valid
-    if (value !== "") dataPairs[key] = value;
-  }
-
-  return dataPairs;
-}
-
-/**
- * @private
- * Gets the URL of the image used as a preview for the game in the conversation.
- * @param {puppeteer.Page} page Page containing the URL to be extrapolated
- * @returns {Promise<String>} URL (String) of the image or null if failed to get it
- */
-async function getGamePreviewSource(page) {
-  // Wait for the selector or return an empty value
-  try {
-    await page.waitForSelector(selectorK.GAME_IMAGES);
-  } catch {
-    return null;
-  }
-
-  const src = await page.evaluate(
-    /* istanbul ignore next */
-    (selector) => {
-      // Get the firs image available
-      const img = document.querySelector(selector);
-
-      if (img) return img.getAttribute("src");
-      else return null;
-    },
-    selectorK.GAME_IMAGES
-  );
-
-  // Check if the URL is valid
-  return urlHelper.isStringAValidURL(src) ? src : null;
-}
-
-/**
- * @private
- * Extrapolates and cleans the title from the page passed by parameter.
- * @param {puppeteer.Page} page Page containing the title to be extrapolated
- * @returns {Promise<String>} Game title
- */
-async function getGameTitle(page) {
-  // Get the game/mod name (without square brackets)
-  const titleHTML = await page.evaluate(
-    /* istanbul ignore next */
-    (selector) => document.querySelector(selector).innerHTML,
-    selectorK.GAME_TITLE
-  );
-  const structuredTitle = HTMLParser.parse(titleHTML);
-
-  // The last element **shoud be** the title without prefixes (engines, status, other...)
-  const gameTitle = structuredTitle.childNodes.pop().rawText;
-  const endTitleIndex = gameTitle.indexOf("[");
-  return gameTitle.substring(0, endTitleIndex).trim();
-}
-
-/**
- * @private
- * Get the alphabetically sorted list of tags associated with the game.
- * @param {puppeteer.Page} page Page containing the tags to be extrapolated
- * @returns {Promise<String[]>} List of uppercase tags
- */
-async function getGameTags(page) {
-  const tags = [];
-
-  // Get the game tags
-  for (const handle of await page.$$(selectorK.GAME_TAGS)) {
-    const tag = await page.evaluate(
-      /* istanbul ignore next */
-      (element) => element.innerText,
-      handle
-    );
-    tags.push(tag.toUpperCase());
-  }
-  return tags.sort();
-}
-
-/**
- * @private
- * Process the game title prefixes to extract information such as game status,
- * graphics engine used, and whether it is a mod or original game.
- * @param {puppeteer.Page} page Page containing the prefixes to be extrapolated
- * @param {GameInfo} info Object to assign the identified information to
- * @returns {Promise<GameInfo>} GameInfo object passed in to which the identified information has been added
- */
-async function parsePrefixes(page, info) {
-  // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete'
-  info.status = "ONGOING";
-  for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) {
-    const value = await page.evaluate(
-      /* istanbul ignore next */
-      (element) => element.innerText,
-      handle
-    );
-
-    // Clean the prefix
-    const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim();
-
-    // Getting infos...
-    if (shared.statuses.includes(prefix)) info.status = prefix;
-    else if (shared.engines.includes(prefix)) info.engine = prefix;
-    // This is not a game but a mod
-    else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true;
-  }
-  return info;
-}
-
-/**
- * @private
- * Get the last changelog available for the game.
- * @param {puppeteer.Page} page Page containing the changelog
- * @returns {Promise<String>} Changelog for the last version or a empty string if no changelog is found
- */
-async function getLastChangelog(page) {
-  // Gets the first post, where are listed all the game's informations
-  const post = (await page.$$(selectorK.THREAD_POSTS))[0];
-
-  const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG);
-  if (!spoiler) return "";
-
-  const changelogHTML = await page.evaluate(
-    /* istanbul ignore next */
-    (e) => e.innerText,
-    spoiler
-  );
-  let parsedText = HTMLParser.parse(changelogHTML).structuredText;
-
-  // Clean the text
-  if (parsedText.startsWith("Spoiler"))
-    parsedText = parsedText.replace("Spoiler", "");
-  if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", "");
-  return parsedText.trim();
-}
-
-/**
- * @private
- * Get game download links for different platforms.
- * @param {puppeteer.Page} page Page containing the links to be extrapolated
- * @returns {Promise<GameDownload[]>} List of objects used for game download
- * @deprecated
- */
-/* istanbul ignore next */
-// skipcq: JS-0128
-async function getGameDownloadLink(page) {
-  // Most used hosting platforms
-  const hostingPlatforms = [
-    "MEGA",
-    "NOPY",
-    "FILESUPLOAD",
-    "MIXDROP",
-    "UPLOADHAVEN",
-    "PIXELDRAIN",
-    "FILESFM",
-  ];
-
-  // Supported OS platforms
-  const platformOS = ["WIN", "LINUX", "MAC", "ALL"];
-
-  // Gets the <span> which contains the download links
-  const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER);
-  if (temp.length === 0) return [];
-
-  // Look for the container that contains the links
-  // It is necessary because the same css selector
-  // also identifies other elements on the page
-  let container = null;
-  for (const candidate of temp) {
-    if (container !== null) break;
-    const upperText = (
-      await page.evaluate(
-        /* istanbul ignore next */
-        (e) => e.innerText,
-        candidate
-      )
-    ).toUpperCase();
-
-    // Search if the container contains the name of a hosting platform
-    for (const p of hostingPlatforms) {
-      if (upperText.includes(p)) {
-        container = candidate;
-        break;
-      }
-    }
-  }
-  if (container === null) return [];
-
-  // Extract the HTML text from the container
-  const searchText = (
-    await page.evaluate(
-      /* istanbul ignore next */
-      (e) => e.innerHTML,
-      container
-    )
-  ).toLowerCase();
-
-  // Parse the download links
-  const downloadData = [];
-  for (const platform of platformOS) {
-    const data = extractGameHostingData(platform, searchText);
-    downloadData.push(...data);
-  }
-  return downloadData;
-}
-
-/**
- * @private
- * From the HTML text it extracts the game download links for the specified operating system.
- * @param {String} platform Name of the operating system to look for a compatible link to.
- * It can only be *WIN/LINUX/MAC/ALL*
- * @param {String} text HTML string to extract links from
- * @returns {GameDownload[]} List of game download links for the selected platform
- * @deprecated
- */
-/* istanbul ignore next */
-function extractGameHostingData(platform, text) {
-  const PLATFORM_BOLD_OPEN = "<b>";
-  const CONTAINER_SPAN_CLOSE = "</span>";
-  const LINK_OPEN = "<a";
-  const LINK_CLOSE = "</a>";
-  const HREF_START = "href='";
-  const HREF_END = "'";
-  const TAG_CLOSE = ">";
-
-  // Identify the individual platforms
-  let startIndex = text.indexOf(platform.toLowerCase());
-  if (startIndex === -1) return [];
-  else startIndex += platform.length;
-
-  // Find the <b>platform</b>
-  let endIndex =
-    text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length;
-
-  // Find the end of the container
-  if (endIndex === -1)
-    endIndex =
-      text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) +
-      CONTAINER_SPAN_CLOSE.length;
-
-  text = text.substring(startIndex, endIndex);
-
-  const downloadData = [];
-  const linkTags = text.split(LINK_OPEN);
-  for (const tag of linkTags) {
-    // Ignore non-link string
-    if (!tag.includes(HREF_START)) continue;
-
-    // Find the hosting platform name
-    startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length;
-    endIndex = tag.indexOf(LINK_CLOSE, startIndex);
-    const hosting = tag.substring(startIndex, endIndex);
-
-    // Find the 'href' attribute
-    startIndex = tag.indexOf(HREF_START) + HREF_START.length;
-    endIndex = tag.indexOf(HREF_END, startIndex);
-    const link = tag.substring(startIndex, endIndex);
-
-    if (urlHelper.isStringAValidURL(link)) {
-      const gd = new GameDownload();
-      gd.hosting = hosting.toUpperCase();
-      gd.link = link;
-      gd.supportedOS = platform.toUpperCase();
-
-      downloadData.push(gd);
-    }
-  }
-  return downloadData;
-}
-
-//#endregion Private methods

app/scripts/game-searcher.js

@@ -1,132 +0,0 @@
-"use strict";
-
-// Public modules from npm
-const puppeteer = require("puppeteer"); // skipcq: JS-0128
-
-// Modules from file
-const shared = require("./shared.js");
-const urlK = require("./constants/url.js");
-const selectorK = require("./constants/css-selector.js");
-const { preparePage } = require("./puppeteer-helper.js");
-const { isF95URL } = require("./url-helper.js");
-
-/**
- * @protected
- * Search the F95Zone portal to find possible conversations regarding the game you are looking for.
- * @param {puppeteer.Browser} browser Browser object used for navigation
- * @param {String} gamename Name of the game to search for
- * @returns {Promise<String[]>} List of URL of possible games obtained from the preliminary research on the F95 portal
- */
-module.exports.getSearchGameResults = async function (browser, gamename) {
-  shared.logger.info(`Searching ${gamename} on F95Zone`);
-
-  const page = await preparePage(browser); // Set new isolated page
-  await page.setCookie(...shared.cookies); // Set cookies to avoid login
-  await page.goto(urlK.F95_SEARCH_URL, {
-    waitUntil: shared.WAIT_STATEMENT,
-  }); // Go to the search form and wait for it
-
-  // Explicitly wait for the required items to load
-  await Promise.all([
-    page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX),
-    page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX),
-    page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION),
-    page.waitForSelector(selectorK.SEARCH_BUTTON),
-  ]);
-
-  await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire
-  await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles
-  await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod
-  await Promise.all([
-    page.click(selectorK.SEARCH_BUTTON), // Execute search
-    page.waitForNavigation({
-      waitUntil: shared.WAIT_STATEMENT,
-    }), // Wait for page to load
-  ]);
-
-  // Select all conversation titles
-  const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY);
-
-  // For each element found extract the info about the conversation
-  shared.logger.info("Extracting info from conversations");
-  const results = [];
-  for (const element of resultsThread) {
-    const gameUrl = await getOnlyGameThreads(page, element);
-    if (gameUrl !== null) results.push(gameUrl);
-  }
-  shared.logger.info(`Find ${results.length} conversations`);
-  await page.close(); // Close the page
-
-  return results;
-};
-
-//#region Private methods
-/**
- * @private
- * Return the link of a conversation if it is a game or a mod.
- * @param {puppeteer.Page} page Page containing the conversation to be analyzed
- * @param {puppeteer.ElementHandle} divHandle Element of the conversation to be analyzed
- * @return {Promise<String>} URL of the game/mod or null if the URL is not of a game
- */
-async function getOnlyGameThreads(page, divHandle) {
-  // Obtain the elements containing the basic information
-  const titleHandle = await divHandle.$(selectorK.THREAD_TITLE);
-  const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP);
-
-  // Get the forum where the thread was posted
-  const forum = await getMembershipForum(page, forumHandle);
-  if (forum !== "GAMES" && forum !== "MODS") return null;
-
-  // Get the URL of the thread from the title
-  return await getThreadURL(page, titleHandle);
-}
-
-/**
- * @private
- * Obtain the membership forum of the thread passed throught 'handle'.
- * @param {puppeteer.Page} page Page containing the conversation to be analyzed
- * @param {puppeteer.ElementHandle} handle Handle containing the forum membership
- * @returns {Promise<String>} Uppercase membership category
- */
-async function getMembershipForum(page, handle) {
-  // The link can be something like:
-  // + /forums/request.NUMBER/
-  // + /forums/game-recommendations-identification.NUMBER/
-  // + /forums/games.NUMBER/ <-- We need this
-
-  let link = await page.evaluate(
-    /* istanbul ignore next */
-    (e) => e.getAttribute("href"),
-    handle
-  );
-
-  // Parse link
-  link = link.replace("/forums/", "");
-  const endIndex = link.indexOf(".");
-  const forum = link.substring(0, endIndex);
-
-  return forum.toUpperCase();
-}
-
-/**
- * @private
- * Obtain the URL of the thread passed through 'handle'.
- * @param {puppeteer.Page} page Page containing the conversation to be analyzed
- * @param {puppeteer.ElementHandle} handle Handle containing the thread title
- * @returns {Promise<String>} URL of the thread
- */
-async function getThreadURL(page, handle) {
-  const relativeURLThread = await page.evaluate(
-    /* istanbul ignore next */
-    (e) => e.querySelector("a").href,
-    handle
-  );
-
-  // Some game already have a full URL...
-  if (isF95URL(relativeURLThread)) return relativeURLThread;
-
-  // ... else compose the URL and return
-  const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString();
-  return urlThread;
-}
-//#endregion Private methods

app/scripts/network-helper.js

@@ -0,0 +1,243 @@
+"use strict";
+
+// Public modules from npm
+const axios = require("axios").default;
+const ky = require("ky-universal").create({
+    throwHttpErrors: false,
+});
+const cheerio = require("cheerio");
+const axiosCookieJarSupport = require("axios-cookiejar-support").default;
+const tough = require("tough-cookie");
+
+// Modules from file
+const shared = require("./shared.js");
+const f95url = require("./constants/url.js");
+const f95selector = require("./constants/css-selector.js");
+const LoginResult = require("./classes/login-result.js");
+
+// Global variables
+const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + 
+    "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15";
+axiosCookieJarSupport(axios);
+const cookieJar = new tough.CookieJar();
+
+const commonConfig = {
+    headers: {
+        "User-Agent": userAgent,
+        "Connection": "keep-alive"
+    },
+    withCredentials: true,
+    jar: cookieJar // Used to store the token in the PC
+};
+
+/**
+ * @protected
+ * Gets the HTML code of a page.
+ * @param {String} url URL to fetch
+ * @returns {Promise<String>} HTML code or `null` if an error arise
+ */
+module.exports.fetchHTML = async function (url) {
+    // Fetch the response of the platform
+    const response = await exports.fetchGETResponse(url);
+    if (!response) {
+        shared.logger.warn(`Unable to fetch HTML for ${url}`);
+        return null;
+    }
+    return response.data;
+};
+
+/**
+ * @protected
+ * It authenticates to the platform using the credentials 
+ * and token obtained previously. Save cookies on your 
+ * device after authentication.
+ * @param {Credentials} credentials Platform access credentials
+ * @param {Boolea} force Specifies whether the request should be forced, ignoring any saved cookies
+ * @returns {Promise<LoginResult>} Result of the operation
+ */
+module.exports.authenticate = async function (credentials, force) {
+    shared.logger.info(`Authenticating with user ${credentials.username}`);
+    if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
+
+    // Secure the URL
+    const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL);
+
+    // Prepare the parameters to send to the platform to authenticate
+    const params = new URLSearchParams();
+    params.append("login", credentials.username);
+    params.append("url", "");
+    params.append("password", credentials.password);
+    params.append("password_confirm", "");
+    params.append("additional_security", "");
+    params.append("remember", "1");
+    params.append("_xfRedirect", "https://f95zone.to/");
+    params.append("website_code", "");
+    params.append("_xfToken", credentials.token);
+
+    try {
+        // Try to log-in
+        let config = Object.assign({}, commonConfig);
+        if (force) delete config.jar;
+        const response = await axios.post(secureURL, params, config);
+
+        // Parse the response HTML
+        const $ = cheerio.load(response.data);
+
+        // Get the error message (if any) and remove the new line chars
+        const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, "");
+
+        // Return the result of the authentication
+        if (errorMessage === "") return new LoginResult(true, "Authentication successful");
+        else return new LoginResult(false, errorMessage);
+    } catch (e) {
+        shared.logger.error(`Error ${e.message} occurred while authenticating to ${secureURL}`);
+        return new LoginResult(false, `Error ${e.message} while authenticating`);
+    }
+};
+
+/**
+ * Obtain the token used to authenticate the user to the platform.
+ * @returns {Promise<String>} Token or `null` if an error arise
+ */
+module.exports.getF95Token = async function() {
+    // Fetch the response of the platform
+    const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL);
+    if (!response) {
+        shared.logger.warn("Unable to get the token for the session");
+        return null;
+    }
+
+    // The response is a HTML page, we need to find the <input> with name "_xfToken"
+    const $ = cheerio.load(response.data);
+    const token = $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value");
+    return token;
+};
+
+/**
+ * @protected
+ * Gets the basic data used for game data processing 
+ * (such as graphics engines and progress statuses)
+ * @deprecated
+ */
+/* istanbul ignore next */
+module.exports.fetchPlatformData = async function() {
+    // Fetch the response of the platform
+    const response = await exports.fetchGETResponse(f95url.F95_LATEST_UPDATES);
+    if (!response) {
+        shared.logger.warn("Unable to get the token for the session");
+        return;
+    }
+
+    // The response is a HTML page, we need to find 
+    // the base data, used when scraping the games
+    const $ = cheerio.load(response.data);
+
+    // Extract the elements
+    const engineElements = $("body").find(f95selector.BD_ENGINE_ID_SELECTOR);
+    const statusesElements = $("body").find(f95selector.BD_STATUS_ID_SELECTOR);
+
+    // Extract the raw text
+    engineElements.each(function extractEngineNames(idx, el) {
+        const engine = cheerio.load(el).text().trim();
+        shared.engines.push(engine);
+    });
+
+    statusesElements.each(function extractEngineNames(idx, el) {
+        const status = cheerio.load(el).text().trim();
+        shared.statuses.push(status);
+    });
+};
+
+//#region Utility methods
+/**
+ * @protected
+ * Performs a GET request to a specific URL and returns the response.
+ * If the request generates an error (for example 400) `null` is returned.
+ * @param {String} url 
+ */
+module.exports.fetchGETResponse = async function(url) {
+    // Secure the URL
+    const secureURL = exports.enforceHttpsUrl(url);
+
+    try {
+        // Fetch and return the response
+        return await axios.get(secureURL, commonConfig);
+    } catch (e) {
+        shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`);
+        return null;
+    }
+};
+
+/**
+ * @protected
+ * Enforces the scheme of the URL is https and returns the new URL.
+ * @param {String} url 
+ * @returns {String} Secure URL or `null` if the argument is not a string
+ */
+module.exports.enforceHttpsUrl = function (url) {
+    return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
+};
+
+/**
+ * @protected
+ * Check if the url belongs to the domain of the F95 platform.
+ * @param {String} url URL to check
+ * @returns {Boolean} true if the url belongs to the domain, false otherwise
+ */
+module.exports.isF95URL = function (url) {
+    if (url.toString().startsWith(f95url.F95_BASE_URL)) return true;
+    else return false;
+};
+
+/**
+ * @protected
+ * Checks if the string passed by parameter has a 
+ * properly formatted and valid path to a URL (HTTP/HTTPS).
+ * @param {String} url String to check for correctness
+ * @returns {Boolean} true if the string is a valid URL, false otherwise
+ */
+module.exports.isStringAValidURL = function (url) {
+    // Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc)
+    const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/;
+    const regex = new RegExp(expression);
+    if (url.match(regex)) return true;
+    else return false;
+};
+
+/**
+ * @protected
+ * Check if a particular URL is valid and reachable on the web.
+ * @param {String} url URL to check
+ * @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false
+ * @returns {Promise<Boolean>} true if the URL exists, false otherwise
+ */
+module.exports.urlExists = async function (url, checkRedirect) {
+    if (!exports.isStringAValidURL(url)) {
+        return false;
+    }
+
+    const response = await ky.head(url);
+    let valid = response !== undefined && !/4\d\d/.test(response.status);
+
+    if (!valid) return false;
+
+    if (checkRedirect) {
+        const redirectUrl = await exports.getUrlRedirect(url);
+        if (redirectUrl === url) valid = true;
+        else valid = false;
+    }
+
+    return valid;
+};
+
+/**
+ * @protected
+ * Check if the URL has a redirect to another page.
+ * @param {String} url URL to check for redirect
+ * @returns {Promise<String>} Redirect URL or the passed URL
+ */
+module.exports.getUrlRedirect = async function (url) {
+    const response = await ky.head(url);
+    return response.url;
+};
+//#endregion Utility methods

app/scripts/puppeteer-helper.js

@@ -1,60 +0,0 @@
-"use strict";
-
-// Public modules from npm
-const puppeteer = require("puppeteer");
-
-// Modules from file
-const shared = require("./shared.js");
-
-/**
- * @protected
- * Create a Chromium instance used to navigate with Puppeteer.
- * By default the browser is headless.
- * @returns {Promise<puppeteer.Browser>} Created browser
- */
-module.exports.prepareBrowser = async function () {
-  // Create a headless browser
-  let browser = null;
-  if (shared.chromiumLocalPath) {
-    browser = await puppeteer.launch({
-      executablePath: shared.chromiumLocalPath,
-      headless: !shared.debug, // Use GUI when debug = true
-    });
-  } else {
-    browser = await puppeteer.launch({
-      headless: !shared.debug, // Use GUI when debug = true
-    });
-  }
-
-  return browser;
-};
-
-/**
- * @protected
- * Prepare a page used to navigate the browser.
- * The page is set up to reject image download requests. The user agent is also changed.
- * @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created
- * @returns {Promise<puppeteer.Page>} New page
- */
-module.exports.preparePage = async function (browser) {
-  // Create new page in the browser argument
-  const page = await browser.newPage();
-
-  // Block image download
-  await page.setRequestInterception(true);
-  page.on("request", (request) => {
-    if (request.resourceType() === "image") request.abort();
-    else if (request.resourceType === "font") request.abort();
-    // else if (request.resourceType() == 'stylesheet') request.abort();
-    // else if(request.resourceType == 'media') request.abort();
-    else request.continue();
-  });
-
-  // Set custom user-agent
-  const userAgent =
-    "Mozilla/5.0 (X11; Linux x86_64)" +
-    "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
-  await page.setUserAgent(userAgent);
-
-  return page;
-};

app/scripts/scraper.js

@@ -0,0 +1,371 @@
+"use strict";
+
+// Public modules from npm
+const cheerio = require("cheerio");
+
+// Modules from file
+const { fetchHTML, getUrlRedirect } = require("./network-helper.js");
+const shared = require("./shared.js");
+const GameInfo = require("./classes/game-info.js");
+const f95Selector = require("./constants/css-selector.js");
+
+/**
+ * @protected
+ * Get information from the game's main page.
+ * @param {String} url URL of the game/mod to extract data from
+ * @return {Promise<GameInfo>} Complete information about the game you are
+ * looking for
+ */
+module.exports.getGameInfo = async function (url) {
+    shared.logger.info("Obtaining game info");
+
+    // Fetch HTML and prepare Cheerio
+    const html = await fetchHTML(url);
+    const $ = cheerio.load(html);
+    const body = $("body");
+    const mainPost = $(f95Selector.GS_POSTS).first();
+
+    // Extract data
+    const titleData = extractInfoFromTitle(body);
+    const tags = extractTags(body);
+    const prefixesData = parseGamePrefixes(body);
+    const src = extractPreviewSource(body);
+    const changelog = extractChangelog(mainPost);
+    const structuredData = extractStructuredData(body);
+    const parsedInfos = parseMainPostText(structuredData["description"]);
+    const overview = getOverview(structuredData["description"], prefixesData.mod);
+    
+    // Obtain the updated URL
+    const redirectUrl = await getUrlRedirect(url);
+
+    // Fill in the GameInfo element with the information obtained
+    const info = new GameInfo();
+    info.name = titleData.name;
+    info.author = titleData.author;
+    info.isMod = prefixesData.mod;
+    info.engine = prefixesData.engine;
+    info.status = prefixesData.status;
+    info.tags = tags;
+    info.url = redirectUrl;
+    info.language = parsedInfos.Language;
+    info.overview = overview;
+    info.supportedOS = parsedInfos.SupportedOS;
+    info.censored = parsedInfos.Censored;
+    info.lastUpdate = parsedInfos.LastUpdate;
+    info.previewSrc = src;
+    info.changelog = changelog;
+    info.version = titleData.version;
+    
+    shared.logger.info(`Founded data for ${info.name}`);
+    return info;
+};
+
+//#region Private methods
+/**
+ * @private
+ * Parse the game prefixes obtaining the engine used, 
+ * the advancement status and if the game is actually a game or a mod.
+ * @param {cheerio.Cheerio} body Page `body` selector
+ * @returns {Object.<string, object>} Dictionary of values with keys `engine`, `status`, `mod`
+ */
+function parseGamePrefixes(body) {
+    shared.logger.trace("Parsing prefixes...");
+
+    // Local variables
+    let mod = false,
+        engine = null,
+        status = null;
+
+    // Obtain the title prefixes
+    const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES);
+    
+    prefixeElements.each(function parseGamePrefix(idx, el) {
+        // Obtain the prefix text
+        let prefix = cheerio.load(el).text().trim();
+
+        // Remove the square brackets
+        prefix = prefix.replace("[", "").replace("]", "");
+
+        // Check what the prefix indicates
+        if (isEngine(prefix)) engine = prefix;
+        else if (isStatus(prefix)) status = prefix;
+        else if (isMod(prefix)) mod = true;
+    });
+
+    // If the status is not set, then the game in in development (Ongoing)
+    if (!status) status = "Ongoing";
+
+    return {
+        engine,
+        status,
+        mod
+    };
+}
+
+/**
+ * @private
+ * Extracts all the possible informations from the title.
+ * @param {cheerio.Cheerio} body Page `body` selector
+ * @returns {Object.<string, string>} Dictionary of values with keys `name`, `author`, `version`
+ */
+function extractInfoFromTitle(body) {
+    shared.logger.trace("Extracting information from title...");
+    const title = body
+        .find(f95Selector.GT_TITLE)
+        .text()
+        .trim();
+
+    // From the title we can extract: Name, author and version
+    // [PREFIXES] TITLE [VERSION] [AUTHOR]
+    const matches = title.match(/\[(.*?)\]/g);
+
+    // Get the title name
+    let name = title;
+    matches.forEach(function replaceElementsInTitle(e) {
+        name = name.replace(e, "");
+    });
+    name = name.trim();
+
+    // The regex [[\]]+ remove the square brackets
+
+    // The version is the penultimate element. 
+    // If the matches are less than 2, than the title 
+    // is malformes and only the author is fetched
+    // (usually the author is always present)
+    let version = null;
+    if (matches.length >= 2) version = matches[matches.length - 2].replace(/[[\]]+/g, "").trim();
+    else shared.logger.trace(`Malformed title: ${title}`);
+
+    // Last element
+    const author = matches[matches.length - 1].replace(/[[\]]+/g, "").trim();
+
+    return {
+        name,
+        version,
+        author,
+    };
+}
+
+/**
+ * @private
+ * Gets the tags used to classify the game.
+ * @param {cheerio.Cheerio} body Page `body` selector
+ * @returns {String[]} List of tags
+ */
+function extractTags(body) {
+    shared.logger.trace("Extracting tags...");
+
+    // Get the game tags
+    const tagResults = body.find(f95Selector.GT_TAGS);
+    return tagResults.map(function parseGameTags(idx, el) {
+        return cheerio.load(el).text().trim();
+    }).get();
+}
+
+/**
+ * @private
+ * Gets the URL of the image used as a preview.
+ * @param {cheerio.Cheerio} body Page `body` selector
+ * @returns {String} URL of the image
+ */
+function extractPreviewSource(body) {
+    shared.logger.trace("Extracting image preview source...");
+    const image = body.find(f95Selector.GT_IMAGES);
+    
+    // The "src" attribute is rendered only in a second moment, 
+    // we need the "static" src value saved in the attribute "data-src"
+    const source = image ? image.attr("data-src") : null;
+    return source;
+}
+
+/**
+ * @private
+ * Gets the changelog of the latest version.
+ * @param {cheerio.Cheerio} mainPost main post selector
+ * @returns {String} Changelog of the last version or `null` if no changelog is fetched
+ */
+function extractChangelog(mainPost) {
+    shared.logger.trace("Extracting last changelog...");
+
+    // Obtain changelog
+    let changelog = mainPost.find(f95Selector.GT_LAST_CHANGELOG).text().trim();
+
+    // Clean changelog
+    changelog = changelog.replace("Spoiler", "");
+    changelog = changelog.replace(/\n+/g, "\n");
+
+    // Return changelog
+    return changelog ? changelog : null;
+}
+
+/**
+ * @private
+ * Process the main post text to get all the useful
+ * information in the format *DESCRIPTOR : VALUE*.
+ * Gets "standard" values such as: `Language`, `SupportedOS`, `Censored`, and `LastUpdate`.
+ * All non-canonical values are instead grouped together as a dictionary with the key `Various`.
+ * @param {String} text Structured text of the post
+ * @returns {Object.<string, object>} Dictionary of information
+ */
+function parseMainPostText(text) {
+    shared.logger.trace("Parsing main post raw text...");
+
+    const data = {};
+
+    // The information searched in the game post are one per line
+    const splittedText = text.split("\n");
+    for (const line of splittedText) {
+        if (!line.includes(":")) continue;
+
+        // Create pair key/value
+        const splitted = line.split(":");
+        const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch
+        const value = splitted[1].trim();
+
+        // Add pair to the dict if valid
+        if (value !== "") data[key] = value;
+    }
+
+    // Parse the standard pairs
+    const parsedDict = {};
+
+    // Check if the game is censored
+    if (data.CENSORED) {
+        const censored = data.CENSORED.toUpperCase() === "NO" ? false : true;
+        parsedDict["Censored"] = censored;
+        delete data.CENSORED;
+    }
+
+    // Last update of the main post
+    if (data.UPDATED) {
+        parsedDict["LastUpdate"] = new Date(data.UPDATED);
+        delete data.UPDATED;
+    }
+    else if (data.THREAD_UPDATED) {
+        parsedDict["LastUpdate"] = new Date(data.THREAD_UPDATED);
+        delete data.THREAD_UPDATED;
+    }
+
+    // Parse the supported OS
+    if (data.OS) {
+        const listOS = [];
+
+        // Usually the string is something like "Windows, Linux, Mac"
+        const splitted = data.OS.split(",");
+        splitted.forEach(function (os) {
+            listOS.push(os.trim());
+        });
+
+        parsedDict["SupportedOS"] = listOS;
+        delete data.OS;
+    }
+
+    // Rename the key for the language
+    if (data.LANGUAGE) {
+        parsedDict["Language"] = data.LANGUAGE;
+        delete data.LANGUAGE;
+    }
+
+    // What remains is added to a sub dictionary
+    parsedDict["Various"] = data;
+    
+    return parsedDict;
+}
+
+/**
+ * @private
+ * Extracts and processes the JSON-LD values found at the bottom of the page.
+ * @param {cheerio.Cheerio} body Page `body` selector
+ * @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
+ */
+function extractStructuredData(body) {
+    shared.logger.trace("Extracting JSON-LD data...");
+    const structuredDataElements = body.find(f95Selector.GT_JSONLD);
+    const json = structuredDataElements.map(function parseScriptTag(idx, el) {
+        // Get the element HTML
+        const html = cheerio.load(el).html().trim();
+
+        // Obtain the JSON-LD
+        const data = html
+            .replace("<script type=\"application/ld+json\">", "")
+            .replace("</script>", "");
+
+        // Convert the string to an object
+        const json = JSON.parse(data);
+
+        // Return only the data of the game
+        if (json["@type"] === "Book") return json;
+    }).get();
+    return json[0] ? json[0] : null;
+}
+
+/**
+ * @private
+ * Get the game description from its web page.
+ * Different processing depending on whether the game is a mod or not.
+ * @param {String} text Structured text extracted from the game's web page
+ * @param {Boolean} mod Specify if it is a game or a mod
+ * @returns {String} Game description
+ */
+function getOverview(text, mod) {
+    shared.logger.trace("Extracting game overview...");
+
+    // Get overview (different parsing for game and mod)
+    const overviewEndIndex = mod ? text.indexOf("Updated") : text.indexOf("Thread Updated");
+    return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim();
+}
+
+/**
+ * @private
+ * Check if the prefix is a game's engine.
+ * @param {String} prefix Prefix to check
+ * @return {Boolean}
+ */
+function isEngine(prefix) {
+    const engines = toUpperCaseArray(shared.engines);
+    return engines.includes(prefix.toUpperCase());
+}
+
+/**
+ * @private
+ * Check if the prefix is a game's status.
+ * @param {String} prefix Prefix to check
+ * @return {Boolean}
+ */
+function isStatus(prefix) {
+    const statuses = toUpperCaseArray(shared.statuses);
+    return statuses.includes(prefix.toUpperCase());
+}
+
+/**
+ * @private
+ * Check if the prefix indicates a mod.
+ * @param {String} prefix Prefix to check
+ * @return {Boolean}
+ */
+function isMod(prefix) {
+    const modPrefixes = ["MOD", "CHEAT MOD"];
+    return modPrefixes.includes(prefix.toUpperCase());
+}
+
+/**
+ * @private
+ * Makes an array of strings uppercase.
+ * @param {String[]} a 
+ * @returns {String[]}
+ */
+function toUpperCaseArray(a) {
+    // If the array is empty, return
+    if(a.length === 0) return [];
+
+    /**
+     * Makes a string uppercase.
+     * @param {String} s 
+     * @returns {String}
+     */
+    function toUpper(s) {
+        return s.toUpperCase();
+    }
+    return a.map(toUpper);
+}
+//#endregion Private methods

app/scripts/searcher.js

@@ -0,0 +1,95 @@
+"use strict";
+
+// Public modules from npm
+const cheerio = require("cheerio");
+
+// Modules from file
+const { fetchHTML } = require("./network-helper.js");
+const shared = require("./shared.js");
+const f95Selector = require("./constants/css-selector.js");
+const { F95_BASE_URL } = require("./constants/url.js");
+
+//#region Public methods
+/**
+ * @protected
+ * Search for a game on F95Zone and return a list of URLs, one for each search result.
+ * @param {String} name Game name
+ * @returns {Promise<String[]>} URLs of results
+ */
+module.exports.searchGame = async function (name) {
+    shared.logger.info(`Searching games with name ${name}`);
+
+    // Replace the whitespaces with +
+    const searchName = encodeURIComponent(name.toUpperCase());
+    
+    // Prepare the URL (only title, search in the "Games" section, order by relevance)
+    const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`;
+
+    // Fetch and parse the result URLs
+    return await fetchResultURLs(url);
+};
+
+/**
+ * @protected
+ * Search for a mod on F95Zone and return a list of URLs, one for each search result.
+ * @param {String} name Mod name
+ * @returns {Promise<String[]>} URLs of results
+ */
+module.exports.searchMod = async function (name) {
+    shared.logger.info(`Searching mods with name ${name}`);
+    
+    // Replace the whitespaces with +
+    const searchName = encodeURIComponent(name.toUpperCase());
+
+    // Prepare the URL (only title, search in the "Mods" section, order by relevance)
+    const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`;
+
+    // Fetch and parse the result URLs
+    return await fetchResultURLs(url);
+};
+//#endregion Public methods
+
+//#region Private methods
+/**
+ * @private
+ * Gets the URLs of the threads resulting from the F95Zone search.
+ * @param {String} url Search URL
+ * @return {Promise<String[]>} List of URLs
+ */
+async function fetchResultURLs(url) {
+    shared.logger.trace(`Fetching ${url}...`);
+
+    // Fetch HTML and prepare Cheerio
+    const html = await fetchHTML(url);
+    const $ = cheerio.load(html);
+
+    // Here we get all the DIV that are the body of the various query results
+    const results = $("body").find(f95Selector.GS_RESULT_BODY);
+
+    // Than we extract the URLs
+    const urls = results.map((idx, el) => {
+        const elementSelector = $(el);
+        return extractLinkFromResult(elementSelector);
+    }).get();
+
+    return urls;
+}
+
+/**
+ * @private
+ * Look for the URL to the thread referenced by the item.
+ * @param {cheerio.Cheerio} selector Element to search
+ * @returns {String} URL to thread
+ */
+function extractLinkFromResult(selector) {
+    shared.logger.trace("Extracting thread link from result...");
+    
+    const partialLink = selector
+        .find(f95Selector.GS_RESULT_THREAD_TITLE)
+        .attr("href")
+        .trim();
+
+    // Compose and return the URL
+    return new URL(partialLink, F95_BASE_URL).toString();
+}
+//#endregion Private methods

app/scripts/shared.js

@@ -1,173 +1,80 @@
+/* istanbul ignore file */
 "use strict";
 
-// Core modules
+// Public modules from npm
-const { join } = require("path");
-
 const log4js = require("log4js");
 
 /**
  * Class containing variables shared between modules.
  */
 class Shared {
-  //#region Properties
+    //#region Properties
-  /**
+    /**
-   * Shows log messages and other useful functions for module debugging.
+     * Indicates whether a user is logged in to the F95Zone platform or not.
-   * @type Boolean
+     * @type Boolean
-   */
+     */
-  static #_debug = false;
+    static #_isLogged = false;
-  /**
+    /**
-   * Indicates whether a user is logged in to the F95Zone platform or not.
+     * List of possible game engines used for development.
-   * @type Boolean
+     * @type String[]
-   */
+     */
-  static #_isLogged = false;
+    static #_engines = ["ADRIFT", "Flash", "HTML", "Java", "Others", "QSP", "RAGS", "RPGM", "Ren'Py", "Tads", "Unity", "Unreal Engine", "WebGL", "Wolf RPG"];
-  /**
+    /**
-   * List of cookies obtained from the F95Zone platform.
+     * List of possible development statuses that a game can assume.
-   * @type Object[]
+     * @type String[]
-   */
+     */
-  static #_cookies = null;
+    static #_statuses = ["Completed", "Onhold", "Abandoned"];
-  /**
+    /**
-   * List of possible game engines used for development.
+     * Logger object used to write to both file and console.
-   * @type String[]
+     * @type log4js.Logger
-   */
+     */
-  static #_engines = null;
+    static #_logger = log4js.getLogger();
-  /**
+    //#endregion Properties
-   * List of possible development statuses that a game can assume.
-   * @type String[]
-   */
-  static #_statuses = null;
-  /**
-   * Wait instruction for the browser created by puppeteer.
-   * @type String
-   */
-  static WAIT_STATEMENT = "domcontentloaded";
-  /**
-   * Path to the directory to save the cache generated by the API.
-   * @type String
-   */
-  static #_cacheDir = "./f95cache";
-  /**
-   * If true, it opens a new browser for each request to
-   * the F95Zone platform, otherwise it reuses the same.
-   * @type Boolean
-   */
-  static #_isolation = false;
-  /**
-   * Logger object used to write to both file and console.
-   * @type log4js.Logger
-   */
-  static #_logger = log4js.getLogger();
-  //#endregion Properties
 
-  //#region Getters
+    //#region Getters
-  /**
+    /**
-   * Shows log messages and other useful functions for module debugging.
-   * @returns {Boolean}
-   */
-  static get debug() {
-    return this.#_debug;
-  }
-  /**
    * Indicates whether a user is logged in to the F95Zone platform or not.
    * @returns {Boolean}
    */
-  static get isLogged() {
+    static get isLogged() {
-    return this.#_isLogged;
+        return this.#_isLogged;
-  }
+    }
-  /**
+    /**
-   * List of cookies obtained from the F95Zone platform.
-   * @returns {Object[]}
-   */
-  static get cookies() {
-    return this.#_cookies;
-  }
-  /**
    * List of possible game engines used for development.
    * @returns {String[]}
    */
-  static get engines() {
+    static get engines() {
-    return this.#_engines;
+        return this.#_engines;
-  }
+    }
-  /**
+    /**
    * List of possible development states that a game can assume.
    * @returns {String[]}
    */
-  static get statuses() {
+    static get statuses() {
-    return this.#_statuses;
+        return this.#_statuses;
-  }
+    }
-  /**
+    /**
-   * Directory to save the API cache.
-   * @returns {String}
-   */
-  static get cacheDir() {
-    return this.#_cacheDir;
-  }
-  /**
-   * Path to the F95 platform cache.
-   * @returns {String}
-   */
-  static get cookiesCachePath() {
-    return join(this.#_cacheDir, "cookies.json");
-  }
-  /**
-   * Path to the game engine cache.
-   * @returns {String}
-   */
-  static get enginesCachePath() {
-    return join(this.#_cacheDir, "engines.json");
-  }
-  /**
-   * Path to the cache of possible game states.
-   * @returns {String}
-   */
-  static get statusesCachePath() {
-    return join(this.#_cacheDir, "statuses.json");
-  }
-  /**
-   * If true, it opens a new browser for each request
-   * to the F95Zone platform, otherwise it reuses the same.
-   * @returns {Boolean}
-   */
-  static get isolation() {
-    return this.#_isolation;
-  }
-  /**
    * Logger object used to write to both file and console.
    * @returns {log4js.Logger}
    */
-  static get logger() {
+    static get logger() {
-    return this.#_logger;
+        return this.#_logger;
-  }
+    }
-  //#endregion Getters
+    //#endregion Getters
 
-  //#region Setters
+    //#region Setters
-  static set cookies(val) {
+    static set engines(val) {
-    this.#_cookies = val;
+        this.#_engines = val;
-  }
+    }
 
-  static set engines(val) {
+    static set statuses(val) {
-    this.#_engines = val;
+        this.#_statuses = val;
-  }
+    }
 
-  static set statuses(val) {
+    static set isLogged(val) {
-    this.#_statuses = val;
+        this.#_isLogged = val;
-  }
+    }
-
+    //#endregion Setters
-  static set cacheDir(val) {
-    this.#_cacheDir = val;
-  }
-
-  static set debug(val) {
-    this.#_debug = val;
-  }
-
-  static set isLogged(val) {
-    this.#_isLogged = val;
-  }
-
-  static set isolation(val) {
-    this.#_isolation = val;
-  }
-  //#endregion Setters
 }
 
 module.exports = Shared;

app/scripts/url-helper.js

@@ -1,72 +0,0 @@
-"use strict";
-
-// Public modules from npm
-const ky = require("ky-universal").create({
-  throwHttpErrors: false,
-});
-
-// Modules from file
-const { F95_BASE_URL } = require("./constants/url.js");
-
-/**
- * @protected
- * Check if the url belongs to the domain of the F95 platform.
- * @param {String} url URL to check
- * @returns {Boolean} true if the url belongs to the domain, false otherwise
- */
-module.exports.isF95URL = function (url) {
-  if (url.toString().startsWith(F95_BASE_URL)) return true;
-  else return false;
-};
-
-/**
- * @protected
- * Checks if the string passed by parameter has a properly formatted and valid path to a URL.
- * @param {String} url String to check for correctness
- * @returns {Boolean} true if the string is a valid URL, false otherwise
- */
-module.exports.isStringAValidURL = function (url) {
-  try {
-    new URL(url); // skipcq: JS-0078
-    return true;
-  } catch (err) {
-    return false;
-  }
-};
-
-/**
- * @protected
- * Check if a particular URL is valid and reachable on the web.
- * @param {String} url URL to check
- * @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false
- * @returns {Promise<Boolean>} true if the URL exists, false otherwise
- */
-module.exports.urlExists = async function (url, checkRedirect) {
-  if (!exports.isStringAValidURL(url)) {
-    return false;
-  }
-
-  const response = await ky.head(url);
-  let valid = response !== undefined && !/4\d\d/.test(response.status);
-
-  if (!valid) return false;
-
-  if (checkRedirect) {
-    const redirectUrl = await exports.getUrlRedirect(url);
-    if (redirectUrl === url) valid = true;
-    else valid = false;
-  }
-
-  return valid;
-};
-
-/**
- * @protected
- * Check if the URL has a redirect to another page.
- * @param {String} url URL to check for redirect
- * @returns {Promise<String>} Redirect URL or the passed URL
- */
-module.exports.getUrlRedirect = async function (url) {
-  const response = await ky.head(url);
-  return response.url;
-};

app/scripts/user-scraper.js

@@ -0,0 +1,123 @@
+"use strict";
+
+// Public modules from npm
+const cheerio = require("cheerio");
+
+// Modules from file
+const networkHelper = require("./network-helper.js");
+const f95Selector = require("./constants/css-selector.js");
+const f95url = require("./constants/url.js");
+const UserData = require("./classes/user-data.js");
+
+/**
+ * @protected
+ * Gets user data, such as username, url of watched threads, and profile picture url.
+ * @return {Promise<UserData>} User data
+ */
+module.exports.getUserData = async function() {
+    // Fetch data
+    const data = await fetchUsernameAndAvatar();
+    const urls = await fetchWatchedThreadURLs();
+
+    // Create object
+    const ud = new UserData();
+    ud.username = data.username;
+    ud.avatarSrc = data.source;
+    ud.watchedThreads = urls;
+
+    return ud;
+};
+
+//#region Private methods
+/**
+ * @private
+ * It connects to the page and extracts the name 
+ * of the currently logged in user and the URL 
+ * of their profile picture.
+ * @return {Promise<Object.<string, string>>}
+ */
+async function fetchUsernameAndAvatar() {
+    // Fetch page
+    const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL);
+
+    // Load HTML response
+    const $ = cheerio.load(html);
+    const body = $("body");
+
+    // Fetch username
+    const username = body.find(f95Selector.UD_USERNAME_ELEMENT).first().text().trim();
+
+    // Fetch user avatar image source
+    const source = body.find(f95Selector.UD_AVATAR_PIC).first().attr("src");
+
+    return {
+        username,
+        source
+    };
+}
+
+/**
+ * @private
+ * Gets the list of URLs of threads watched by the user.
+ * @returns {Promise<String[]>} List of URLs
+ */
+async function fetchWatchedThreadURLs() {
+    // Local variables
+    let currentURL = f95url.F95_WATCHED_THREADS;
+    const wathcedThreadURLs = [];
+
+    do {
+        // Fetch page
+        const html = await networkHelper.fetchHTML(currentURL);
+
+        // Load HTML response
+        const $ = cheerio.load(html);
+        const body = $("body");
+
+        // Find the URLs
+        const urls = fetchPageURLs(body);
+        wathcedThreadURLs.push(...urls);
+
+        // Find the next page (if any)
+        currentURL = fetchNextPageURL(body);
+    }
+    while (currentURL);
+
+    return wathcedThreadURLs;
+}
+
+/**
+ * @private
+ * Gets the URLs of the watched threads on the page.
+ * @param {cheerio.Cheerio} body Page `body` selector
+ * @returns {String[]}
+ */
+function fetchPageURLs(body) {
+    const elements = body.find(f95Selector.WT_URLS);
+
+    return elements.map(function extractURLs(idx, e) {
+        // Obtain the link (replace "unread" only for the unread threads)
+        const partialLink = e.attribs.href.replace("unread", "");
+
+        // Compose and return the URL
+        return new URL(partialLink, f95url.F95_BASE_URL).toString();
+    }).get();
+}
+
+/**
+ * @private
+ * Gets the URL of the next page containing the watched threads 
+ * or `null` if that page does not exist.
+ * @param {cheerio.Cheerio} body Page `body` selector
+ * @returns {String}
+ */
+function fetchNextPageURL(body) {
+    const element = body.find(f95Selector.WT_NEXT_PAGE).first();
+
+    // No element found
+    if(element.length === 0) return null;
+
+    // Compose and return the URL
+    return new URL(element.attr("href"), f95url.F95_BASE_URL).toString();
+}
+//#endregion Private methods

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "f95api",
-  "version": "1.3.5",
+  "version": "1.5.0",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -285,17 +285,7 @@
     "@types/node": {
       "version": "14.11.2",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz",
-      "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==",
+      "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA=="
-      "optional": true
-    },
-    "@types/yauzl": {
-      "version": "2.9.1",
-      "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz",
-      "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==",
-      "optional": true,
-      "requires": {
-        "@types/node": "*"
-      }
     },
     "abort-controller": {
       "version": "3.0.0",
@@ -317,11 +307,6 @@
       "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==",
       "dev": true
     },
-    "agent-base": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz",
-      "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g=="
-    },
     "aggregate-error": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
@@ -423,6 +408,23 @@
       "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==",
       "dev": true
     },
+    "axios": {
+      "version": "0.21.0",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz",
+      "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==",
+      "requires": {
+        "follow-redirects": "^1.10.0"
+      }
+    },
+    "axios-cookiejar-support": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
+      "integrity": "sha512-IZJxnAJ99XxiLqNeMOqrPbfR7fRyIfaoSLdPUf4AMQEGkH8URs0ghJK/xtqBsD+KsSr3pKl4DEQjCn834pHMig==",
+      "requires": {
+        "is-redirect": "^1.0.0",
+        "pify": "^5.0.0"
+      }
+    },
     "babel-eslint": {
       "version": "10.1.0",
       "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz",
@@ -435,17 +437,21 @@
         "@babel/types": "^7.7.0",
         "eslint-visitor-keys": "^1.0.0",
         "resolve": "^1.12.0"
+      },
+      "dependencies": {
+        "eslint-visitor-keys": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+          "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+          "dev": true
+        }
       }
     },
     "balanced-match": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
-      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
+      "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
-    },
+      "dev": true
-    "base64-js": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
-      "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
     },
     "binary-extensions": {
       "version": "2.1.0",
@@ -453,20 +459,16 @@
       "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
       "dev": true
     },
-    "bl": {
+    "boolbase": {
-      "version": "4.0.3",
+      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
-      "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
-      "requires": {
-        "buffer": "^5.5.0",
-        "inherits": "^2.0.4",
-        "readable-stream": "^3.4.0"
-      }
     },
     "brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
       "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
       "requires": {
         "balanced-match": "^1.0.0",
         "concat-map": "0.0.1"
@@ -487,20 +489,6 @@
       "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==",
       "dev": true
     },
-    "buffer": {
-      "version": "5.6.0",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
-      "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
-      "requires": {
-        "base64-js": "^1.0.2",
-        "ieee754": "^1.1.4"
-      }
-    },
-    "buffer-crc32": {
-      "version": "0.2.13",
-      "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
-      "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
-    },
     "caching-transform": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz",
@@ -556,6 +544,19 @@
       "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=",
       "dev": true
     },
+    "cheerio": {
+      "version": "1.0.0-rc.3",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
+      "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
+      "requires": {
+        "css-select": "~1.2.0",
+        "dom-serializer": "~0.1.1",
+        "entities": "~1.1.1",
+        "htmlparser2": "^3.9.1",
+        "lodash": "^4.15.0",
+        "parse5": "^3.0.1"
+      }
+    },
     "chokidar": {
       "version": "3.4.2",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz",
@@ -572,11 +573,6 @@
         "readdirp": "~3.4.0"
       }
     },
-    "chownr": {
-      "version": "1.1.4",
-      "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
-      "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
-    },
     "clean-stack": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
@@ -618,7 +614,8 @@
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
-      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
+      "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
+      "dev": true
     },
     "convert-source-map": {
       "version": "1.7.0",
@@ -640,6 +637,22 @@
         "which": "^2.0.1"
       }
     },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "~1.0.0",
+        "css-what": "2.1",
+        "domutils": "1.5.1",
+        "nth-check": "~1.0.1"
+      }
+    },
+    "css-what": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
+      "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
+    },
     "data-uri-to-buffer": {
       "version": "3.0.1",
       "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz",
@@ -697,11 +710,6 @@
         "object-keys": "^1.0.12"
       }
     },
-    "devtools-protocol": {
-      "version": "0.0.799653",
-      "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz",
-      "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg=="
-    },
     "diff": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@@ -717,6 +725,37 @@
         "esutils": "^2.0.2"
       }
     },
+    "dom-serializer": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
+      "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
+      "requires": {
+        "domelementtype": "^1.3.0",
+        "entities": "^1.1.1"
+      }
+    },
+    "domelementtype": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
+    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "requires": {
+        "domelementtype": "1"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "requires": {
+        "dom-serializer": "0",
+        "domelementtype": "1"
+      }
+    },
     "dotenv": {
       "version": "8.2.0",
       "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
@@ -729,14 +768,6 @@
       "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
       "dev": true
     },
-    "end-of-stream": {
-      "version": "1.4.4",
-      "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
-      "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
-      "requires": {
-        "once": "^1.4.0"
-      }
-    },
     "enquirer": {
       "version": "2.3.6",
       "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
@@ -746,6 +777,11 @@
         "ansi-colors": "^4.1.1"
       }
     },
+    "entities": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
+    },
     "es-abstract": {
       "version": "1.17.6",
       "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz",
@@ -888,12 +924,6 @@
           "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
           "dev": true
         },
-        "eslint-visitor-keys": {
-          "version": "2.0.0",
-          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
-          "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
-          "dev": true
-        },
         "globals": {
           "version": "12.4.0",
           "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
@@ -949,12 +979,20 @@
       "dev": true,
       "requires": {
         "eslint-visitor-keys": "^1.1.0"
+      },
+      "dependencies": {
+        "eslint-visitor-keys": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+          "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+          "dev": true
+        }
       }
     },
     "eslint-visitor-keys": {
-      "version": "1.3.0",
+      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
-      "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+      "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
       "dev": true
     },
     "espree": {
@@ -966,6 +1004,14 @@
         "acorn": "^7.4.0",
         "acorn-jsx": "^5.2.0",
         "eslint-visitor-keys": "^1.3.0"
+      },
+      "dependencies": {
+        "eslint-visitor-keys": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+          "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+          "dev": true
+        }
       }
     },
     "esprima": {
@@ -1025,17 +1071,6 @@
       "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
       "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="
     },
-    "extract-zip": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
-      "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
-      "requires": {
-        "@types/yauzl": "^2.9.1",
-        "debug": "^4.1.1",
-        "get-stream": "^5.1.0",
-        "yauzl": "^2.10.0"
-      }
-    },
     "fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -1054,14 +1089,6 @@
       "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
       "dev": true
     },
-    "fd-slicer": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
-      "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
-      "requires": {
-        "pend": "~1.2.0"
-      }
-    },
     "fetch-blob": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.1.tgz",
@@ -1100,6 +1127,7 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
       "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
+      "dev": true,
       "requires": {
         "locate-path": "^5.0.0",
         "path-exists": "^4.0.0"
@@ -1141,6 +1169,11 @@
       "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz",
       "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA=="
     },
+    "follow-redirects": {
+      "version": "1.13.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz",
+      "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA=="
+    },
     "foreground-child": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz",
@@ -1157,11 +1190,6 @@
       "integrity": "sha512-Xu2Qh8yqYuDhQGOhD5iJGninErSfI9A3FrriD3tjUgV5VbJFeH8vfgZ9HnC6jWN80QDVNQK5vmxRAmEAp7Mevw==",
       "dev": true
     },
-    "fs-constants": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
-      "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
-    },
     "fs-extra": {
       "version": "8.1.0",
       "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
@@ -1175,7 +1203,8 @@
     "fs.realpath": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
+      "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
+      "dev": true
     },
     "fsevents": {
       "version": "2.1.3",
@@ -1220,18 +1249,11 @@
       "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
       "dev": true
     },
-    "get-stream": {
-      "version": "5.2.0",
-      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
-      "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
-      "requires": {
-        "pump": "^3.0.0"
-      }
-    },
     "glob": {
       "version": "7.1.6",
       "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
       "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
+      "dev": true,
       "requires": {
         "fs.realpath": "^1.0.0",
         "inflight": "^1.0.4",
@@ -1301,7 +1323,8 @@
     "he": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
-      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="
+      "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
+      "dev": true
     },
     "html-escaper": {
       "version": "2.0.2",
@@ -1309,19 +1332,42 @@
       "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==",
       "dev": true
     },
-    "https-proxy-agent": {
+    "htmlparser2": {
-      "version": "4.0.0",
+      "version": "3.10.1",
-      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
-      "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==",
+      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
       "requires": {
-        "agent-base": "5",
+        "domelementtype": "^1.3.1",
-        "debug": "4"
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^3.1.1"
       }
     },
-    "ieee754": {
+    "ignore": {
-      "version": "1.1.13",
+      "version": "4.0.6",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
-      "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
+      "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+      "dev": true
+    },
+    "import-fresh": {
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
+      "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==",
+      "dev": true,
+      "requires": {
+        "parent-module": "^1.0.0",
+        "resolve-from": "^4.0.0"
+      },
+      "dependencies": {
+        "resolve-from": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
+          "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
+          "dev": true
+        }
+      }
     },
     "ignore": {
       "version": "4.0.6",
@@ -1363,6 +1409,7 @@
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
       "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
+      "dev": true,
       "requires": {
         "once": "^1.3.0",
         "wrappy": "1"
@@ -1445,6 +1492,11 @@
       "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
       "dev": true
     },
+    "is-redirect": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz",
+      "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ="
+    },
     "is-regex": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz",
@@ -1699,6 +1751,7 @@
       "version": "5.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
       "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
+      "dev": true,
       "requires": {
         "p-locate": "^4.1.0"
       }
@@ -1706,8 +1759,7 @@
     "lodash": {
       "version": "4.17.20",
       "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
-      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
-      "dev": true
     },
     "lodash.flattendeep": {
       "version": "4.4.0",
@@ -1801,6 +1853,7 @@
       "version": "3.0.4",
       "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
       "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
+      "dev": true,
       "requires": {
         "brace-expansion": "^1.1.7"
       }
@@ -1820,11 +1873,6 @@
         "minimist": "^1.2.5"
       }
     },
-    "mkdirp-classic": {
-      "version": "0.5.3",
-      "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
-      "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
-    },
     "mocha": {
       "version": "8.1.3",
       "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.1.3.tgz",
@@ -2065,10 +2113,10 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
-    "nan": {
+    "natural-compare": {
-      "version": "2.14.1",
+      "version": "1.4.0",
-      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
+      "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
-      "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==",
+      "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
       "dev": true
     },
     "natural-compare": {
@@ -2086,14 +2134,6 @@
         "fetch-blob": "^2.1.1"
       }
     },
-    "node-html-parser": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-1.3.1.tgz",
-      "integrity": "sha512-AwYVI6GyEKj9NGoyMfSx4j5l7Axf7obQgLWGxtasLjED6RggTTQoq5ZRzjwSUfgSZ+Mv8Nzbi3pID0gFGqNUsA==",
-      "requires": {
-        "he": "1.2.0"
-      }
-    },
     "node-preload": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz",
@@ -2109,6 +2149,14 @@
       "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
       "dev": true
     },
+    "nth-check": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+      "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
+      "requires": {
+        "boolbase": "~1.0.0"
+      }
+    },
     "nyc": {
       "version": "15.1.0",
       "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz",
@@ -2172,6 +2220,7 @@
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
       "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
+      "dev": true,
       "requires": {
         "wrappy": "1"
       }
@@ -2194,6 +2243,7 @@
       "version": "2.3.0",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
       "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
+      "dev": true,
       "requires": {
         "p-try": "^2.0.0"
       }
@@ -2202,6 +2252,7 @@
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
       "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
+      "dev": true,
       "requires": {
         "p-limit": "^2.2.0"
       }
@@ -2218,7 +2269,8 @@
     "p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
-      "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
+      "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
+      "dev": true
     },
     "package-hash": {
       "version": "4.0.0",
@@ -2241,15 +2293,25 @@
         "callsites": "^3.0.0"
       }
     },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "path-exists": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
-      "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
+      "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+      "dev": true
     },
     "path-is-absolute": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
-      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
+      "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
+      "dev": true
     },
     "path-key": {
       "version": "3.1.1",
@@ -2269,21 +2331,22 @@
       "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
       "dev": true
     },
-    "pend": {
-      "version": "1.2.0",
-      "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
-      "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
-    },
     "picomatch": {
       "version": "2.2.2",
       "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz",
       "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
       "dev": true
     },
+    "pify": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz",
+      "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA=="
+    },
     "pkg-dir": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
       "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
+      "dev": true,
       "requires": {
         "find-up": "^4.0.0"
       }
@@ -2306,7 +2369,8 @@
     "progress": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
-      "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
+      "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
+      "dev": true
     },
     "promise.allsettled": {
       "version": "1.0.2",
@@ -2321,43 +2385,15 @@
         "iterate-value": "^1.0.0"
       }
     },
-    "proxy-from-env": {
+    "psl": {
-      "version": "1.1.0",
+      "version": "1.8.0",
-      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
-      "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
+      "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
-    },
-    "pump": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
-      "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
-      "requires": {
-        "end-of-stream": "^1.1.0",
-        "once": "^1.3.1"
-      }
     },
     "punycode": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
-      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
-      "dev": true
-    },
-    "puppeteer": {
-      "version": "5.3.1",
-      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz",
-      "integrity": "sha512-YTM1RaBeYrj6n7IlRXRYLqJHF+GM7tasbvrNFx6w1S16G76NrPq7oYFKLDO+BQsXNtS8kW2GxWCXjIMPvfDyaQ==",
-      "requires": {
-        "debug": "^4.1.0",
-        "devtools-protocol": "0.0.799653",
-        "extract-zip": "^2.0.0",
-        "https-proxy-agent": "^4.0.0",
-        "pkg-dir": "^4.2.0",
-        "progress": "^2.0.1",
-        "proxy-from-env": "^1.0.0",
-        "rimraf": "^3.0.2",
-        "tar-fs": "^2.0.0",
-        "unbzip2-stream": "^1.3.3",
-        "ws": "^7.2.3"
-      }
     },
     "randombytes": {
       "version": "2.1.0",
@@ -2438,6 +2474,7 @@
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
       "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
+      "dev": true,
       "requires": {
         "glob": "^7.1.3"
       }
@@ -2490,13 +2527,23 @@
       "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
       "dev": true
     },
-    "sleep": {
+    "slice-ansi": {
-      "version": "6.3.0",
+      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/sleep/-/sleep-6.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz",
-      "integrity": "sha512-+WgYl951qdUlb1iS97UvQ01pkauoBK9ML9I/CMPg41v0Ze4EyMlTgFTDDo32iYj98IYqxIjDMRd+L71lawFfpQ==",
+      "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==",
       "dev": true,
       "requires": {
-        "nan": "^2.14.1"
+        "ansi-styles": "^3.2.0",
+        "astral-regex": "^1.0.0",
+        "is-fullwidth-code-point": "^2.0.0"
+      },
+      "dependencies": {
+        "is-fullwidth-code-point": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
+          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
+          "dev": true
+        }
       }
     },
     "slice-ansi": {
@@ -2689,29 +2736,6 @@
         }
       }
     },
-    "tar-fs": {
-      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz",
-      "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==",
-      "requires": {
-        "chownr": "^1.1.1",
-        "mkdirp-classic": "^0.5.2",
-        "pump": "^3.0.0",
-        "tar-stream": "^2.0.0"
-      }
-    },
-    "tar-stream": {
-      "version": "2.1.4",
-      "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz",
-      "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==",
-      "requires": {
-        "bl": "^4.0.3",
-        "end-of-stream": "^1.4.1",
-        "fs-constants": "^1.0.0",
-        "inherits": "^2.0.3",
-        "readable-stream": "^3.1.1"
-      }
-    },
     "test-exclude": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz",
@@ -2729,11 +2753,6 @@
       "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
       "dev": true
     },
-    "through": {
-      "version": "2.3.8",
-      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
-      "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
-    },
     "to-fast-properties": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
@@ -2749,6 +2768,16 @@
         "is-number": "^7.0.0"
       }
     },
+    "tough-cookie": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
+      "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
+      "requires": {
+        "psl": "^1.1.33",
+        "punycode": "^2.1.1",
+        "universalify": "^0.1.2"
+      }
+    },
     "type-check": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@@ -2779,15 +2808,6 @@
         "is-typedarray": "^1.0.0"
       }
     },
-    "unbzip2-stream": {
-      "version": "1.4.3",
-      "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
-      "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
-      "requires": {
-        "buffer": "^5.2.1",
-        "through": "^2.3.8"
-      }
-    },
     "universalify": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@@ -2814,9 +2834,9 @@
       "dev": true
     },
     "v8-compile-cache": {
-      "version": "2.1.1",
+      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz",
-      "integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==",
+      "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==",
       "dev": true
     },
     "which": {
@@ -2929,7 +2949,17 @@
     "wrappy": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
-      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
+      "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
+      "dev": true
+    },
+    "write": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz",
+      "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==",
+      "dev": true,
+      "requires": {
+        "mkdirp": "^0.5.1"
+      }
     },
     "write": {
       "version": "1.0.3",
@@ -2952,11 +2982,6 @@
         "typedarray-to-buffer": "^3.1.5"
       }
     },
-    "ws": {
-      "version": "7.3.1",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz",
-      "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA=="
-    },
     "y18n": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
@@ -3129,15 +3154,6 @@
           }
         }
       }
-    },
-    "yauzl": {
-      "version": "2.10.0",
-      "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
-      "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
-      "requires": {
-        "buffer-crc32": "~0.2.3",
-        "fd-slicer": "~1.1.0"
-      }
     }
   }
 }

package.json

@@ -1,7 +1,7 @@
 {
   "main": "./app/index.js",
   "name": "f95api",
-  "version": "1.3.5",
+  "version": "1.5.0",
   "author": {
     "name": "Millennium Earl"
   },
@@ -19,7 +19,10 @@
     "scraping",
     "login",
     "game",
-    "games"
+    "games",
+    "data",
+    "userdata",
+    "user data"
   ],
   "scripts": {
     "unit-test-mocha": "nyc --reporter=text mocha './test/index-test.js'",
@@ -31,11 +34,13 @@
     "node": ">=10.0"
   },
   "dependencies": {
+    "axios": "^0.21.0",
+    "axios-cookiejar-support": "^1.0.1",
+    "cheerio": "^1.0.0-rc.3",
     "ky": "^0.24.0",
     "ky-universal": "^0.8.2",
     "log4js": "^6.3.0",
-    "node-html-parser": "^1.3.1",
+    "tough-cookie": "^4.0.0"
-    "puppeteer": "^5.3.1"
   },
   "devDependencies": {
     "babel-eslint": "^10.1.0",
@@ -44,7 +49,6 @@
     "eslint": "^7.12.1",
     "lodash": "^4.17.20",
     "mocha": "^8.1.3",
-    "nyc": "^15.1.0",
+    "nyc": "^15.1.0"
-    "sleep": "^6.3.0"
   }
 }

test/index-test.js

@@ -1,290 +1,36 @@
 "use strict";
 
-// Core modules
+// Test suite
-const fs = require("fs");
+const api = require("./suites/api-test.js").suite;
+const credentials = require("./suites/credentials-test.js").suite;
+const network = require("./suites/network-helper-test.js").suite;
+const scraper = require("./suites/scraper-test.js").suite;
+const searcher = require("./suites/searcher-test.js").suite;
+const uScraper = require("./suites/user-scraper-test.js").suite;
 
-// Public modules from npm
+describe("Test basic function", function testBasic() {
-const _ = require("lodash");
+    //#region Set-up
-const expect = require("chai").expect;
+    this.timeout(15000); // All tests in this suite get 15 seconds before timeout
-const sleep = require("sleep");
+    //#endregion Set-up
-const dotenv = require("dotenv");
 
-// Modules from file
+    describe("Test credentials class", credentials.bind(this));
-const urlHelper = require("../app/scripts/url-helper.js");
+    describe("Test network helper", network.bind(this));
-const F95API = require("../app/index.js");
-
-// Configure the .env reader
-dotenv.config();
-
-const COOKIES_SAVE_PATH = "./f95cache/cookies.json";
-const ENGINES_SAVE_PATH = "./f95cache/engines.json";
-const STATUSES_SAVE_PATH = "./f95cache/statuses.json";
-const USERNAME = process.env.F95_USERNAME;
-const PASSWORD = process.env.F95_PASSWORD;
-const FAKE_USERNAME = "FakeUsername091276";
-const FAKE_PASSWORD = "fake_password";
-
-//F95API.debug(false);
-
-function randomSleep() {
-  const random = Math.floor(Math.random() * 500) + 50;
-  sleep.msleep(500 + random);
-}
-
-describe("Login without cookies", function () {
-  //#region Set-up
-  this.timeout(30000); // All tests in this suite get 30 seconds before timeout
-
-  before("Set isolation", function () {
-    F95API.setIsolation(true);
-  });
-
-  beforeEach("Remove all cookies", function () {
-    // Runs before each test in this block
-    if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH);
-    if (F95API.isLogged()) F95API.logout();
-  });
-  //#endregion Set-up
-
-  let testOrder = 0;
-
-  it("Test with valid credentials", async function () {
-    // Gain exclusive use of the cookies
-    while (testOrder !== 0) randomSleep();
-
-    const result = await F95API.login(USERNAME, PASSWORD);
-    expect(result.success).to.be.true;
-    expect(result.message).equal("Authentication successful");
-
-    testOrder = 1;
-  });
-  it("Test with invalid username", async function () {
-    // Gain exclusive use of the cookies
-    while (testOrder !== 1) randomSleep();
-
-    const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD);
-    expect(result.success).to.be.false;
-    expect(result.message).to.equal("Incorrect username");
-
-    testOrder = 2;
-  });
-  it("Test with invalid password", async function () {
-    // Gain exclusive use of the cookies
-    while (testOrder !== 2) randomSleep();
-
-    const result = await F95API.login(USERNAME, FAKE_PASSWORD);
-    expect(result.success).to.be.false;
-    expect(result.message).to.equal("Incorrect password");
-
-    testOrder = 3;
-  });
-  it("Test with invalid credentials", async function () {
-    // Gain exclusive use of the cookies
-    while (testOrder !== 3) randomSleep();
-
-    const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD);
-    expect(result.success).to.be.false;
-    expect(result.message).to.equal("Incorrect username"); // It should first check the username
-
-    testOrder = 4;
-  });
 });
 
-describe("Login with cookies", function () {
+describe("Test F95 modules", function testF95Modules() {
-  //#region Set-up
+    //#region Set-up
-  this.timeout(30000); // All tests in this suite get 30 seconds before timeout
+    this.timeout(15000); // All tests in this suite get 15 seconds before timeout
+    //#endregion Set-up
     
-  before("Log in to create cookies then logout", async function () {
+    describe("Test scraper methods", scraper.bind(this));
-    // Runs once before the first test in this block
+    describe("Test searcher methods", searcher.bind(this));
-    if (!fs.existsSync(COOKIES_SAVE_PATH))
+    describe("Test user scraper methods", uScraper.bind(this));
-      await F95API.login(USERNAME, PASSWORD); // Download cookies
-    if (F95API.isLogged()) F95API.logout();
-  });
-  //#endregion Set-up
-
-  it("Test with valid credentials", async function () {
-    const result = await F95API.login(USERNAME, PASSWORD);
-    expect(result.success).to.be.true;
-    expect(result.message).equal("Logged with cookies");
-  });
 });
 
-describe("Load base data without cookies", function () {
+describe("Test complete API", function testAPI() {
-  //#region Set-up
+    //#region Set-up
-  this.timeout(30000); // All tests in this suite get 30 seconds before timeout
+    this.timeout(15000); // All tests in this suite get 15 seconds before timeout
+    //#endregion Set-up
 
-  before("Delete cache if exists", function () {
+    describe("Test API", api.bind(this));
-    // Runs once before the first test in this block
-    if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH);
-    if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH);
-  });
-  //#endregion Set-up
-
-  it("With login", async function () {
-    const loginResult = await F95API.login(USERNAME, PASSWORD);
-    expect(loginResult.success).to.be.true;
-
-    const result = await F95API.loadF95BaseData();
-
-    const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH);
-    const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH);
-
-    expect(result).to.be.true;
-    expect(enginesCacheExists).to.be.true;
-    expect(statusesCacheExists).to.be.true;
-  });
-
-  it("Without login", async function () {
-    if (F95API.isLogged()) F95API.logout();
-    const result = await F95API.loadF95BaseData();
-    expect(result).to.be.false;
-  });
-});
-
-describe("Search game data", function () {
-  //#region Set-up
-  this.timeout(60000); // All tests in this suite get 60 seconds before timeout
-
-  beforeEach("Prepare API", function () {
-    // Runs once before the first test in this block
-    if (F95API.isLogged()) F95API.logout();
-  });
-  //#endregion Set-up
-
-  let testGame = null;
-
-  it("Search game when logged", async function () {
-    const loginResult = await F95API.login(USERNAME, PASSWORD);
-    expect(loginResult.success).to.be.true;
-
-    const loadResult = await F95API.loadF95BaseData();
-    expect(loadResult).to.be.true;
-
-    // This test depend on the data on F95Zone at
-    // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/
-    const gamesList = await F95API.getGameData("Kingdom of Deception", false);
-    expect(gamesList.length, "Should find only the game").to.equal(1);
-    const result = gamesList[0];
-    const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png";
-
-    // Test only the main information
-    expect(result.name).to.equal("Kingdom of Deception");
-    expect(result.author).to.equal("Hreinn Games");
-    expect(result.isMod, "Should be false").to.be.false;
-    expect(result.engine).to.equal("REN'PY");
-    expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image?
-    testGame = Object.assign({}, result);
-  });
-  it("Search game when not logged", async function () {
-    const result = await F95API.getGameData("Kingdom of Deception", false);
-    expect(result, "Without being logged should return null").to.be.null;
-  });
-  it("Test game serialization", function () {
-    const json = JSON.stringify(testGame);
-    const parsedGameInfo = JSON.parse(json);
-    const result = _.isEqual(parsedGameInfo, testGame);
-    expect(result).to.be.true;
-  });
-});
-
-describe("Load user data", function () {
-  //#region Set-up
-  this.timeout(30000); // All tests in this suite get 30 seconds before timeout
-  //#endregion Set-up
-
-  it("Retrieve when logged", async function () {
-    // Login
-    await F95API.login(USERNAME, PASSWORD);
-
-    // Then retrieve user data
-    const data = await F95API.getUserData();
-
-    expect(data).to.exist;
-    expect(data.username).to.equal(USERNAME);
-  });
-  it("Retrieve when not logged", async function () {
-    // Logout
-    if (F95API.isLogged()) F95API.logout();
-
-    // Try to retrieve user data
-    const data = await F95API.getUserData();
-
-    expect(data).to.be.null;
-  });
-});
-
-describe("Check game update", function () {
-  //#region Set-up
-  this.timeout(30000); // All tests in this suite get 30 seconds before timeout
-  //#endregion Set-up
-
-  it("Get online game and verify that no update exists", async function () {
-    const loginResult = await F95API.login(USERNAME, PASSWORD);
-    expect(loginResult.success).to.be.true;
-
-    const loadResult = await F95API.loadF95BaseData();
-    expect(loadResult).to.be.true;
-
-    // This test depend on the data on F95Zone at
-    // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/
-    const result = (await F95API.getGameData("Kingdom of Deception", false))[0];
-
-    const update = await F95API.chekIfGameHasUpdate(result);
-    expect(update).to.be.false;
-  });
-
-  it("Verify that update exists from old URL", async function () {
-    const loginResult = await F95API.login(USERNAME, PASSWORD);
-    expect(loginResult.success).to.be.true;
-
-    // This test depend on the data on F95Zone at
-    // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/
-    const url =
-      "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/";
-    const result = await F95API.getGameDataFromURL(url);
-    result.version = "0.9600";
-
-    const update = await F95API.chekIfGameHasUpdate(result);
-    expect(update).to.be.true;
-  });
-});
-
-describe("Test url-helper", function () {
-  //#region Set-up
-  this.timeout(30000); // All tests in this suite get 30 seconds before timeout
-  //#endregion Set-up
-
-  it("Check if URL exists", async function () {
-    // Check generic URLs...
-    let exists = await urlHelper.urlExists("https://www.google.com/");
-    expect(exists, "Complete valid URL").to.be.true;
-
-    exists = await urlHelper.urlExists("www.google.com");
-    expect(exists, "URl without protocol prefix").to.be.false;
-
-    exists = await urlHelper.urlExists("https://www.google/");
-    expect(exists, "URL without third level domain").to.be.false;
-
-    // Now check for more specific URLs (with redirect)...
-    exists = await urlHelper.urlExists(
-      "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"
-    );
-    expect(exists, "URL with redirect without check").to.be.true;
-
-    exists = await urlHelper.urlExists(
-      "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/",
-      true
-    );
-    expect(exists, "URL with redirect with check").to.be.false;
-  });
-
-  it("Check if URL belong to the platform", async function () {
-    let belong = urlHelper.isF95URL(
-      "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"
-    );
-    expect(belong).to.be.true;
-
-    belong = urlHelper.isF95URL("https://www.google/");
-    expect(belong).to.be.false;
-  });
 });

test/suites/api-test.js

@@ -0,0 +1,64 @@
+"use strict";
+
+// Public module from npm
+const expect = require("chai").expect;
+const dotenv = require("dotenv");
+const {
+    isEqual
+} = require("lodash");
+
+// Modules from file
+const F95API = require("../../app/index.js");
+
+// Configure the .env reader
+dotenv.config();
+
+// Global variables
+const USERNAME = process.env.F95_USERNAME;
+const PASSWORD = process.env.F95_PASSWORD;
+
+module.exports.suite = function suite() {
+    // Global suite variables
+    const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/";
+
+    it("Test login", async function testLogin() {
+        const result = await F95API.login(USERNAME, PASSWORD);
+        expect(result.success).to.be.true;
+        expect(F95API.isLogged()).to.be.true;
+    });
+
+    it("Test user data fetching", async function testUserDataFetch() {
+        const userdata = await F95API.getUserData();
+        expect(userdata.username).to.be.equal(USERNAME);
+    });
+
+    it("Test game update checking", async function testGameUpdateCheck() {
+        // We force the creation of a GameInfo object, 
+        // knowing that the checkIfGameHasUpdate() function 
+        // only needs the game URL
+        const info = new F95API.GameInfo();
+
+        // The gameURL identifies a game for which we know there is an update
+        info.url = gameURL;
+
+        // Check for updates
+        const update = await F95API.checkIfGameHasUpdate(info);
+        expect(update).to.be.true;
+    });
+
+    it("Test game data fetching", async function testGameDataFetch() {
+        // Search a game by name
+        const gameList = await F95API.getGameData("perverted education", false);
+
+        // We know that there is only one game with the selected name
+        expect(gameList.length).to.be.equal(1, `There should be only one game, not ${gameList.length}`);
+        const game = gameList[0];
+
+        // Than we fetch a game from URL
+        const gameFromURL = await F95API.getGameDataFromURL(game.url);
+        
+        // The two games must be equal
+        const equal = isEqual(game, gameFromURL);
+        expect(equal).to.be.true;
+    });
+};

test/suites/credentials-test.js

@@ -0,0 +1,52 @@
+"use strict";
+
+// Public module from npm
+const expect = require("chai").expect;
+
+// Modules from file
+const Credentials = require("../../app/scripts/classes/credentials.js");
+
+module.exports.suite = function suite() {
+    it("Check token formatting", async function testValidToken() {
+        // Token example:
+        // 1604309951,0338213c00fcbd894fd9415e6ba08403
+        // 1604309986,ebdb75502337699381f0f55c86353555
+        // 1604310008,2d50d55808e5ec3a157ec01953da9d26
+
+        // Fetch token (is a GET request, we don't need the credentials)
+        const cred = new Credentials(null, null);
+        await cred.fetchToken();
+
+        // Parse token for assert
+        const splitted = cred.token.split(",");
+        const unique = splitted[0];
+        const hash = splitted[1];
+        expect(splitted.length).to.be.equal(2, "The token consists of two parts");
+
+        // Check type of parts
+        expect(isNumeric(unique)).to.be.true;
+        expect(isNumeric(hash)).to.be.false;
+
+        // The second part is most probably the MD5 hash of something
+        expect(hash.length).to.be.equal(32, "Hash should have 32 hex chars");
+    });
+};
+
+//#region Private methods
+/**
+ * @private
+ * Check if a string is a number
+ * @param {String} str 
+ * @author Dan, Ben Aston
+ * @see https://preview.tinyurl.com/y46jqwkt
+ */
+function isNumeric(str) {
+    // We only process strings!  
+    if (typeof str != "string") return false;
+
+    // Use type coercion to parse the _entirety_ of the string 
+    // (`parseFloat` alone does not do this) and ensure strings 
+    // of whitespace fail
+    return !isNaN(str) && !isNaN(parseFloat(str));
+}
+//#endregion

test/suites/network-helper-test.js

@@ -0,0 +1,107 @@
+"use strict";
+
+// Public module from npm
+const expect = require("chai").expect;
+const dotenv = require("dotenv");
+
+// Modules from file
+const Credentials = require("../../app/scripts/classes/credentials.js");
+const networkHelper = require("../../app/scripts/network-helper.js");
+const {
+    F95_SEARCH_URL
+} = require("../../app/scripts/constants/url.js");
+
+// Configure the .env reader
+dotenv.config();
+
+// Global variables
+const USERNAME = process.env.F95_USERNAME;
+const PASSWORD = process.env.F95_PASSWORD;
+const FAKE_USERNAME = "Fake_Username091276";
+const FAKE_PASSWORD = "fake_password";
+
+module.exports.suite = function suite() {
+    // Global suite variables
+    const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/";
+    
+    it("Check if URL exists", async function checkURLExistence() {
+        // Check generic URLs...
+        let exists = await networkHelper.urlExists("https://www.google.com/");
+        expect(exists, "Complete valid URL").to.be.true;
+
+        exists = await networkHelper.urlExists("www.google.com");
+        expect(exists, "URl without protocol prefix").to.be.false;
+
+        exists = await networkHelper.urlExists("https://www.google/");
+        expect(exists, "URL without third level domain").to.be.false;
+
+        // Now check for more specific URLs (with redirect)...
+        exists = await networkHelper.urlExists(gameURL);
+        expect(exists, "URL with redirect without check").to.be.true;
+
+        exists = await networkHelper.urlExists(gameURL, true);
+        expect(exists, "URL with redirect with check").to.be.false;
+    });
+
+    it("Check if URL belong to the platform", function checkIfURLIsF95() {
+        let belong = networkHelper.isF95URL(gameURL);
+        expect(belong).to.be.true;
+
+        belong = networkHelper.isF95URL("https://www.google/");
+        expect(belong).to.be.false;
+    });
+
+    it("Enforce secure URLs", function testSecureURLEnforcement() {
+        // This URL is already secure, should remain the same
+        let enforced = networkHelper.enforceHttpsUrl(gameURL);
+        expect(enforced).to.be.equal(gameURL, "The game URL is already secure");
+
+        // This URL is not secure
+        enforced = networkHelper.enforceHttpsUrl("http://www.google.com");
+        expect(enforced).to.be.equal("https://www.google.com", "The URL was without SSL/TLS (HTTPs)");
+
+        // Finally, we check when we pass a invalid URL
+        enforced = networkHelper.enforceHttpsUrl("http://invalidurl");
+        expect(enforced).to.be.null;
+    });
+
+    it("Check URL redirect", async function checkURLRedirect() {
+        // gameURL is an old URL it has been verified that it generates a redirect
+        const redirectURL = await networkHelper.getUrlRedirect(gameURL);
+        expect(redirectURL).to.not.be.equal(gameURL, "The original URL has redirect");
+
+        // If we recheck the new URL, we find that no redirect happens
+        const secondRedirectURL = await networkHelper.getUrlRedirect(redirectURL);
+        expect(secondRedirectURL).to.be.equal(redirectURL, "The URL has no redirect");
+    });
+
+    it("Check response to GET request", async function testGETResponse() {
+        // We should be able to fetch a game page
+        let response = await networkHelper.fetchGETResponse(gameURL);
+        expect(response.status).to.be.equal(200, "The operation must be successful");
+
+        // We should NOT be able to fetch the search page (we must be logged)
+        response = await networkHelper.fetchGETResponse(F95_SEARCH_URL);
+        expect(response).to.be.null;
+    });
+
+    it("Test for authentication to platform", async function testAuthentication() {
+        // Try to authenticate with valid credentials
+        const creds = new Credentials(USERNAME, PASSWORD);
+        await creds.fetchToken();
+        const validResult = await networkHelper.authenticate(creds);
+        expect(validResult.success).to.be.true;
+
+        // Now we use fake credentials
+        const fakeCreds = new Credentials(FAKE_USERNAME, FAKE_PASSWORD);
+        await fakeCreds.fetchToken();
+        const invalidResult = await networkHelper.authenticate(fakeCreds, true);
+        expect(invalidResult.success).to.be.false;
+    });
+
+    it("Test fetching HTML", async function testFetchHTML() {
+        // This should return the HTML code of the page
+        const html = await networkHelper.fetchHTML(gameURL);
+        expect(html.startsWith("<!DOCTYPE html>")).to.be.true;
+    });
+};

test/suites/scraper-test.js

@@ -0,0 +1,42 @@
+"use strict";
+
+// Public module from npm
+const expect = require("chai").expect;
+const { isEqual } = require("lodash");
+const GameInfo = require("../../app/scripts/classes/game-info.js");
+
+// Modules from file
+const scraper = require("../../app/scripts/scraper.js");
+
+module.exports.suite = function suite() {
+    // Global suite variables
+    const gameURL = "https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/";
+    const previewSrc = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png";
+    
+    it("Search game", async function () {
+        // This test depend on the data on F95Zone at gameURL
+        const result = await scraper.getGameInfo(gameURL);
+        
+        // Test only the main information
+        expect(result.name).to.equal("Kingdom of Deception");
+        expect(result.author).to.equal("Hreinn Games");
+        expect(result.isMod, "Should be false").to.be.false;
+        expect(result.engine).to.equal("Ren'Py");
+        expect(result.previewSrc).to.equal(previewSrc, "Preview not equals");
+    });
+
+    it("Test game serialization", async function testGameSerialization() {
+        // This test depend on the data on F95Zone at gameURL
+        const testGame = await scraper.getGameInfo(gameURL);
+
+        // Serialize...
+        const json = JSON.stringify(testGame);
+
+        // Deserialize...
+        const parsedGameInfo = GameInfo.fromJSON(json);
+
+        // Compare with lodash
+        const result = isEqual(parsedGameInfo, testGame);
+        expect(result).to.be.true;
+    });
+};

test/suites/searcher-test.js

@@ -0,0 +1,64 @@
+"use strict";
+
+// Public module from npm
+const expect = require("chai").expect;
+const dotenv = require("dotenv");
+
+// Modules from file
+const Credentials = require("../../app/scripts/classes/credentials.js");
+const searcher = require("../../app/scripts/searcher.js");
+const {
+    authenticate
+} = require("../../app/scripts/network-helper.js");
+
+// Configure the .env reader
+dotenv.config();
+
+// Global variables
+const USERNAME = process.env.F95_USERNAME;
+const PASSWORD = process.env.F95_PASSWORD;
+
+module.exports.suite = function suite() {
+    // TODO: 
+    // This method should delete the store F95Zone cookies, 
+    // but what if the other tests require them?
+    
+    // it("Search game when not logged", async function searchGameWhenNotLogged() {
+    //     // Search for a game that we know has only one result
+    //     // but without logging in first
+    //     const urls = await searcher.searchGame("kingdom of deception");
+    //     expect(urls.lenght).to.be.equal(0, "There should not be any URL");
+    // });
+
+    it("Search game", async function searchGame() {
+        // Authenticate
+        const result = await auth();
+        expect(result.success, "Authentication should be successful").to.be.true;
+
+        // Search for a game that we know has only one result
+        const urls = await searcher.searchGame("kingdom of deception");
+        expect(urls.length).to.be.equal(1, `There should be only one game result instead of ${urls.length}`);
+    });
+
+    it("Search mod", async function searchMod() {
+        // Authenticate
+        const result = await auth();
+        expect(result.success, "Authentication should be successful").to.be.true;
+
+        // Search for a mod that we know has only one result
+        const urls = await searcher.searchMod("kingdom of deception jdmod");
+        expect(urls.length).to.be.equal(1, `There should be only one mod result instead of ${urls.length}`);
+    });
+};
+
+//#region Private methods
+/**
+ * @private
+ * Simple wrapper for authentication.
+ */
+async function auth() {
+    const creds = new Credentials(USERNAME, PASSWORD);
+    await creds.fetchToken();
+    return await authenticate(creds);
+}
+//#endregion Private methods

test/suites/user-scraper-test.js

@@ -0,0 +1,54 @@
+"use strict";
+
+// Public module from npm
+const expect = require("chai").expect;
+const dotenv = require("dotenv");
+
+// Modules from file
+const Credentials = require("../../app/scripts/classes/credentials.js");
+const uScraper = require("../../app/scripts/user-scraper.js");
+const {
+    authenticate
+} = require("../../app/scripts/network-helper.js");
+
+// Configure the .env reader
+dotenv.config();
+
+// Global variables
+const USERNAME = process.env.F95_USERNAME;
+const PASSWORD = process.env.F95_PASSWORD;
+
+module.exports.suite = function suite() {
+    // TODO: 
+    // This method should delete the store F95Zone cookies, 
+    // but what if the other tests require them?
+
+    // it("Fetch data when not logged", async function fetchUserDataWhenLogged() {
+    //     const data = await uScraper.getUserData();
+    //     expect(data.username).to.be.equal("");
+    //     expect(data.avatarSrc).to.be.equal("");
+    //     expect(data.watchedThreads.length).to.be.equal(0);
+    // });
+
+    it("Fetch data when logged", async function fetchUserDataWhenNotLogged() {
+        // Authenticate
+        const result = await auth();
+        expect(result.success, "Authentication should be successful").to.be.true;
+
+        // We test only for the username, the other test data depends on the user logged
+        const data = await uScraper.getUserData();
+        expect(data.username).to.be.equal(USERNAME);
+    });
+};
+
+//#region Private methods
+/**
+ * @private
+ * Simple wrapper for authentication.
+ */
+async function auth() {
+    const creds = new Credentials(USERNAME, PASSWORD);
+    await creds.fetchToken();
+    return await authenticate(creds);
+}
+//#endregion Private methods

test/user-test.js

@@ -1,15 +0,0 @@
-const F95API = require("../app/index.js");
-
-F95API.debug(true);
-main();
-
-async function main() {
-  const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK");
-
-  if (loginResult.success) {
-    await F95API.loadF95BaseData();
-    const gameData = await F95API.getGameData("a struggle with sin", false);
-    console.log(gameData);
-  }
-  F95API.logout();
-}