Fixed unit tests, better game search and improved speed

.gitignore

@@ -1,3 +1,4 @@
 node_modules/
 f95cache/
 .nyc_output/
+.env

app/index.js

@@ -113,12 +113,8 @@ module.exports.login = async function (username, password) {
   if (shared.debug)
     console.log("No saved sessions or expired session, login on the platform");
 
-  let browser = null;
+  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  if (shared.isolation) browser = await prepareBrowser();
+  let browser = shared.isolation ? await prepareBrowser() : _browser;
-  else {
-    if (_browser === null) _browser = await prepareBrowser();
-    browser = _browser;
-  }
 
   let result = await loginF95(browser, username, password);
   shared.isLogged = result.success;
@@ -141,7 +137,7 @@ module.exports.login = async function (username, password) {
  * @returns {Promise<Boolean>} Result of the operation
  */
 module.exports.loadF95BaseData = async function () {
-  if (!shared.isLogged) {
+  if (!shared.isLogged || !shared.cookies) {
     console.warn("User not authenticated, unable to continue");
     return false;
   }
@@ -149,12 +145,8 @@ module.exports.loadF95BaseData = async function () {
   if (shared.debug) console.log("Loading base data...");
 
   // Prepare a new web page
-  let browser = null;
+  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  if (shared.isolation) browser = await prepareBrowser();
+  let browser = shared.isolation ? await prepareBrowser() : _browser;
-  else {
-    if (_browser === null) _browser = await prepareBrowser();
-    browser = _browser;
-  }
 
   let page = await preparePage(browser); // Set new isolated page
   await page.setCookie(...shared.cookies); // Set cookies to avoid login
@@ -194,14 +186,25 @@ module.exports.loadF95BaseData = async function () {
  * @returns {Promise<Boolean>} true if an update is available, false otherwise
  */
 module.exports.chekIfGameHasUpdate = async function (info) {
-  if (!shared.isLogged) {
+  if (!shared.isLogged || !shared.cookies) {
     console.warn("user not authenticated, unable to continue");
     return info.version;
   }
 
   // F95 change URL at every game update,
-  // so if the URL is the same no update is available
+  // so if the URL is different an update is available
-  return !(await urlExists(info.f95url, true));
+  let exists = await urlExists(info.f95url, true);
+  if (!exists) return true;
+
+  // Parse version from title
+  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
+  let browser = shared.isolation ? await prepareBrowser() : _browser;
+
+  let onlineVersion = await scraper.getGameVersionFromTitle(browser, info);
+
+  if (shared.isolation) await browser.close();
+
+  return onlineVersion.toUpperCase() !== info.version.toUpperCase();
 };
 /**
  * @public
@@ -213,18 +216,14 @@ module.exports.chekIfGameHasUpdate = async function (info) {
  * an identified game (in the case of homonymy). If no games were found, null is returned
  */
 module.exports.getGameData = async function (name, includeMods) {
-  if (!shared.isLogged) {
+  if (!shared.isLogged || !shared.cookies) {
     console.warn("user not authenticated, unable to continue");
     return null;
   }
 
   // Gets the search results of the game being searched for
-  let browser = null;
+  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  if (shared.isolation) browser = await prepareBrowser();
+  let browser = shared.isolation ? await prepareBrowser() : _browser;
-  else {
-    if (_browser === null) _browser = await prepareBrowser();
-    browser = _browser;
-  }
   let urlList = await searcher.getSearchGameResults(browser, name);
 
   // Process previous partial results
@@ -254,7 +253,7 @@ module.exports.getGameData = async function (name, includeMods) {
  * @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
  */
 module.exports.getGameDataFromURL = async function (url) {
-  if (!shared.isLogged) {
+  if (!shared.isLogged || !shared.cookies) {
     console.warn("user not authenticated, unable to continue");
     return null;
   }
@@ -264,12 +263,8 @@ module.exports.getGameDataFromURL = async function (url) {
   if (!isF95URL(url)) throw url + " is not a valid F95Zone URL";
 
   // Gets the search results of the game being searched for
-  let browser = null;
+  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  if (shared.isolation) browser = await prepareBrowser();
+  let browser = shared.isolation ? await prepareBrowser() : _browser;
-  else {
-    if (_browser === null) _browser = await prepareBrowser();
-    browser = _browser;
-  }
 
   // Get game data
   let result = await scraper.getGameInfo(browser, url);
@@ -284,18 +279,14 @@ module.exports.getGameDataFromURL = async function (url) {
  * @returns {Promise<UserData>} Data of the user currently logged in or null if an error arise
  */
 module.exports.getUserData = async function () {
-  if (!shared.isLogged) {
+  if (!shared.isLogged || !shared.cookies) {
     console.warn("user not authenticated, unable to continue");
     return null;
   }
 
   // Prepare a new web page
-  let browser = null;
+  if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
-  if (shared.isolation) browser = await prepareBrowser();
+  let browser = shared.isolation ? await prepareBrowser() : _browser;
-  else {
-    if (_browser === null) _browser = await prepareBrowser();
-    browser = _browser;
-  }
   let page = await preparePage(browser); // Set new isolated page
   await page.setCookie(...shared.cookies); // Set cookies to avoid login
   await page.goto(constURLs.F95_BASE_URL); // Go to base page
@@ -333,8 +324,8 @@ module.exports.getUserData = async function () {
  * Logout from the current user and gracefully close shared browser.
  * You **must** be logged in to the portal before calling this method.
  */
-module.exports.logout = function () {
+module.exports.logout = async function () {
-  if (!shared.isLogged) {
+  if (!shared.isLogged || !shared.cookies) {
     console.warn("user not authenticated, unable to continue");
     return;
   }
@@ -342,7 +333,8 @@ module.exports.logout = function () {
 
   // Gracefully close shared browser
   if (!shared.isolation && _browser !== null) {
-    _browser.close().then(() => (_browser = null));
+    await _browser.close();
+    _browser = null;
   }
 };
 //#endregion
@@ -483,6 +475,7 @@ async function loginF95(browser, username, password) {
   await page.waitForSelector(selectors.USERNAME_INPUT);
   await page.waitForSelector(selectors.PASSWORD_INPUT);
   await page.waitForSelector(selectors.LOGIN_BUTTON);
+
   await page.type(selectors.USERNAME_INPUT, username); // Insert username
   await page.type(selectors.PASSWORD_INPUT, password); // Insert password
   await page.click(selectors.LOGIN_BUTTON); // Click on the login button

app/scripts/classes/game-download.js

@@ -1,3 +1,5 @@
+/* istanbul ignore file */
+
 "use strict";
 
 // Core modules

app/scripts/game-scraper.js

@@ -71,6 +71,36 @@ module.exports.getGameInfo = async function (browser, url) {
   return info;
 };
 
+/**
+ * Obtain the game version without parsing again all the data of the game.
+ * @param {puppeteer.Browser} browser Browser object used for navigation
+ * @param {GameInfo} info Information about the game
+ * @returns {Promise<String>} Online version of the game
+ */
+module.exports.getGameVersionFromTitle = async function(browser, info) {
+  let page = await preparePage(browser); // Set new isolated page
+  await page.setCookie(...shared.cookies); // Set cookies to avoid login
+  await page.goto(info.f95url, {
+    waitUntil: shared.WAIT_STATEMENT,
+  }); // Go to the game page and wait until it loads
+
+  // Get the title
+  let titleHTML = await page.evaluate(
+    /* istanbul ignore next */
+    (selector) =>
+    document.querySelector(selector).innerHTML,
+    selectors.GAME_TITLE
+  );
+  let title = HTMLParser.parse(titleHTML).childNodes.pop().rawText;
+
+  // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR]
+  let startIndex = title.indexOf("[") + 1;
+  let endIndex = title.indexOf("]", startIndex);
+  let version = title.substring(startIndex, endIndex).trim().toUpperCase();
+  if(version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence
+  return version;
+}
+
 //#region Private methods
 /**
  * @private

app/scripts/game-searcher.js

@@ -70,7 +70,7 @@ async function getOnlyGameThreads(page, divHandle) {
     
     // Get the forum where the thread was posted
     let forum = await getMembershipForum(page, forumHandle);
-    if(forum !== "GAMES") return null;
+    if(forum !== "GAMES" && forum != "MODS") return null;
 
     // Get the URL of the thread from the title
     return await getThreadURL(page, titleHandle);

app/scripts/puppeteer-helper.js

@@ -36,7 +36,8 @@ module.exports.preparePage = async function(browser) {
     await page.setRequestInterception(true);
     page.on('request', (request) => {
         if (request.resourceType() === 'image') request.abort();
-        // else if(request.resourceType == 'font') request.abort();
+        else if(request.resourceType == 'font') request.abort();
+        // else if (request.resourceType() == 'stylesheet') request.abort();
         // else if(request.resourceType == 'media') request.abort();
         else request.continue();
     });

app/scripts/urls-helper.js

@@ -30,6 +30,8 @@ module.exports.isStringAValidURL = function (url) {
     new URL(url);
     return true;
   } catch (err) {
+    console.error(err);
+    console.log(url);
     return false;
   }
 };

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "f95api",
-  "version": "1.1.2",
+  "version": "1.2.3",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -612,6 +612,12 @@
       "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==",
       "dev": true
     },
+    "dotenv": {
+      "version": "8.2.0",
+      "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
+      "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==",
+      "dev": true
+    },
     "emoji-regex": {
       "version": "8.0.0",
       "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
@@ -1596,6 +1602,12 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
       "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
     },
+    "nan": {
+      "version": "2.14.1",
+      "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
+      "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==",
+      "dev": true
+    },
     "node-fetch": {
       "version": "3.0.0-beta.9",
       "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0-beta.9.tgz",
@@ -1963,6 +1975,15 @@
       "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
       "dev": true
     },
+    "sleep": {
+      "version": "6.3.0",
+      "resolved": "https://registry.npmjs.org/sleep/-/sleep-6.3.0.tgz",
+      "integrity": "sha512-+WgYl951qdUlb1iS97UvQ01pkauoBK9ML9I/CMPg41v0Ze4EyMlTgFTDDo32iYj98IYqxIjDMRd+L71lawFfpQ==",
+      "dev": true,
+      "requires": {
+        "nan": "^2.14.1"
+      }
+    },
     "source-map": {
       "version": "0.5.7",
       "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",

package.json

@@ -1,7 +1,7 @@
 {
   "main": "./app/index.js",
   "name": "f95api",
-  "version": "1.1.2",
+  "version": "1.2.3",
   "author": {
     "name": "Millennium Earl"
   },
@@ -22,9 +22,9 @@
     "games"
   ],
   "scripts": {
-    "unit-test-mocha": "nyc --reporter=text mocha",
+    "unit-test-mocha": "nyc --reporter=text mocha './test/index-test.js'",
     "report-coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov -t 38ad72bf-a29d-4c2e-9827-96cbe037afd2",
-    "test": "node ./test/test.js",
+    "test": "node ./test/user-test.js",
     "deploy": "npm pack"
   },
   "engines": {
@@ -38,7 +38,9 @@
   },
   "devDependencies": {
     "chai": "^4.2.0",
+    "dotenv": "^8.2.0",
     "mocha": "^8.1.3",
-    "nyc": "^15.1.0"
+    "nyc": "^15.1.0",
+    "sleep": "^6.3.0"
   }
 }

test/index-test.js

@@ -3,47 +3,81 @@
 const expect = require("chai").expect;
 const F95API = require("../app/index");
 const fs = require("fs");
+const sleep = require('sleep');
+const dotenv = require('dotenv');
+dotenv.config();
 
 const COOKIES_SAVE_PATH = "./f95cache/cookies.json";
 const ENGINES_SAVE_PATH = "./f95cache/engines.json";
 const STATUSES_SAVE_PATH = "./f95cache/statuses.json";
-const USERNAME = "MillenniumEarl";
+const USERNAME = process.env.F95_USERNAME;
-const PASSWORD = "f9vTcRNuvxj4YpK";
+const PASSWORD = process.env.F95_PASSWORD;
 const FAKE_USERNAME = "FakeUsername091276";
 const FAKE_PASSWORD = "fake_password";
 
-F95API.setIsolation(true);
+F95API.debug(false);
+
+function randomSleep() {
+  let random = Math.floor(Math.random() * 500) + 50;
+  sleep.msleep(500 + random);
+}
 
 describe("Login without cookies", function () {
   //#region Set-up
   this.timeout(30000); // All tests in this suite get 30 seconds before timeout
 
+  before("Set isolation", function() {
+    F95API.setIsolation(true);
+  })
+
   beforeEach("Remove all cookies", function () {
     // Runs before each test in this block
     if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH);
-    F95API.logout();
+    if (F95API.isLogged()) F95API.logout();
   });
   //#endregion Set-up
 
+  let testOrder = 0;
+
   it("Test with valid credentials", async function () {
+    // Gain exclusive use of the cookies
+    while (testOrder !== 0) randomSleep();
+    
     const result = await F95API.login(USERNAME, PASSWORD);
     expect(result.success).to.be.true;
     expect(result.message).equal("Authentication successful");
+
+    testOrder = 1;
   });
   it("Test with invalid username", async function () {
+    // Gain exclusive use of the cookies
+    while (testOrder !== 1) randomSleep();
+
     const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD);
     expect(result.success).to.be.false;
     expect(result.message).to.equal("Incorrect username");
+
+    testOrder = 2;
   });
   it("Test with invalid password", async function () {
+    // Gain exclusive use of the cookies
+    while (testOrder !== 2) randomSleep();
+
     const result = await F95API.login(USERNAME, FAKE_PASSWORD);
     expect(result.success).to.be.false;
     expect(result.message).to.equal("Incorrect password");
+
+    testOrder = 3;
   });
   it("Test with invalid credentials", async function () {
+    // Gain exclusive use of the cookies
+    while (testOrder !== 3) randomSleep();
+
     const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD);
     expect(result.success).to.be.false;
     expect(result.message).to.equal("Incorrect username"); // It should first check the username
+
+    testOrder = 4;
   });
 });
 
@@ -55,7 +89,7 @@ describe("Login with cookies", function () {
     // Runs once before the first test in this block
     if (!fs.existsSync(COOKIES_SAVE_PATH))
       await F95API.login(USERNAME, PASSWORD); // Download cookies
-    F95API.logout();
+    if (F95API.isLogged()) F95API.logout();
   });
   //#endregion Set-up
 
@@ -92,7 +126,7 @@ describe("Load base data without cookies", function () {
   });
 
   it("Without login", async function () {
-    F95API.logout();
+    if (F95API.isLogged()) F95API.logout();
     let result = await F95API.loadF95BaseData();
     expect(result).to.be.false;
   });
@@ -104,7 +138,7 @@ describe("Search game data", function () {
 
   beforeEach("Prepare API", function () {
     // Runs once before the first test in this block
-    F95API.logout();
+    if (F95API.isLogged()) F95API.logout();
   });
   //#endregion Set-up
 
@@ -117,7 +151,9 @@ describe("Search game data", function () {
 
     // This test depend on the data on F95Zone at
     // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/
-    const result = (await F95API.getGameData("Kingdom of Deception", false))[0];
+    const gamesList = await F95API.getGameData("Kingdom of Deception", false);
+    expect(gamesList.length, "Should find only the game").to.equal(1);
+    const result = gamesList[0];
     let src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png";
 
     // Test only the main information
@@ -125,7 +161,7 @@ describe("Search game data", function () {
     expect(result.author).to.equal("Hreinn Games");
     expect(result.isMod, "Should be false").to.be.false;
     expect(result.engine).to.equal("REN'PY");
-    // expect(result.previewSource).to.equal(src); could be null -> Why sometimes doesn't get the image?
+    expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image?
   });
   it("Search game when not logged", async function () {
     const result = await F95API.getGameData("Kingdom of Deception", false);
@@ -150,7 +186,7 @@ describe("Load user data", function () {
   });
   it("Retrieve when not logged", async function () {
     // Logout
-    F95API.logout();
+    if (F95API.isLogged()) F95API.logout();
 
     // Try to retrieve user data
     let data = await F95API.getUserData();
@@ -164,7 +200,7 @@ describe("Check game update", function () {
   this.timeout(30000); // All tests in this suite get 30 seconds before timeout
   //#endregion Set-up
 
-  it("Get game update", async function () {
+  it("Get online game and verify that no update exists", async function () {
     const loginResult = await F95API.login(USERNAME, PASSWORD);
     expect(loginResult.success).to.be.true;
 
@@ -178,4 +214,18 @@ describe("Check game update", function () {
     let update = await F95API.chekIfGameHasUpdate(result);
     expect(update).to.be.false;
   });
+
+  it("Verify that update exists from old URL", async function () {
+    const loginResult = await F95API.login(USERNAME, PASSWORD);
+    expect(loginResult.success).to.be.true;
+
+    // This test depend on the data on F95Zone at
+    // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/
+    let url = "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/";
+    const result = await F95API.getGameDataFromURL(url);
+    result.version = "0.9600";
+
+    let update = await F95API.chekIfGameHasUpdate(result);
+    expect(update).to.be.true;
+  });
 });