From ce649f2b9f6b7b6c9f60298bd17270453dbbfe83 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 12 Oct 2020 14:11:36 +0200 Subject: [PATCH] Fixed unit tests, better game search and improved speed --- .gitignore | 3 +- app/index.js | 75 +++++++++++++--------------- app/scripts/classes/game-download.js | 2 + app/scripts/game-scraper.js | 30 +++++++++++ app/scripts/game-searcher.js | 2 +- app/scripts/puppeteer-helper.js | 3 +- app/scripts/urls-helper.js | 2 + package-lock.json | 23 ++++++++- package.json | 10 ++-- test/index-test.js | 72 ++++++++++++++++++++++---- test/{test.js => user-test.js} | 0 11 files changed, 162 insertions(+), 60 deletions(-) rename test/{test.js => user-test.js} (100%) diff --git a/.gitignore b/.gitignore index 1006417..299d660 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ node_modules/ f95cache/ -.nyc_output/ \ No newline at end of file +.nyc_output/ +.env \ No newline at end of file diff --git a/app/index.js b/app/index.js index fad52e3..9d5478d 100644 --- a/app/index.js +++ b/app/index.js @@ -113,12 +113,8 @@ module.exports.login = async function (username, password) { if (shared.debug) console.log("No saved sessions or expired session, login on the platform"); - let browser = null; - if (shared.isolation) browser = await prepareBrowser(); - else { - if (_browser === null) _browser = await prepareBrowser(); - browser = _browser; - } + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + let browser = shared.isolation ? await prepareBrowser() : _browser; let result = await loginF95(browser, username, password); shared.isLogged = result.success; @@ -141,7 +137,7 @@ module.exports.login = async function (username, password) { * @returns {Promise} Result of the operation */ module.exports.loadF95BaseData = async function () { - if (!shared.isLogged) { + if (!shared.isLogged || !shared.cookies) { console.warn("User not authenticated, unable to continue"); return false; } @@ -149,12 +145,8 @@ module.exports.loadF95BaseData = async function () { if (shared.debug) console.log("Loading base data..."); // Prepare a new web page - let browser = null; - if (shared.isolation) browser = await prepareBrowser(); - else { - if (_browser === null) _browser = await prepareBrowser(); - browser = _browser; - } + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + let browser = shared.isolation ? await prepareBrowser() : _browser; let page = await preparePage(browser); // Set new isolated page await page.setCookie(...shared.cookies); // Set cookies to avoid login @@ -194,14 +186,25 @@ module.exports.loadF95BaseData = async function () { * @returns {Promise} true if an update is available, false otherwise */ module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged) { + if (!shared.isLogged || !shared.cookies) { console.warn("user not authenticated, unable to continue"); return info.version; } // F95 change URL at every game update, - // so if the URL is the same no update is available - return !(await urlExists(info.f95url, true)); + // so if the URL is different an update is available + let exists = await urlExists(info.f95url, true); + if (!exists) return true; + + // Parse version from title + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + let browser = shared.isolation ? await prepareBrowser() : _browser; + + let onlineVersion = await scraper.getGameVersionFromTitle(browser, info); + + if (shared.isolation) await browser.close(); + + return onlineVersion.toUpperCase() !== info.version.toUpperCase(); }; /** * @public @@ -213,18 +216,14 @@ module.exports.chekIfGameHasUpdate = async function (info) { * an identified game (in the case of homonymy). If no games were found, null is returned */ module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged) { + if (!shared.isLogged || !shared.cookies) { console.warn("user not authenticated, unable to continue"); return null; } // Gets the search results of the game being searched for - let browser = null; - if (shared.isolation) browser = await prepareBrowser(); - else { - if (_browser === null) _browser = await prepareBrowser(); - browser = _browser; - } + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + let browser = shared.isolation ? await prepareBrowser() : _browser; let urlList = await searcher.getSearchGameResults(browser, name); // Process previous partial results @@ -254,7 +253,7 @@ module.exports.getGameData = async function (name, includeMods) { * @returns {Promise} Information about the game. If no game was found, null is returned */ module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged) { + if (!shared.isLogged || !shared.cookies) { console.warn("user not authenticated, unable to continue"); return null; } @@ -264,12 +263,8 @@ module.exports.getGameDataFromURL = async function (url) { if (!isF95URL(url)) throw url + " is not a valid F95Zone URL"; // Gets the search results of the game being searched for - let browser = null; - if (shared.isolation) browser = await prepareBrowser(); - else { - if (_browser === null) _browser = await prepareBrowser(); - browser = _browser; - } + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + let browser = shared.isolation ? await prepareBrowser() : _browser; // Get game data let result = await scraper.getGameInfo(browser, url); @@ -284,18 +279,14 @@ module.exports.getGameDataFromURL = async function (url) { * @returns {Promise} Data of the user currently logged in or null if an error arise */ module.exports.getUserData = async function () { - if (!shared.isLogged) { + if (!shared.isLogged || !shared.cookies) { console.warn("user not authenticated, unable to continue"); return null; } // Prepare a new web page - let browser = null; - if (shared.isolation) browser = await prepareBrowser(); - else { - if (_browser === null) _browser = await prepareBrowser(); - browser = _browser; - } + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + let browser = shared.isolation ? await prepareBrowser() : _browser; let page = await preparePage(browser); // Set new isolated page await page.setCookie(...shared.cookies); // Set cookies to avoid login await page.goto(constURLs.F95_BASE_URL); // Go to base page @@ -333,8 +324,8 @@ module.exports.getUserData = async function () { * Logout from the current user and gracefully close shared browser. * You **must** be logged in to the portal before calling this method. */ -module.exports.logout = function () { - if (!shared.isLogged) { +module.exports.logout = async function () { + if (!shared.isLogged || !shared.cookies) { console.warn("user not authenticated, unable to continue"); return; } @@ -342,7 +333,8 @@ module.exports.logout = function () { // Gracefully close shared browser if (!shared.isolation && _browser !== null) { - _browser.close().then(() => (_browser = null)); + await _browser.close(); + _browser = null; } }; //#endregion @@ -483,13 +475,14 @@ async function loginF95(browser, username, password) { await page.waitForSelector(selectors.USERNAME_INPUT); await page.waitForSelector(selectors.PASSWORD_INPUT); await page.waitForSelector(selectors.LOGIN_BUTTON); + await page.type(selectors.USERNAME_INPUT, username); // Insert username await page.type(selectors.PASSWORD_INPUT, password); // Insert password await page.click(selectors.LOGIN_BUTTON); // Click on the login button await page.waitForNavigation({ waitUntil: shared.WAIT_STATEMENT, }); // Wait for page to load - + // Prepare result let result = new LoginResult(); diff --git a/app/scripts/classes/game-download.js b/app/scripts/classes/game-download.js index 0dc9c37..b1f14af 100644 --- a/app/scripts/classes/game-download.js +++ b/app/scripts/classes/game-download.js @@ -1,3 +1,5 @@ +/* istanbul ignore file */ + "use strict"; // Core modules diff --git a/app/scripts/game-scraper.js b/app/scripts/game-scraper.js index 0134290..50bb80d 100644 --- a/app/scripts/game-scraper.js +++ b/app/scripts/game-scraper.js @@ -71,6 +71,36 @@ module.exports.getGameInfo = async function (browser, url) { return info; }; +/** + * Obtain the game version without parsing again all the data of the game. + * @param {puppeteer.Browser} browser Browser object used for navigation + * @param {GameInfo} info Information about the game + * @returns {Promise} Online version of the game + */ +module.exports.getGameVersionFromTitle = async function(browser, info) { + let page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(info.f95url, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the game page and wait until it loads + + // Get the title + let titleHTML = await page.evaluate( + /* istanbul ignore next */ + (selector) => + document.querySelector(selector).innerHTML, + selectors.GAME_TITLE + ); + let title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; + + // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] + let startIndex = title.indexOf("[") + 1; + let endIndex = title.indexOf("]", startIndex); + let version = title.substring(startIndex, endIndex).trim().toUpperCase(); + if(version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence + return version; +} + //#region Private methods /** * @private diff --git a/app/scripts/game-searcher.js b/app/scripts/game-searcher.js index ea6b3bd..57d0f95 100644 --- a/app/scripts/game-searcher.js +++ b/app/scripts/game-searcher.js @@ -70,7 +70,7 @@ async function getOnlyGameThreads(page, divHandle) { // Get the forum where the thread was posted let forum = await getMembershipForum(page, forumHandle); - if(forum !== "GAMES") return null; + if(forum !== "GAMES" && forum != "MODS") return null; // Get the URL of the thread from the title return await getThreadURL(page, titleHandle); diff --git a/app/scripts/puppeteer-helper.js b/app/scripts/puppeteer-helper.js index d346892..5d14222 100644 --- a/app/scripts/puppeteer-helper.js +++ b/app/scripts/puppeteer-helper.js @@ -36,7 +36,8 @@ module.exports.preparePage = async function(browser) { await page.setRequestInterception(true); page.on('request', (request) => { if (request.resourceType() === 'image') request.abort(); - // else if(request.resourceType == 'font') request.abort(); + else if(request.resourceType == 'font') request.abort(); + // else if (request.resourceType() == 'stylesheet') request.abort(); // else if(request.resourceType == 'media') request.abort(); else request.continue(); }); diff --git a/app/scripts/urls-helper.js b/app/scripts/urls-helper.js index 459d846..ad327f2 100644 --- a/app/scripts/urls-helper.js +++ b/app/scripts/urls-helper.js @@ -30,6 +30,8 @@ module.exports.isStringAValidURL = function (url) { new URL(url); return true; } catch (err) { + console.error(err); + console.log(url); return false; } }; diff --git a/package-lock.json b/package-lock.json index 17bf550..b73eae9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "f95api", - "version": "1.1.2", + "version": "1.2.3", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -612,6 +612,12 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, + "dotenv": { + "version": "8.2.0", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", + "integrity": "sha512-8sJ78ElpbDJBHNeBzUbUVLsqKdccaa/BXF1uPTw3GrvQTBgrQrtObr2mUrE38vzYd8cEv+m/JBfDLioYcfXoaw==", + "dev": true + }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -1596,6 +1602,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "nan": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", + "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", + "dev": true + }, "node-fetch": { "version": "3.0.0-beta.9", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0-beta.9.tgz", @@ -1963,6 +1975,15 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, + "sleep": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/sleep/-/sleep-6.3.0.tgz", + "integrity": "sha512-+WgYl951qdUlb1iS97UvQ01pkauoBK9ML9I/CMPg41v0Ze4EyMlTgFTDDo32iYj98IYqxIjDMRd+L71lawFfpQ==", + "dev": true, + "requires": { + "nan": "^2.14.1" + } + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", diff --git a/package.json b/package.json index 5144a3e..6389722 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "main": "./app/index.js", "name": "f95api", - "version": "1.1.2", + "version": "1.2.3", "author": { "name": "Millennium Earl" }, @@ -22,9 +22,9 @@ "games" ], "scripts": { - "unit-test-mocha": "nyc --reporter=text mocha", + "unit-test-mocha": "nyc --reporter=text mocha './test/index-test.js'", "report-coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov -t 38ad72bf-a29d-4c2e-9827-96cbe037afd2", - "test": "node ./test/test.js", + "test": "node ./test/user-test.js", "deploy": "npm pack" }, "engines": { @@ -38,7 +38,9 @@ }, "devDependencies": { "chai": "^4.2.0", + "dotenv": "^8.2.0", "mocha": "^8.1.3", - "nyc": "^15.1.0" + "nyc": "^15.1.0", + "sleep": "^6.3.0" } } diff --git a/test/index-test.js b/test/index-test.js index f0caf9c..e27a36b 100644 --- a/test/index-test.js +++ b/test/index-test.js @@ -3,47 +3,81 @@ const expect = require("chai").expect; const F95API = require("../app/index"); const fs = require("fs"); +const sleep = require('sleep'); +const dotenv = require('dotenv'); +dotenv.config(); const COOKIES_SAVE_PATH = "./f95cache/cookies.json"; const ENGINES_SAVE_PATH = "./f95cache/engines.json"; const STATUSES_SAVE_PATH = "./f95cache/statuses.json"; -const USERNAME = "MillenniumEarl"; -const PASSWORD = "f9vTcRNuvxj4YpK"; +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; const FAKE_USERNAME = "FakeUsername091276"; const FAKE_PASSWORD = "fake_password"; -F95API.setIsolation(true); +F95API.debug(false); + +function randomSleep() { + let random = Math.floor(Math.random() * 500) + 50; + sleep.msleep(500 + random); +} describe("Login without cookies", function () { //#region Set-up this.timeout(30000); // All tests in this suite get 30 seconds before timeout + before("Set isolation", function() { + F95API.setIsolation(true); + }) + beforeEach("Remove all cookies", function () { // Runs before each test in this block if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); - F95API.logout(); + if (F95API.isLogged()) F95API.logout(); }); //#endregion Set-up + let testOrder = 0; + it("Test with valid credentials", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 0) randomSleep(); + const result = await F95API.login(USERNAME, PASSWORD); expect(result.success).to.be.true; expect(result.message).equal("Authentication successful"); + + testOrder = 1; }); it("Test with invalid username", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 1) randomSleep(); + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); expect(result.success).to.be.false; expect(result.message).to.equal("Incorrect username"); + + testOrder = 2; }); it("Test with invalid password", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 2) randomSleep(); + const result = await F95API.login(USERNAME, FAKE_PASSWORD); expect(result.success).to.be.false; expect(result.message).to.equal("Incorrect password"); + + testOrder = 3; }); it("Test with invalid credentials", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 3) randomSleep(); + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); expect(result.success).to.be.false; expect(result.message).to.equal("Incorrect username"); // It should first check the username + + testOrder = 4; }); }); @@ -55,7 +89,7 @@ describe("Login with cookies", function () { // Runs once before the first test in this block if (!fs.existsSync(COOKIES_SAVE_PATH)) await F95API.login(USERNAME, PASSWORD); // Download cookies - F95API.logout(); + if (F95API.isLogged()) F95API.logout(); }); //#endregion Set-up @@ -92,7 +126,7 @@ describe("Load base data without cookies", function () { }); it("Without login", async function () { - F95API.logout(); + if (F95API.isLogged()) F95API.logout(); let result = await F95API.loadF95BaseData(); expect(result).to.be.false; }); @@ -104,7 +138,7 @@ describe("Search game data", function () { beforeEach("Prepare API", function () { // Runs once before the first test in this block - F95API.logout(); + if (F95API.isLogged()) F95API.logout(); }); //#endregion Set-up @@ -117,7 +151,9 @@ describe("Search game data", function () { // This test depend on the data on F95Zone at // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; + const gamesList = await F95API.getGameData("Kingdom of Deception", false); + expect(gamesList.length, "Should find only the game").to.equal(1); + const result = gamesList[0]; let src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; // Test only the main information @@ -125,7 +161,7 @@ describe("Search game data", function () { expect(result.author).to.equal("Hreinn Games"); expect(result.isMod, "Should be false").to.be.false; expect(result.engine).to.equal("REN'PY"); - // expect(result.previewSource).to.equal(src); could be null -> Why sometimes doesn't get the image? + expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? }); it("Search game when not logged", async function () { const result = await F95API.getGameData("Kingdom of Deception", false); @@ -150,7 +186,7 @@ describe("Load user data", function () { }); it("Retrieve when not logged", async function () { // Logout - F95API.logout(); + if (F95API.isLogged()) F95API.logout(); // Try to retrieve user data let data = await F95API.getUserData(); @@ -164,7 +200,7 @@ describe("Check game update", function () { this.timeout(30000); // All tests in this suite get 30 seconds before timeout //#endregion Set-up - it("Get game update", async function () { + it("Get online game and verify that no update exists", async function () { const loginResult = await F95API.login(USERNAME, PASSWORD); expect(loginResult.success).to.be.true; @@ -178,4 +214,18 @@ describe("Check game update", function () { let update = await F95API.chekIfGameHasUpdate(result); expect(update).to.be.false; }); + + it("Verify that update exists from old URL", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; + + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ + let url = "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; + const result = await F95API.getGameDataFromURL(url); + result.version = "0.9600"; + + let update = await F95API.chekIfGameHasUpdate(result); + expect(update).to.be.true; + }); }); diff --git a/test/test.js b/test/user-test.js similarity index 100% rename from test/test.js rename to test/user-test.js