Code coverage of 90%, added browser isolation and possibility to set cache dir

pull/4/head
MillenniumEarl 2020-10-02 14:01:51 +02:00
parent 0773f77c2c
commit f637c4759d
12 changed files with 259 additions and 67 deletions

View File

@ -2,7 +2,6 @@
// Core modules // Core modules
const fs = require('fs'); const fs = require('fs');
const path = require('path');
// Public modules from npm // Public modules from npm
const urlExist = require('url-exist'); const urlExist = require('url-exist');
@ -19,19 +18,15 @@ const {
prepareBrowser, prepareBrowser,
preparePage preparePage
} = require('./scripts/puppeteer-helper.js'); } = require('./scripts/puppeteer-helper.js');
const GameInfo = require('./scripts/classes/game-info.js').GameInfo; const GameInfo = require('./scripts/classes/game-info.js');
const LoginResult = require('./scripts/classes/login-result.js').LoginResult; const LoginResult = require('./scripts/classes/login-result.js');
const UserData = require('./scripts/classes/user-data.js').UserData; const UserData = require('./scripts/classes/user-data.js');
//#region Directories //#region Expose classes
const CACHE_PATH = './f95cache'; module.exports.GameInfo = GameInfo;
const COOKIES_SAVE_PATH = path.join(CACHE_PATH, 'cookies.json'); module.exports.LoginResult = LoginResult;
const ENGINES_SAVE_PATH = path.join(CACHE_PATH, 'engines.json'); module.exports.UserData = UserData;
const STATUSES_SAVE_PATH = path.join(CACHE_PATH, 'statuses.json'); //#endregion Expose classes
// Create directory if it doesn't exist
if (!fs.existsSync(CACHE_PATH)) fs.mkdirSync(CACHE_PATH);
//#endregion Directories
//#region Exposed properties //#region Exposed properties
/** /**
@ -44,8 +39,24 @@ module.exports.debug = function (value) {
module.exports.isLogged = function () { module.exports.isLogged = function () {
return shared.isLogged; return shared.isLogged;
}; };
module.exports.isolation = function(value) {
shared.isolation = value;
}
module.exports.getCacheDir = function() {
return shared.cacheDir;
}
module.exports.setCacheDir = function(value) {
shared.cacheDir = value;
// Create directory if it doesn't exist
if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
}
//#endregion Exposed properties //#endregion Exposed properties
//#region Global variables
var _browser = null;
//#endregion
//#region Export methods //#region Export methods
/** /**
* @public * @public
@ -77,7 +88,14 @@ module.exports.login = async function (username, password) {
// Else, log in throught browser // Else, log in throught browser
if (shared.debug) console.log('No saved sessions or expired session, login on the platform'); if (shared.debug) console.log('No saved sessions or expired session, login on the platform');
let browser = await prepareBrowser();
let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let result = await loginF95(browser, username, password); let result = await loginF95(browser, username, password);
shared.isLogged = result.success; shared.isLogged = result.success;
@ -88,7 +106,7 @@ module.exports.login = async function (username, password) {
} else { } else {
console.warn('Error during authentication: ' + result.message); console.warn('Error during authentication: ' + result.message);
} }
await browser.close(); if (shared.isolation) await browser.close();
return result; return result;
} }
/** /**
@ -107,7 +125,13 @@ module.exports.loadF95BaseData = async function () {
if (shared.debug) console.log('Loading base data...'); if (shared.debug) console.log('Loading base data...');
// Prepare a new web page // Prepare a new web page
let browser = await prepareBrowser(); let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let page = await preparePage(browser); // Set new isolated page let page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login await page.setCookie(...shared.cookies); // Set cookies to avoid login
@ -119,18 +143,18 @@ module.exports.loadF95BaseData = async function () {
// Obtain engines (disc/online) // Obtain engines (disc/online)
await page.waitForSelector(constSelectors.ENGINE_ID_SELECTOR); await page.waitForSelector(constSelectors.ENGINE_ID_SELECTOR);
shared.engines = await loadValuesFromLatestPage(page, shared.engines = await loadValuesFromLatestPage(page,
ENGINES_SAVE_PATH, shared.enginesCachePath,
constSelectors.ENGINE_ID_SELECTOR, constSelectors.ENGINE_ID_SELECTOR,
'engines'); 'engines');
// Obtain statuses (disc/online) // Obtain statuses (disc/online)
await page.waitForSelector(constSelectors.STATUS_ID_SELECTOR); await page.waitForSelector(constSelectors.STATUS_ID_SELECTOR);
shared.statuses = await loadValuesFromLatestPage(page, shared.statuses = await loadValuesFromLatestPage(page,
STATUSES_SAVE_PATH, shared.statusesCachePath,
constSelectors.STATUS_ID_SELECTOR, constSelectors.STATUS_ID_SELECTOR,
'statuses'); 'statuses');
await browser.close(); if (shared.isolation) await browser.close();
if (shared.debug) console.log('Base data loaded'); if (shared.debug) console.log('Base data loaded');
return true; return true;
} }
@ -169,7 +193,12 @@ module.exports.getGameData = async function (name, includeMods) {
} }
// Gets the search results of the game being searched for // Gets the search results of the game being searched for
let browser = await prepareBrowser(); let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let urlList = await getSearchGameResults(browser, name); let urlList = await getSearchGameResults(browser, name);
// Process previous partial results // Process previous partial results
@ -187,7 +216,7 @@ module.exports.getGameData = async function (name, includeMods) {
else result.push(info); else result.push(info);
} }
await browser.close(); if (shared.isolation) await browser.close();
return result; return result;
} }
/** /**
@ -202,7 +231,12 @@ module.exports.getUserData = async function () {
} }
// Prepare a new web page // Prepare a new web page
let browser = await prepareBrowser(); let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let page = await preparePage(browser); // Set new isolated page let page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(constURLs.F95_BASE_URL); // Go to base page await page.goto(constURLs.F95_BASE_URL); // Go to base page
@ -227,7 +261,7 @@ module.exports.getUserData = async function () {
ud.watchedThreads = await threads; ud.watchedThreads = await threads;
await page.close(); await page.close();
await browser.close(); if (shared.isolation) await browser.close();
return ud; return ud;
} }
@ -247,9 +281,9 @@ module.exports.logout = function() {
*/ */
function loadCookies() { function loadCookies() {
// Check the existence of the cookie file // Check the existence of the cookie file
if (fs.existsSync(COOKIES_SAVE_PATH)) { if (fs.existsSync(shared.cookiesCachePath)) {
// Read cookies // Read cookies
let cookiesJSON = fs.readFileSync(COOKIES_SAVE_PATH); let cookiesJSON = fs.readFileSync(shared.cookiesCachePath);
let cookies = JSON.parse(cookiesJSON); let cookies = JSON.parse(cookiesJSON);
// Check if the cookies have expired // Check if the cookies have expired
@ -376,7 +410,7 @@ async function loginF95(browser, username, password) {
// Save cookies to avoid re-auth // Save cookies to avoid re-auth
if (result.success) { if (result.success) {
let c = await page.cookies(); let c = await page.cookies();
fs.writeFileSync(COOKIES_SAVE_PATH, JSON.stringify(c)); fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
result.message = 'Authentication successful'; result.message = 'Authentication successful';
} }
// Obtain the error message // Obtain the error message

View File

@ -1,3 +1,5 @@
'use strict';
class GameDownload { class GameDownload {
constructor() { constructor() {
/** /**
@ -30,7 +32,7 @@ class GameDownload {
} }
} }
module.exports.GameDownload = GameDownload; module.exports = GameDownload;
function downloadMEGA(url){ function downloadMEGA(url){

View File

@ -1,7 +1,10 @@
'use strict';
const UNKNOWN = 'Unknown'; const UNKNOWN = 'Unknown';
class GameInfo { class GameInfo {
constructor() { constructor() {
//#region Properties
/** /**
* Game name * Game name
* @type String * @type String
@ -68,9 +71,13 @@ class GameInfo {
*/ */
this.gameDir = UNKNOWN; this.gameDir = UNKNOWN;
/** /**
* * Information on game file download links,
* including information on hosting platforms
* and operating system supported by the specific link
* @type GameDownload[]
*/ */
this.downloadInfo = []; this.downloadInfo = [];
//#endregion Properties
} }
/** /**
@ -89,7 +96,8 @@ class GameInfo {
lastUpdate: this.lastUpdate, lastUpdate: this.lastUpdate,
lastPlayed: this.lastPlayed, lastPlayed: this.lastPlayed,
isMod: this.isMod, isMod: this.isMod,
gameDir: this.gameDir gameDir: this.gameDir,
downloadInfo: this.downloadInfo
} }
} }
@ -102,4 +110,4 @@ class GameInfo {
return Object.assign(new GameInfo(), json); return Object.assign(new GameInfo(), json);
} }
} }
module.exports.GameInfo = GameInfo; module.exports = GameInfo;

View File

@ -1,3 +1,5 @@
'use strict';
/** /**
* Object obtained in response to an attempt to login to the portal. * Object obtained in response to an attempt to login to the portal.
*/ */
@ -15,4 +17,4 @@ class LoginResult {
this.message = ''; this.message = '';
} }
} }
module.exports.LoginResult = LoginResult; module.exports = LoginResult;

View File

@ -1,3 +1,5 @@
'use strict';
/** /**
* Class containing the data of the user currently connected to the F95Zone platform. * Class containing the data of the user currently connected to the F95Zone platform.
*/ */
@ -21,4 +23,4 @@ class UserData {
} }
} }
module.exports.UserData = UserData; module.exports = UserData;

View File

@ -1,3 +1,5 @@
'use strict';
// Public modules from npm // Public modules from npm
const HTMLParser = require('node-html-parser'); const HTMLParser = require('node-html-parser');
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');
@ -7,8 +9,8 @@ const urlExist = require('url-exist');
const shared = require('./shared.js'); const shared = require('./shared.js');
const selectors = require('./costants/css-selectors.js'); const selectors = require('./costants/css-selectors.js');
const { preparePage } = require('./puppeteer-helper.js'); const { preparePage } = require('./puppeteer-helper.js');
const GameDownload = require('./classes/game-download.js').GameDownload; const GameDownload = require('./classes/game-download.js');
const GameInfo = require('./classes/game-info.js').GameInfo; const GameInfo = require('./classes/game-info.js');
const { isStringAValidURL, isF95URL } = require('./urls-helper.js'); const { isStringAValidURL, isF95URL } = require('./urls-helper.js');
/** /**
@ -154,8 +156,8 @@ async function getGamePreviewSource(page) {
// Get the firs image available // Get the firs image available
let img = document.querySelector(selector); let img = document.querySelector(selector);
if (img === null || img === undefined) return null; if (img) return img.getAttribute('src');
else return img.getAttribute('src'); else return null;
}, selectors.GAME_IMAGES); }, selectors.GAME_IMAGES);
// Check if the URL is valid // Check if the URL is valid

View File

@ -1,3 +1,5 @@
'use strict';
// Public modules from npm // Public modules from npm
const puppeteer = require('puppeteer'); const puppeteer = require('puppeteer');

View File

@ -1,71 +1,155 @@
'use strict';
// Core modules
const { join } = require('path');
/** /**
* Class containing variables shared between modules. * Class containing variables shared between modules.
*/ */
class Shared { class Shared {
//#region Properties
/** /**
* Shows log messages and other useful functions for module debugging. * Shows log messages and other useful functions for module debugging.
* @type Boolean
*/ */
static _debug = false; static _debug = false;
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
* @type Boolean
*/
static _isLogged = false; static _isLogged = false;
/**
* List of cookies obtained from the F95Zone platform.
* @type Object[]
*/
static _cookies = null; static _cookies = null;
/**
* List of possible game engines used for development.
* @type String[]
*/
static _engines = null; static _engines = null;
/**
* List of possible development statuses that a game can assume.
* @type String[]
*/
static _statuses = null; static _statuses = null;
/**
* Wait instruction for the browser created by puppeteer.
* @type String
*/
static WAIT_STATEMENT = 'domcontentloaded'; static WAIT_STATEMENT = 'domcontentloaded';
/**
* Path to the directory to save the cache generated by the API.
* @type String
*/
static _cacheDir = './f95cache';
/**
* If true, it opens a new browser for each request to
* the F95Zone platform, otherwise it reuses the same.
* @type Boolean
*/
static _isolation = false;
//#endregion Properties
static set debug(val) { //#region Getters
this._debug = val;
}
/** /**
* Shows log messages and other useful functions for module debugging. * Shows log messages and other useful functions for module debugging.
* @returns {boolean} * @returns {Boolean}
*/ */
static get debug() { static get debug() {
return this._debug; return this._debug;
} }
static set isLogged(val) {
this._isLogged = val;
}
/** /**
* @returns {boolean} * @returns {boolean}
*/ */
static get isLogged() { static get isLogged() {
return this._isLogged; return this._isLogged;
} }
static set cookies(val) {
this._cookies = val;
}
/** /**
* @returns {object[]} * @returns {object[]}
*/ */
static get cookies() { static get cookies() {
return this._cookies; return this._cookies;
} }
/**
* @returns {String[]}
*/
static get engines() {
return this._engines;
}
/**
* @returns {String[]}
*/
static get statuses() {
return this._statuses;
}
/**
* Directory to save the API cache.
* @returns {String}
*/
static get cacheDir() {
return this._cacheDir;
}
/**
* Path to the F95 platform cache.
* @returns {String}
*/
static get cookiesCachePath() {
return join(this._cacheDir, 'cookies.json');
}
/**
* Path to the game engine cache.
* @returns {String}
*/
static get enginesCachePath() {
return join(this._cacheDir, 'engines.json');
}
/**
* Path to the cache of possible game states.
* @returns {String}
*/
static get statusesCachePath() {
return join(this._cacheDir, 'statuses.json');
}
/**
* If true, it opens a new browser for each request
* to the F95Zone platform, otherwise it reuses the same.
* @returns {Boolean}
*/
static get isolation() {
return this._isolation;
}
//#endregion Getters
//#region Setters
static set cookies(val) {
this._cookies = val;
}
static set engines(val) { static set engines(val) {
this._engines = val; this._engines = val;
} }
/**
* @returns {string[]}
*/
static get engines() {
return this._engines;
}
static set statuses(val) { static set statuses(val) {
this._statuses = val; this._statuses = val;
} }
/**
* @returns {string[]} static set cacheDir(val) {
*/ this._cacheDir = val;
static get statuses() {
return this._statuses;
} }
static set debug(val) {
this._debug = val;
}
static set isLogged(val) {
this._isLogged = val;
}
static set isolation(val) {
this._isolation = val;
}
//#endregion Setters
} }
module.exports = Shared; module.exports = Shared;

View File

@ -1,7 +1,8 @@
'use strict';
// Modules from file // Modules from file
const { F95_BASE_URL } = require('./costants/urls.js'); const { F95_BASE_URL } = require('./costants/urls.js');
/** /**
* @protected * @protected
* Check if the url belongs to the domain of the F95 platform. * Check if the url belongs to the domain of the F95 platform.

BIN
f95api-1.0.0.tgz Normal file

Binary file not shown.

View File

@ -3,9 +3,16 @@
"version": "1.0.0", "version": "1.0.0",
"description": "Unofficial Node JS module for scraping F95Zone platform", "description": "Unofficial Node JS module for scraping F95Zone platform",
"main": "./app/index.js", "main": "./app/index.js",
"repository": {
"type": "git",
"url": "https://github.com/MillenniumEarl/F95API.git"
},
"license": "UNLICENSED",
"private": true,
"scripts": { "scripts": {
"unit-test": "nyc --reporter=text mocha", "unit-test": "nyc --reporter=text mocha",
"test": "node ./app/test.js" "test": "node ./app/test.js",
"deploy": "npm pack"
}, },
"keywords": [ "keywords": [
"f95zone", "f95zone",

View File

@ -1,7 +1,6 @@
const expect = require("chai").expect; const expect = require("chai").expect;
const F95API = require("../app/index"); const F95API = require("../app/index");
const fs = require("fs"); const fs = require("fs");
const { debug } = require("console");
const COOKIES_SAVE_PATH = "./f95cache/cookies.json"; const COOKIES_SAVE_PATH = "./f95cache/cookies.json";
const ENGINES_SAVE_PATH = "./f95cache/engines.json"; const ENGINES_SAVE_PATH = "./f95cache/engines.json";
@ -11,6 +10,8 @@ const PASSWORD = "f9vTcRNuvxj4YpK";
const FAKE_USERNAME = "FakeUsername091276"; const FAKE_USERNAME = "FakeUsername091276";
const FAKE_PASSWORD = "fake_password"; const FAKE_PASSWORD = "fake_password";
F95API.isolation(true);
describe("Login without cookies", function () { describe("Login without cookies", function () {
//#region Set-up //#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout this.timeout(30000); // All tests in this suite get 30 seconds before timeout
@ -48,7 +49,7 @@ describe("Login with cookies", function () {
//#region Set-up //#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout this.timeout(30000); // All tests in this suite get 30 seconds before timeout
before("Log in to create cookies", async function () { before("Log in to create cookies then logout", async function () {
// Runs once before the first test in this block // Runs once before the first test in this block
if (!fs.existsSync(COOKIES_SAVE_PATH)) await F95API.login(USERNAME, PASSWORD); // Download cookies if (!fs.existsSync(COOKIES_SAVE_PATH)) await F95API.login(USERNAME, PASSWORD); // Download cookies
F95API.logout(); F95API.logout();
@ -128,3 +129,50 @@ describe("Search game data", function () {
expect(result, "Without being logged should return null").to.be.null; expect(result, "Without being logged should return null").to.be.null;
}); });
}); });
describe("Load user data", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
it("Retrieve when logged", async function () {
// Login
await F95API.login(USERNAME, PASSWORD);
// Then retrieve user data
let data = await F95API.getUserData();
expect(data).to.exist;
expect(data.username).to.equal(USERNAME);
});
it("Retrieve when not logged", async function () {
// Logout
F95API.logout();
// Try to retrieve user data
let data = await F95API.getUserData();
expect(data).to.be.null;
});
});
describe("Check game version", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
it("Get game version", async function () {
const loginResult = await F95API.login(USERNAME, PASSWORD);
expect(loginResult.success).to.be.true;
const loadResult = await F95API.loadF95BaseData();
expect(loadResult).to.be.true;
// This test depend on the data on F95Zone at
// https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/
const result = (await F95API.getGameData("Kingdom of Deception", false))[0];
let version = await F95API.getGameVersion(result);
expect(version).to.be.equal(result.version);
});
});