MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Code coverage of 90%, added browser isolation and possibility to set cache dir

pull/4/head
MillenniumEarl 2020-10-02 14:01:51 +02:00
parent 0773f77c2c
commit f637c4759d
12 changed files with 259 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
app/index.js
View File

@ -2,7 +2,6 @@
// Core modules
const fs = require('fs');
const path = require('path');
// Public modules from npm
const urlExist = require('url-exist');
@ -19,19 +18,15 @@ const {
prepareBrowser,
preparePage
} = require('./scripts/puppeteer-helper.js');
const GameInfo = require('./scripts/classes/game-info.js').GameInfo;
const LoginResult = require('./scripts/classes/login-result.js').LoginResult;
const UserData = require('./scripts/classes/user-data.js').UserData;
const GameInfo = require('./scripts/classes/game-info.js');
const LoginResult = require('./scripts/classes/login-result.js');
const UserData = require('./scripts/classes/user-data.js');
//#region Directories
const CACHE_PATH = './f95cache';
const COOKIES_SAVE_PATH = path.join(CACHE_PATH, 'cookies.json');
const ENGINES_SAVE_PATH = path.join(CACHE_PATH, 'engines.json');
const STATUSES_SAVE_PATH = path.join(CACHE_PATH, 'statuses.json');
// Create directory if it doesn't exist
if (!fs.existsSync(CACHE_PATH)) fs.mkdirSync(CACHE_PATH);
//#endregion Directories
//#region Expose classes
module.exports.GameInfo = GameInfo;
module.exports.LoginResult = LoginResult;
module.exports.UserData = UserData;
//#endregion Expose classes
//#region Exposed properties
/**
@ -44,8 +39,24 @@ module.exports.debug = function (value) {
module.exports.isLogged = function () {
return shared.isLogged;
};
module.exports.isolation = function(value) {
shared.isolation = value;
}
module.exports.getCacheDir = function() {
return shared.cacheDir;
}
module.exports.setCacheDir = function(value) {
shared.cacheDir = value;
// Create directory if it doesn't exist
if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
}
//#endregion Exposed properties
//#region Global variables
var _browser = null;
//#endregion
//#region Export methods
/**
* @public
@ -77,7 +88,14 @@ module.exports.login = async function (username, password) {
// Else, log in throught browser
if (shared.debug) console.log('No saved sessions or expired session, login on the platform');
let browser = await prepareBrowser();
let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let result = await loginF95(browser, username, password);
shared.isLogged = result.success;
@ -88,7 +106,7 @@ module.exports.login = async function (username, password) {
} else {
console.warn('Error during authentication: ' + result.message);
}
await browser.close();
if (shared.isolation) await browser.close();
return result;
}
/**
@ -107,7 +125,13 @@ module.exports.loadF95BaseData = async function () {
if (shared.debug) console.log('Loading base data...');
// Prepare a new web page
let browser = await prepareBrowser();
let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
@ -119,18 +143,18 @@ module.exports.loadF95BaseData = async function () {
// Obtain engines (disc/online)
await page.waitForSelector(constSelectors.ENGINE_ID_SELECTOR);
shared.engines = await loadValuesFromLatestPage(page,
ENGINES_SAVE_PATH,
shared.enginesCachePath,
constSelectors.ENGINE_ID_SELECTOR,
'engines');
// Obtain statuses (disc/online)
await page.waitForSelector(constSelectors.STATUS_ID_SELECTOR);
shared.statuses = await loadValuesFromLatestPage(page,
STATUSES_SAVE_PATH,
shared.statusesCachePath,
constSelectors.STATUS_ID_SELECTOR,
'statuses');
await browser.close();
if (shared.isolation) await browser.close();
if (shared.debug) console.log('Base data loaded');
return true;
}
@ -169,7 +193,12 @@ module.exports.getGameData = async function (name, includeMods) {
}
// Gets the search results of the game being searched for
let browser = await prepareBrowser();
let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let urlList = await getSearchGameResults(browser, name);
// Process previous partial results
@ -187,7 +216,7 @@ module.exports.getGameData = async function (name, includeMods) {
else result.push(info);
}
await browser.close();
if (shared.isolation) await browser.close();
return result;
}
/**
@ -202,7 +231,12 @@ module.exports.getUserData = async function () {
}
// Prepare a new web page
let browser = await prepareBrowser();
let browser = null;
if (shared.isolation) browser = await prepareBrowser();
else {
if (_browser === null) _browser = await prepareBrowser();
browser = _browser;
}
let page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(constURLs.F95_BASE_URL); // Go to base page
@ -227,7 +261,7 @@ module.exports.getUserData = async function () {
ud.watchedThreads = await threads;
await page.close();
await browser.close();
if (shared.isolation) await browser.close();
return ud;
}
@ -247,9 +281,9 @@ module.exports.logout = function() {
*/
function loadCookies() {
// Check the existence of the cookie file
if (fs.existsSync(COOKIES_SAVE_PATH)) {
if (fs.existsSync(shared.cookiesCachePath)) {
// Read cookies
let cookiesJSON = fs.readFileSync(COOKIES_SAVE_PATH);
let cookiesJSON = fs.readFileSync(shared.cookiesCachePath);
let cookies = JSON.parse(cookiesJSON);
// Check if the cookies have expired
@ -376,7 +410,7 @@ async function loginF95(browser, username, password) {
// Save cookies to avoid re-auth
if (result.success) {
let c = await page.cookies();
fs.writeFileSync(COOKIES_SAVE_PATH, JSON.stringify(c));
fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
result.message = 'Authentication successful';
}
// Obtain the error message

4
app/scripts/classes/game-download.js
View File

@ -1,3 +1,5 @@
'use strict';
class GameDownload {
constructor() {
/**
@ -30,7 +32,7 @@ class GameDownload {
}
}
module.exports.GameDownload = GameDownload;
module.exports = GameDownload;
function downloadMEGA(url){

14
app/scripts/classes/game-info.js
View File

@ -1,7 +1,10 @@
'use strict';
const UNKNOWN = 'Unknown';
class GameInfo {
constructor() {
//#region Properties
/**
* Game name
* @type String
@ -68,9 +71,13 @@ class GameInfo {
*/
this.gameDir = UNKNOWN;
/**
*
* Information on game file download links,
* including information on hosting platforms
* and operating system supported by the specific link
* @type GameDownload[]
*/
this.downloadInfo = [];
//#endregion Properties
}
/**
@ -89,7 +96,8 @@ class GameInfo {
lastUpdate: this.lastUpdate,
lastPlayed: this.lastPlayed,
isMod: this.isMod,
gameDir: this.gameDir
gameDir: this.gameDir,
downloadInfo: this.downloadInfo
}
}
@ -102,4 +110,4 @@ class GameInfo {
return Object.assign(new GameInfo(), json);
}
}
module.exports.GameInfo = GameInfo;
module.exports = GameInfo;

4
app/scripts/classes/login-result.js
View File

@ -1,3 +1,5 @@
'use strict';
/**
* Object obtained in response to an attempt to login to the portal.
*/
@ -15,4 +17,4 @@ class LoginResult {
this.message = '';
}
}
module.exports.LoginResult = LoginResult;
module.exports = LoginResult;

4
app/scripts/classes/user-data.js
View File

@ -1,3 +1,5 @@
'use strict';
/**
* Class containing the data of the user currently connected to the F95Zone platform.
*/
@ -21,4 +23,4 @@ class UserData {
}
}
module.exports.UserData = UserData;
module.exports = UserData;

10
app/scripts/game-scraper.js
View File

@ -1,3 +1,5 @@
'use strict';
// Public modules from npm
const HTMLParser = require('node-html-parser');
const puppeteer = require('puppeteer');
@ -7,8 +9,8 @@ const urlExist = require('url-exist');
const shared = require('./shared.js');
const selectors = require('./costants/css-selectors.js');
const { preparePage } = require('./puppeteer-helper.js');
const GameDownload = require('./classes/game-download.js').GameDownload;
const GameInfo = require('./classes/game-info.js').GameInfo;
const GameDownload = require('./classes/game-download.js');
const GameInfo = require('./classes/game-info.js');
const { isStringAValidURL, isF95URL } = require('./urls-helper.js');
/**
@ -154,8 +156,8 @@ async function getGamePreviewSource(page) {
// Get the firs image available
let img = document.querySelector(selector);
if (img === null || img === undefined) return null;
else return img.getAttribute('src');
if (img) return img.getAttribute('src');
else return null;
}, selectors.GAME_IMAGES);
// Check if the URL is valid

2
app/scripts/puppeteer-helper.js
View File

@ -1,3 +1,5 @@
'use strict';
// Public modules from npm
const puppeteer = require('puppeteer');

138
app/scripts/shared.js
View File

@ -1,71 +1,155 @@
'use strict';
// Core modules
const { join } = require('path');
/**
* Class containing variables shared between modules.
*/
class Shared {
//#region Properties
/**
* Shows log messages and other useful functions for module debugging.
* @type Boolean
*/
static _debug = false;
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
* @type Boolean
*/
static _isLogged = false;
/**
* List of cookies obtained from the F95Zone platform.
* @type Object[]
*/
static _cookies = null;
/**
* List of possible game engines used for development.
* @type String[]
*/
static _engines = null;
/**
* List of possible development statuses that a game can assume.
* @type String[]
*/
static _statuses = null;
/**
* Wait instruction for the browser created by puppeteer.
* @type String
*/
static WAIT_STATEMENT = 'domcontentloaded';
/**
* Path to the directory to save the cache generated by the API.
* @type String
*/
static _cacheDir = './f95cache';
/**
* If true, it opens a new browser for each request to
* the F95Zone platform, otherwise it reuses the same.
* @type Boolean
*/
static _isolation = false;
//#endregion Properties
static set debug(val) {
this._debug = val;
}
//#region Getters
/**
* Shows log messages and other useful functions for module debugging.
* @returns {boolean}
* @returns {Boolean}
*/
static get debug() {
return this._debug;
}
static set isLogged(val) {
this._isLogged = val;
}
/**
* @returns {boolean}
*/
static get isLogged() {
return this._isLogged;
}
static set cookies(val) {
this._cookies = val;
}
/**
* @returns {object[]}
*/
static get cookies() {
return this._cookies;
}
/**
* @returns {String[]}
*/
static get engines() {
return this._engines;
}
/**
* @returns {String[]}
*/
static get statuses() {
return this._statuses;
}
/**
* Directory to save the API cache.
* @returns {String}
*/
static get cacheDir() {
return this._cacheDir;
}
/**
* Path to the F95 platform cache.
* @returns {String}
*/
static get cookiesCachePath() {
return join(this._cacheDir, 'cookies.json');
}
/**
* Path to the game engine cache.
* @returns {String}
*/
static get enginesCachePath() {
return join(this._cacheDir, 'engines.json');
}
/**
* Path to the cache of possible game states.
* @returns {String}
*/
static get statusesCachePath() {
return join(this._cacheDir, 'statuses.json');
}
/**
* If true, it opens a new browser for each request
* to the F95Zone platform, otherwise it reuses the same.
* @returns {Boolean}
*/
static get isolation() {
return this._isolation;
}
//#endregion Getters
//#region Setters
static set cookies(val) {
this._cookies = val;
}
static set engines(val) {
this._engines = val;
}
/**
* @returns {string[]}
*/
static get engines() {
return this._engines;
}
static set statuses(val) {
this._statuses = val;
}
/**
* @returns {string[]}
*/
static get statuses() {
return this._statuses;
static set cacheDir(val) {
this._cacheDir = val;
}
static set debug(val) {
this._debug = val;
}
static set isLogged(val) {
this._isLogged = val;
}
static set isolation(val) {
this._isolation = val;
}
//#endregion Setters
}
module.exports = Shared;

3
app/scripts/urls-helper.js
View File

@ -1,7 +1,8 @@
'use strict';
// Modules from file
const { F95_BASE_URL } = require('./costants/urls.js');
/**
* @protected
* Check if the url belongs to the domain of the F95 platform.

BIN
f95api-1.0.0.tgz Normal file
View File

Binary file not shown.

9
package.json
View File

@ -3,9 +3,16 @@
"version": "1.0.0",
"description": "Unofficial Node JS module for scraping F95Zone platform",
"main": "./app/index.js",
"repository": {
"type": "git",
"url": "https://github.com/MillenniumEarl/F95API.git"
},
"license": "UNLICENSED",
"private": true,
"scripts": {
"unit-test": "nyc --reporter=text mocha",
"test": "node ./app/test.js"
"test": "node ./app/test.js",
"deploy": "npm pack"
},
"keywords": [
"f95zone",

52
test/index-test.js
View File

@ -1,7 +1,6 @@
const expect = require("chai").expect;
const F95API = require("../app/index");
const fs = require("fs");
const { debug } = require("console");
const COOKIES_SAVE_PATH = "./f95cache/cookies.json";
const ENGINES_SAVE_PATH = "./f95cache/engines.json";
@ -11,6 +10,8 @@ const PASSWORD = "f9vTcRNuvxj4YpK";
const FAKE_USERNAME = "FakeUsername091276";
const FAKE_PASSWORD = "fake_password";
F95API.isolation(true);
describe("Login without cookies", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
@ -48,7 +49,7 @@ describe("Login with cookies", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
before("Log in to create cookies", async function () {
before("Log in to create cookies then logout", async function () {
// Runs once before the first test in this block
if (!fs.existsSync(COOKIES_SAVE_PATH)) await F95API.login(USERNAME, PASSWORD); // Download cookies
F95API.logout();
@ -128,3 +129,50 @@ describe("Search game data", function () {
expect(result, "Without being logged should return null").to.be.null;
});
});
describe("Load user data", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
it("Retrieve when logged", async function () {
// Login
await F95API.login(USERNAME, PASSWORD);
// Then retrieve user data
let data = await F95API.getUserData();
expect(data).to.exist;
expect(data.username).to.equal(USERNAME);
});
it("Retrieve when not logged", async function () {
// Logout
F95API.logout();
// Try to retrieve user data
let data = await F95API.getUserData();
expect(data).to.be.null;
});
});
describe("Check game version", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
it("Get game version", async function () {
const loginResult = await F95API.login(USERNAME, PASSWORD);
expect(loginResult.success).to.be.true;
const loadResult = await F95API.loadF95BaseData();
expect(loadResult).to.be.true;
// This test depend on the data on F95Zone at
// https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/
const result = (await F95API.getGameData("Kingdom of Deception", false))[0];
let version = await F95API.getGameVersion(result);
expect(version).to.be.equal(result.version);
});
});