MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #5 from MillenniumEarl/deepsource-transform-c3007dfe 

Format code with prettier
pull/6/head
Millennium Earl 2020-10-02 17:56:10 +02:00 committed by GitHub
parent ca2a59e0c3 a52a8e32f4
commit b660ba0021
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 530 additions and 487 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

232
app/index.js
View File

@ -1,26 +1,24 @@
'use strict';
"use strict";
// Core modules
const fs = require('fs');
const fs = require("fs");
// Public modules from npm
const urlExist = require('url-exist');
const urlExist = require("url-exist");
// Modules from file
const shared = require('./scripts/shared.js');
const constURLs = require('./scripts/costants/urls.js');
const constSelectors = require('./scripts/costants/css-selectors.js');
const {
isStringAValidURL
} = require('./scripts/urls-helper.js');
const gameScraper = require('./scripts/game-scraper.js');
const shared = require("./scripts/shared.js");
const constURLs = require("./scripts/costants/urls.js");
const constSelectors = require("./scripts/costants/css-selectors.js");
const { isStringAValidURL } = require("./scripts/urls-helper.js");
const gameScraper = require("./scripts/game-scraper.js");
const {
prepareBrowser,
preparePage
} = require('./scripts/puppeteer-helper.js');
const GameInfo = require('./scripts/classes/game-info.js');
const LoginResult = require('./scripts/classes/login-result.js');
const UserData = require('./scripts/classes/user-data.js');
preparePage,
} = require("./scripts/puppeteer-helper.js");
const GameInfo = require("./scripts/classes/game-info.js");
const LoginResult = require("./scripts/classes/login-result.js");
const UserData = require("./scripts/classes/user-data.js");
//#region Expose classes
module.exports.GameInfo = GameInfo;
@ -35,7 +33,7 @@ module.exports.UserData = UserData;
*/
module.exports.debug = function (value) {
shared.debug = value;
}
};
/**
* @public
* Indicates whether a user is logged in to the F95Zone platform or not.
@ -50,28 +48,28 @@ module.exports.isLogged = function () {
* to the F95Zone platform, otherwise it reuses the same.
* @returns {String}
*/
module.exports.setIsolation = function(value) {
module.exports.setIsolation = function (value) {
shared.isolation = value;
}
};
/**
* @public
* Path to the cache directory
* @returns {String}
*/
module.exports.getCacheDir = function() {
module.exports.getCacheDir = function () {
return shared.cacheDir;
}
};
/**
* @public
* Set path to the cache directory
* @returns {String}
*/
module.exports.setCacheDir = function(value) {
module.exports.setCacheDir = function (value) {
shared.cacheDir = value;
// Create directory if it doesn't exist
if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
}
};
//#endregion Exposed properties
//#region Global variables
@ -92,23 +90,24 @@ module.exports.login = async function (username, password) {
if (shared.debug) console.log("Already logged in");
let result = new LoginResult();
result.success = true;
result.message = 'Already logged in';
result.message = "Already logged in";
return result;
}
// If cookies are loaded, use them to authenticate
shared.cookies = loadCookies();
if (shared.cookies !== null) {
if (shared.debug) console.log('Valid session, no need to re-authenticate');
if (shared.debug) console.log("Valid session, no need to re-authenticate");
shared.isLogged = true;
let result = new LoginResult();
result.success = true;
result.message = 'Logged with cookies';
result.message = "Logged with cookies";
return result;
}
// Else, log in throught browser
if (shared.debug) console.log('No saved sessions or expired session, login on the platform');
if (shared.debug)
console.log("No saved sessions or expired session, login on the platform");
let browser = null;
if (shared.isolation) browser = await prepareBrowser();
@ -123,13 +122,13 @@ module.exports.login = async function (username, password) {
if (result.success) {
// Reload cookies
shared.cookies = loadCookies();
if (shared.debug) console.log('User logged in through the platform');
if (shared.debug) console.log("User logged in through the platform");
} else {
console.warn('Error during authentication: ' + result.message);
console.warn("Error during authentication: " + result.message);
}
if (shared.isolation) await browser.close();
return result;
}
};
/**
* @public
* This method loads the main data from the F95 portal
@ -139,11 +138,11 @@ module.exports.login = async function (username, password) {
*/
module.exports.loadF95BaseData = async function () {
if (!shared.isLogged) {
console.warn('User not authenticated, unable to continue');
console.warn("User not authenticated, unable to continue");
return false;
}
if (shared.debug) console.log('Loading base data...');
if (shared.debug) console.log("Loading base data...");
// Prepare a new web page
let browser = null;
@ -158,27 +157,31 @@ module.exports.loadF95BaseData = async function () {
// Go to latest update page and wait for it to load
await page.goto(constURLs.F95_LATEST_UPDATES, {
waitUntil: shared.WAIT_STATEMENT
waitUntil: shared.WAIT_STATEMENT,
});
// Obtain engines (disc/online)
await page.waitForSelector(constSelectors.ENGINE_ID_SELECTOR);
shared.engines = await loadValuesFromLatestPage(page,
shared.engines = await loadValuesFromLatestPage(
page,
shared.enginesCachePath,
constSelectors.ENGINE_ID_SELECTOR,
'engines');
"engines"
);
// Obtain statuses (disc/online)
await page.waitForSelector(constSelectors.STATUS_ID_SELECTOR);
shared.statuses = await loadValuesFromLatestPage(page,
shared.statuses = await loadValuesFromLatestPage(
page,
shared.statusesCachePath,
constSelectors.STATUS_ID_SELECTOR,
'statuses');
"statuses"
);
if (shared.isolation) await browser.close();
if (shared.debug) console.log('Base data loaded');
if (shared.debug) console.log("Base data loaded");
return true;
}
};
/**
* @public
* Returns the currently online version of the specified game.
@ -188,7 +191,7 @@ module.exports.loadF95BaseData = async function () {
*/
module.exports.getGameVersion = async function (info) {
if (!shared.isLogged) {
console.warn('user not authenticated, unable to continue');
console.warn("user not authenticated, unable to continue");
return info.version;
}
@ -197,7 +200,7 @@ module.exports.getGameVersion = async function (info) {
// F95 change URL at every game update, so if the URL is the same no update is available
if (urlExists) return info.version;
else return await module.exports.getGameData(info.name, info.isMod).version;
}
};
/**
* @public
* Starting from the name, it gets all the information about the game you are looking for.
@ -209,7 +212,7 @@ module.exports.getGameVersion = async function (info) {
*/
module.exports.getGameData = async function (name, includeMods) {
if (!shared.isLogged) {
console.warn('user not authenticated, unable to continue');
console.warn("user not authenticated, unable to continue");
return null;
}
@ -239,7 +242,7 @@ module.exports.getGameData = async function (name, includeMods) {
if (shared.isolation) await browser.close();
return result;
}
};
/**
* @public
* Gets the data of the currently logged in user.
@ -248,7 +251,7 @@ module.exports.getGameData = async function (name, includeMods) {
*/
module.exports.getUserData = async function () {
if (!shared.isLogged) {
console.warn('user not authenticated, unable to continue');
console.warn("user not authenticated, unable to continue");
return null;
}
@ -269,13 +272,17 @@ module.exports.getUserData = async function () {
let threads = getUserWatchedGameThreads(browser);
let username = await page.evaluate( /* istanbul ignore next */ (selector) =>
let username = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerText,
constSelectors.USERNAME_ELEMENT);
constSelectors.USERNAME_ELEMENT
);
let avatarSrc = await page.evaluate( /* istanbul ignore next */ (selector) =>
document.querySelector(selector).getAttribute('src'),
constSelectors.AVATAR_PIC);
let avatarSrc = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).getAttribute("src"),
constSelectors.AVATAR_PIC
);
let ud = new UserData();
ud.username = username;
@ -286,19 +293,19 @@ module.exports.getUserData = async function () {
if (shared.isolation) await browser.close();
return ud;
}
};
/**
* @public
* Logout from the current user.
* You **must** be logged in to the portal before calling this method.
*/
module.exports.logout = function() {
module.exports.logout = function () {
if (!shared.isLogged) {
console.warn('user not authenticated, unable to continue');
console.warn("user not authenticated, unable to continue");
return;
}
shared.isLogged = false;
}
};
//#endregion
//#region Private methods
@ -324,7 +331,6 @@ function loadCookies() {
// Cookies loaded and verified
return cookies;
} else return null;
}
/**
@ -338,14 +344,17 @@ function isCookieExpired(cookie) {
let expiredCookies = false;
// Ignore cookies that never expire
let expirationUnixTimestamp = cookie['expire'];
let expirationUnixTimestamp = cookie["expire"];
if (expirationUnixTimestamp !== '-1') {
if (expirationUnixTimestamp !== "-1") {
// Convert UNIX epoch timestamp to normal Date
let expirationDate = new Date(expirationUnixTimestamp * 1000);
if (expirationDate < Date.now()) {
if (shared.debug) console.log('Cookie ' + cookie['name'] + ' expired, you need to re-authenticate');
if (shared.debug)
console.log(
"Cookie " + cookie["name"] + " expired, you need to re-authenticate"
);
expiredCookies = true;
}
}
@ -366,17 +375,27 @@ function isCookieExpired(cookie) {
* @param {String} elementRequested Required element (engines or states) used to detail log messages
* @returns {Promise<String[]>} List of required values in uppercase
*/
async function loadValuesFromLatestPage(page, path, selector, elementRequested) {
async function loadValuesFromLatestPage(
page,
path,
selector,
elementRequested
) {
// If the values already exist they are loaded from disk without having to connect to F95
if (shared.debug) console.log('Load ' + elementRequested + ' from disk...');
if (shared.debug) console.log("Load " + elementRequested + " from disk...");
if (fs.existsSync(path)) {
let valueJSON = fs.readFileSync(path);
return JSON.parse(valueJSON);
}
// Otherwise, connect and download the data from the portal
if (shared.debug) console.log('No ' + elementRequested + ' cached, downloading...');
let values = await getValuesFromLatestPage(page, selector, 'Getting ' + elementRequested + ' from page');
if (shared.debug)
console.log("No " + elementRequested + " cached, downloading...");
let values = await getValuesFromLatestPage(
page,
selector,
"Getting " + elementRequested + " from page"
);
fs.writeFileSync(path, JSON.stringify(values));
return values;
}
@ -397,7 +416,9 @@ async function getValuesFromLatestPage(page, selector, logMessage) {
let elements = await page.$$(selector);
for (let element of elements) {
let text = await element.evaluate( /* istanbul ignore next */ e => e.innerText);
let text = await element.evaluate(
/* istanbul ignore next */ (e) => e.innerText
);
// Save as upper text for better match if used in query
result.push(text.toUpperCase());
@ -427,37 +448,47 @@ async function loginF95(browser, username, password) {
await page.type(constSelectors.PASSWORD_INPUT, password); // Insert password
await page.click(constSelectors.LOGIN_BUTTON); // Click on the login button
await page.waitForNavigation({
waitUntil: shared.WAIT_STATEMENT
waitUntil: shared.WAIT_STATEMENT,
}); // Wait for page to load
// Prepare result
let result = new LoginResult();
// Check if the user is logged in
result.success = await page.evaluate( /* istanbul ignore next */ (selector) =>
result.success = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector) !== null,
constSelectors.AVATAR_INFO);
constSelectors.AVATAR_INFO
);
// Save cookies to avoid re-auth
if (result.success) {
let c = await page.cookies();
fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
result.message = 'Authentication successful';
result.message = "Authentication successful";
}
// Obtain the error message
else if (await page.evaluate( /* istanbul ignore next */ (selector) =>
else if (
await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector) !== null,
constSelectors.LOGIN_MESSAGE_ERROR)) {
let errorMessage = await page.evaluate( /* istanbul ignore next */ (selector) =>
constSelectors.LOGIN_MESSAGE_ERROR
)
) {
let errorMessage = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerText,
constSelectors.LOGIN_MESSAGE_ERROR);
constSelectors.LOGIN_MESSAGE_ERROR
);
if (errorMessage === 'Incorrect password. Please try again.') {
result.message = 'Incorrect password';
} else if (errorMessage === "The requested user '" + username + "' could not be found.") {
result.message = 'Incorrect username';
if (errorMessage === "Incorrect password. Please try again.") {
result.message = "Incorrect password";
} else if (
errorMessage ===
"The requested user '" + username + "' could not be found."
) {
result.message = "Incorrect username";
} else result.message = errorMessage;
} else result.message = "Unknown error";
await page.close(); // Close the page
@ -483,9 +514,11 @@ async function getUserWatchedGameThreads(browser) {
await page.waitForSelector(constSelectors.FILTER_THREADS_BUTTON);
// Set the filters
await page.evaluate( /* istanbul ignore next */ (selector) =>
document.querySelector(selector).removeAttribute('checked'),
constSelectors.UNREAD_THREAD_CHECKBOX); // Also read the threads already read
await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).removeAttribute("checked"),
constSelectors.UNREAD_THREAD_CHECKBOX
); // Also read the threads already read
await page.click(constSelectors.ONLY_GAMES_THREAD_OPTION);
@ -499,23 +532,26 @@ async function getUserWatchedGameThreads(browser) {
do {
// Get all the URLs
for (let handle of await page.$$(constSelectors.WATCHED_THREAD_URLS)) {
let src = await page.evaluate( /* istanbul ignore next */ (element) => element.href, handle);
let src = await page.evaluate(
/* istanbul ignore next */ (element) => element.href,
handle
);
// If 'unread' is left, it will redirect to the last unread post
let url = new URL(src.replace('/unread', ''));
let url = new URL(src.replace("/unread", ""));
urls.push(url);
}
nextPageExists = await page.evaluate( /* istanbul ignore next */ (selector) =>
document.querySelector(selector),
constSelectors.WATCHED_THREAD_NEXT_PAGE);
nextPageExists = await page.evaluate(
/* istanbul ignore next */ (selector) => document.querySelector(selector),
constSelectors.WATCHED_THREAD_NEXT_PAGE
);
// Click to next page
if (nextPageExists) {
await page.click(constSelectors.WATCHED_THREAD_NEXT_PAGE);
await page.waitForSelector(constSelectors.WATCHED_THREAD_URLS);
}
}
while (nextPageExists);
} while (nextPageExists);
await page.close();
return urls;
@ -531,12 +567,12 @@ async function getUserWatchedGameThreads(browser) {
* @returns {Promise<URL[]>} List of URL of possible games obtained from the preliminary research on the F95 portal
*/
async function getSearchGameResults(browser, gamename) {
if (shared.debug) console.log('Searching ' + gamename + ' on F95Zone');
if (shared.debug) console.log("Searching " + gamename + " on F95Zone");
let page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(constURLs.F95_SEARCH_URL, {
waitUntil: shared.WAIT_STATEMENT
waitUntil: shared.WAIT_STATEMENT,
}); // Go to the search form and wait for it
// Explicitly wait for the required items to load
@ -544,18 +580,18 @@ async function getSearchGameResults(browser, gamename) {
await page.waitForSelector(constSelectors.TITLE_ONLY_CHECKBOX);
await page.waitForSelector(constSelectors.SEARCH_BUTTON);
await page.type(constSelectors.SEARCH_FORM_TEXTBOX, gamename) // Type the game we desire
await page.click(constSelectors.TITLE_ONLY_CHECKBOX) // Select only the thread with the game in the titles
await page.type(constSelectors.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire
await page.click(constSelectors.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles
await page.click(constSelectors.SEARCH_BUTTON); // Execute search
await page.waitForNavigation({
waitUntil: shared.WAIT_STATEMENT
waitUntil: shared.WAIT_STATEMENT,
}); // Wait for page to load
// Select all conversation titles
let threadTitleList = await page.$$(constSelectors.THREAD_TITLE);
// For each title extract the info about the conversation
if (shared.debug) console.log('Extracting info from conversation titles');
if (shared.debug) console.log("Extracting info from conversation titles");
let results = [];
for (let title of threadTitleList) {
let gameUrl = await getOnlyGameThreads(page, title);
@ -563,7 +599,7 @@ async function getSearchGameResults(browser, gamename) {
// Append the game's informations
if (gameUrl !== null) results.push(gameUrl);
}
if (shared.debug) console.log('Find ' + results.length + ' conversations');
if (shared.debug) console.log("Find " + results.length + " conversations");
await page.close(); // Close the page
return results;
@ -576,17 +612,23 @@ async function getSearchGameResults(browser, gamename) {
* @return {Promise<URL>} URL of the game/mod
*/
async function getOnlyGameThreads(page, titleHandle) {
const GAME_RECOMMENDATION_PREFIX = 'RECOMMENDATION';
const GAME_RECOMMENDATION_PREFIX = "RECOMMENDATION";
// Get the URL of the thread from the title
let relativeURLThread = await page.evaluate( /* istanbul ignore next */ (element) => element.querySelector('a').href, titleHandle);
let relativeURLThread = await page.evaluate(
/* istanbul ignore next */ (element) => element.querySelector("a").href,
titleHandle
);
let url = new URL(relativeURLThread, constURLs.F95_BASE_URL);
// Parse prefixes to ignore game recommendation
for (let element of await titleHandle.$$('span[dir="auto"]')) {
// Elaborate the prefixes
let prefix = await page.evaluate( /* istanbul ignore next */ element => element.textContent.toUpperCase(), element);
prefix = prefix.replace('[', '').replace(']', '');
let prefix = await page.evaluate(
/* istanbul ignore next */ (element) => element.textContent.toUpperCase(),
element
);
prefix = prefix.replace("[", "").replace("]", "");
// This is not a game nor a mod, we can exit
if (prefix === GAME_RECOMMENDATION_PREFIX) return null;

5
test/index-test.js
View File

@ -1,4 +1,4 @@
"use strict"
"use strict";
const expect = require("chai").expect;
const F95API = require("../app/index");
@ -53,7 +53,8 @@ describe("Login with cookies", function () {
before("Log in to create cookies then logout", async function () {
// Runs once before the first test in this block
if (!fs.existsSync(COOKIES_SAVE_PATH)) await F95API.login(USERNAME, PASSWORD); // Download cookies
if (!fs.existsSync(COOKIES_SAVE_PATH))
await F95API.login(USERNAME, PASSWORD); // Download cookies
F95API.logout();
});
//#endregion Set-up