F95API/app/index.js

610 lines
19 KiB
JavaScript
Raw Normal View History

"use strict";
// Core modules
const fs = require("fs");
2020-09-29 15:11:43 +00:00
// Modules from file
const shared = require("./scripts/shared.js");
const urlK = require("./scripts/constants/url.js");
const selectorK = require("./scripts/constants/css-selector.js");
const urlHelper = require("./scripts/url-helper.js");
const scraper = require("./scripts/game-scraper.js");
const {
prepareBrowser,
preparePage,
} = require("./scripts/puppeteer-helper.js");
const searcher = require("./scripts/game-searcher.js");
2020-10-12 08:27:48 +00:00
// Classes from file
const GameInfo = require("./scripts/classes/game-info.js");
const LoginResult = require("./scripts/classes/login-result.js");
const UserData = require("./scripts/classes/user-data.js");
2020-09-29 15:11:43 +00:00
2020-10-03 16:16:45 +00:00
//#region Export classes
module.exports.GameInfo = GameInfo;
module.exports.LoginResult = LoginResult;
module.exports.UserData = UserData;
2020-10-03 16:16:45 +00:00
//#endregion Export classes
2020-09-29 15:11:43 +00:00
2020-10-03 16:16:45 +00:00
//#region Export properties
2020-09-29 15:11:43 +00:00
/**
2020-10-02 15:43:14 +00:00
* Shows log messages and other useful functions for module debugging.
* @param {Boolean} value
2020-09-29 15:11:43 +00:00
*/
module.exports.debug = function (value) {
shared.debug = value;
// Configure logger
2020-10-21 13:43:11 +00:00
shared.logger.level = value ? "debug" : "warn";
};
2020-10-02 15:43:14 +00:00
/**
* @public
* Indicates whether a user is logged in to the F95Zone platform or not.
* @returns {String}
*/
module.exports.isLogged = function () {
return shared.isLogged;
2020-09-29 15:11:43 +00:00
};
2020-10-02 15:43:14 +00:00
/**
* @public
* If true, it opens a new browser for each request
2020-10-02 15:43:14 +00:00
* to the F95Zone platform, otherwise it reuses the same.
* @returns {String}
*/
module.exports.setIsolation = function (value) {
shared.isolation = value;
};
2020-10-02 15:43:14 +00:00
/**
* @public
* Path to the cache directory
* @returns {String}
*/
module.exports.getCacheDir = function () {
return shared.cacheDir;
};
2020-10-02 15:43:14 +00:00
/**
* @public
* Set path to the cache directory
* @returns {String}
*/
module.exports.setCacheDir = function (value) {
shared.cacheDir = value;
// Create directory if it doesn't exist
if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
};
2020-10-13 13:40:10 +00:00
/**
* @public
* Set local chromium path.
* @returns {String}
*/
module.exports.setChromiumPath = function (value) {
shared.chromiumLocalPath = value;
};
2020-10-03 16:16:45 +00:00
//#endregion Export properties
2020-09-29 15:11:43 +00:00
//#region Global variables
var _browser = null;
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
//#endregion
2020-09-29 15:11:43 +00:00
//#region Export methods
/**
* @public
* Log in to the F95Zone platform.
* This **must** be the first operation performed before accessing any other script functions.
* @param {String} username Username used for login
* @param {String} password Password used for login
* @returns {Promise<LoginResult>} Result of the operation
*/
module.exports.login = async function (username, password) {
if (shared.isLogged) {
shared.logger.info("Already logged in");
const result = new LoginResult(true, "Already logged in");
return result;
}
// If cookies are loaded, use them to authenticate
shared.cookies = loadCookies();
if (shared.cookies !== null) {
shared.logger.info("Valid session, no need to re-authenticate");
shared.isLogged = true;
const result = new LoginResult(true, "Logged with cookies");
return result;
}
2020-09-29 15:11:43 +00:00
// Else, log in throught browser
shared.logger.info(
"No saved sessions or expired session, login on the platform"
);
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
2020-10-16 07:21:19 +00:00
const browser = shared.isolation ? await prepareBrowser() : _browser;
2020-10-16 07:21:19 +00:00
const result = await loginF95(browser, username, password);
shared.isLogged = result.success;
2020-09-29 15:11:43 +00:00
if (result.success) {
// Reload cookies
shared.cookies = loadCookies();
shared.logger.info("User logged in through the platform");
} else {
shared.logger.warn("Error during authentication: " + result.message);
}
if (shared.isolation) await browser.close();
return result;
};
2020-09-29 15:11:43 +00:00
/**
* @public
* This method loads the main data from the F95 portal
* used to provide game information. You **must** be logged
2020-09-29 15:11:43 +00:00
* in to the portal before calling this method.
* @returns {Promise<Boolean>} Result of the operation
*/
module.exports.loadF95BaseData = async function () {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return false;
}
shared.logger.info("Loading base data...");
// Prepare a new web page
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
2020-10-16 07:21:19 +00:00
const browser = shared.isolation ? await prepareBrowser() : _browser;
2020-10-16 07:21:19 +00:00
const page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
// Go to latest update page and wait for it to load
await page.goto(urlK.F95_LATEST_UPDATES, {
waitUntil: shared.WAIT_STATEMENT,
});
2020-10-21 06:32:35 +00:00
// Obtain engines (disk/online)
await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR);
shared.engines = await loadValuesFromLatestPage(
page,
shared.enginesCachePath,
selectorK.ENGINE_ID_SELECTOR,
"engines"
);
2020-10-21 06:32:35 +00:00
// Obtain statuses (disk/online)
await page.waitForSelector(selectorK.STATUS_ID_SELECTOR);
shared.statuses = await loadValuesFromLatestPage(
page,
shared.statusesCachePath,
selectorK.STATUS_ID_SELECTOR,
"statuses"
);
await page.close();
if (shared.isolation) await browser.close();
shared.logger.info("Base data loaded");
return true;
};
2020-09-29 15:11:43 +00:00
/**
* @public
2020-10-10 09:33:54 +00:00
* Chek if exists a new version of the game.
2020-09-29 15:11:43 +00:00
* You **must** be logged in to the portal before calling this method.
* @param {GameInfo} info Information about the game to get the version for
2020-10-10 09:33:54 +00:00
* @returns {Promise<Boolean>} true if an update is available, false otherwise
2020-09-29 15:11:43 +00:00
*/
2020-10-10 09:33:54 +00:00
module.exports.chekIfGameHasUpdate = async function (info) {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return false;
}
2020-09-29 15:11:43 +00:00
// F95 change URL at every game update,
// so if the URL is different an update is available
const exists = await urlHelper.urlExists(info.f95url, true);
if (!exists) return true;
// Parse version from title
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
2020-10-16 07:21:19 +00:00
const browser = shared.isolation ? await prepareBrowser() : _browser;
2020-10-16 07:21:19 +00:00
const onlineVersion = await scraper.getGameVersionFromTitle(browser, info);
if (shared.isolation) await browser.close();
return onlineVersion.toUpperCase() !== info.version.toUpperCase();
};
2020-09-29 15:11:43 +00:00
/**
* @public
* Starting from the name, it gets all the information about the game you are looking for.
* You **must** be logged in to the portal before calling this method.
* @param {String} name Name of the game searched
* @param {Boolean} includeMods Indicates whether to also take mods into account when searching
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
* an identified game (in the case of homonymy of titles)
2020-09-29 15:11:43 +00:00
*/
module.exports.getGameData = async function (name, includeMods) {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
// Gets the search results of the game being searched for
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
2020-10-16 07:21:19 +00:00
const browser = shared.isolation ? await prepareBrowser() : _browser;
const urlList = await searcher.getSearchGameResults(browser, name);
// Process previous partial results
2020-10-16 07:21:19 +00:00
const promiseList = [];
for (const url of urlList) {
// Start looking for information
2020-10-12 08:27:48 +00:00
promiseList.push(scraper.getGameInfo(browser, url));
}
// Filter for mods
2020-10-16 07:21:19 +00:00
const result = [];
for (const info of await Promise.all(promiseList)) {
// Ignore empty results
2020-10-12 08:27:48 +00:00
if (!info) continue;
// Skip mods if not required
if (info.isMod && !includeMods) continue;
// Else save data
result.push(info);
}
if (shared.isolation) await browser.close();
return result;
};
2020-10-10 09:45:43 +00:00
/**
* @public
* Starting from the url, it gets all the information about the game you are looking for.
* You **must** be logged in to the portal before calling this method.
* @param {String} url URL of the game to obtain information of
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
*/
module.exports.getGameDataFromURL = async function (url) {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
2020-10-10 09:45:43 +00:00
return null;
}
// Check URL
const exists = await urlHelper.urlExists(url);
if (!exists) throw new URIError(url + " is not a valid URL");
if (!urlHelper.isF95URL(url))
throw new Error(url + " is not a valid F95Zone URL");
2020-10-10 09:45:43 +00:00
// Gets the search results of the game being searched for
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
2020-10-16 07:21:19 +00:00
const browser = shared.isolation ? await prepareBrowser() : _browser;
2020-10-10 09:45:43 +00:00
// Get game data
2020-10-16 07:21:19 +00:00
const result = await scraper.getGameInfo(browser, url);
2020-10-10 09:45:43 +00:00
if (shared.isolation) await browser.close();
return result;
};
2020-09-29 15:11:43 +00:00
/**
* @public
* Gets the data of the currently logged in user.
2020-10-02 15:43:14 +00:00
* You **must** be logged in to the portal before calling this method.
* @returns {Promise<UserData>} Data of the user currently logged in
2020-09-29 15:11:43 +00:00
*/
module.exports.getUserData = async function () {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
// Prepare a new web page
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
2020-10-16 07:21:19 +00:00
const browser = shared.isolation ? await prepareBrowser() : _browser;
const page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(urlK.F95_BASE_URL); // Go to base page
// Explicitly wait for the required items to load
await Promise.all([
page.waitForSelector(selectorK.USERNAME_ELEMENT),
page.waitForSelector(selectorK.AVATAR_PIC),
]);
2020-10-16 07:21:19 +00:00
const threads = getUserWatchedGameThreads(browser);
2020-10-16 07:21:19 +00:00
const username = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerText,
selectorK.USERNAME_ELEMENT
);
2020-10-16 07:21:19 +00:00
const avatarSrc = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).getAttribute("src"),
selectorK.AVATAR_PIC
);
2020-10-16 07:21:19 +00:00
const ud = new UserData();
ud.username = username;
ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null;
ud.watchedThreads = await threads;
await page.close();
if (shared.isolation) await browser.close();
return ud;
};
2020-10-02 15:43:14 +00:00
/**
* @public
* Logout from the current user and gracefully close shared browser.
2020-10-02 15:43:14 +00:00
* You **must** be logged in to the portal before calling this method.
*/
module.exports.logout = async function () {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return;
}
// Logout
shared.isLogged = false;
// Gracefully close shared browser
2020-10-08 20:39:03 +00:00
if (!shared.isolation && _browser !== null) {
await _browser.close();
_browser = null;
}
};
2020-09-29 15:11:43 +00:00
//#endregion
//#region Private methods
//#region Cookies functions
/**
* @private
* Loads and verifies the expiration of previously stored cookies from disk
* if they exist, otherwise it returns null.
* @return {object[]} List of dictionaries or null if cookies don't exist
*/
function loadCookies() {
// Check the existence of the cookie file
if (fs.existsSync(shared.cookiesCachePath)) {
// Read cookies
2020-10-16 07:21:19 +00:00
const cookiesJSON = fs.readFileSync(shared.cookiesCachePath);
const cookies = JSON.parse(cookiesJSON);
// Check if the cookies have expired
2020-10-16 07:21:19 +00:00
for (const cookie of cookies) {
if (isCookieExpired(cookie)) return null;
}
2020-09-29 15:11:43 +00:00
// Cookies loaded and verified
return cookies;
} else return null;
2020-09-29 15:11:43 +00:00
}
/**
* @private
* Check the validity of a cookie.
* @param {object} cookie Cookies to verify the validity. It's a dictionary
* @returns {Boolean} true if the cookie has expired, false otherwise
*/
function isCookieExpired(cookie) {
// Local variables
let expiredCookies = false;
// Ignore cookies that never expire
const expirationUnixTimestamp = cookie.expire;
if (expirationUnixTimestamp !== "-1") {
// Convert UNIX epoch timestamp to normal Date
2020-10-16 07:21:19 +00:00
const expirationDate = new Date(expirationUnixTimestamp * 1000);
if (expirationDate < Date.now()) {
shared.logger.warn(
"Cookie " + cookie.name + " expired, you need to re-authenticate"
);
expiredCookies = true;
2020-09-29 15:11:43 +00:00
}
}
2020-09-29 15:11:43 +00:00
return expiredCookies;
2020-09-29 15:11:43 +00:00
}
//#endregion Cookies functions
//#region Latest Updates page parserer
/**
* @private
* If present, it reads the file containing the searched values (engines or states)
* from the disk, otherwise it connects to the F95 portal (at the page
2020-09-29 15:11:43 +00:00
* https://f95zone.to/latest) and downloads them.
* @param {puppeteer.Page} page Page used to locate the required elements
* @param {String} path Path to disk of the JSON file containing the data to read / write
* @param {String} selector CSS selector of the required elements
* @param {String} elementRequested Required element (engines or states) used to detail log messages
* @returns {Promise<String[]>} List of required values in uppercase
*/
async function loadValuesFromLatestPage(
page,
path,
selector,
elementRequested
) {
// If the values already exist they are loaded from disk without having to connect to F95
shared.logger.info("Load " + elementRequested + " from disk...");
if (fs.existsSync(path)) {
2020-10-16 07:21:19 +00:00
const valueJSON = fs.readFileSync(path);
return JSON.parse(valueJSON);
}
// Otherwise, connect and download the data from the portal
shared.logger.info("No " + elementRequested + " cached, downloading...");
2020-10-16 07:21:19 +00:00
const values = await getValuesFromLatestPage(
page,
selector,
"Getting " + elementRequested + " from page"
);
fs.writeFileSync(path, JSON.stringify(values));
return values;
2020-09-29 15:11:43 +00:00
}
/**
* @private
* Gets all the textual values of the elements present
* in the F95 portal page and identified by the selector
2020-09-29 15:11:43 +00:00
* passed by parameter
* @param {puppeteer.Page} page Page used to locate items specified by the selector
* @param {String} selector CSS selector
* @param {String} logMessage Log message indicating which items the selector is requesting
* @return {Promise<String[]>} List of uppercase strings indicating the textual values of the elements identified by the selector
*/
async function getValuesFromLatestPage(page, selector, logMessage) {
shared.logger.info(logMessage);
2020-09-29 15:11:43 +00:00
2020-10-16 07:21:19 +00:00
const result = [];
const elements = await page.$$(selector);
2020-09-29 15:11:43 +00:00
2020-10-16 07:21:19 +00:00
for (const element of elements) {
const text = await element.evaluate(
/* istanbul ignore next */ (e) => e.innerText
);
2020-09-29 15:11:43 +00:00
// Save as upper text for better match if used in query
result.push(text.toUpperCase());
}
return result;
2020-09-29 15:11:43 +00:00
}
//#endregion
//#region User
/**
* @private
* Log in to the F95Zone portal and, if successful, save the cookies.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @param {String} username Username to use during login
* @param {String} password Password to use during login
* @returns {Promise<LoginResult>} Result of the operation
*/
async function loginF95(browser, username, password) {
2020-10-16 07:21:19 +00:00
const page = await preparePage(browser); // Set new isolated page
await page.goto(urlK.F95_LOGIN_URL); // Go to login page
// Explicitly wait for the required items to load
await Promise.all([
page.waitForSelector(selectorK.USERNAME_INPUT),
page.waitForSelector(selectorK.PASSWORD_INPUT),
page.waitForSelector(selectorK.LOGIN_BUTTON),
]);
await page.type(selectorK.USERNAME_INPUT, username); // Insert username
await page.type(selectorK.PASSWORD_INPUT, password); // Insert password
2020-10-14 13:12:35 +00:00
await Promise.all([
page.click(selectorK.LOGIN_BUTTON), // Click on the login button
2020-10-14 13:12:35 +00:00
page.waitForNavigation({
waitUntil: shared.WAIT_STATEMENT,
}), // Wait for page to load
2020-10-14 13:12:35 +00:00
]);
// Prepare result
let message = "";
// Check if the user is logged in
2020-10-21 13:33:08 +00:00
const success = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector) !== null,
selectorK.AVATAR_INFO
);
2020-10-21 13:33:08 +00:00
const errorMessageExists = await page.evaluate(
/* istanbul ignore next */
(selector) => document.querySelector(selector) !== null,
selectorK.LOGIN_MESSAGE_ERROR
);
// Save cookies to avoid re-auth
if (success) {
2020-10-16 07:21:19 +00:00
const c = await page.cookies();
fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
message = "Authentication successful";
} else if (errorMessageExists) {
2020-10-16 07:21:19 +00:00
const errorMessage = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerText,
selectorK.LOGIN_MESSAGE_ERROR
);
if (errorMessage === "Incorrect password. Please try again.") {
message = "Incorrect password";
} else if (
errorMessage ===
"The requested user '" + username + "' could not be found."
) {
// The escaped quotes are important!
message = "Incorrect username";
} else message = errorMessage;
} else message = "Unknown error";
await page.close(); // Close the page
return new LoginResult(success, message);
2020-09-29 15:11:43 +00:00
}
/**
* @private
* Gets the list of URLs of threads the user follows.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @returns {Promise<String[]>} URL list
2020-09-29 15:11:43 +00:00
*/
async function getUserWatchedGameThreads(browser) {
2020-10-16 07:21:19 +00:00
const page = await preparePage(browser); // Set new isolated page
await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page
// Explicitly wait for the required items to load
await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON);
// Show the popup
await Promise.all([
page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON),
page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX),
page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION),
page.waitForSelector(selectorK.FILTER_THREADS_BUTTON),
]);
// Set the filters
await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).removeAttribute("checked"),
selectorK.UNREAD_THREAD_CHECKBOX
); // Also read the threads already read
// Filter the threads
await page.click(selectorK.ONLY_GAMES_THREAD_OPTION);
await page.click(selectorK.FILTER_THREADS_BUTTON);
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
// Get the threads urls
2020-10-16 07:21:19 +00:00
const urls = [];
let nextPageExists = false;
do {
// Get all the URLs
for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) {
2020-10-16 07:21:19 +00:00
const src = await page.evaluate(
/* istanbul ignore next */ (element) => element.href,
handle
);
// If 'unread' is left, it will redirect to the last unread post
const url = src.replace("/unread", "");
urls.push(url);
}
nextPageExists = await page.evaluate(
/* istanbul ignore next */ (selector) => document.querySelector(selector),
selectorK.WATCHED_THREAD_NEXT_PAGE
);
// Click to next page
if (nextPageExists) {
await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE);
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
}
} while (nextPageExists);
await page.close();
return urls;
2020-09-29 15:11:43 +00:00
}
//#endregion User
//#endregion Private methods