MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #44 from MillenniumEarl/Use-plain-HTML-scraper 

Switch to plain HTML scraper
pull/45/head
Millennium Earl 2020-11-02 16:23:55 +01:00 committed by GitHub
parent 1830e61aea a8d2ab9c4c
commit 151a0e7d3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 2002 additions and 2247 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
.eslintrc.json
View File

@ -1,20 +1,38 @@
{
"env": {
"browser": true,
"commonjs": true,
"es2021": true,
"node": true
},
"extends": "eslint:recommended",
"parser": "./node_modules/babel-eslint",
"parserOptions": {
"ecmaVersion": 12
},
"rules": {
"indent": ["error", 4],
"linebreak-style": ["error", "windows"],
"quotes": ["error", "double"],
"semi": ["error", "always"],
"no-unused-vars": ["error", "after-used"]
}
"env": {
"browser": true,
"commonjs": true,
"es2021": true,
"node": true,
"mocha": true
},
"extends": "eslint:recommended",
"parser": "babel-eslint",
"parserOptions": {
"ecmaVersion": 12
},
"rules": {
"indent": [
"error",
4
],
"linebreak-style": [
"error",
"windows"
],
"quotes": [
"error",
"double"
],
"semi": [
"error",
"always"
],
"no-unused-vars": [
"error",
{
"args": "after-used"
}
]
}
}

69
README.md
View File

@ -8,6 +8,75 @@
Unofficial Node JS module for scraping F95Zone platform
These APIs have been developed to support this application and allow you to obtain data on games and mods on the platform [F95zone](https://f95zone.to/) (**NSFW**)
A simple usage example can be found in [app/example.js](https://github.com/MillenniumEarl/F95API/blob/master/app/example.js)
**Attention**: Two-factor authentication is not supported
# Data scraping
Games/mods can be obtained by name or URL
```javascript
// The name is case insensitive
let listOfFoundGames = await F95API.getGameData("your game name");
let listOfFoundMods = await F95API.getGameData("your mod name", true);
let specificGame = await F95API.getGameDataFromURL("the URL of your game");
```
While user data (after authenticating) with
```javascript
let authResult = await F95API.login(username, password);
let loggedUserData = await F95API.getUserData();
```
# Classes
## Games and mods
Information about games and mods is stored in a GameInfo object with the following fields:
```
name: The game name
author: The game developer
url: The URL that leads to the game thread on F95Zone
overview: Description of the game
language: Main language of the game
supportedOS: List of supported OS (Windows/Linux/Mac/Android...)
censored: Are the NSFW parts censored?
engine: Game engine (Unity, Ren'Py, RPGM...)
status: Completed/Abandoned/Ongoing/Onhold
tags: List of tags
previewSrc: Source URL of the game description image
version: Version of the game
lastUpdate: Date of the last update (it's a Date object)
isMod: Is it a game or a mod?
changelog: Latest changelog available
```
The serialization in JSON format of this object is possible through `JSON.stringfy()` while the deserialization must happen through the static method `GameInfo.fromJSON()`.
## User data
User data (after authentication) can be stored in a UserData object, consisting of the following fields:
```
username: Name of the logged in user
avatarSrc: Source URL of the user's profile picture
watchedThreads: List of URLs of threads followed by the user
```
## Login results
The outcome of the authentication process is represented by the LoginResult object:
```
success: Was the authentication successful?;
message: Possible error message (unrecognized user, wrong password ...) or authentication successful message
```
# Logging
To log the behavior of the application [log4js](https://github.com/log4js-node/log4js-node) is used with a default level of "warn". This option can be changed with the `loggerLevel` property.
# Guidelines for errors
- If you can, return a meaningful value

56
app/example.js Normal file
View File

@ -0,0 +1,56 @@
/*
to use this example, create an .env file
in the project root with the following values:
F95_USERNAME = YOUR_USERNAME
F95_PASSWORD = YOUR_PASSWORD
*/
"use strict";
// Public modules from npm
const dotenv = require("dotenv");
// Modules from file
const F95API = require("./index.js");
// Configure the .env reader
dotenv.config();
main();
async function main() {
// Local variables
const gameList = [
"kingdom of deception",
"perverted education",
"corrupted kingdoms",
"summertime saga",
"brothel king"
];
// Log in the platform
console.log("Authenticating...");
const result = await F95API.login(process.env.F95_USERNAME, process.env.F95_PASSWORD);
console.log(`Authentication result: ${result.message}`);
// Get user data
console.log("Fetching user data...");
const userdata = await F95API.getUserData();
console.log(`${userdata.username} follows ${userdata.watchedThreads.length} threads`);
for(const gamename of gameList) {
console.log(`Searching '${gamename}'...`);
const found = await F95API.getGameData(gamename, false);
// If no game is found
if (found.length === 0) {
console.log(`No data found for '${gamename}'`);
continue;
}
// Extract first game
const gamedata = found[0];
console.log(`Found ${gamedata.name} (${gamedata.version}) by ${gamedata.author}`);
}
}

600
app/index.js
View File

@ -1,21 +1,14 @@
"use strict";
// Core modules
const fs = require("fs");
// Modules from file
const shared = require("./scripts/shared.js");
const urlK = require("./scripts/constants/url.js");
const selectorK = require("./scripts/constants/css-selector.js");
const urlHelper = require("./scripts/url-helper.js");
const scraper = require("./scripts/game-scraper.js");
const {
prepareBrowser,
preparePage,
} = require("./scripts/puppeteer-helper.js");
const searcher = require("./scripts/game-searcher.js");
const networkHelper = require("./scripts/network-helper.js");
const scraper = require("./scripts/scraper.js");
const searcher = require("./scripts/searcher.js");
const uScraper = require("./scripts/user-scraper.js");
// Classes from file
const Credentials = require("./scripts/classes/credentials.js");
const GameInfo = require("./scripts/classes/game-info.js");
const LoginResult = require("./scripts/classes/login-result.js");
const UserData = require("./scripts/classes/user-data.js");
@ -28,63 +21,24 @@ module.exports.UserData = UserData;
//#region Export properties
/**
* Shows log messages and other useful functions for module debugging.
* @param {Boolean} value
* @public
* Set the logger level for module debugging.
*/
module.exports.debug = function (value) {
shared.debug = value;
// Configure logger
shared.logger.level = value ? "debug" : "warn";
};
/* istambul ignore next */
module.exports.loggerLevel = shared.logger.level;
exports.loggerLevel = "warn"; // By default log only the warn messages
/**
* @public
* Indicates whether a user is logged in to the F95Zone platform or not.
* @returns {String}
*/
module.exports.isLogged = function () {
return shared.isLogged;
};
/**
* @public
* If true, it opens a new browser for each request
* to the F95Zone platform, otherwise it reuses the same.
* @returns {String}
*/
module.exports.setIsolation = function (value) {
shared.isolation = value;
};
/**
* @public
* Path to the cache directory
* @returns {String}
*/
module.exports.getCacheDir = function () {
return shared.cacheDir;
};
/**
* @public
* Set path to the cache directory
* @returns {String}
*/
module.exports.setCacheDir = function (value) {
shared.cacheDir = value;
// Create directory if it doesn't exist
if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir);
};
/**
* @public
* Set local chromium path.
* @returns {String}
*/
module.exports.setChromiumPath = function (value) {
shared.chromiumLocalPath = value;
/* istambul ignore next */
module.exports.isLogged = function isLogged() {
return shared.isLogged;
};
//#endregion Export properties
//#region Global variables
var _browser = null;
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
//#endregion
@ -98,92 +52,25 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue";
* @returns {Promise<LoginResult>} Result of the operation
*/
module.exports.login = async function (username, password) {
if (shared.isLogged) {
shared.logger.info("Already logged in");
const result = new LoginResult(true, "Already logged in");
if (shared.isLogged) {
shared.logger.info(`${username} already authenticated`);
return new LoginResult(true, `${username} already authenticated`);
}
shared.logger.trace("Fetching token...");
const creds = new Credentials(username, password);
await creds.fetchToken();
shared.logger.trace(`Authentication for ${username}`);
const result = await networkHelper.authenticate(creds);
shared.isLogged = result.success;
if (result.success) shared.logger.info("User logged in through the platform");
else shared.logger.warn(`Error during authentication: ${result.message}`);
return result;
}
// If cookies are loaded, use them to authenticate
shared.cookies = loadCookies();
if (shared.cookies !== null) {
shared.logger.info("Valid session, no need to re-authenticate");
shared.isLogged = true;
const result = new LoginResult(true, "Logged with cookies");
return result;
}
// Else, log in throught browser
shared.logger.info(
"No saved sessions or expired session, login on the platform"
);
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
const browser = shared.isolation ? await prepareBrowser() : _browser;
const result = await loginF95(browser, username, password);
shared.isLogged = result.success;
if (result.success) {
// Reload cookies
shared.cookies = loadCookies();
shared.logger.info("User logged in through the platform");
} else {
shared.logger.warn("Error during authentication: " + result.message);
}
if (shared.isolation) await browser.close();
return result;
};
/**
* @public
* This method loads the main data from the F95 portal
* used to provide game information. You **must** be logged
* in to the portal before calling this method.
* @returns {Promise<Boolean>} Result of the operation
*/
module.exports.loadF95BaseData = async function () {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return false;
}
shared.logger.info("Loading base data...");
// Prepare a new web page
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
const browser = shared.isolation ? await prepareBrowser() : _browser;
const page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
// Go to latest update page and wait for it to load
await page.goto(urlK.F95_LATEST_UPDATES, {
waitUntil: shared.WAIT_STATEMENT,
});
// Obtain engines (disk/online)
await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR);
shared.engines = await loadValuesFromLatestPage(
page,
shared.enginesCachePath,
selectorK.ENGINE_ID_SELECTOR,
"engines"
);
// Obtain statuses (disk/online)
await page.waitForSelector(selectorK.STATUS_ID_SELECTOR);
shared.statuses = await loadValuesFromLatestPage(
page,
shared.statusesCachePath,
selectorK.STATUS_ID_SELECTOR,
"statuses"
);
await page.close();
if (shared.isolation) await browser.close();
shared.logger.info("Base data loaded");
return true;
};
/**
* @public
* Chek if exists a new version of the game.
@ -191,68 +78,54 @@ module.exports.loadF95BaseData = async function () {
* @param {GameInfo} info Information about the game to get the version for
* @returns {Promise<Boolean>} true if an update is available, false otherwise
*/
module.exports.chekIfGameHasUpdate = async function (info) {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return false;
}
module.exports.checkIfGameHasUpdate = async function (info) {
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return false;
}
// F95 change URL at every game update,
// so if the URL is different an update is available
const exists = await urlHelper.urlExists(info.f95url, true);
if (!exists) return true;
// F95 change URL at every game update,
// so if the URL is different an update is available
const exists = await networkHelper.urlExists(info.url, true);
if (!exists) return true;
// Parse version from title
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
const browser = shared.isolation ? await prepareBrowser() : _browser;
// Parse version from title
const onlineVersion = await scraper.getGameInfo(info.url).version;
const onlineVersion = await scraper.getGameVersionFromTitle(browser, info);
if (shared.isolation) await browser.close();
return onlineVersion.toUpperCase() !== info.version.toUpperCase();
// Compare the versions
return onlineVersion.toUpperCase() !== info.version.toUpperCase();
};
/**
* @public
* Starting from the name, it gets all the information about the game you are looking for.
* You **must** be logged in to the portal before calling this method.
* @param {String} name Name of the game searched
* @param {Boolean} includeMods Indicates whether to also take mods into account when searching
* @param {Boolean} mod Indicate if you are looking for mods or games
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
* an identified game (in the case of homonymy of titles)
*/
module.exports.getGameData = async function (name, includeMods) {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
module.exports.getGameData = async function (name, mod) {
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
// Gets the search results of the game being searched for
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
const browser = shared.isolation ? await prepareBrowser() : _browser;
const urlList = await searcher.getSearchGameResults(browser, name);
// Gets the search results of the game/mod being searched for
let urls = [];
if(mod) urls = await searcher.searchMod(name);
else urls = await searcher.searchGame(name);
// Process previous partial results
const promiseList = [];
for (const url of urlList) {
// Start looking for information
promiseList.push(scraper.getGameInfo(browser, url));
}
// Filter for mods
const result = [];
for (const info of await Promise.all(promiseList)) {
// Ignore empty results
if (!info) continue;
// Skip mods if not required
if (info.isMod && !includeMods) continue;
// Else save data
result.push(info);
}
if (shared.isolation) await browser.close();
return result;
// Process previous partial results
const results = [];
for (const url of urls) {
// Start looking for information
const info = await scraper.getGameInfo(url);
results.push(info);
}
return results;
};
/**
* @public
* Starting from the url, it gets all the information about the game you are looking for.
@ -261,27 +134,20 @@ module.exports.getGameData = async function (name, includeMods) {
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
*/
module.exports.getGameDataFromURL = async function (url) {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
// Check URL
const exists = await urlHelper.urlExists(url);
if (!exists) throw new URIError(url + " is not a valid URL");
if (!urlHelper.isF95URL(url))
throw new Error(url + " is not a valid F95Zone URL");
// Check URL validity
const exists = await networkHelper.urlExists(url);
if (!exists) throw new URIError(`${url} is not a valid URL`);
if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
// Gets the search results of the game being searched for
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
const browser = shared.isolation ? await prepareBrowser() : _browser;
// Get game data
const result = await scraper.getGameInfo(browser, url);
if (shared.isolation) await browser.close();
return result;
// Get game data
return await scraper.getGameInfo(url);
};
/**
* @public
* Gets the data of the currently logged in user.
@ -289,321 +155,11 @@ module.exports.getGameDataFromURL = async function (url) {
* @returns {Promise<UserData>} Data of the user currently logged in
*/
module.exports.getUserData = async function () {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
// Prepare a new web page
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
const browser = shared.isolation ? await prepareBrowser() : _browser;
const page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(urlK.F95_BASE_URL); // Go to base page
// Explicitly wait for the required items to load
await Promise.all([
page.waitForSelector(selectorK.USERNAME_ELEMENT),
page.waitForSelector(selectorK.AVATAR_PIC),
]);
const threads = getUserWatchedGameThreads(browser);
const username = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerText,
selectorK.USERNAME_ELEMENT
);
const avatarSrc = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).getAttribute("src"),
selectorK.AVATAR_PIC
);
const ud = new UserData();
ud.username = username;
ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null;
ud.watchedThreads = await threads;
await page.close();
if (shared.isolation) await browser.close();
return ud;
};
/**
* @public
* Logout from the current user and gracefully close shared browser.
* You **must** be logged in to the portal before calling this method.
*/
module.exports.logout = async function () {
if (!shared.isLogged || !shared.cookies) {
shared.logger.warn(USER_NOT_LOGGED);
return;
}
// Logout
shared.isLogged = false;
// Gracefully close shared browser
if (!shared.isolation && _browser !== null) {
await _browser.close();
_browser = null;
}
return await uScraper.getUserData();
};
//#endregion
//#region Private methods
//#region Cookies functions
/**
* @private
* Loads and verifies the expiration of previously stored cookies from disk
* if they exist, otherwise it returns null.
* @return {object[]} List of dictionaries or null if cookies don't exist
*/
function loadCookies() {
// Check the existence of the cookie file
if (fs.existsSync(shared.cookiesCachePath)) {
// Read cookies
const cookiesJSON = fs.readFileSync(shared.cookiesCachePath);
const cookies = JSON.parse(cookiesJSON);
// Check if the cookies have expired
for (const cookie of cookies) {
if (isCookieExpired(cookie)) return null;
}
// Cookies loaded and verified
return cookies;
} else return null;
}
/**
* @private
* Check the validity of a cookie.
* @param {object} cookie Cookies to verify the validity. It's a dictionary
* @returns {Boolean} true if the cookie has expired, false otherwise
*/
function isCookieExpired(cookie) {
// Local variables
let expiredCookies = false;
// Ignore cookies that never expire
const expirationUnixTimestamp = cookie.expire;
if (expirationUnixTimestamp !== "-1") {
// Convert UNIX epoch timestamp to normal Date
const expirationDate = new Date(expirationUnixTimestamp * 1000);
if (expirationDate < Date.now()) {
shared.logger.warn(
"Cookie " + cookie.name + " expired, you need to re-authenticate"
);
expiredCookies = true;
}
}
return expiredCookies;
}
//#endregion Cookies functions
//#region Latest Updates page parserer
/**
* @private
* If present, it reads the file containing the searched values (engines or states)
* from the disk, otherwise it connects to the F95 portal (at the page
* https://f95zone.to/latest) and downloads them.
* @param {puppeteer.Page} page Page used to locate the required elements
* @param {String} path Path to disk of the JSON file containing the data to read / write
* @param {String} selector CSS selector of the required elements
* @param {String} elementRequested Required element (engines or states) used to detail log messages
* @returns {Promise<String[]>} List of required values in uppercase
*/
async function loadValuesFromLatestPage(
page,
path,
selector,
elementRequested
) {
// If the values already exist they are loaded from disk without having to connect to F95
shared.logger.info("Load " + elementRequested + " from disk...");
if (fs.existsSync(path)) {
const valueJSON = fs.readFileSync(path);
return JSON.parse(valueJSON);
}
// Otherwise, connect and download the data from the portal
shared.logger.info("No " + elementRequested + " cached, downloading...");
const values = await getValuesFromLatestPage(
page,
selector,
"Getting " + elementRequested + " from page"
);
fs.writeFileSync(path, JSON.stringify(values));
return values;
}
/**
* @private
* Gets all the textual values of the elements present
* in the F95 portal page and identified by the selector
* passed by parameter
* @param {puppeteer.Page} page Page used to locate items specified by the selector
* @param {String} selector CSS selector
* @param {String} logMessage Log message indicating which items the selector is requesting
* @return {Promise<String[]>} List of uppercase strings indicating the textual values of the elements identified by the selector
*/
async function getValuesFromLatestPage(page, selector, logMessage) {
shared.logger.info(logMessage);
const result = [];
const elements = await page.$$(selector);
for (const element of elements) {
const text = await element.evaluate(
/* istanbul ignore next */ (e) => e.innerText
);
// Save as upper text for better match if used in query
result.push(text.toUpperCase());
}
return result;
}
//#endregion
//#region User
/**
* @private
* Log in to the F95Zone portal and, if successful, save the cookies.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @param {String} username Username to use during login
* @param {String} password Password to use during login
* @returns {Promise<LoginResult>} Result of the operation
*/
async function loginF95(browser, username, password) {
const page = await preparePage(browser); // Set new isolated page
await page.goto(urlK.F95_LOGIN_URL); // Go to login page
// Explicitly wait for the required items to load
await Promise.all([
page.waitForSelector(selectorK.USERNAME_INPUT),
page.waitForSelector(selectorK.PASSWORD_INPUT),
page.waitForSelector(selectorK.LOGIN_BUTTON),
]);
await page.type(selectorK.USERNAME_INPUT, username); // Insert username
await page.type(selectorK.PASSWORD_INPUT, password); // Insert password
await Promise.all([
page.click(selectorK.LOGIN_BUTTON), // Click on the login button
page.waitForNavigation({
waitUntil: shared.WAIT_STATEMENT,
}), // Wait for page to load
]);
// Prepare result
let message = "";
// Check if the user is logged in
const success = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector) !== null,
selectorK.AVATAR_INFO
);
const errorMessageExists = await page.evaluate(
/* istanbul ignore next */
(selector) => document.querySelector(selector) !== null,
selectorK.LOGIN_MESSAGE_ERROR
);
// Save cookies to avoid re-auth
if (success) {
const c = await page.cookies();
fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
message = "Authentication successful";
} else if (errorMessageExists) {
const errorMessage = await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerText,
selectorK.LOGIN_MESSAGE_ERROR
);
if (errorMessage === "Incorrect password. Please try again.") {
message = "Incorrect password";
} else if (
errorMessage ===
"The requested user '" + username + "' could not be found."
) {
// The escaped quotes are important!
message = "Incorrect username";
} else message = errorMessage;
} else message = "Unknown error";
await page.close(); // Close the page
return new LoginResult(success, message);
}
/**
* @private
* Gets the list of URLs of threads the user follows.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @returns {Promise<String[]>} URL list
*/
async function getUserWatchedGameThreads(browser) {
const page = await preparePage(browser); // Set new isolated page
await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page
// Explicitly wait for the required items to load
await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON);
// Show the popup
await Promise.all([
page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON),
page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX),
page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION),
page.waitForSelector(selectorK.FILTER_THREADS_BUTTON),
]);
// Set the filters
await page.evaluate(
/* istanbul ignore next */ (selector) =>
document.querySelector(selector).removeAttribute("checked"),
selectorK.UNREAD_THREAD_CHECKBOX
); // Also read the threads already read
// Filter the threads
await page.click(selectorK.ONLY_GAMES_THREAD_OPTION);
await page.click(selectorK.FILTER_THREADS_BUTTON);
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
// Get the threads urls
const urls = [];
let nextPageExists = false;
do {
// Get all the URLs
for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) {
const src = await page.evaluate(
/* istanbul ignore next */ (element) => element.href,
handle
);
// If 'unread' is left, it will redirect to the last unread post
const url = src.replace("/unread", "");
urls.push(url);
}
nextPageExists = await page.evaluate(
/* istanbul ignore next */ (selector) => document.querySelector(selector),
selectorK.WATCHED_THREAD_NEXT_PAGE
);
// Click to next page
if (nextPageExists) {
await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE);
await page.waitForSelector(selectorK.WATCHED_THREAD_URLS);
}
} while (nextPageExists);
await page.close();
return urls;
}
//#endregion User
//#endregion Private methods

18
app/scripts/classes/credentials.js Normal file
View File

@ -0,0 +1,18 @@
"use strict";
// Modules from file
const { getF95Token } = require("../network-helper.js");
class Credentials {
constructor(username, password) {
this.username = username;
this.password = password;
this.token = null;
}
async fetchToken() {
this.token = await getF95Token();
}
}
module.exports = Credentials;

110
app/scripts/classes/game-download.js
View File

@ -1,110 +0,0 @@
/* istanbul ignore file */
"use strict";
// Core modules
const fs = require("fs");
// Public modules from npm
// const { File } = require('megajs');
// Modules from file
const { prepareBrowser, preparePage } = require("../puppeteer-helper.js");
const shared = require("../shared.js");
class GameDownload {
constructor() {
/**
* @public
* Platform that hosts game files
* @type String
*/
this.hosting = "";
/**
* @public
* Link to game files
* @type String
*/
this.link = null;
/**
* @public
* Operating systems supported by the game version indicated in this class.
* Can be *WINDOWS/LINUX/MACOS*
* @type String[]
*/
this.supportedOS = [];
}
/**
* @public
* Download the game data in the indicated path.
* Supported hosting platforms: MEGA, NOPY
* @param {String} path Save path
* @return {Promise<Boolean>} Result of the operation
*/
async download(path) {
if (this.link.includes("mega.nz"))
return await downloadMEGA(this.link, path);
else if (this.link.includes("nopy.to"))
return await downloadNOPY(this.link, path);
}
}
module.exports = GameDownload;
async function downloadMEGA(url, savepath) {
// The URL is masked
const browser = await prepareBrowser();
const page = await preparePage(browser);
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(url);
await page.waitForSelector("a.host_link");
// Obtain the link for the unmasked page and click it
const link = await page.$("a.host_link");
await link.click();
await page.goto(url, {
waitUntil: shared.WAIT_STATEMENT,
}); // Go to the game page and wait until it loads
// Obtain the URL after the redirect
const downloadURL = page.url();
// Close browser and page
await page.close();
await browser.close();
const stream = fs.createWriteStream(savepath);
const file = File.fromURL(downloadURL);
file.download().pipe(stream);
return fs.existsSync(savepath);
}
async function downloadNOPY(url, savepath) {
// Prepare browser
const browser = await prepareBrowser();
const page = await preparePage(browser);
await page.goto(url);
await page.waitForSelector("#download");
// Set the save path
await page._client.send("Page.setDownloadBehavior", {
behavior: "allow",
downloadPath: path.basename(path.dirname(savepath)), // It's a directory
});
// Obtain the download button and click it
const downloadButton = await page.$("#download");
await downloadButton.click();
// Await for all the connections to close
await page.waitForNavigation({
waitUntil: "networkidle0",
timeout: 0, // Disable timeout
});
// Close browser and page
await page.close();
await browser.close();
return fs.existsSync(savepath);
}

240
app/scripts/classes/game-info.js
View File

@ -1,119 +1,135 @@
"use strict";
class GameInfo {
constructor() {
//#region Properties
/**
* Game name
* @type String
*/
this.name = null;
/**
* Game author
* @type String
*/
this.author = null;
/**
* URL to the game's official conversation on the F95Zone portal
* @type String
*/
this.f95url = null;
/**
* Game description
* @type String
*/
this.overview = null;
/**
* List of tags associated with the game
* @type String[]
*/
this.tags = [];
/**
* Graphics engine used for game development
* @type String
*/
this.engine = null;
/**
* Progress of the game
* @type String
*/
this.status = null;
/**
* Game description image URL
* @type String
*/
this.previewSource = null;
/**
* Game version
* @type String
*/
this.version = null;
/**
* Last time the game underwent updates
* @type String
*/
this.lastUpdate = null;
/**
* Last time the local copy of the game was run
* @type String
*/
this.lastPlayed = null;
/**
* Specifies if the game is original or a mod
* @type Boolean
*/
this.isMod = false;
/**
* Changelog for the last version.
* @type String
*/
this.changelog = null;
/**
* Directory containing the local copy of the game
* @type String
*/
this.gameDir = null;
/**
* Information on game file download links,
* including information on hosting platforms
* and operating system supported by the specific link
* @type GameDownload[]
*/
this.downloadInfo = [];
//#endregion Properties
}
constructor() {
//#region Properties
/**
* Game name
* @type String
*/
this.name = null;
/**
* Game author
* @type String
*/
this.author = null;
/**
* URL to the game's official conversation on the F95Zone portal
* @type String
*/
this.url = null;
/**
* Game description
* @type String
*/
this.overview = null;
/**
* Game language.
* @type String
*/
this.language = null;
/**
* List of supported OS.
* @type
*/
this.supportedOS = [];
/**
* Specify whether the game has censorship
* measures regarding NSFW scenes.
* @type Boolean
*/
this.censored = null;
/**
* List of tags associated with the game
* @type String[]
*/
this.tags = [];
/**
* Graphics engine used for game development
* @type String
*/
this.engine = null;
/**
* Progress of the game
* @type String
*/
this.status = null;
/**
* Game description image URL
* @type String
*/
this.previewSrc = null;
/**
* Game version
* @type String
*/
this.version = null;
/**
* Last time the game underwent updates
* @type String
*/
this.lastUpdate = null;
/**
* Last time the local copy of the game was run
* @type String
*/
this.lastPlayed = null;
/**
* Specifies if the game is original or a mod
* @type Boolean
*/
this.isMod = false;
/**
* Changelog for the last version.
* @type String
*/
this.changelog = null;
/**
* Directory containing the local copy of the game
* @type String
*/
this.gameDir = null;
//#endregion Properties
}
/**
* Converts the object to a dictionary used for JSON serialization
*/
/* istanbul ignore next */
toJSON() {
return {
name: this.name,
author: this.author,
f95url: this.f95url,
overview: this.overview,
engine: this.engine,
status: this.status,
previewSource: this.previewSource,
version: this.version,
lastUpdate: this.lastUpdate,
lastPlayed: this.lastPlayed,
isMod: this.isMod,
changelog: this.changelog,
gameDir: this.gameDir,
downloadInfo: this.downloadInfo,
};
}
/**
* Converts the object to a dictionary used for JSON serialization.
*/
/* istanbul ignore next */
toJSON() {
return {
name: this.name,
author: this.author,
url: this.url,
overview: this.overview,
language: this.language,
supportedOS: this.supportedOS,
censored: this.censored,
engine: this.engine,
status: this.status,
tags: this.tags,
previewSrc: this.previewSrc,
version: this.version,
lastUpdate: this.lastUpdate,
lastPlayed: this.lastPlayed,
isMod: this.isMod,
changelog: this.changelog,
gameDir: this.gameDir,
};
}
/**
* Return a new GameInfo from a JSON string
* @param {String} json JSON string used to create the new object
* @returns {GameInfo}
*/
/* istanbul ignore next */
static fromJSON(json) {
return Object.assign(new GameInfo(), json);
}
/**
* Return a new GameInfo from a JSON string.
* @param {String} json JSON string used to create the new object
* @returns {GameInfo}
*/
static fromJSON(json) {
// Convert string
const temp = Object.assign(new GameInfo(), JSON.parse(json));
// JSON cannot transform a string to a date implicitly
temp.lastUpdate = new Date(temp.lastUpdate);
return temp;
}
}
module.exports = GameInfo;

24
app/scripts/classes/login-result.js
View File

@ -4,17 +4,17 @@
* Object obtained in response to an attempt to login to the portal.
*/
class LoginResult {
constructor(success, message) {
/**
* Result of the login operation
* @type Boolean
*/
this.success = success;
/**
* Login response message
* @type String
*/
this.message = message;
}
constructor(success, message) {
/**
* Result of the login operation
* @type Boolean
*/
this.success = success;
/**
* Login response message
* @type String
*/
this.message = message;
}
}
module.exports = LoginResult;

34
app/scripts/classes/user-data.js
View File

@ -4,23 +4,23 @@
* Class containing the data of the user currently connected to the F95Zone platform.
*/
class UserData {
constructor() {
/**
* User username.
* @type String
*/
this.username = "";
/**
* Path to the user's profile picture.
* @type String
*/
this.avatarSrc = null;
/**
* List of followed thread URLs.
* @type URL[]
*/
this.watchedThreads = [];
}
constructor() {
/**
* User name.
* @type String
*/
this.username = "";
/**
* Path to the user's profile picture.
* @type String
*/
this.avatarSrc = null;
/**
* List of followed thread URLs.
* @type String[]
*/
this.watchedThreads = [];
}
}
module.exports = UserData;

52
app/scripts/constants/css-selector.js
View File

@ -1,33 +1,23 @@
module.exports = Object.freeze({
AVATAR_INFO: "span.avatar",
AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]',
ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span',
FILTER_THREADS_BUTTON: 'button[class="button--primary button"]',
GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]',
GAME_TAGS: "a.tagItem",
GAME_TITLE: "h1.p-title-value",
GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]',
LOGIN_BUTTON: "button.button--icon--login",
LOGIN_MESSAGE_ERROR:
"div.blockMessage.blockMessage--error.blockMessage--iconic",
ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]',
PASSWORD_INPUT: 'input[name="password"]',
SEARCH_BUTTON: "form.block > * button.button--icon--search",
SEARCH_FORM_TEXTBOX: 'input[name="keywords"][type="search"]',
SEARCH_ONLY_GAMES_OPTION: 'select[name="c[nodes][]"] > option[value="1"]',
STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span',
THREAD_POSTS:
"article.message-body:first-child > div.bbWrapper:first-of-type",
THREAD_TITLE: "h3.contentRow-title",
TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]',
UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]',
USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText',
USERNAME_INPUT: 'input[name="login"]',
WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next",
WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]',
DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]',
SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main",
SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)",
THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type",
BD_ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span",
BD_STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span",
GT_IMAGES: "img:not([title])[data-src^=\"https://attachments.f95zone.to\"][data-url=\"\"]",
GT_TAGS: "a.tagItem",
GT_TITLE: "h1.p-title-value",
GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]",
GT_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type",
GT_JSONLD: "script[type=\"application/ld+json\"]",
WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
WT_NEXT_PAGE: "a.pageNav-jump--next",
WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]",
WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]",
GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type",
GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a",
GS_RESULT_BODY: "div.contentRow-main",
GS_MEMBERSHIP: "li > a:not(.username)",
GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]",
UD_USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText",
UD_AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]",
LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic",
});

10
app/scripts/constants/url.js
View File

@ -1,7 +1,7 @@
module.exports = Object.freeze({
F95_BASE_URL: "https://f95zone.to",
F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
F95_LATEST_UPDATES: "https://f95zone.to/latest",
F95_LOGIN_URL: "https://f95zone.to/login",
F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
F95_BASE_URL: "https://f95zone.to",
F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
F95_LATEST_UPDATES: "https://f95zone.to/latest",
F95_LOGIN_URL: "https://f95zone.to/login/login",
F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
});

467
app/scripts/game-scraper.js
View File

@ -1,467 +0,0 @@
"use strict";
// Public modules from npm
const HTMLParser = require("node-html-parser");
const puppeteer = require("puppeteer"); // skipcq: JS-0128
// Modules from file
const shared = require("./shared.js");
const selectorK = require("./constants/css-selector.js");
const { preparePage } = require("./puppeteer-helper.js");
const GameDownload = require("./classes/game-download.js");
const GameInfo = require("./classes/game-info.js");
const urlHelper = require("./url-helper.js");
/**
* @protected
* Get information from the game's main page.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @param {String} url URL (String) of the game/mod to extract data from
* @return {Promise<GameInfo>} Complete information about the game you are
* looking for
*/
module.exports.getGameInfo = async function (browser, url) {
shared.logger.info("Obtaining game info");
// Verify the correctness of the URL
const exists = await urlHelper.urlExists(url);
if (!exists) throw new URIError(`${url} is not a valid URL`);
if (!urlHelper.isF95URL(url))
throw new Error(`${url} is not a valid F95Zone URL`);
const page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(url, {
waitUntil: shared.WAIT_STATEMENT,
}); // Go to the game page and wait until it loads
// It asynchronously searches for the elements and
// then waits at the end to compile the object to be returned
let info = new GameInfo();
const title = getGameTitle(page);
const author = getGameAuthor(page);
const tags = getGameTags(page);
const redirectUrl = urlHelper.getUrlRedirect(url);
info = await parsePrefixes(page, info); // Fill status/engines/isMod
const structuredText = await getMainPostStructuredText(page);
const overview = getOverview(structuredText, info.isMod);
const parsedInfos = parseConversationPage(structuredText);
const previewSource = getGamePreviewSource(page);
const changelog = getLastChangelog(page);
// Fill in the GameInfo element with the information obtained
info.name = await title;
info.author = await author;
info.tags = await tags;
info.f95url = await redirectUrl;
info.overview = overview;
info.lastUpdate = info.isMod
? parsedInfos.UPDATED
: parsedInfos.THREAD_UPDATED;
info.previewSource = await previewSource;
info.changelog = await changelog;
info.version = await exports.getGameVersionFromTitle(browser, info);
//let downloadData = getGameDownloadLink(page);
//info.downloadInfo = await downloadData;
/* Downloading games without going directly to
* the platform appears to be prohibited by
* the guidelines. It is therefore useless to
* keep the links for downloading the games. */
await page.close(); // Close the page
shared.logger.info("Founded data for " + info.name);
return info;
};
/**
* Obtain the game version without parsing again all the data of the game.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @param {GameInfo} info Information about the game
* @returns {Promise<String>} Online version of the game
*/
module.exports.getGameVersionFromTitle = async function (browser, info) {
const page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(info.f95url, {
waitUntil: shared.WAIT_STATEMENT,
}); // Go to the game page and wait until it loads
// Get the title
const titleHTML = await page.evaluate(
/* istanbul ignore next */
(selector) => document.querySelector(selector).innerHTML,
selectorK.GAME_TITLE
);
const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText;
// The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR]
const startIndex = title.indexOf("[") + 1;
const endIndex = title.indexOf("]", startIndex);
let version = title.substring(startIndex, endIndex).trim().toUpperCase();
if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence
await page.close();
return cleanFSString(version);
};
//#region Private methods
/**
* Clean a string from invalid File System chars.
* @param {String} s
* @returns {String}
*/
function cleanFSString(s) {
const rx = /[/\\?%*:|"<>]/g;
return s.replace(rx, "");
}
/**
* @private
* Get the game description from its web page.
* Different processing depending on whether the game is a mod or not.
* @param {String} text Structured text extracted from the game's web page
* @param {Boolean} isMod Specify if it is a game or a mod
* @returns {Promise<String>} Game description
*/
function getOverview(text, isMod) {
// Get overview (different parsing for game and mod)
let overviewEndIndex;
if (isMod) overviewEndIndex = text.indexOf("Updated");
else overviewEndIndex = text.indexOf("Thread Updated");
return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim();
}
/**
* @private
* Extrapolate the page structure by removing the element tags
* and leaving only the text and its spacing.
* @param {puppeteer.Page} page Page containing the text
* @returns {Promise<String>} Structured text
*/
async function getMainPostStructuredText(page) {
// Gets the first post, where are listed all the game's informations
const post = (await page.$$(selectorK.THREAD_POSTS))[0];
// The info are plain text so we need to parse the HTML code
const bodyHTML = await page.evaluate(
/* istanbul ignore next */
(mainPost) => mainPost.innerHTML,
post
);
return HTMLParser.parse(bodyHTML).structuredText;
}
/**
* @private
* Extrapolates and cleans the author from the page passed by parameter.
* @param {puppeteer.Page} page Page containing the author to be extrapolated
* @returns {Promise<String>} Game author
*/
async function getGameAuthor(page) {
// Get the game/mod name (without square brackets)
const titleHTML = await page.evaluate(
/* istanbul ignore next */
(selector) => document.querySelector(selector).innerHTML,
selectorK.GAME_TITLE
);
const structuredTitle = HTMLParser.parse(titleHTML);
// The last element **shoud be** the title without prefixes (engines, status, other...)
const gameTitle = structuredTitle.childNodes.pop().rawText;
// The last square brackets contain the author
const startTitleIndex = gameTitle.lastIndexOf("[") + 1;
return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim();
}
/**
* @private
* Process the post text to get all the useful
* information in the format *DESCRIPTOR : VALUE*.
* @param {String} text Structured text of the post
* @returns {Object} Dictionary of information
*/
function parseConversationPage(text) {
const dataPairs = {};
// The information searched in the game post are one per line
const splittedText = text.split("\n");
for (const line of splittedText) {
if (!line.includes(":")) continue;
// Create pair key/value
const splitted = line.split(":");
const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch
const value = splitted[1].trim();
// Add pair to the dict if valid
if (value !== "") dataPairs[key] = value;
}
return dataPairs;
}
/**
* @private
* Gets the URL of the image used as a preview for the game in the conversation.
* @param {puppeteer.Page} page Page containing the URL to be extrapolated
* @returns {Promise<String>} URL (String) of the image or null if failed to get it
*/
async function getGamePreviewSource(page) {
// Wait for the selector or return an empty value
try {
await page.waitForSelector(selectorK.GAME_IMAGES);
} catch {
return null;
}
const src = await page.evaluate(
/* istanbul ignore next */
(selector) => {
// Get the firs image available
const img = document.querySelector(selector);
if (img) return img.getAttribute("src");
else return null;
},
selectorK.GAME_IMAGES
);
// Check if the URL is valid
return urlHelper.isStringAValidURL(src) ? src : null;
}
/**
* @private
* Extrapolates and cleans the title from the page passed by parameter.
* @param {puppeteer.Page} page Page containing the title to be extrapolated
* @returns {Promise<String>} Game title
*/
async function getGameTitle(page) {
// Get the game/mod name (without square brackets)
const titleHTML = await page.evaluate(
/* istanbul ignore next */
(selector) => document.querySelector(selector).innerHTML,
selectorK.GAME_TITLE
);
const structuredTitle = HTMLParser.parse(titleHTML);
// The last element **shoud be** the title without prefixes (engines, status, other...)
const gameTitle = structuredTitle.childNodes.pop().rawText;
const endTitleIndex = gameTitle.indexOf("[");
return gameTitle.substring(0, endTitleIndex).trim();
}
/**
* @private
* Get the alphabetically sorted list of tags associated with the game.
* @param {puppeteer.Page} page Page containing the tags to be extrapolated
* @returns {Promise<String[]>} List of uppercase tags
*/
async function getGameTags(page) {
const tags = [];
// Get the game tags
for (const handle of await page.$$(selectorK.GAME_TAGS)) {
const tag = await page.evaluate(
/* istanbul ignore next */
(element) => element.innerText,
handle
);
tags.push(tag.toUpperCase());
}
return tags.sort();
}
/**
* @private
* Process the game title prefixes to extract information such as game status,
* graphics engine used, and whether it is a mod or original game.
* @param {puppeteer.Page} page Page containing the prefixes to be extrapolated
* @param {GameInfo} info Object to assign the identified information to
* @returns {Promise<GameInfo>} GameInfo object passed in to which the identified information has been added
*/
async function parsePrefixes(page, info) {
// The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete'
info.status = "ONGOING";
for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) {
const value = await page.evaluate(
/* istanbul ignore next */
(element) => element.innerText,
handle
);
// Clean the prefix
const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim();
// Getting infos...
if (shared.statuses.includes(prefix)) info.status = prefix;
else if (shared.engines.includes(prefix)) info.engine = prefix;
// This is not a game but a mod
else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true;
}
return info;
}
/**
* @private
* Get the last changelog available for the game.
* @param {puppeteer.Page} page Page containing the changelog
* @returns {Promise<String>} Changelog for the last version or a empty string if no changelog is found
*/
async function getLastChangelog(page) {
// Gets the first post, where are listed all the game's informations
const post = (await page.$$(selectorK.THREAD_POSTS))[0];
const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG);
if (!spoiler) return "";
const changelogHTML = await page.evaluate(
/* istanbul ignore next */
(e) => e.innerText,
spoiler
);
let parsedText = HTMLParser.parse(changelogHTML).structuredText;
// Clean the text
if (parsedText.startsWith("Spoiler"))
parsedText = parsedText.replace("Spoiler", "");
if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", "");
return parsedText.trim();
}
/**
* @private
* Get game download links for different platforms.
* @param {puppeteer.Page} page Page containing the links to be extrapolated
* @returns {Promise<GameDownload[]>} List of objects used for game download
* @deprecated
*/
/* istanbul ignore next */
// skipcq: JS-0128
async function getGameDownloadLink(page) {
// Most used hosting platforms
const hostingPlatforms = [
"MEGA",
"NOPY",
"FILESUPLOAD",
"MIXDROP",
"UPLOADHAVEN",
"PIXELDRAIN",
"FILESFM",
];
// Supported OS platforms
const platformOS = ["WIN", "LINUX", "MAC", "ALL"];
// Gets the <span> which contains the download links
const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER);
if (temp.length === 0) return [];
// Look for the container that contains the links
// It is necessary because the same css selector
// also identifies other elements on the page
let container = null;
for (const candidate of temp) {
if (container !== null) break;
const upperText = (
await page.evaluate(
/* istanbul ignore next */
(e) => e.innerText,
candidate
)
).toUpperCase();
// Search if the container contains the name of a hosting platform
for (const p of hostingPlatforms) {
if (upperText.includes(p)) {
container = candidate;
break;
}
}
}
if (container === null) return [];
// Extract the HTML text from the container
const searchText = (
await page.evaluate(
/* istanbul ignore next */
(e) => e.innerHTML,
container
)
).toLowerCase();
// Parse the download links
const downloadData = [];
for (const platform of platformOS) {
const data = extractGameHostingData(platform, searchText);
downloadData.push(...data);
}
return downloadData;
}
/**
* @private
* From the HTML text it extracts the game download links for the specified operating system.
* @param {String} platform Name of the operating system to look for a compatible link to.
* It can only be *WIN/LINUX/MAC/ALL*
* @param {String} text HTML string to extract links from
* @returns {GameDownload[]} List of game download links for the selected platform
* @deprecated
*/
/* istanbul ignore next */
function extractGameHostingData(platform, text) {
const PLATFORM_BOLD_OPEN = "<b>";
const CONTAINER_SPAN_CLOSE = "</span>";
const LINK_OPEN = "<a";
const LINK_CLOSE = "</a>";
const HREF_START = "href='";
const HREF_END = "'";
const TAG_CLOSE = ">";
// Identify the individual platforms
let startIndex = text.indexOf(platform.toLowerCase());
if (startIndex === -1) return [];
else startIndex += platform.length;
// Find the <b>platform</b>
let endIndex =
text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length;
// Find the end of the container
if (endIndex === -1)
endIndex =
text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) +
CONTAINER_SPAN_CLOSE.length;
text = text.substring(startIndex, endIndex);
const downloadData = [];
const linkTags = text.split(LINK_OPEN);
for (const tag of linkTags) {
// Ignore non-link string
if (!tag.includes(HREF_START)) continue;
// Find the hosting platform name
startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length;
endIndex = tag.indexOf(LINK_CLOSE, startIndex);
const hosting = tag.substring(startIndex, endIndex);
// Find the 'href' attribute
startIndex = tag.indexOf(HREF_START) + HREF_START.length;
endIndex = tag.indexOf(HREF_END, startIndex);
const link = tag.substring(startIndex, endIndex);
if (urlHelper.isStringAValidURL(link)) {
const gd = new GameDownload();
gd.hosting = hosting.toUpperCase();
gd.link = link;
gd.supportedOS = platform.toUpperCase();
downloadData.push(gd);
}
}
return downloadData;
}
//#endregion Private methods

132
app/scripts/game-searcher.js
View File

@ -1,132 +0,0 @@
"use strict";
// Public modules from npm
const puppeteer = require("puppeteer"); // skipcq: JS-0128
// Modules from file
const shared = require("./shared.js");
const urlK = require("./constants/url.js");
const selectorK = require("./constants/css-selector.js");
const { preparePage } = require("./puppeteer-helper.js");
const { isF95URL } = require("./url-helper.js");
/**
* @protected
* Search the F95Zone portal to find possible conversations regarding the game you are looking for.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @param {String} gamename Name of the game to search for
* @returns {Promise<String[]>} List of URL of possible games obtained from the preliminary research on the F95 portal
*/
module.exports.getSearchGameResults = async function (browser, gamename) {
shared.logger.info(`Searching ${gamename} on F95Zone`);
const page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(urlK.F95_SEARCH_URL, {
waitUntil: shared.WAIT_STATEMENT,
}); // Go to the search form and wait for it
// Explicitly wait for the required items to load
await Promise.all([
page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX),
page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX),
page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION),
page.waitForSelector(selectorK.SEARCH_BUTTON),
]);
await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire
await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles
await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod
await Promise.all([
page.click(selectorK.SEARCH_BUTTON), // Execute search
page.waitForNavigation({
waitUntil: shared.WAIT_STATEMENT,
}), // Wait for page to load
]);
// Select all conversation titles
const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY);
// For each element found extract the info about the conversation
shared.logger.info("Extracting info from conversations");
const results = [];
for (const element of resultsThread) {
const gameUrl = await getOnlyGameThreads(page, element);
if (gameUrl !== null) results.push(gameUrl);
}
shared.logger.info(`Find ${results.length} conversations`);
await page.close(); // Close the page
return results;
};
//#region Private methods
/**
* @private
* Return the link of a conversation if it is a game or a mod.
* @param {puppeteer.Page} page Page containing the conversation to be analyzed
* @param {puppeteer.ElementHandle} divHandle Element of the conversation to be analyzed
* @return {Promise<String>} URL of the game/mod or null if the URL is not of a game
*/
async function getOnlyGameThreads(page, divHandle) {
// Obtain the elements containing the basic information
const titleHandle = await divHandle.$(selectorK.THREAD_TITLE);
const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP);
// Get the forum where the thread was posted
const forum = await getMembershipForum(page, forumHandle);
if (forum !== "GAMES" && forum !== "MODS") return null;
// Get the URL of the thread from the title
return await getThreadURL(page, titleHandle);
}
/**
* @private
* Obtain the membership forum of the thread passed throught 'handle'.
* @param {puppeteer.Page} page Page containing the conversation to be analyzed
* @param {puppeteer.ElementHandle} handle Handle containing the forum membership
* @returns {Promise<String>} Uppercase membership category
*/
async function getMembershipForum(page, handle) {
// The link can be something like:
// + /forums/request.NUMBER/
// + /forums/game-recommendations-identification.NUMBER/
// + /forums/games.NUMBER/ <-- We need this
let link = await page.evaluate(
/* istanbul ignore next */
(e) => e.getAttribute("href"),
handle
);
// Parse link
link = link.replace("/forums/", "");
const endIndex = link.indexOf(".");
const forum = link.substring(0, endIndex);
return forum.toUpperCase();
}
/**
* @private
* Obtain the URL of the thread passed through 'handle'.
* @param {puppeteer.Page} page Page containing the conversation to be analyzed
* @param {puppeteer.ElementHandle} handle Handle containing the thread title
* @returns {Promise<String>} URL of the thread
*/
async function getThreadURL(page, handle) {
const relativeURLThread = await page.evaluate(
/* istanbul ignore next */
(e) => e.querySelector("a").href,
handle
);
// Some game already have a full URL...
if (isF95URL(relativeURLThread)) return relativeURLThread;
// ... else compose the URL and return
const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString();
return urlThread;
}
//#endregion Private methods

243
app/scripts/network-helper.js Normal file
View File

@ -0,0 +1,243 @@
"use strict";
// Public modules from npm
const axios = require("axios").default;
const ky = require("ky-universal").create({
throwHttpErrors: false,
});
const cheerio = require("cheerio");
const axiosCookieJarSupport = require("axios-cookiejar-support").default;
const tough = require("tough-cookie");
// Modules from file
const shared = require("./shared.js");
const f95url = require("./constants/url.js");
const f95selector = require("./constants/css-selector.js");
const LoginResult = require("./classes/login-result.js");
// Global variables
const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " +
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15";
axiosCookieJarSupport(axios);
const cookieJar = new tough.CookieJar();
const commonConfig = {
headers: {
"User-Agent": userAgent,
"Connection": "keep-alive"
},
withCredentials: true,
jar: cookieJar // Used to store the token in the PC
};
/**
* @protected
* Gets the HTML code of a page.
* @param {String} url URL to fetch
* @returns {Promise<String>} HTML code or `null` if an error arise
*/
module.exports.fetchHTML = async function (url) {
// Fetch the response of the platform
const response = await exports.fetchGETResponse(url);
if (!response) {
shared.logger.warn(`Unable to fetch HTML for ${url}`);
return null;
}
return response.data;
};
/**
* @protected
* It authenticates to the platform using the credentials
* and token obtained previously. Save cookies on your
* device after authentication.
* @param {Credentials} credentials Platform access credentials
* @param {Boolea} force Specifies whether the request should be forced, ignoring any saved cookies
* @returns {Promise<LoginResult>} Result of the operation
*/
module.exports.authenticate = async function (credentials, force) {
shared.logger.info(`Authenticating with user ${credentials.username}`);
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
// Secure the URL
const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL);
// Prepare the parameters to send to the platform to authenticate
const params = new URLSearchParams();
params.append("login", credentials.username);
params.append("url", "");
params.append("password", credentials.password);
params.append("password_confirm", "");
params.append("additional_security", "");
params.append("remember", "1");
params.append("_xfRedirect", "https://f95zone.to/");
params.append("website_code", "");
params.append("_xfToken", credentials.token);
try {
// Try to log-in
let config = Object.assign({}, commonConfig);
if (force) delete config.jar;
const response = await axios.post(secureURL, params, config);
// Parse the response HTML
const $ = cheerio.load(response.data);
// Get the error message (if any) and remove the new line chars
const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, "");
// Return the result of the authentication
if (errorMessage === "") return new LoginResult(true, "Authentication successful");
else return new LoginResult(false, errorMessage);
} catch (e) {
shared.logger.error(`Error ${e.message} occurred while authenticating to ${secureURL}`);
return new LoginResult(false, `Error ${e.message} while authenticating`);
}
};
/**
* Obtain the token used to authenticate the user to the platform.
* @returns {Promise<String>} Token or `null` if an error arise
*/
module.exports.getF95Token = async function() {
// Fetch the response of the platform
const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL);
if (!response) {
shared.logger.warn("Unable to get the token for the session");
return null;
}
// The response is a HTML page, we need to find the <input> with name "_xfToken"
const $ = cheerio.load(response.data);
const token = $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value");
return token;
};
/**
* @protected
* Gets the basic data used for game data processing
* (such as graphics engines and progress statuses)
* @deprecated
*/
/* istanbul ignore next */
module.exports.fetchPlatformData = async function() {
// Fetch the response of the platform
const response = await exports.fetchGETResponse(f95url.F95_LATEST_UPDATES);
if (!response) {
shared.logger.warn("Unable to get the token for the session");
return;
}
// The response is a HTML page, we need to find
// the base data, used when scraping the games
const $ = cheerio.load(response.data);
// Extract the elements
const engineElements = $("body").find(f95selector.BD_ENGINE_ID_SELECTOR);
const statusesElements = $("body").find(f95selector.BD_STATUS_ID_SELECTOR);
// Extract the raw text
engineElements.each(function extractEngineNames(idx, el) {
const engine = cheerio.load(el).text().trim();
shared.engines.push(engine);
});
statusesElements.each(function extractEngineNames(idx, el) {
const status = cheerio.load(el).text().trim();
shared.statuses.push(status);
});
};
//#region Utility methods
/**
* @protected
* Performs a GET request to a specific URL and returns the response.
* If the request generates an error (for example 400) `null` is returned.
* @param {String} url
*/
module.exports.fetchGETResponse = async function(url) {
// Secure the URL
const secureURL = exports.enforceHttpsUrl(url);
try {
// Fetch and return the response
return await axios.get(secureURL, commonConfig);
} catch (e) {
shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`);
return null;
}
};
/**
* @protected
* Enforces the scheme of the URL is https and returns the new URL.
* @param {String} url
* @returns {String} Secure URL or `null` if the argument is not a string
*/
module.exports.enforceHttpsUrl = function (url) {
return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
};
/**
* @protected
* Check if the url belongs to the domain of the F95 platform.
* @param {String} url URL to check
* @returns {Boolean} true if the url belongs to the domain, false otherwise
*/
module.exports.isF95URL = function (url) {
if (url.toString().startsWith(f95url.F95_BASE_URL)) return true;
else return false;
};
/**
* @protected
* Checks if the string passed by parameter has a
* properly formatted and valid path to a URL (HTTP/HTTPS).
* @param {String} url String to check for correctness
* @returns {Boolean} true if the string is a valid URL, false otherwise
*/
module.exports.isStringAValidURL = function (url) {
// Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc)
const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/;
const regex = new RegExp(expression);
if (url.match(regex)) return true;
else return false;
};
/**
* @protected
* Check if a particular URL is valid and reachable on the web.
* @param {String} url URL to check
* @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false
* @returns {Promise<Boolean>} true if the URL exists, false otherwise
*/
module.exports.urlExists = async function (url, checkRedirect) {
if (!exports.isStringAValidURL(url)) {
return false;
}
const response = await ky.head(url);
let valid = response !== undefined && !/4\d\d/.test(response.status);
if (!valid) return false;
if (checkRedirect) {
const redirectUrl = await exports.getUrlRedirect(url);
if (redirectUrl === url) valid = true;
else valid = false;
}
return valid;
};
/**
* @protected
* Check if the URL has a redirect to another page.
* @param {String} url URL to check for redirect
* @returns {Promise<String>} Redirect URL or the passed URL
*/
module.exports.getUrlRedirect = async function (url) {
const response = await ky.head(url);
return response.url;
};
//#endregion Utility methods

60
app/scripts/puppeteer-helper.js
View File

@ -1,60 +0,0 @@
"use strict";
// Public modules from npm
const puppeteer = require("puppeteer");
// Modules from file
const shared = require("./shared.js");
/**
* @protected
* Create a Chromium instance used to navigate with Puppeteer.
* By default the browser is headless.
* @returns {Promise<puppeteer.Browser>} Created browser
*/
module.exports.prepareBrowser = async function () {
// Create a headless browser
let browser = null;
if (shared.chromiumLocalPath) {
browser = await puppeteer.launch({
executablePath: shared.chromiumLocalPath,
headless: !shared.debug, // Use GUI when debug = true
});
} else {
browser = await puppeteer.launch({
headless: !shared.debug, // Use GUI when debug = true
});
}
return browser;
};
/**
* @protected
* Prepare a page used to navigate the browser.
* The page is set up to reject image download requests. The user agent is also changed.
* @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created
* @returns {Promise<puppeteer.Page>} New page
*/
module.exports.preparePage = async function (browser) {
// Create new page in the browser argument
const page = await browser.newPage();
// Block image download
await page.setRequestInterception(true);
page.on("request", (request) => {
if (request.resourceType() === "image") request.abort();
else if (request.resourceType === "font") request.abort();
// else if (request.resourceType() == 'stylesheet') request.abort();
// else if(request.resourceType == 'media') request.abort();
else request.continue();
});
// Set custom user-agent
const userAgent =
"Mozilla/5.0 (X11; Linux x86_64)" +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
await page.setUserAgent(userAgent);
return page;
};

371
app/scripts/scraper.js Normal file
View File

@ -0,0 +1,371 @@
"use strict";
// Public modules from npm
const cheerio = require("cheerio");
// Modules from file
const { fetchHTML, getUrlRedirect } = require("./network-helper.js");
const shared = require("./shared.js");
const GameInfo = require("./classes/game-info.js");
const f95Selector = require("./constants/css-selector.js");
/**
* @protected
* Get information from the game's main page.
* @param {String} url URL of the game/mod to extract data from
* @return {Promise<GameInfo>} Complete information about the game you are
* looking for
*/
module.exports.getGameInfo = async function (url) {
shared.logger.info("Obtaining game info");
// Fetch HTML and prepare Cheerio
const html = await fetchHTML(url);
const $ = cheerio.load(html);
const body = $("body");
const mainPost = $(f95Selector.GS_POSTS).first();
// Extract data
const titleData = extractInfoFromTitle(body);
const tags = extractTags(body);
const prefixesData = parseGamePrefixes(body);
const src = extractPreviewSource(body);
const changelog = extractChangelog(mainPost);
const structuredData = extractStructuredData(body);
const parsedInfos = parseMainPostText(structuredData["description"]);
const overview = getOverview(structuredData["description"], prefixesData.mod);
// Obtain the updated URL
const redirectUrl = await getUrlRedirect(url);
// Fill in the GameInfo element with the information obtained
const info = new GameInfo();
info.name = titleData.name;
info.author = titleData.author;
info.isMod = prefixesData.mod;
info.engine = prefixesData.engine;
info.status = prefixesData.status;
info.tags = tags;
info.url = redirectUrl;
info.language = parsedInfos.Language;
info.overview = overview;
info.supportedOS = parsedInfos.SupportedOS;
info.censored = parsedInfos.Censored;
info.lastUpdate = parsedInfos.LastUpdate;
info.previewSrc = src;
info.changelog = changelog;
info.version = titleData.version;
shared.logger.info(`Founded data for ${info.name}`);
return info;
};
//#region Private methods
/**
* @private
* Parse the game prefixes obtaining the engine used,
* the advancement status and if the game is actually a game or a mod.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, object>} Dictionary of values with keys `engine`, `status`, `mod`
*/
function parseGamePrefixes(body) {
shared.logger.trace("Parsing prefixes...");
// Local variables
let mod = false,
engine = null,
status = null;
// Obtain the title prefixes
const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES);
prefixeElements.each(function parseGamePrefix(idx, el) {
// Obtain the prefix text
let prefix = cheerio.load(el).text().trim();
// Remove the square brackets
prefix = prefix.replace("[", "").replace("]", "");
// Check what the prefix indicates
if (isEngine(prefix)) engine = prefix;
else if (isStatus(prefix)) status = prefix;
else if (isMod(prefix)) mod = true;
});
// If the status is not set, then the game in in development (Ongoing)
if (!status) status = "Ongoing";
return {
engine,
status,
mod
};
}
/**
* @private
* Extracts all the possible informations from the title.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, string>} Dictionary of values with keys `name`, `author`, `version`
*/
function extractInfoFromTitle(body) {
shared.logger.trace("Extracting information from title...");
const title = body
.find(f95Selector.GT_TITLE)
.text()
.trim();
// From the title we can extract: Name, author and version
// [PREFIXES] TITLE [VERSION] [AUTHOR]
const matches = title.match(/\[(.*?)\]/g);
// Get the title name
let name = title;
matches.forEach(function replaceElementsInTitle(e) {
name = name.replace(e, "");
});
name = name.trim();
// The regex [[\]]+ remove the square brackets
// The version is the penultimate element.
// If the matches are less than 2, than the title
// is malformes and only the author is fetched
// (usually the author is always present)
let version = null;
if (matches.length >= 2) version = matches[matches.length - 2].replace(/[[\]]+/g, "").trim();
else shared.logger.trace(`Malformed title: ${title}`);
// Last element
const author = matches[matches.length - 1].replace(/[[\]]+/g, "").trim();
return {
name,
version,
author,
};
}
/**
* @private
* Gets the tags used to classify the game.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String[]} List of tags
*/
function extractTags(body) {
shared.logger.trace("Extracting tags...");
// Get the game tags
const tagResults = body.find(f95Selector.GT_TAGS);
return tagResults.map(function parseGameTags(idx, el) {
return cheerio.load(el).text().trim();
}).get();
}
/**
* @private
* Gets the URL of the image used as a preview.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String} URL of the image
*/
function extractPreviewSource(body) {
shared.logger.trace("Extracting image preview source...");
const image = body.find(f95Selector.GT_IMAGES);
// The "src" attribute is rendered only in a second moment,
// we need the "static" src value saved in the attribute "data-src"
const source = image ? image.attr("data-src") : null;
return source;
}
/**
* @private
* Gets the changelog of the latest version.
* @param {cheerio.Cheerio} mainPost main post selector
* @returns {String} Changelog of the last version or `null` if no changelog is fetched
*/
function extractChangelog(mainPost) {
shared.logger.trace("Extracting last changelog...");
// Obtain changelog
let changelog = mainPost.find(f95Selector.GT_LAST_CHANGELOG).text().trim();
// Clean changelog
changelog = changelog.replace("Spoiler", "");
changelog = changelog.replace(/\n+/g, "\n");
// Return changelog
return changelog ? changelog : null;
}
/**
* @private
* Process the main post text to get all the useful
* information in the format *DESCRIPTOR : VALUE*.
* Gets "standard" values such as: `Language`, `SupportedOS`, `Censored`, and `LastUpdate`.
* All non-canonical values are instead grouped together as a dictionary with the key `Various`.
* @param {String} text Structured text of the post
* @returns {Object.<string, object>} Dictionary of information
*/
function parseMainPostText(text) {
shared.logger.trace("Parsing main post raw text...");
const data = {};
// The information searched in the game post are one per line
const splittedText = text.split("\n");
for (const line of splittedText) {
if (!line.includes(":")) continue;
// Create pair key/value
const splitted = line.split(":");
const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch
const value = splitted[1].trim();
// Add pair to the dict if valid
if (value !== "") data[key] = value;
}
// Parse the standard pairs
const parsedDict = {};
// Check if the game is censored
if (data.CENSORED) {
const censored = data.CENSORED.toUpperCase() === "NO" ? false : true;
parsedDict["Censored"] = censored;
delete data.CENSORED;
}
// Last update of the main post
if (data.UPDATED) {
parsedDict["LastUpdate"] = new Date(data.UPDATED);
delete data.UPDATED;
}
else if (data.THREAD_UPDATED) {
parsedDict["LastUpdate"] = new Date(data.THREAD_UPDATED);
delete data.THREAD_UPDATED;
}
// Parse the supported OS
if (data.OS) {
const listOS = [];
// Usually the string is something like "Windows, Linux, Mac"
const splitted = data.OS.split(",");
splitted.forEach(function (os) {
listOS.push(os.trim());
});
parsedDict["SupportedOS"] = listOS;
delete data.OS;
}
// Rename the key for the language
if (data.LANGUAGE) {
parsedDict["Language"] = data.LANGUAGE;
delete data.LANGUAGE;
}
// What remains is added to a sub dictionary
parsedDict["Various"] = data;
return parsedDict;
}
/**
* @private
* Extracts and processes the JSON-LD values found at the bottom of the page.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
*/
function extractStructuredData(body) {
shared.logger.trace("Extracting JSON-LD data...");
const structuredDataElements = body.find(f95Selector.GT_JSONLD);
const json = structuredDataElements.map(function parseScriptTag(idx, el) {
// Get the element HTML
const html = cheerio.load(el).html().trim();
// Obtain the JSON-LD
const data = html
.replace("<script type=\"application/ld+json\">", "")
.replace("</script>", "");
// Convert the string to an object
const json = JSON.parse(data);
// Return only the data of the game
if (json["@type"] === "Book") return json;
}).get();
return json[0] ? json[0] : null;
}
/**
* @private
* Get the game description from its web page.
* Different processing depending on whether the game is a mod or not.
* @param {String} text Structured text extracted from the game's web page
* @param {Boolean} mod Specify if it is a game or a mod
* @returns {String} Game description
*/
function getOverview(text, mod) {
shared.logger.trace("Extracting game overview...");
// Get overview (different parsing for game and mod)
const overviewEndIndex = mod ? text.indexOf("Updated") : text.indexOf("Thread Updated");
return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim();
}
/**
* @private
* Check if the prefix is a game's engine.
* @param {String} prefix Prefix to check
* @return {Boolean}
*/
function isEngine(prefix) {
const engines = toUpperCaseArray(shared.engines);
return engines.includes(prefix.toUpperCase());
}
/**
* @private
* Check if the prefix is a game's status.
* @param {String} prefix Prefix to check
* @return {Boolean}
*/
function isStatus(prefix) {
const statuses = toUpperCaseArray(shared.statuses);
return statuses.includes(prefix.toUpperCase());
}
/**
* @private
* Check if the prefix indicates a mod.
* @param {String} prefix Prefix to check
* @return {Boolean}
*/
function isMod(prefix) {
const modPrefixes = ["MOD", "CHEAT MOD"];
return modPrefixes.includes(prefix.toUpperCase());
}
/**
* @private
* Makes an array of strings uppercase.
* @param {String[]} a
* @returns {String[]}
*/
function toUpperCaseArray(a) {
// If the array is empty, return
if(a.length === 0) return [];
/**
* Makes a string uppercase.
* @param {String} s
* @returns {String}
*/
function toUpper(s) {
return s.toUpperCase();
}
return a.map(toUpper);
}
//#endregion Private methods

95
app/scripts/searcher.js Normal file
View File

@ -0,0 +1,95 @@
"use strict";
// Public modules from npm
const cheerio = require("cheerio");
// Modules from file
const { fetchHTML } = require("./network-helper.js");
const shared = require("./shared.js");
const f95Selector = require("./constants/css-selector.js");
const { F95_BASE_URL } = require("./constants/url.js");
//#region Public methods
/**
* @protected
* Search for a game on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Game name
* @returns {Promise<String[]>} URLs of results
*/
module.exports.searchGame = async function (name) {
shared.logger.info(`Searching games with name ${name}`);
// Replace the whitespaces with +
const searchName = encodeURIComponent(name.toUpperCase());
// Prepare the URL (only title, search in the "Games" section, order by relevance)
const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`;
// Fetch and parse the result URLs
return await fetchResultURLs(url);
};
/**
* @protected
* Search for a mod on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Mod name
* @returns {Promise<String[]>} URLs of results
*/
module.exports.searchMod = async function (name) {
shared.logger.info(`Searching mods with name ${name}`);
// Replace the whitespaces with +
const searchName = encodeURIComponent(name.toUpperCase());
// Prepare the URL (only title, search in the "Mods" section, order by relevance)
const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`;
// Fetch and parse the result URLs
return await fetchResultURLs(url);
};
//#endregion Public methods
//#region Private methods
/**
* @private
* Gets the URLs of the threads resulting from the F95Zone search.
* @param {String} url Search URL
* @return {Promise<String[]>} List of URLs
*/
async function fetchResultURLs(url) {
shared.logger.trace(`Fetching ${url}...`);
// Fetch HTML and prepare Cheerio
const html = await fetchHTML(url);
const $ = cheerio.load(html);
// Here we get all the DIV that are the body of the various query results
const results = $("body").find(f95Selector.GS_RESULT_BODY);
// Than we extract the URLs
const urls = results.map((idx, el) => {
const elementSelector = $(el);
return extractLinkFromResult(elementSelector);
}).get();
return urls;
}
/**
* @private
* Look for the URL to the thread referenced by the item.
* @param {cheerio.Cheerio} selector Element to search
* @returns {String} URL to thread
*/
function extractLinkFromResult(selector) {
shared.logger.trace("Extracting thread link from result...");
const partialLink = selector
.find(f95Selector.GS_RESULT_THREAD_TITLE)
.attr("href")
.trim();
// Compose and return the URL
return new URL(partialLink, F95_BASE_URL).toString();
}
//#endregion Private methods

199
app/scripts/shared.js
View File

@ -1,173 +1,80 @@
/* istanbul ignore file */
"use strict";
// Core modules
const { join } = require("path");
// Public modules from npm
const log4js = require("log4js");
/**
* Class containing variables shared between modules.
*/
class Shared {
//#region Properties
/**
* Shows log messages and other useful functions for module debugging.
* @type Boolean
*/
static #_debug = false;
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
* @type Boolean
*/
static #_isLogged = false;
/**
* List of cookies obtained from the F95Zone platform.
* @type Object[]
*/
static #_cookies = null;
/**
* List of possible game engines used for development.
* @type String[]
*/
static #_engines = null;
/**
* List of possible development statuses that a game can assume.
* @type String[]
*/
static #_statuses = null;
/**
* Wait instruction for the browser created by puppeteer.
* @type String
*/
static WAIT_STATEMENT = "domcontentloaded";
/**
* Path to the directory to save the cache generated by the API.
* @type String
*/
static #_cacheDir = "./f95cache";
/**
* If true, it opens a new browser for each request to
* the F95Zone platform, otherwise it reuses the same.
* @type Boolean
*/
static #_isolation = false;
/**
* Logger object used to write to both file and console.
* @type log4js.Logger
*/
static #_logger = log4js.getLogger();
//#endregion Properties
//#region Properties
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
* @type Boolean
*/
static #_isLogged = false;
/**
* List of possible game engines used for development.
* @type String[]
*/
static #_engines = ["ADRIFT", "Flash", "HTML", "Java", "Others", "QSP", "RAGS", "RPGM", "Ren'Py", "Tads", "Unity", "Unreal Engine", "WebGL", "Wolf RPG"];
/**
* List of possible development statuses that a game can assume.
* @type String[]
*/
static #_statuses = ["Completed", "Onhold", "Abandoned"];
/**
* Logger object used to write to both file and console.
* @type log4js.Logger
*/
static #_logger = log4js.getLogger();
//#endregion Properties
//#region Getters
/**
* Shows log messages and other useful functions for module debugging.
* @returns {Boolean}
*/
static get debug() {
return this.#_debug;
}
/**
//#region Getters
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
* @returns {Boolean}
*/
static get isLogged() {
return this.#_isLogged;
}
/**
* List of cookies obtained from the F95Zone platform.
* @returns {Object[]}
*/
static get cookies() {
return this.#_cookies;
}
/**
static get isLogged() {
return this.#_isLogged;
}
/**
* List of possible game engines used for development.
* @returns {String[]}
*/
static get engines() {
return this.#_engines;
}
/**
static get engines() {
return this.#_engines;
}
/**
* List of possible development states that a game can assume.
* @returns {String[]}
*/
static get statuses() {
return this.#_statuses;
}
/**
* Directory to save the API cache.
* @returns {String}
*/
static get cacheDir() {
return this.#_cacheDir;
}
/**
* Path to the F95 platform cache.
* @returns {String}
*/
static get cookiesCachePath() {
return join(this.#_cacheDir, "cookies.json");
}
/**
* Path to the game engine cache.
* @returns {String}
*/
static get enginesCachePath() {
return join(this.#_cacheDir, "engines.json");
}
/**
* Path to the cache of possible game states.
* @returns {String}
*/
static get statusesCachePath() {
return join(this.#_cacheDir, "statuses.json");
}
/**
* If true, it opens a new browser for each request
* to the F95Zone platform, otherwise it reuses the same.
* @returns {Boolean}
*/
static get isolation() {
return this.#_isolation;
}
/**
static get statuses() {
return this.#_statuses;
}
/**
* Logger object used to write to both file and console.
* @returns {log4js.Logger}
*/
static get logger() {
return this.#_logger;
}
//#endregion Getters
static get logger() {
return this.#_logger;
}
//#endregion Getters
//#region Setters
static set cookies(val) {
this.#_cookies = val;
}
//#region Setters
static set engines(val) {
this.#_engines = val;
}
static set engines(val) {
this.#_engines = val;
}
static set statuses(val) {
this.#_statuses = val;
}
static set statuses(val) {
this.#_statuses = val;
}
static set cacheDir(val) {
this.#_cacheDir = val;
}
static set debug(val) {
this.#_debug = val;
}
static set isLogged(val) {
this.#_isLogged = val;
}
static set isolation(val) {
this.#_isolation = val;
}
//#endregion Setters
static set isLogged(val) {
this.#_isLogged = val;
}
//#endregion Setters
}
module.exports = Shared;

72
app/scripts/url-helper.js
View File

@ -1,72 +0,0 @@
"use strict";
// Public modules from npm
const ky = require("ky-universal").create({
throwHttpErrors: false,
});
// Modules from file
const { F95_BASE_URL } = require("./constants/url.js");
/**
* @protected
* Check if the url belongs to the domain of the F95 platform.
* @param {String} url URL to check
* @returns {Boolean} true if the url belongs to the domain, false otherwise
*/
module.exports.isF95URL = function (url) {
if (url.toString().startsWith(F95_BASE_URL)) return true;
else return false;
};
/**
* @protected
* Checks if the string passed by parameter has a properly formatted and valid path to a URL.
* @param {String} url String to check for correctness
* @returns {Boolean} true if the string is a valid URL, false otherwise
*/
module.exports.isStringAValidURL = function (url) {
try {
new URL(url); // skipcq: JS-0078
return true;
} catch (err) {
return false;
}
};
/**
* @protected
* Check if a particular URL is valid and reachable on the web.
* @param {String} url URL to check
* @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false
* @returns {Promise<Boolean>} true if the URL exists, false otherwise
*/
module.exports.urlExists = async function (url, checkRedirect) {
if (!exports.isStringAValidURL(url)) {
return false;
}
const response = await ky.head(url);
let valid = response !== undefined && !/4\d\d/.test(response.status);
if (!valid) return false;
if (checkRedirect) {
const redirectUrl = await exports.getUrlRedirect(url);
if (redirectUrl === url) valid = true;
else valid = false;
}
return valid;
};
/**
* @protected
* Check if the URL has a redirect to another page.
* @param {String} url URL to check for redirect
* @returns {Promise<String>} Redirect URL or the passed URL
*/
module.exports.getUrlRedirect = async function (url) {
const response = await ky.head(url);
return response.url;
};

123
app/scripts/user-scraper.js Normal file
View File

@ -0,0 +1,123 @@
"use strict";
// Public modules from npm
const cheerio = require("cheerio");
// Modules from file
const networkHelper = require("./network-helper.js");
const f95Selector = require("./constants/css-selector.js");
const f95url = require("./constants/url.js");
const UserData = require("./classes/user-data.js");
/**
* @protected
* Gets user data, such as username, url of watched threads, and profile picture url.
* @return {Promise<UserData>} User data
*/
module.exports.getUserData = async function() {
// Fetch data
const data = await fetchUsernameAndAvatar();
const urls = await fetchWatchedThreadURLs();
// Create object
const ud = new UserData();
ud.username = data.username;
ud.avatarSrc = data.source;
ud.watchedThreads = urls;
return ud;
};
//#region Private methods
/**
* @private
* It connects to the page and extracts the name
* of the currently logged in user and the URL
* of their profile picture.
* @return {Promise<Object.<string, string>>}
*/
async function fetchUsernameAndAvatar() {
// Fetch page
const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL);
// Load HTML response
const $ = cheerio.load(html);
const body = $("body");
// Fetch username
const username = body.find(f95Selector.UD_USERNAME_ELEMENT).first().text().trim();
// Fetch user avatar image source
const source = body.find(f95Selector.UD_AVATAR_PIC).first().attr("src");
return {
username,
source
};
}
/**
* @private
* Gets the list of URLs of threads watched by the user.
* @returns {Promise<String[]>} List of URLs
*/
async function fetchWatchedThreadURLs() {
// Local variables
let currentURL = f95url.F95_WATCHED_THREADS;
const wathcedThreadURLs = [];
do {
// Fetch page
const html = await networkHelper.fetchHTML(currentURL);
// Load HTML response
const $ = cheerio.load(html);
const body = $("body");
// Find the URLs
const urls = fetchPageURLs(body);
wathcedThreadURLs.push(...urls);
// Find the next page (if any)
currentURL = fetchNextPageURL(body);
}
while (currentURL);
return wathcedThreadURLs;
}
/**
* @private
* Gets the URLs of the watched threads on the page.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String[]}
*/
function fetchPageURLs(body) {
const elements = body.find(f95Selector.WT_URLS);
return elements.map(function extractURLs(idx, e) {
// Obtain the link (replace "unread" only for the unread threads)
const partialLink = e.attribs.href.replace("unread", "");
// Compose and return the URL
return new URL(partialLink, f95url.F95_BASE_URL).toString();
}).get();
}
/**
* @private
* Gets the URL of the next page containing the watched threads
* or `null` if that page does not exist.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String}
*/
function fetchNextPageURL(body) {
const element = body.find(f95Selector.WT_NEXT_PAGE).first();
// No element found
if(element.length === 0) return null;
// Compose and return the URL
return new URL(element.attr("href"), f95url.F95_BASE_URL).toString();
}
//#endregion Private methods

494
package-lock.json generated
View File

@ -1,6 +1,6 @@
{
"name": "f95api",
"version": "1.3.5",
"version": "1.5.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@ -285,17 +285,7 @@
"@types/node": {
"version": "14.11.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz",
"integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==",
"optional": true
},
"@types/yauzl": {
"version": "2.9.1",
"resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz",
"integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==",
"optional": true,
"requires": {
"@types/node": "*"
}
"integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA=="
},
"abort-controller": {
"version": "3.0.0",
@ -317,11 +307,6 @@
"integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==",
"dev": true
},
"agent-base": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz",
"integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g=="
},
"aggregate-error": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
@ -423,6 +408,23 @@
"integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==",
"dev": true
},
"axios": {
"version": "0.21.0",
"resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz",
"integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==",
"requires": {
"follow-redirects": "^1.10.0"
}
},
"axios-cookiejar-support": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
"integrity": "sha512-IZJxnAJ99XxiLqNeMOqrPbfR7fRyIfaoSLdPUf4AMQEGkH8URs0ghJK/xtqBsD+KsSr3pKl4DEQjCn834pHMig==",
"requires": {
"is-redirect": "^1.0.0",
"pify": "^5.0.0"
}
},
"babel-eslint": {
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz",
@ -435,17 +437,21 @@
"@babel/types": "^7.7.0",
"eslint-visitor-keys": "^1.0.0",
"resolve": "^1.12.0"
},
"dependencies": {
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"dev": true
}
}
},
"balanced-match": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c="
},
"base64-js": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz",
"integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g=="
"integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
"dev": true
},
"binary-extensions": {
"version": "2.1.0",
@ -453,20 +459,16 @@
"integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==",
"dev": true
},
"bl": {
"version": "4.0.3",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
"integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
"requires": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
}
"boolbase": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
},
"brace-expansion": {
"version": "1.1.11",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
"integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
"dev": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -487,20 +489,6 @@
"integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==",
"dev": true
},
"buffer": {
"version": "5.6.0",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz",
"integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==",
"requires": {
"base64-js": "^1.0.2",
"ieee754": "^1.1.4"
}
},
"buffer-crc32": {
"version": "0.2.13",
"resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
"integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
},
"caching-transform": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz",
@ -556,6 +544,19 @@
"integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=",
"dev": true
},
"cheerio": {
"version": "1.0.0-rc.3",
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
"integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
"requires": {
"css-select": "~1.2.0",
"dom-serializer": "~0.1.1",
"entities": "~1.1.1",
"htmlparser2": "^3.9.1",
"lodash": "^4.15.0",
"parse5": "^3.0.1"
}
},
"chokidar": {
"version": "3.4.2",
"resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz",
@ -572,11 +573,6 @@
"readdirp": "~3.4.0"
}
},
"chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg=="
},
"clean-stack": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
@ -618,7 +614,8 @@
"concat-map": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s="
"integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
"dev": true
},
"convert-source-map": {
"version": "1.7.0",
@ -640,6 +637,22 @@
"which": "^2.0.1"
}
},
"css-select": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
"integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
"requires": {
"boolbase": "~1.0.0",
"css-what": "2.1",
"domutils": "1.5.1",
"nth-check": "~1.0.1"
}
},
"css-what": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
"integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
},
"data-uri-to-buffer": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz",
@ -697,11 +710,6 @@
"object-keys": "^1.0.12"
}
},
"devtools-protocol": {
"version": "0.0.799653",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz",
"integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg=="
},
"diff": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz",
@ -717,6 +725,37 @@
"esutils": "^2.0.2"
}
},
"dom-serializer": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
"integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
"requires": {
"domelementtype": "^1.3.0",
"entities": "^1.1.1"
}
},
"domelementtype": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
"integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
},
"domhandler": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
"integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
"requires": {
"domelementtype": "1"
}
},
"domutils": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
"integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
"requires": {
"dom-serializer": "0",
"domelementtype": "1"
}
},
"dotenv": {
"version": "8.2.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz",
@ -729,14 +768,6 @@
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==",
"dev": true
},
"end-of-stream": {
"version": "1.4.4",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz",
"integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==",
"requires": {
"once": "^1.4.0"
}
},
"enquirer": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz",
@ -746,6 +777,11 @@
"ansi-colors": "^4.1.1"
}
},
"entities": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
},
"es-abstract": {
"version": "1.17.6",
"resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz",
@ -888,12 +924,6 @@
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
"dev": true
},
"eslint-visitor-keys": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
"integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
"dev": true
},
"globals": {
"version": "12.4.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
@ -949,12 +979,20 @@
"dev": true,
"requires": {
"eslint-visitor-keys": "^1.1.0"
},
"dependencies": {
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"dev": true
}
}
},
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
"integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
"dev": true
},
"espree": {
@ -966,6 +1004,14 @@
"acorn": "^7.4.0",
"acorn-jsx": "^5.2.0",
"eslint-visitor-keys": "^1.3.0"
},
"dependencies": {
"eslint-visitor-keys": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
"integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
"dev": true
}
}
},
"esprima": {
@ -1025,17 +1071,6 @@
"resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
"integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ=="
},
"extract-zip": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz",
"integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==",
"requires": {
"@types/yauzl": "^2.9.1",
"debug": "^4.1.1",
"get-stream": "^5.1.0",
"yauzl": "^2.10.0"
}
},
"fast-deep-equal": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@ -1054,14 +1089,6 @@
"integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=",
"dev": true
},
"fd-slicer": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz",
"integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=",
"requires": {
"pend": "~1.2.0"
}
},
"fetch-blob": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.1.tgz",
@ -1100,6 +1127,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
"integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
"dev": true,
"requires": {
"locate-path": "^5.0.0",
"path-exists": "^4.0.0"
@ -1141,6 +1169,11 @@
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz",
"integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA=="
},
"follow-redirects": {
"version": "1.13.0",
"resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz",
"integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA=="
},
"foreground-child": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz",
@ -1157,11 +1190,6 @@
"integrity": "sha512-Xu2Qh8yqYuDhQGOhD5iJGninErSfI9A3FrriD3tjUgV5VbJFeH8vfgZ9HnC6jWN80QDVNQK5vmxRAmEAp7Mevw==",
"dev": true
},
"fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow=="
},
"fs-extra": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz",
@ -1175,7 +1203,8 @@
"fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8="
"integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=",
"dev": true
},
"fsevents": {
"version": "2.1.3",
@ -1220,18 +1249,11 @@
"integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
"dev": true
},
"get-stream": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz",
"integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==",
"requires": {
"pump": "^3.0.0"
}
},
"glob": {
"version": "7.1.6",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz",
"integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==",
"dev": true,
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
@ -1301,7 +1323,8 @@
"he": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz",
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw=="
"integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==",
"dev": true
},
"html-escaper": {
"version": "2.0.2",
@ -1309,19 +1332,42 @@
"integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==",
"dev": true
},
"https-proxy-agent": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz",
"integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==",
"htmlparser2": {
"version": "3.10.1",
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
"integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
"requires": {
"agent-base": "5",
"debug": "4"
"domelementtype": "^1.3.1",
"domhandler": "^2.3.0",
"domutils": "^1.5.1",
"entities": "^1.1.1",
"inherits": "^2.0.1",
"readable-stream": "^3.1.1"
}
},
"ieee754": {
"version": "1.1.13",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
"integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
"ignore": {
"version": "4.0.6",
"resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
"integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
"dev": true
},
"import-fresh": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz",
"integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==",
"dev": true,
"requires": {
"parent-module": "^1.0.0",
"resolve-from": "^4.0.0"
},
"dependencies": {
"resolve-from": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz",
"integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
"dev": true
}
}
},
"ignore": {
"version": "4.0.6",
@ -1363,6 +1409,7 @@
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
"integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=",
"dev": true,
"requires": {
"once": "^1.3.0",
"wrappy": "1"
@ -1445,6 +1492,11 @@
"integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
"dev": true
},
"is-redirect": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz",
"integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ="
},
"is-regex": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz",
@ -1699,6 +1751,7 @@
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
"integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
"dev": true,
"requires": {
"p-locate": "^4.1.0"
}
@ -1706,8 +1759,7 @@
"lodash": {
"version": "4.17.20",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==",
"dev": true
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
},
"lodash.flattendeep": {
"version": "4.4.0",
@ -1801,6 +1853,7 @@
"version": "3.0.4",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
"integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
"dev": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -1820,11 +1873,6 @@
"minimist": "^1.2.5"
}
},
"mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A=="
},
"mocha": {
"version": "8.1.3",
"resolved": "https://registry.npmjs.org/mocha/-/mocha-8.1.3.tgz",
@ -2065,10 +2113,10 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"nan": {
"version": "2.14.1",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz",
"integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==",
"natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
"integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=",
"dev": true
},
"natural-compare": {
@ -2086,14 +2134,6 @@
"fetch-blob": "^2.1.1"
}
},
"node-html-parser": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-1.3.1.tgz",
"integrity": "sha512-AwYVI6GyEKj9NGoyMfSx4j5l7Axf7obQgLWGxtasLjED6RggTTQoq5ZRzjwSUfgSZ+Mv8Nzbi3pID0gFGqNUsA==",
"requires": {
"he": "1.2.0"
}
},
"node-preload": {
"version": "0.2.1",
"resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz",
@ -2109,6 +2149,14 @@
"integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
"dev": true
},
"nth-check": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
"integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
"requires": {
"boolbase": "~1.0.0"
}
},
"nyc": {
"version": "15.1.0",
"resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz",
@ -2172,6 +2220,7 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
"dev": true,
"requires": {
"wrappy": "1"
}
@ -2194,6 +2243,7 @@
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz",
"integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==",
"dev": true,
"requires": {
"p-try": "^2.0.0"
}
@ -2202,6 +2252,7 @@
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
"integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
"dev": true,
"requires": {
"p-limit": "^2.2.0"
}
@ -2218,7 +2269,8 @@
"p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ=="
"integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==",
"dev": true
},
"package-hash": {
"version": "4.0.0",
@ -2241,15 +2293,25 @@
"callsites": "^3.0.0"
}
},
"parse5": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
"integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
"requires": {
"@types/node": "*"
}
},
"path-exists": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
"integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="
"integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
"dev": true
},
"path-is-absolute": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18="
"integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
"dev": true
},
"path-key": {
"version": "3.1.1",
@ -2269,21 +2331,22 @@
"integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=",
"dev": true
},
"pend": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz",
"integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA="
},
"picomatch": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz",
"integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
"dev": true
},
"pify": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz",
"integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA=="
},
"pkg-dir": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
"integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
"dev": true,
"requires": {
"find-up": "^4.0.0"
}
@ -2306,7 +2369,8 @@
"progress": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="
"integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
"dev": true
},
"promise.allsettled": {
"version": "1.0.2",
@ -2321,43 +2385,15 @@
"iterate-value": "^1.0.0"
}
},
"proxy-from-env": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz",
"integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="
},
"pump": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz",
"integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==",
"requires": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
"psl": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
"integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
},
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
"dev": true
},
"puppeteer": {
"version": "5.3.1",
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz",
"integrity": "sha512-YTM1RaBeYrj6n7IlRXRYLqJHF+GM7tasbvrNFx6w1S16G76NrPq7oYFKLDO+BQsXNtS8kW2GxWCXjIMPvfDyaQ==",
"requires": {
"debug": "^4.1.0",
"devtools-protocol": "0.0.799653",
"extract-zip": "^2.0.0",
"https-proxy-agent": "^4.0.0",
"pkg-dir": "^4.2.0",
"progress": "^2.0.1",
"proxy-from-env": "^1.0.0",
"rimraf": "^3.0.2",
"tar-fs": "^2.0.0",
"unbzip2-stream": "^1.3.3",
"ws": "^7.2.3"
}
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
},
"randombytes": {
"version": "2.1.0",
@ -2438,6 +2474,7 @@
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz",
"integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==",
"dev": true,
"requires": {
"glob": "^7.1.3"
}
@ -2490,13 +2527,23 @@
"integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==",
"dev": true
},
"sleep": {
"version": "6.3.0",
"resolved": "https://registry.npmjs.org/sleep/-/sleep-6.3.0.tgz",
"integrity": "sha512-+WgYl951qdUlb1iS97UvQ01pkauoBK9ML9I/CMPg41v0Ze4EyMlTgFTDDo32iYj98IYqxIjDMRd+L71lawFfpQ==",
"slice-ansi": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz",
"integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==",
"dev": true,
"requires": {
"nan": "^2.14.1"
"ansi-styles": "^3.2.0",
"astral-regex": "^1.0.0",
"is-fullwidth-code-point": "^2.0.0"
},
"dependencies": {
"is-fullwidth-code-point": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
"integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
"dev": true
}
}
},
"slice-ansi": {
@ -2689,29 +2736,6 @@
}
}
},
"tar-fs": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz",
"integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==",
"requires": {
"chownr": "^1.1.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.0.0"
}
},
"tar-stream": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz",
"integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==",
"requires": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
}
},
"test-exclude": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz",
@ -2729,11 +2753,6 @@
"integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=",
"dev": true
},
"through": {
"version": "2.3.8",
"resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
"integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU="
},
"to-fast-properties": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz",
@ -2749,6 +2768,16 @@
"is-number": "^7.0.0"
}
},
"tough-cookie": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
"integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
"requires": {
"psl": "^1.1.33",
"punycode": "^2.1.1",
"universalify": "^0.1.2"
}
},
"type-check": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",
@ -2779,15 +2808,6 @@
"is-typedarray": "^1.0.0"
}
},
"unbzip2-stream": {
"version": "1.4.3",
"resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz",
"integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==",
"requires": {
"buffer": "^5.2.1",
"through": "^2.3.8"
}
},
"universalify": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz",
@ -2814,9 +2834,9 @@
"dev": true
},
"v8-compile-cache": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.1.tgz",
"integrity": "sha512-8OQ9CL+VWyt3JStj7HX7/ciTL2V3Rl1Wf5OL+SNTm0yK1KvtReVulksyeRnCANHHuUxHlQig+JJDlUhBt1NQDQ==",
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz",
"integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==",
"dev": true
},
"which": {
@ -2929,7 +2949,17 @@
"wrappy": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8="
"integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
"dev": true
},
"write": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz",
"integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==",
"dev": true,
"requires": {
"mkdirp": "^0.5.1"
}
},
"write": {
"version": "1.0.3",
@ -2952,11 +2982,6 @@
"typedarray-to-buffer": "^3.1.5"
}
},
"ws": {
"version": "7.3.1",
"resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz",
"integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA=="
},
"y18n": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz",
@ -3129,15 +3154,6 @@
}
}
}
},
"yauzl": {
"version": "2.10.0",
"resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz",
"integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=",
"requires": {
"buffer-crc32": "~0.2.3",
"fd-slicer": "~1.1.0"
}
}
}
}

16
package.json
View File

@ -1,7 +1,7 @@
{
"main": "./app/index.js",
"name": "f95api",
"version": "1.3.5",
"version": "1.5.0",
"author": {
"name": "Millennium Earl"
},
@ -19,7 +19,10 @@
"scraping",
"login",
"game",
"games"
"games",
"data",
"userdata",
"user data"
],
"scripts": {
"unit-test-mocha": "nyc --reporter=text mocha './test/index-test.js'",
@ -31,11 +34,13 @@
"node": ">=10.0"
},
"dependencies": {
"axios": "^0.21.0",
"axios-cookiejar-support": "^1.0.1",
"cheerio": "^1.0.0-rc.3",
"ky": "^0.24.0",
"ky-universal": "^0.8.2",
"log4js": "^6.3.0",
"node-html-parser": "^1.3.1",
"puppeteer": "^5.3.1"
"tough-cookie": "^4.0.0"
},
"devDependencies": {
"babel-eslint": "^10.1.0",
@ -44,7 +49,6 @@
"eslint": "^7.12.1",
"lodash": "^4.17.20",
"mocha": "^8.1.3",
"nyc": "^15.1.0",
"sleep": "^6.3.0"
"nyc": "^15.1.0"
}
}

304
test/index-test.js
View File

@ -1,290 +1,36 @@
"use strict";
// Core modules
const fs = require("fs");
// Test suite
const api = require("./suites/api-test.js").suite;
const credentials = require("./suites/credentials-test.js").suite;
const network = require("./suites/network-helper-test.js").suite;
const scraper = require("./suites/scraper-test.js").suite;
const searcher = require("./suites/searcher-test.js").suite;
const uScraper = require("./suites/user-scraper-test.js").suite;
// Public modules from npm
const _ = require("lodash");
const expect = require("chai").expect;
const sleep = require("sleep");
const dotenv = require("dotenv");
describe("Test basic function", function testBasic() {
//#region Set-up
this.timeout(15000); // All tests in this suite get 15 seconds before timeout
//#endregion Set-up
// Modules from file
const urlHelper = require("../app/scripts/url-helper.js");
const F95API = require("../app/index.js");
// Configure the .env reader
dotenv.config();
const COOKIES_SAVE_PATH = "./f95cache/cookies.json";
const ENGINES_SAVE_PATH = "./f95cache/engines.json";
const STATUSES_SAVE_PATH = "./f95cache/statuses.json";
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
const FAKE_USERNAME = "FakeUsername091276";
const FAKE_PASSWORD = "fake_password";
//F95API.debug(false);
function randomSleep() {
const random = Math.floor(Math.random() * 500) + 50;
sleep.msleep(500 + random);
}
describe("Login without cookies", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
before("Set isolation", function () {
F95API.setIsolation(true);
});
beforeEach("Remove all cookies", function () {
// Runs before each test in this block
if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH);
if (F95API.isLogged()) F95API.logout();
});
//#endregion Set-up
let testOrder = 0;
it("Test with valid credentials", async function () {
// Gain exclusive use of the cookies
while (testOrder !== 0) randomSleep();
const result = await F95API.login(USERNAME, PASSWORD);
expect(result.success).to.be.true;
expect(result.message).equal("Authentication successful");
testOrder = 1;
});
it("Test with invalid username", async function () {
// Gain exclusive use of the cookies
while (testOrder !== 1) randomSleep();
const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD);
expect(result.success).to.be.false;
expect(result.message).to.equal("Incorrect username");
testOrder = 2;
});
it("Test with invalid password", async function () {
// Gain exclusive use of the cookies
while (testOrder !== 2) randomSleep();
const result = await F95API.login(USERNAME, FAKE_PASSWORD);
expect(result.success).to.be.false;
expect(result.message).to.equal("Incorrect password");
testOrder = 3;
});
it("Test with invalid credentials", async function () {
// Gain exclusive use of the cookies
while (testOrder !== 3) randomSleep();
const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD);
expect(result.success).to.be.false;
expect(result.message).to.equal("Incorrect username"); // It should first check the username
testOrder = 4;
});
describe("Test credentials class", credentials.bind(this));
describe("Test network helper", network.bind(this));
});
describe("Login with cookies", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
describe("Test F95 modules", function testF95Modules() {
//#region Set-up
this.timeout(15000); // All tests in this suite get 15 seconds before timeout
//#endregion Set-up
before("Log in to create cookies then logout", async function () {
// Runs once before the first test in this block
if (!fs.existsSync(COOKIES_SAVE_PATH))
await F95API.login(USERNAME, PASSWORD); // Download cookies
if (F95API.isLogged()) F95API.logout();
});
//#endregion Set-up
it("Test with valid credentials", async function () {
const result = await F95API.login(USERNAME, PASSWORD);
expect(result.success).to.be.true;
expect(result.message).equal("Logged with cookies");
});
describe("Test scraper methods", scraper.bind(this));
describe("Test searcher methods", searcher.bind(this));
describe("Test user scraper methods", uScraper.bind(this));
});
describe("Load base data without cookies", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
describe("Test complete API", function testAPI() {
//#region Set-up
this.timeout(15000); // All tests in this suite get 15 seconds before timeout
//#endregion Set-up
before("Delete cache if exists", function () {
// Runs once before the first test in this block
if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH);
if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH);
});
//#endregion Set-up
it("With login", async function () {
const loginResult = await F95API.login(USERNAME, PASSWORD);
expect(loginResult.success).to.be.true;
const result = await F95API.loadF95BaseData();
const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH);
const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH);
expect(result).to.be.true;
expect(enginesCacheExists).to.be.true;
expect(statusesCacheExists).to.be.true;
});
it("Without login", async function () {
if (F95API.isLogged()) F95API.logout();
const result = await F95API.loadF95BaseData();
expect(result).to.be.false;
});
});
describe("Search game data", function () {
//#region Set-up
this.timeout(60000); // All tests in this suite get 60 seconds before timeout
beforeEach("Prepare API", function () {
// Runs once before the first test in this block
if (F95API.isLogged()) F95API.logout();
});
//#endregion Set-up
let testGame = null;
it("Search game when logged", async function () {
const loginResult = await F95API.login(USERNAME, PASSWORD);
expect(loginResult.success).to.be.true;
const loadResult = await F95API.loadF95BaseData();
expect(loadResult).to.be.true;
// This test depend on the data on F95Zone at
// https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/
const gamesList = await F95API.getGameData("Kingdom of Deception", false);
expect(gamesList.length, "Should find only the game").to.equal(1);
const result = gamesList[0];
const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png";
// Test only the main information
expect(result.name).to.equal("Kingdom of Deception");
expect(result.author).to.equal("Hreinn Games");
expect(result.isMod, "Should be false").to.be.false;
expect(result.engine).to.equal("REN'PY");
expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image?
testGame = Object.assign({}, result);
});
it("Search game when not logged", async function () {
const result = await F95API.getGameData("Kingdom of Deception", false);
expect(result, "Without being logged should return null").to.be.null;
});
it("Test game serialization", function () {
const json = JSON.stringify(testGame);
const parsedGameInfo = JSON.parse(json);
const result = _.isEqual(parsedGameInfo, testGame);
expect(result).to.be.true;
});
});
describe("Load user data", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
it("Retrieve when logged", async function () {
// Login
await F95API.login(USERNAME, PASSWORD);
// Then retrieve user data
const data = await F95API.getUserData();
expect(data).to.exist;
expect(data.username).to.equal(USERNAME);
});
it("Retrieve when not logged", async function () {
// Logout
if (F95API.isLogged()) F95API.logout();
// Try to retrieve user data
const data = await F95API.getUserData();
expect(data).to.be.null;
});
});
describe("Check game update", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
it("Get online game and verify that no update exists", async function () {
const loginResult = await F95API.login(USERNAME, PASSWORD);
expect(loginResult.success).to.be.true;
const loadResult = await F95API.loadF95BaseData();
expect(loadResult).to.be.true;
// This test depend on the data on F95Zone at
// https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/
const result = (await F95API.getGameData("Kingdom of Deception", false))[0];
const update = await F95API.chekIfGameHasUpdate(result);
expect(update).to.be.false;
});
it("Verify that update exists from old URL", async function () {
const loginResult = await F95API.login(USERNAME, PASSWORD);
expect(loginResult.success).to.be.true;
// This test depend on the data on F95Zone at
// https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/
const url =
"https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/";
const result = await F95API.getGameDataFromURL(url);
result.version = "0.9600";
const update = await F95API.chekIfGameHasUpdate(result);
expect(update).to.be.true;
});
});
describe("Test url-helper", function () {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
it("Check if URL exists", async function () {
// Check generic URLs...
let exists = await urlHelper.urlExists("https://www.google.com/");
expect(exists, "Complete valid URL").to.be.true;
exists = await urlHelper.urlExists("www.google.com");
expect(exists, "URl without protocol prefix").to.be.false;
exists = await urlHelper.urlExists("https://www.google/");
expect(exists, "URL without third level domain").to.be.false;
// Now check for more specific URLs (with redirect)...
exists = await urlHelper.urlExists(
"https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"
);
expect(exists, "URL with redirect without check").to.be.true;
exists = await urlHelper.urlExists(
"https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/",
true
);
expect(exists, "URL with redirect with check").to.be.false;
});
it("Check if URL belong to the platform", async function () {
let belong = urlHelper.isF95URL(
"https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"
);
expect(belong).to.be.true;
belong = urlHelper.isF95URL("https://www.google/");
expect(belong).to.be.false;
});
describe("Test API", api.bind(this));
});

64
test/suites/api-test.js Normal file
View File

@ -0,0 +1,64 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
const {
isEqual
} = require("lodash");
// Modules from file
const F95API = require("../../app/index.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
// Global suite variables
const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/";
it("Test login", async function testLogin() {
const result = await F95API.login(USERNAME, PASSWORD);
expect(result.success).to.be.true;
expect(F95API.isLogged()).to.be.true;
});
it("Test user data fetching", async function testUserDataFetch() {
const userdata = await F95API.getUserData();
expect(userdata.username).to.be.equal(USERNAME);
});
it("Test game update checking", async function testGameUpdateCheck() {
// We force the creation of a GameInfo object,
// knowing that the checkIfGameHasUpdate() function
// only needs the game URL
const info = new F95API.GameInfo();
// The gameURL identifies a game for which we know there is an update
info.url = gameURL;
// Check for updates
const update = await F95API.checkIfGameHasUpdate(info);
expect(update).to.be.true;
});
it("Test game data fetching", async function testGameDataFetch() {
// Search a game by name
const gameList = await F95API.getGameData("perverted education", false);
// We know that there is only one game with the selected name
expect(gameList.length).to.be.equal(1, `There should be only one game, not ${gameList.length}`);
const game = gameList[0];
// Than we fetch a game from URL
const gameFromURL = await F95API.getGameDataFromURL(game.url);
// The two games must be equal
const equal = isEqual(game, gameFromURL);
expect(equal).to.be.true;
});
};

52
test/suites/credentials-test.js Normal file
View File

@ -0,0 +1,52 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
module.exports.suite = function suite() {
it("Check token formatting", async function testValidToken() {
// Token example:
// 1604309951,0338213c00fcbd894fd9415e6ba08403
// 1604309986,ebdb75502337699381f0f55c86353555
// 1604310008,2d50d55808e5ec3a157ec01953da9d26
// Fetch token (is a GET request, we don't need the credentials)
const cred = new Credentials(null, null);
await cred.fetchToken();
// Parse token for assert
const splitted = cred.token.split(",");
const unique = splitted[0];
const hash = splitted[1];
expect(splitted.length).to.be.equal(2, "The token consists of two parts");
// Check type of parts
expect(isNumeric(unique)).to.be.true;
expect(isNumeric(hash)).to.be.false;
// The second part is most probably the MD5 hash of something
expect(hash.length).to.be.equal(32, "Hash should have 32 hex chars");
});
};
//#region Private methods
/**
* @private
* Check if a string is a number
* @param {String} str
* @author Dan, Ben Aston
* @see https://preview.tinyurl.com/y46jqwkt
*/
function isNumeric(str) {
// We only process strings!
if (typeof str != "string") return false;
// Use type coercion to parse the _entirety_ of the string
// (`parseFloat` alone does not do this) and ensure strings
// of whitespace fail
return !isNaN(str) && !isNaN(parseFloat(str));
}
//#endregion

107
test/suites/network-helper-test.js Normal file
View File

@ -0,0 +1,107 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
const networkHelper = require("../../app/scripts/network-helper.js");
const {
F95_SEARCH_URL
} = require("../../app/scripts/constants/url.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
const FAKE_USERNAME = "Fake_Username091276";
const FAKE_PASSWORD = "fake_password";
module.exports.suite = function suite() {
// Global suite variables
const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/";
it("Check if URL exists", async function checkURLExistence() {
// Check generic URLs...
let exists = await networkHelper.urlExists("https://www.google.com/");
expect(exists, "Complete valid URL").to.be.true;
exists = await networkHelper.urlExists("www.google.com");
expect(exists, "URl without protocol prefix").to.be.false;
exists = await networkHelper.urlExists("https://www.google/");
expect(exists, "URL without third level domain").to.be.false;
// Now check for more specific URLs (with redirect)...
exists = await networkHelper.urlExists(gameURL);
expect(exists, "URL with redirect without check").to.be.true;
exists = await networkHelper.urlExists(gameURL, true);
expect(exists, "URL with redirect with check").to.be.false;
});
it("Check if URL belong to the platform", function checkIfURLIsF95() {
let belong = networkHelper.isF95URL(gameURL);
expect(belong).to.be.true;
belong = networkHelper.isF95URL("https://www.google/");
expect(belong).to.be.false;
});
it("Enforce secure URLs", function testSecureURLEnforcement() {
// This URL is already secure, should remain the same
let enforced = networkHelper.enforceHttpsUrl(gameURL);
expect(enforced).to.be.equal(gameURL, "The game URL is already secure");
// This URL is not secure
enforced = networkHelper.enforceHttpsUrl("http://www.google.com");
expect(enforced).to.be.equal("https://www.google.com", "The URL was without SSL/TLS (HTTPs)");
// Finally, we check when we pass a invalid URL
enforced = networkHelper.enforceHttpsUrl("http://invalidurl");
expect(enforced).to.be.null;
});
it("Check URL redirect", async function checkURLRedirect() {
// gameURL is an old URL it has been verified that it generates a redirect
const redirectURL = await networkHelper.getUrlRedirect(gameURL);
expect(redirectURL).to.not.be.equal(gameURL, "The original URL has redirect");
// If we recheck the new URL, we find that no redirect happens
const secondRedirectURL = await networkHelper.getUrlRedirect(redirectURL);
expect(secondRedirectURL).to.be.equal(redirectURL, "The URL has no redirect");
});
it("Check response to GET request", async function testGETResponse() {
// We should be able to fetch a game page
let response = await networkHelper.fetchGETResponse(gameURL);
expect(response.status).to.be.equal(200, "The operation must be successful");
// We should NOT be able to fetch the search page (we must be logged)
response = await networkHelper.fetchGETResponse(F95_SEARCH_URL);
expect(response).to.be.null;
});
it("Test for authentication to platform", async function testAuthentication() {
// Try to authenticate with valid credentials
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
const validResult = await networkHelper.authenticate(creds);
expect(validResult.success).to.be.true;
// Now we use fake credentials
const fakeCreds = new Credentials(FAKE_USERNAME, FAKE_PASSWORD);
await fakeCreds.fetchToken();
const invalidResult = await networkHelper.authenticate(fakeCreds, true);
expect(invalidResult.success).to.be.false;
});
it("Test fetching HTML", async function testFetchHTML() {
// This should return the HTML code of the page
const html = await networkHelper.fetchHTML(gameURL);
expect(html.startsWith("<!DOCTYPE html>")).to.be.true;
});
};

42
test/suites/scraper-test.js Normal file
View File

@ -0,0 +1,42 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const { isEqual } = require("lodash");
const GameInfo = require("../../app/scripts/classes/game-info.js");
// Modules from file
const scraper = require("../../app/scripts/scraper.js");
module.exports.suite = function suite() {
// Global suite variables
const gameURL = "https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/";
const previewSrc = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png";
it("Search game", async function () {
// This test depend on the data on F95Zone at gameURL
const result = await scraper.getGameInfo(gameURL);
// Test only the main information
expect(result.name).to.equal("Kingdom of Deception");
expect(result.author).to.equal("Hreinn Games");
expect(result.isMod, "Should be false").to.be.false;
expect(result.engine).to.equal("Ren'Py");
expect(result.previewSrc).to.equal(previewSrc, "Preview not equals");
});
it("Test game serialization", async function testGameSerialization() {
// This test depend on the data on F95Zone at gameURL
const testGame = await scraper.getGameInfo(gameURL);
// Serialize...
const json = JSON.stringify(testGame);
// Deserialize...
const parsedGameInfo = GameInfo.fromJSON(json);
// Compare with lodash
const result = isEqual(parsedGameInfo, testGame);
expect(result).to.be.true;
});
};

64
test/suites/searcher-test.js Normal file
View File

@ -0,0 +1,64 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
const searcher = require("../../app/scripts/searcher.js");
const {
authenticate
} = require("../../app/scripts/network-helper.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
// TODO:
// This method should delete the store F95Zone cookies,
// but what if the other tests require them?
// it("Search game when not logged", async function searchGameWhenNotLogged() {
// // Search for a game that we know has only one result
// // but without logging in first
// const urls = await searcher.searchGame("kingdom of deception");
// expect(urls.lenght).to.be.equal(0, "There should not be any URL");
// });
it("Search game", async function searchGame() {
// Authenticate
const result = await auth();
expect(result.success, "Authentication should be successful").to.be.true;
// Search for a game that we know has only one result
const urls = await searcher.searchGame("kingdom of deception");
expect(urls.length).to.be.equal(1, `There should be only one game result instead of ${urls.length}`);
});
it("Search mod", async function searchMod() {
// Authenticate
const result = await auth();
expect(result.success, "Authentication should be successful").to.be.true;
// Search for a mod that we know has only one result
const urls = await searcher.searchMod("kingdom of deception jdmod");
expect(urls.length).to.be.equal(1, `There should be only one mod result instead of ${urls.length}`);
});
};
//#region Private methods
/**
* @private
* Simple wrapper for authentication.
*/
async function auth() {
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
return await authenticate(creds);
}
//#endregion Private methods

54
test/suites/user-scraper-test.js Normal file
View File

@ -0,0 +1,54 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
const uScraper = require("../../app/scripts/user-scraper.js");
const {
authenticate
} = require("../../app/scripts/network-helper.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
// TODO:
// This method should delete the store F95Zone cookies,
// but what if the other tests require them?
// it("Fetch data when not logged", async function fetchUserDataWhenLogged() {
// const data = await uScraper.getUserData();
// expect(data.username).to.be.equal("");
// expect(data.avatarSrc).to.be.equal("");
// expect(data.watchedThreads.length).to.be.equal(0);
// });
it("Fetch data when logged", async function fetchUserDataWhenNotLogged() {
// Authenticate
const result = await auth();
expect(result.success, "Authentication should be successful").to.be.true;
// We test only for the username, the other test data depends on the user logged
const data = await uScraper.getUserData();
expect(data.username).to.be.equal(USERNAME);
});
};
//#region Private methods
/**
* @private
* Simple wrapper for authentication.
*/
async function auth() {
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
return await authenticate(creds);
}
//#endregion Private methods

15
test/user-test.js
View File

@ -1,15 +0,0 @@
const F95API = require("../app/index.js");
F95API.debug(true);
main();
async function main() {
const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK");
if (loginResult.success) {
await F95API.loadF95BaseData();
const gameData = await F95API.getGameData("a struggle with sin", false);
console.log(gameData);
}
F95API.logout();
}