Compare commits

..

20 Commits

Author SHA1 Message Date
Millennium Earl 48d2ff4038
Merge pull request #78 from MillenniumEarl/dependabot/npm_and_yarn/chai-4.3.4
Bump chai from 4.3.3 to 4.3.4
2021-03-15 21:36:25 +01:00
Millennium Earl a014f4526f
Merge pull request #79 from MillenniumEarl/dependabot/npm_and_yarn/eslint-7.22.0
Bump eslint from 7.21.0 to 7.22.0
2021-03-15 21:36:15 +01:00
Millennium Earl 411d59b994
Merge pull request #80 from MillenniumEarl/dependabot/npm_and_yarn/mocha-8.3.2
Bump mocha from 8.3.1 to 8.3.2
2021-03-15 21:33:10 +01:00
dependabot[bot] c019d05364
Bump mocha from 8.3.1 to 8.3.2
Bumps [mocha](https://github.com/mochajs/mocha) from 8.3.1 to 8.3.2.
- [Release notes](https://github.com/mochajs/mocha/releases)
- [Changelog](https://github.com/mochajs/mocha/blob/master/CHANGELOG.md)
- [Commits](https://github.com/mochajs/mocha/compare/v8.3.1...v8.3.2)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-15 07:50:56 +00:00
dependabot[bot] f226242290
Bump eslint from 7.21.0 to 7.22.0
Bumps [eslint](https://github.com/eslint/eslint) from 7.21.0 to 7.22.0.
- [Release notes](https://github.com/eslint/eslint/releases)
- [Changelog](https://github.com/eslint/eslint/blob/master/CHANGELOG.md)
- [Commits](https://github.com/eslint/eslint/compare/v7.21.0...v7.22.0)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-15 07:50:37 +00:00
dependabot[bot] 86d9663da5
Bump chai from 4.3.3 to 4.3.4
Bumps [chai](https://github.com/chaijs/chai) from 4.3.3 to 4.3.4.
- [Release notes](https://github.com/chaijs/chai/releases)
- [Changelog](https://github.com/chaijs/chai/blob/main/History.md)
- [Commits](https://github.com/chaijs/chai/compare/4.3.3...v4.3.4)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-15 07:50:09 +00:00
Millennium Earl 166afaccd8
Merge pull request #76 from MillenniumEarl/dependabot/npm_and_yarn/mocha-8.3.1
Bump mocha from 8.3.0 to 8.3.1
2021-03-08 20:51:36 +01:00
dependabot[bot] c0baa0dc65
Bump mocha from 8.3.0 to 8.3.1
Bumps [mocha](https://github.com/mochajs/mocha) from 8.3.0 to 8.3.1.
- [Release notes](https://github.com/mochajs/mocha/releases)
- [Changelog](https://github.com/mochajs/mocha/blob/master/CHANGELOG.md)
- [Commits](https://github.com/mochajs/mocha/compare/v8.3.0...v8.3.1)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-08 07:39:13 +00:00
Millennium Earl e328b82918
Merge pull request #70 from MillenniumEarl/dependabot/npm_and_yarn/eslint-7.21.0
Bump eslint from 7.20.0 to 7.21.0
2021-03-04 10:42:03 +01:00
Millennium Earl 016c75501a
Merge pull request #72 from MillenniumEarl/dependabot/npm_and_yarn/chai-4.3.3
Bump chai from 4.3.0 to 4.3.3
2021-03-04 10:41:55 +01:00
dependabot[bot] dddf40ff16
Bump chai from 4.3.0 to 4.3.3
Bumps [chai](https://github.com/chaijs/chai) from 4.3.0 to 4.3.3.
- [Release notes](https://github.com/chaijs/chai/releases)
- [Changelog](https://github.com/chaijs/chai/blob/main/History.md)
- [Commits](https://github.com/chaijs/chai/compare/4.3.0...4.3.3)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-04 06:26:04 +00:00
dependabot[bot] 1617c8be88
Bump eslint from 7.20.0 to 7.21.0
Bumps [eslint](https://github.com/eslint/eslint) from 7.20.0 to 7.21.0.
- [Release notes](https://github.com/eslint/eslint/releases)
- [Changelog](https://github.com/eslint/eslint/blob/master/CHANGELOG.md)
- [Commits](https://github.com/eslint/eslint/compare/v7.20.0...v7.21.0)

Signed-off-by: dependabot[bot] <support@github.com>
2021-03-01 08:04:36 +00:00
Millennium Earl 9de98a1f05
Merge pull request #66 from MillenniumEarl/dependabot/npm_and_yarn/mocha-8.3.0
Bump mocha from 8.2.1 to 8.3.0
2021-02-22 08:39:44 +01:00
Millennium Earl 704dd5418b
Merge pull request #69 from MillenniumEarl/dependabot/npm_and_yarn/lodash-4.17.21
Bump lodash from 4.17.20 to 4.17.21
2021-02-22 08:38:58 +01:00
Millennium Earl cb75826fdb
Merge pull request #68 from MillenniumEarl/dependabot/npm_and_yarn/luxon-1.26.0
Bump luxon from 1.25.0 to 1.26.0
2021-02-22 08:38:39 +01:00
Millennium Earl cc163f06fc
Merge pull request #67 from MillenniumEarl/dependabot/npm_and_yarn/eslint-7.20.0
Bump eslint from 7.19.0 to 7.20.0
2021-02-22 08:38:25 +01:00
dependabot[bot] a8f9664424
Bump lodash from 4.17.20 to 4.17.21
Bumps [lodash](https://github.com/lodash/lodash) from 4.17.20 to 4.17.21.
- [Release notes](https://github.com/lodash/lodash/releases)
- [Commits](https://github.com/lodash/lodash/compare/4.17.20...4.17.21)

Signed-off-by: dependabot[bot] <support@github.com>
2021-02-22 07:35:59 +00:00
dependabot[bot] c0e28d53df
Bump luxon from 1.25.0 to 1.26.0
Bumps [luxon](https://github.com/moment/luxon) from 1.25.0 to 1.26.0.
- [Release notes](https://github.com/moment/luxon/releases)
- [Changelog](https://github.com/moment/luxon/blob/master/CHANGELOG.md)
- [Commits](https://github.com/moment/luxon/compare/1.25.0...1.26.0)

Signed-off-by: dependabot[bot] <support@github.com>
2021-02-15 07:23:18 +00:00
dependabot[bot] 8d4e8b09bb
Bump eslint from 7.19.0 to 7.20.0
Bumps [eslint](https://github.com/eslint/eslint) from 7.19.0 to 7.20.0.
- [Release notes](https://github.com/eslint/eslint/releases)
- [Changelog](https://github.com/eslint/eslint/blob/master/CHANGELOG.md)
- [Commits](https://github.com/eslint/eslint/compare/v7.19.0...v7.20.0)

Signed-off-by: dependabot[bot] <support@github.com>
2021-02-15 07:22:56 +00:00
dependabot[bot] dc5b438db4
Bump mocha from 8.2.1 to 8.3.0
Bumps [mocha](https://github.com/mochajs/mocha) from 8.2.1 to 8.3.0.
- [Release notes](https://github.com/mochajs/mocha/releases)
- [Changelog](https://github.com/mochajs/mocha/blob/master/CHANGELOG.md)
- [Commits](https://github.com/mochajs/mocha/compare/v8.2.1...v8.3.0)

Signed-off-by: dependabot[bot] <support@github.com>
2021-02-12 06:28:08 +00:00
82 changed files with 4070 additions and 12089 deletions

16
.deepsource.toml Normal file
View File

@ -0,0 +1,16 @@
version = 1
test_patterns = [
"test/**"
]
[[analyzers]]
name = "javascript"
enabled = true
[analyzers.meta]
environment = [
"nodejs",
"mocha"
]
style_guide = "standard"

View File

@ -1,2 +0,0 @@
node_modules
dist

View File

@ -1,34 +0,0 @@
{
"root": true,
"parser": "@typescript-eslint/parser",
"plugins": [
"@typescript-eslint",
"prettier"
],
"extends": [
"eslint:recommended",
"plugin:@typescript-eslint/eslint-recommended",
"plugin:@typescript-eslint/recommended",
"prettier"
],
"rules": {
"no-inferrable-types": "off",
"no-console": "warn",
"prettier/prettier": "error",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/no-inferrable-types": "off",
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/ban-ts-comment": "warn",
"no-unused-vars": "off",
"no-prototype-builtins": "warn"
},
"overrides": [
{
"files": [
"*/src/example.ts"
],
"rules": {
"no-console": "off"
}
}]
}

38
.eslintrc.json Normal file
View File

@ -0,0 +1,38 @@
{
"env": {
"browser": true,
"commonjs": true,
"es2021": true,
"node": true,
"mocha": true
},
"extends": ["eslint:recommended"],
"parser": "babel-eslint",
"parserOptions": {
"ecmaVersion": 12
},
"rules": {
"indent": [
"error",
4
],
"linebreak-style": [
"error",
"windows"
],
"quotes": [
"error",
"double"
],
"semi": [
"error",
"always"
],
"no-unused-vars": [
"error",
{
"args": "after-used"
}
]
}
}

7
.gitignore vendored
View File

@ -1,6 +1,5 @@
.nyc_output/
dist/
node_modules/
f95cache/
.nyc_output/
.env
coverage.lcov
**/*.js
coverage.lcov

View File

@ -1,14 +1,10 @@
test
coverage.lcov
tsconfig.json
.dist
.nyc_output
.eslintignore
.eslintrc
.eslintrc.json
.gitignore
.gitattribute
.github
.vscode
.deepsource.toml
.env
.prettierrc

View File

@ -1,6 +0,0 @@
{
"semi": true,
"trailingComma": "none",
"singleQuote": false,
"printWidth": 100
}

17
.vscode/launch.json vendored
View File

@ -2,21 +2,10 @@
"configurations": [
{
"type": "node-terminal",
"name": "Test",
"name": "Run Script: test",
"request": "launch",
"command": "npm run test",
"cwd": "${workspaceFolder}",
},
{
"type": "node-terminal",
"name": "Example",
"request": "launch",
"command": "npm run run-example",
"cwd": "${workspaceFolder}",
"skipFiles": [
"${workspaceFolder}/node_modules/**/*",
"<node_internals>/**/*"
]
},
"cwd": "${workspaceFolder}"
}
]
}

View File

@ -1,7 +0,0 @@
{
"files.exclude": {
"**/.nyc_output": true,
"**/dist": true,
"**/node_modules": true
}
}

View File

@ -1,6 +1,6 @@
The MIT License (MIT)
Copyright (c) 2021 Millennium Earl
Copyright (c) 2020 Millennium Earl
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal

View File

@ -1,3 +1,4 @@
[![DeepSource](https://deepsource.io/gh/MillenniumEarl/F95API.svg/?label=active+issues&show_trend=true)](https://deepsource.io/gh/MillenniumEarl/F95API/?ref=repository-badge)
[![CodeFactor](https://www.codefactor.io/repository/github/millenniumearl/f95api/badge)](https://www.codefactor.io/repository/github/millenniumearl/f95api)
[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2FMillenniumEarl%2FF95API.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2FMillenniumEarl%2FF95API?ref=badge_shield)
[![Known Vulnerabilities](https://snyk.io/test/github/MillenniumEarl/F95API/badge.svg)](https://snyk.io/test/github/MillenniumEarl/F95API)
@ -58,7 +59,7 @@ changelog: Latest changelog available
The serialization in JSON format of this object is possible through `JSON.stringify()` while the deserialization must happen through the static method `GameInfo.fromJSON()`.
## User data
User data (after authentication) can be stored in a UserProfile object, consisting of the following fields:
User data (after authentication) can be stored in a UserData object, consisting of the following fields:
```
username: Name of the logged in user

61
app/example.js Normal file
View File

@ -0,0 +1,61 @@
/*
to use this example, create an .env file
in the project root with the following values:
F95_USERNAME = YOUR_USERNAME
F95_PASSWORD = YOUR_PASSWORD
*/
"use strict";
// Public modules from npm
const dotenv = require("dotenv");
// Modules from file
const F95API = require("./index.js");
// Configure the .env reader
dotenv.config();
main();
async function main() {
// Local variables
const gameList = [
"Four Elements Trainer",
"corrupted kingdoms",
"summertime saga"
];
// Log in the platform
console.log("Authenticating...");
const result = await F95API.login(process.env.F95_USERNAME, process.env.F95_PASSWORD);
console.log(`Authentication result: ${result.message}\n`);
// Get user data
console.log("Fetching user data...");
const userdata = await F95API.getUserData();
console.log(`${userdata.username} follows ${userdata.watchedGameThreads.length} threads\n`);
// Get latest game update
const latestUpdates = await F95API.getLatestUpdates({
tags: ["3d game"]
}, 1);
console.log(`"${latestUpdates[0].name}" was the last "3d game" tagged game to be updated\n`);
// Get game data
for(const gamename of gameList) {
console.log(`Searching '${gamename}'...`);
const found = await F95API.getGameData(gamename, false);
// If no game is found
if (found.length === 0) {
console.log(`No data found for '${gamename}'`);
continue;
}
// Extract first game
const gamedata = found[0];
console.log(`Found: ${gamedata.name} (${gamedata.version}) by ${gamedata.author}\n`);
}
}

243
app/index.js Normal file
View File

@ -0,0 +1,243 @@
"use strict";
// Modules from file
const shared = require("./scripts/shared.js");
const networkHelper = require("./scripts/network-helper.js");
const scraper = require("./scripts/scraper.js");
const searcher = require("./scripts/searcher.js");
const uScraper = require("./scripts/user-scraper.js");
const latestFetch = require("./scripts/latest-fetch.js");
const fetchPlatformData = require("./scripts/platform-data.js").fetchPlatformData;
// Classes from file
const Credentials = require("./scripts/classes/credentials.js");
const GameInfo = require("./scripts/classes/game-info.js");
const LoginResult = require("./scripts/classes/login-result.js");
const UserData = require("./scripts/classes/user-data.js");
const PrefixParser = require("./scripts/classes/prefix-parser.js");
//#region Global variables
const USER_NOT_LOGGED = "User not authenticated, unable to continue";
//#endregion
//#region Export classes
module.exports.GameInfo = GameInfo;
module.exports.LoginResult = LoginResult;
module.exports.UserData = UserData;
module.exports.PrefixParser = PrefixParser;
//#endregion Export classes
//#region Export properties
/**
* @public
* Set the logger level for module debugging.
*/
/* istambul ignore next */
module.exports.loggerLevel = shared.logger.level;
exports.loggerLevel = "warn"; // By default log only the warn messages
/**
* @public
* Indicates whether a user is logged in to the F95Zone platform or not.
* @returns {String}
*/
module.exports.isLogged = function isLogged() {
return shared.isLogged;
};
//#endregion Export properties
//#region Export methods
/**
* @public
* Log in to the F95Zone platform.
* This **must** be the first operation performed before accessing any other script functions.
* @param {String} username Username used for login
* @param {String} password Password used for login
* @returns {Promise<LoginResult>} Result of the operation
*/
module.exports.login = async function (username, password) {
/* istanbul ignore next */
if (shared.isLogged) {
shared.logger.info(`${username} already authenticated`);
return new LoginResult(true, `${username} already authenticated`);
}
shared.logger.trace("Fetching token...");
const creds = new Credentials(username, password);
await creds.fetchToken();
shared.logger.trace(`Authentication for ${username}`);
const result = await networkHelper.authenticate(creds);
shared.isLogged = result.success;
// Load platform data
if (result.success) await fetchPlatformData();
/* istambul ignore next */
if (result.success) shared.logger.info("User logged in through the platform");
else shared.logger.warn(`Error during authentication: ${result.message}`);
return result;
};
/**
* @public
* Chek if exists a new version of the game.
* You **must** be logged in to the portal before calling this method.
* @param {GameInfo} info Information about the game to get the version for
* @returns {Promise<Boolean>} true if an update is available, false otherwise
*/
module.exports.checkIfGameHasUpdate = async function (info) {
/* istanbul ignore next */
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return false;
}
// F95 change URL at every game update,
// so if the URL is different an update is available
const exists = await networkHelper.urlExists(info.url, true);
if (!exists) return true;
// Parse version from title
const onlineInfo = await scraper.getGameInfo(info.url);
const onlineVersion = onlineInfo.version;
// Compare the versions
return onlineVersion.toUpperCase() !== info.version.toUpperCase();
};
/**
* @public
* Starting from the name, it gets all the information about the game you are looking for.
* You **must** be logged in to the portal before calling this method.
* @param {String} name Name of the game searched
* @param {Boolean} mod Indicate if you are looking for mods or games
* @returns {Promise<GameInfo[]>} List of information obtained where each item corresponds to
* an identified game (in the case of homonymy of titles)
*/
module.exports.getGameData = async function (name, mod) {
/* istanbul ignore next */
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
// Gets the search results of the game/mod being searched for
const urls = mod ?
await searcher.searchMod(name) :
await searcher.searchGame(name);
// Process previous partial results
const results = [];
for (const url of urls) {
// Start looking for information
const info = await scraper.getGameInfo(url);
if (info) results.push(info);
}
return results;
};
/**
* @public
* Starting from the url, it gets all the information about the game you are looking for.
* You **must** be logged in to the portal before calling this method.
* @param {String} url URL of the game to obtain information of
* @returns {Promise<GameInfo>} Information about the game. If no game was found, null is returned
*/
module.exports.getGameDataFromURL = async function (url) {
/* istanbul ignore next */
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
// Check URL validity
const exists = await networkHelper.urlExists(url);
if (!exists) throw new URIError(`${url} is not a valid URL`);
if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
// Get game data
return await scraper.getGameInfo(url);
};
/**
* @public
* Gets the data of the currently logged in user.
* You **must** be logged in to the portal before calling this method.
* @returns {Promise<UserData>} Data of the user currently logged in
*/
module.exports.getUserData = async function () {
/* istanbul ignore next */
if (!shared.isLogged) {
shared.logger.warn(USER_NOT_LOGGED);
return null;
}
return await uScraper.getUserData();
};
/**
* @public
* Gets the latest updated games that match the specified parameters.
* You **must** be logged in to the portal before calling this method.
* @param {Object} args
* Parameters used for the search.
* @param {String[]} [args.tags]
* List of tags to be included in the search (max 5).
* @param {Number} [args.datelimit]
* Number of days since the game was last updated.
* The entered value will be approximated to the nearest valid one.
* Use `0` to select no time limit.
* @param {String[]} [args.prefixes]
* Prefixes to be included in the search.
* @param {String} [args.sorting]
* Method of sorting the results between (default: `date`):
* `date`, `likes`, `views`, `name`, `rating`
* @param {Number} limit Maximum number of results
* @returns {Promise<GameInfo[]>} List of games
*/
module.exports.getLatestUpdates = async function(args, limit) {
// Check limit value
if(limit <= 0) throw new Error("limit must be greater than 0");
// Prepare the parser
const parser = new PrefixParser();
// Get the closest date limit
let filterDate = 0;
if(args.datelimit) {
const validDate = [365, 180, 90, 30, 14, 7, 3, 1, 0];
filterDate = getNearestValueFromArray(validDate, args.datelimit);
}
// Fetch the games
const query = {
tags: args.tags ? parser.prefixesToIDs(args.tags) : [],
prefixes: args.prefixes ? parser.prefixesToIDs(args.prefixes) : [],
sort: args.sorting ? args.sorting : "date",
date: filterDate,
};
const urls = await latestFetch.fetchLatest(query, limit);
// Get the gamedata from urls
const promiseList = urls.map(u => exports.getGameDataFromURL(u));
return await Promise.all(promiseList);
};
//#endregion
//#region Private Methods
/**
* @private
* Given an array of numbers, get the nearest value for a given `value`.
* @param {Number[]} array List of default values
* @param {Number} value Value to search
*/
function getNearestValueFromArray(array, value) {
// Script taken from:
// https://www.gavsblog.com/blog/find-closest-number-in-array-javascript
array.sort((a, b) => {
return Math.abs(value - a) - Math.abs(value - b);
});
return array[0];
}
//#endregion

View File

@ -0,0 +1,22 @@
"use strict";
// Modules from file
const { getF95Token } = require("../network-helper.js");
class Credentials {
constructor(username, password) {
this.username = username;
this.password = password;
this.token = null;
}
/**
* @public
* Fetch and save the token used to log in to F95Zone.
*/
async fetchToken() {
this.token = await getF95Token();
}
}
module.exports = Credentials;

View File

@ -0,0 +1,129 @@
"use strict";
class GameInfo {
constructor() {
//#region Properties
/**
* Unique ID of the game on the platform.
* @type Number
*/
this.id = -1;
/**
* Game name
* @type String
*/
this.name = null;
/**
* Game author
* @type String
*/
this.author = null;
/**
* URL to the game's official conversation on the F95Zone portal
* @type String
*/
this.url = null;
/**
* Game description
* @type String
*/
this.overview = null;
/**
* Game language.
* @type String
*/
this.language = null;
/**
* List of supported OS.
* @type
*/
this.supportedOS = [];
/**
* Specify whether the game has censorship
* measures regarding NSFW scenes.
* @type Boolean
*/
this.censored = null;
/**
* List of tags associated with the game
* @type String[]
*/
this.tags = [];
/**
* Graphics engine used for game development
* @type String
*/
this.engine = null;
/**
* Development of the game
* @type String
*/
this.status = null;
/**
* Game description image URL
* @type String
*/
this.previewSrc = null;
/**
* Game version
* @type String
*/
this.version = null;
/**
* Last time the game underwent updates
* @type Date
*/
this.lastUpdate = null;
/**
* Specifies if the game is original or a mod
* @type Boolean
*/
this.isMod = false;
/**
* Changelog for the last version.
* @type String
*/
this.changelog = null;
//#endregion Properties
}
/**
* Converts the object to a dictionary used for JSON serialization.
*/
/* istanbul ignore next */
toJSON() {
return {
id: this.id,
name: this.name,
author: this.author,
url: this.url,
overview: this.overview,
language: this.language,
supportedOS: this.supportedOS,
censored: this.censored,
engine: this.engine,
status: this.status,
tags: this.tags,
previewSrc: this.previewSrc,
version: this.version,
lastUpdate: this.lastUpdate,
isMod: this.isMod,
changelog: this.changelog,
};
}
/**
* Return a new GameInfo from a JSON string.
* @param {String} json JSON string used to create the new object
* @returns {GameInfo}
*/
static fromJSON(json) {
// Convert string
const temp = Object.assign(new GameInfo(), JSON.parse(json));
// JSON cannot transform a string to a date implicitly
temp.lastUpdate = new Date(temp.lastUpdate);
return temp;
}
}
module.exports = GameInfo;

View File

@ -0,0 +1,20 @@
"use strict";
/**
* Object obtained in response to an attempt to login to the portal.
*/
class LoginResult {
constructor(success, message) {
/**
* Result of the login operation
* @type Boolean
*/
this.success = success;
/**
* Login response message
* @type String
*/
this.message = message;
}
}
module.exports = LoginResult;

View File

@ -0,0 +1,103 @@
"use strict";
// Modules from file
const shared = require("../shared.js");
/**
* Convert prefixes and platform tags from string to ID and vice versa.
*/
class PrefixParser {
constructor() {
}
//#region Private methods
/**
* @private
* Gets the key associated with a given value from a dictionary.
* @param {Object} object Dictionary to search
* @param {Any} value Value associated with the key
* @returns {String|undefined} Key found or undefined
*/
_getKeyByValue(object, value) {
return Object.keys(object).find(key => object[key] === value);
}
/**
* @private
* Makes an array of strings uppercase.
* @param {String[]} a
* @returns {String[]}
*/
_toUpperCaseArray(a) {
/**
* Makes a string uppercase.
* @param {String} s
* @returns {String}
*/
function toUpper(s) {
return s.toUpperCase();
}
return a.map(toUpper);
}
/**
* @private
* Check if `dict` contains `value` as a value.
* @param {Object.<number, string>} dict
* @param {String} value
*/
_valueInDict(dict, value) {
const array = Object.values(dict);
const upperArr = this._toUpperCaseArray(array);
const element = value.toUpperCase();
return upperArr.includes(element);
}
//#endregion Private methods
/**
* @public
* Convert a list of prefixes to their respective IDs.
* @param {String[]} prefixes
*/
prefixesToIDs(prefixes) {
const ids = [];
for(const p of prefixes) {
// Check what dict contains the value
let dict = null;
if (this._valueInDict(shared.statuses, p)) dict = shared.statuses;
else if (this._valueInDict(shared.engines, p)) dict = shared.engines;
else if (this._valueInDict(shared.tags, p)) dict = shared.tags;
else if (this._valueInDict(shared.others, p)) dict = shared.others;
else continue;
// Extract the key from the dict
const key = this._getKeyByValue(dict, p);
if(key) ids.push(parseInt(key));
}
return ids;
}
/**
* @public
* It converts a list of IDs into their respective prefixes.
* @param {number[]} ids
*/
idsToPrefixes(ids) {
const prefixes = [];
for(const id of ids) {
// Check what dict contains the key
let dict = null;
if (Object.keys(shared.statuses).includes(id.toString())) dict = shared.statuses;
else if (Object.keys(shared.engines).includes(id.toString())) dict = shared.engines;
else if (Object.keys(shared.tags).includes(id.toString())) dict = shared.tags;
else if (Object.keys(shared.others).includes(id.toString())) dict = shared.others;
else continue;
// Check if the key exists in the dict
if (id in dict) prefixes.push(dict[id]);
}
return prefixes;
}
}
module.exports = PrefixParser;

View File

@ -0,0 +1,141 @@
"use strict";
// Core modules
const fs = require("fs");
const promisify = require("util").promisify;
// Public modules from npm
const md5 = require("md5");
// Promisifed functions
const areadfile = promisify(fs.readFile);
const awritefile = promisify(fs.writeFile);
const aunlinkfile = promisify(fs.unlink);
class Session {
constructor(path) {
/**
* Max number of days the session is valid.
*/
this.SESSION_TIME = 1;
/**
* Path of the session map file on disk.
*/
this._path = path;
/**
* Indicates if the session is mapped on disk.
*/
this._isMapped = fs.existsSync(this._path);
/**
* Date of creation of the session.
*/
this._created = new Date(Date.now());
/**
* MD5 hash of the username and the password.
*/
this._hash = null;
}
//#region Private Methods
/**
* @private
* Get the difference in days between two dates.
* @param {Date} a
* @param {Date} b
*/
_dateDiffInDays(a, b) {
const MS_PER_DAY = 1000 * 60 * 60 * 24;
// Discard the time and time-zone information.
const utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate());
const utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate());
return Math.floor((utc2 - utc1) / MS_PER_DAY);
}
/**
* @private
* Convert the object to a dictionary serializable in JSON.
*/
_toJSON() {
return {
created: this._created,
hash: this._hash,
};
}
//#endregion Private Methods
//#region Public Methods
create(username, password) {
// First, create the hash of the credentials
const value = `${username}%%%${password}`;
this._hash = md5(value);
// Update the creation date
this._created = new Date(Date.now());
}
/**
* @public
* Save the session to disk.
*/
async save() {
// Update the creation date
this._created = new Date(Date.now());
// Convert data
const json = this._toJSON();
const data = JSON.stringify(json);
// Write data
await awritefile(this._path, data);
}
/**
* @public
* Load the session from disk.
*/
async load() {
// Read data
const data = await areadfile(this._path);
const json = JSON.parse(data);
// Assign values
this._created = json.created;
this._hash = json.hash;
}
/**
* @public
* Delete the session from disk.
*/
async delete() {
await aunlinkfile(this._path);
}
/**
* @public
* Check if the session is valid.
*/
isValid(username, password) {
// Get the number of days from the file creation
const diff = this._dateDiffInDays(new Date(Date.now()), this._created);
// The session is valid if the number of days is minor than SESSION_TIME
let valid = diff < this.SESSION_TIME;
if(valid) {
// Check the hash
const value = `${username}%%%${password}`;
valid = md5(value) === this._hash;
}
return valid;
}
//#endregion Public Methods
}
module.exports = Session;

View File

@ -0,0 +1,26 @@
"use strict";
/**
* Class containing the data of the user currently connected to the F95Zone platform.
*/
class UserData {
constructor() {
/**
* User name.
* @type String
*/
this.username = "";
/**
* Path to the user's profile picture.
* @type String
*/
this.avatarSrc = null;
/**
* List of followed game thread URLs.
* @type String[]
*/
this.watchedGameThreads = [];
}
}
module.exports = UserData;

View File

@ -0,0 +1,21 @@
module.exports = Object.freeze({
GT_IMAGES: "img:not([title])[data-src^=\"https://attachments.f95zone.to\"][data-url=\"\"]",
GT_TAGS: "a.tagItem",
GT_TITLE: "h1.p-title-value",
GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]",
GT_LAST_CHANGELOG: "b:contains('Changelog') + br + div > div",
GT_JSONLD: "script[type=\"application/ld+json\"]",
WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
WT_NEXT_PAGE: "a.pageNav-jump--next",
WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]",
WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]",
GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type",
GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a",
GS_RESULT_BODY: "div.contentRow-main",
GS_MEMBERSHIP: "li > a:not(.username)",
GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]",
UD_USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText",
UD_AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]",
LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic",
LU_TAGS_SCRIPT: "script:contains('latestUpdates')",
});

View File

@ -0,0 +1,7 @@
module.exports = Object.freeze({
F95_BASE_URL: "https://f95zone.to",
F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
F95_LATEST_UPDATES: "https://f95zone.to/latest",
F95_LOGIN_URL: "https://f95zone.to/login/login",
F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
});

121
app/scripts/latest-fetch.js Normal file
View File

@ -0,0 +1,121 @@
"use strict";
// Modules from file
const { fetchGETResponse } = require("./network-helper.js");
const f95url = require("./constants/url.js");
/**
* @public
* Gets the URLs of the latest updated games that match the passed parameters.
* You *must* be logged.
* @param {Object} query
* Query used for the search
* @param {Number[]} [query.tags]
* List of tags to be included in the search. Max. 5 tags
* @param {Number[]} [query.prefixes]
* List of prefixes to be included in the search.
* @param {String} [query.sort]
* Sorting type between (default: `date`):
*`date`, `likes`, `views`, `name`, `rating`
* @param {Number} [query.date]
* Date limit in days, to be understood as "less than".
* Possible values:
* `365`, `180`, `90`, `30`, `14`, `7`, `3`, `1`.
* Use `1` to indicate "today" or set no value to indicate "anytime"
* @param {Number} limit
* Maximum number of items to get. Default: 30
* @returns {Promise<String[]>} URLs of the fetched games
*/
module.exports.fetchLatest = async function(query, limit = 30) {
// Local variables
const threadURL = new URL("threads/", f95url.F95_BASE_URL).href;
const resultURLs = [];
let fetchedResults = 0;
let page = 1;
let noMorePages = false;
do {
// Prepare the URL
const url = parseLatestURL(query, page);
// Fetch the response (application/json)
const response = await fetchGETResponse(url);
// Save the URLs
for(const result of response.data.msg.data) {
if(fetchedResults < limit) {
const gameURL = new URL(result.thread_id, threadURL).href;
resultURLs.push(gameURL);
fetchedResults += 1;
}
}
// Increment page and check for it's existence
page += 1;
if (page > response.data.msg.pagination.total) noMorePages = true;
}
while (fetchedResults < limit && !noMorePages);
return resultURLs;
};
/**
* @private
* Parse the URL with the passed parameters.
* @param {Object} query
* Query used for the search
* @param {Number[]} [query.tags]
* List of tags to be included in the search. Max. 5 tags
* @param {Number[]} [query.prefixes]
* List of prefixes to be included in the search.
* @param {String} [query.sort]
* Sorting type between (default: `date`):
* `date`, `likes`, `views`, `title`, `rating`
* @param {Number} [query.date]
* Date limit in days, to be understood as "less than".
* Possible values:
* `365`, `180`, `90`, `30`, `14`, `7`, `3`, `1`.
* Use `1` to indicate "today" or set no value to indicate "anytime"
* @param {Number} [page]
* Index of the page to be obtained. Default: 1.
*/
function parseLatestURL(query, page = 1) {
// Create the URL
const url = new URL("https://f95zone.to/new_latest.php");
url.searchParams.set("cmd", "list");
url.searchParams.set("cat", "games");
// Add the parameters
if (query.tags) {
if (query.tags.length > 5)
throw new Error(`Too many tags: ${query.tags.length} instead of 5`);
for(const tag of query.tags) {
url.searchParams.append("tags[]", tag);
}
}
if (query.prefixes) {
for (const p of query.prefixes) {
url.searchParams.append("prefixes[]", p);
}
}
if(query.sort) {
const validSort = ["date", "likes", "views", "title", "rating"];
if (!validSort.includes(query.sort))
throw new Error(`Invalid sort parameter: ${query.sort}`);
url.searchParams.set("sort", query.sort);
}
if (query.date) {
const validDate = [365, 180, 90, 30, 14, 7, 3, 1];
if (!validDate.includes(query.date))
throw new Error(`Invalid date parameter: ${query.date}`);
url.searchParams.set("date", query.date);
}
if (page) url.searchParams.set("page", page);
return url.toString();
}

View File

@ -0,0 +1,238 @@
"use strict";
// Public modules from npm
const axios = require("axios").default;
const cheerio = require("cheerio");
const axiosCookieJarSupport = require("axios-cookiejar-support").default;
const tough = require("tough-cookie");
// Modules from file
const shared = require("./shared.js");
const f95url = require("./constants/url.js");
const f95selector = require("./constants/css-selector.js");
const LoginResult = require("./classes/login-result.js");
// Global variables
const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " +
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15";
axiosCookieJarSupport(axios);
const commonConfig = {
headers: {
"User-Agent": userAgent,
"Connection": "keep-alive"
},
withCredentials: true,
jar: new tough.CookieJar() // Used to store the token in the PC
};
/**
* @protected
* Gets the HTML code of a page.
* @param {String} url URL to fetch
* @returns {Promise<String>} HTML code or `null` if an error arise
*/
module.exports.fetchHTML = async function (url) {
// Local variables
let returnValue = null;
// Fetch the response of the platform
const response = await exports.fetchGETResponse(url);
// Manage response
/* istambul ignore next */
if (!response) {
shared.logger.warn(`Unable to fetch HTML for ${url}`);
}
/* istambul ignore next */
else if (!response.headers["content-type"].includes("text/html")) {
// The response is not a HTML page
shared.logger.warn(`The ${url} returned a ${response.headers["content-type"]} response`);
}
returnValue = response.data;
return returnValue;
};
/**
* @protected
* It authenticates to the platform using the credentials
* and token obtained previously. Save cookies on your
* device after authentication.
* @param {Credentials} credentials Platform access credentials
* @param {Boolean} force Specifies whether the request should be forced, ignoring any saved cookies
* @returns {Promise<LoginResult>} Result of the operation
*/
module.exports.authenticate = async function (credentials, force) {
shared.logger.info(`Authenticating with user ${credentials.username}`);
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
// Secure the URL
const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL);
// Prepare the parameters to send to the platform to authenticate
const params = new URLSearchParams();
params.append("login", credentials.username);
params.append("url", "");
params.append("password", credentials.password);
params.append("password_confirm", "");
params.append("additional_security", "");
params.append("remember", "1");
params.append("_xfRedirect", "https://f95zone.to/");
params.append("website_code", "");
params.append("_xfToken", credentials.token);
try {
// Try to log-in
let config = Object.assign({}, commonConfig);
if (force) delete config.jar;
const response = await axios.post(secureURL, params, config);
// Parse the response HTML
const $ = cheerio.load(response.data);
// Get the error message (if any) and remove the new line chars
const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, "");
// Return the result of the authentication
const result = errorMessage.trim() === "";
const message = errorMessage.trim() === "" ? "Authentication successful" : errorMessage;
return new LoginResult(result, message);
} catch (e) {
shared.logger.error(`Error ${e.message} occurred while authenticating to ${secureURL}`);
return new LoginResult(false, `Error ${e.message} while authenticating`);
}
};
/**
* Obtain the token used to authenticate the user to the platform.
* @returns {Promise<String>} Token or `null` if an error arise
*/
module.exports.getF95Token = async function() {
// Fetch the response of the platform
const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL);
/* istambul ignore next */
if (!response) {
shared.logger.warn("Unable to get the token for the session");
return null;
}
// The response is a HTML page, we need to find the <input> with name "_xfToken"
const $ = cheerio.load(response.data);
const token = $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value");
return token;
};
//#region Utility methods
/**
* @protected
* Performs a GET request to a specific URL and returns the response.
* If the request generates an error (for example 400) `null` is returned.
* @param {String} url
*/
module.exports.fetchGETResponse = async function(url) {
// Secure the URL
const secureURL = exports.enforceHttpsUrl(url);
try {
// Fetch and return the response
return await axios.get(secureURL, commonConfig);
} catch (e) {
shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`);
return null;
}
};
/**
* @protected
* Enforces the scheme of the URL is https and returns the new URL.
* @param {String} url
* @returns {String} Secure URL or `null` if the argument is not a string
*/
module.exports.enforceHttpsUrl = function (url) {
return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
};
/**
* @protected
* Check if the url belongs to the domain of the F95 platform.
* @param {String} url URL to check
* @returns {Boolean} true if the url belongs to the domain, false otherwise
*/
module.exports.isF95URL = function (url) {
if (url.toString().startsWith(f95url.F95_BASE_URL)) return true;
else return false;
};
/**
* @protected
* Checks if the string passed by parameter has a
* properly formatted and valid path to a URL (HTTP/HTTPS).
* @param {String} url String to check for correctness
* @returns {Boolean} true if the string is a valid URL, false otherwise
*/
module.exports.isStringAValidURL = function (url) {
// Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc)
const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/;
const regex = new RegExp(expression);
if (url.match(regex)) return true;
else return false;
};
/**
* @protected
* Check if a particular URL is valid and reachable on the web.
* @param {String} url URL to check
* @param {Boolean} [checkRedirect]
* If true, the function will consider redirects a violation and return false.
* Default: false
* @returns {Promise<Boolean>} true if the URL exists, false otherwise
*/
module.exports.urlExists = async function (url, checkRedirect = false) {
// Local variables
let valid = false;
if (exports.isStringAValidURL(url)) {
valid = await _axiosUrlExists(url);
if (valid && checkRedirect) {
const redirectUrl = await exports.getUrlRedirect(url);
valid = redirectUrl === url;
}
}
return valid;
};
/**
* @protected
* Check if the URL has a redirect to another page.
* @param {String} url URL to check for redirect
* @returns {Promise<String>} Redirect URL or the passed URL
*/
module.exports.getUrlRedirect = async function (url) {
const response = await axios.head(url);
return response.config.url;
};
//#endregion Utility methods
//#region Private methods
/**
* @private
* Check with Axios if a URL exists.
* @param {String} url
*/
async function _axiosUrlExists(url) {
// Local variables
let valid = false;
try {
const response = await axios.head(url, {timeout: 3000});
valid = response && !/4\d\d/.test(response.status);
} catch (error) {
if (error.code === "ENOTFOUND") valid = false;
else if (error.code === "ETIMEDOUT") valid = false;
else throw error;
}
return valid;
}
//#endregion

View File

@ -0,0 +1,126 @@
"use strict";
// Core modules
const {readFileSync, writeFileSync, existsSync} = require("fs");
// Public modules from npm
const cheerio = require("cheerio");
// Modules from file
const shared = require("./shared.js");
const f95url = require("./constants/url.js");
const f95selector = require("./constants/css-selector.js");
const {fetchHTML} = require("./network-helper.js");
/**
* @protected
* Gets the basic data used for game data processing
* (such as graphics engines and progress statuses)
*/
module.exports.fetchPlatformData = async function () {
// Check if the data are cached
if (!_readCache(shared.cachePath)) {
// Load the HTML
const html = await fetchHTML(f95url.F95_LATEST_UPDATES);
// Parse data
const data = _parseLatestPlatformHTML(html);
// Assign data
_assignLatestPlatformData(data);
// Cache data
_saveCache(shared.cachePath);
}
};
//#region Private methods
/**
* @private
* Read the platform cache (if available)
* @param {String} path Path to cache
*/
function _readCache(path) {
// Local variables
let returnValue = false;
if (existsSync(path)) {
const data = readFileSync(path);
const json = JSON.parse(data);
shared.engines = json.engines;
shared.statuses = json.statuses;
shared.tags = json.tags;
shared.others = json.others;
returnValue = true;
}
return returnValue;
}
/**
* @private
* Save the current platform variables to disk.
* @param {String} path Path to cache
*/
function _saveCache(path) {
const saveDict = {
engines: shared.engines,
statuses: shared.statuses,
tags: shared.tags,
others: shared.others,
};
const json = JSON.stringify(saveDict);
writeFileSync(path, json);
}
/**
* @private
* Given the HTML code of the response from the F95Zone,
* parse it and return the result.
* @param {String} html
* @returns {Object.<string, object>} Parsed data
*/
function _parseLatestPlatformHTML(html) {
const $ = cheerio.load(html);
// Clean the JSON string
const unparsedText = $(f95selector.LU_TAGS_SCRIPT).html().trim();
const startIndex = unparsedText.indexOf("{");
const endIndex = unparsedText.lastIndexOf("}");
const parsedText = unparsedText.substring(startIndex, endIndex + 1);
return JSON.parse(parsedText);
}
/**
* @private
* Assign to the local variables the values from the F95Zone.
* @param {Object.<string, object>} data
*/
function _assignLatestPlatformData(data) {
// Local variables
const scrapedData = {};
// Extract and parse the data
const prefixes = data.prefixes.games.map(e => {
return {
element: e.name,
data: e.prefixes
};
});
// Parse and assign the values that are NOT tags
for (const p of prefixes) {
// Prepare the dict
const dict = {};
for (const e of p.data) dict[parseInt(e.id)] = e.name.replace("&#039;", "'");
// Save the property
scrapedData[p.element] = dict;
}
// Save the values
shared.engines = Object.assign({}, scrapedData["Engine"]);
shared.statuses = Object.assign({}, scrapedData["Status"]);
shared.others = Object.assign({}, scrapedData["Other"]);
shared.tags = data.tags;
}
//#endregion

417
app/scripts/scraper.js Normal file
View File

@ -0,0 +1,417 @@
"use strict";
// Public modules from npm
const cheerio = require("cheerio");
const {DateTime} = require("luxon");
// Modules from file
const { fetchHTML, getUrlRedirect } = require("./network-helper.js");
const shared = require("./shared.js");
const GameInfo = require("./classes/game-info.js");
const f95Selector = require("./constants/css-selector.js");
/**
* @protected
* Get information from the game's main page.
* @param {String} url URL of the game/mod to extract data from
* @return {Promise<GameInfo>} Complete information about the game you are
* looking for or `null` if is impossible to parse information
*/
module.exports.getGameInfo = async function (url) {
shared.logger.info("Obtaining game info");
// Fetch HTML and prepare Cheerio
const html = await fetchHTML(url);
if(!html) return null;
const $ = cheerio.load(html);
const body = $("body");
const mainPost = $(f95Selector.GS_POSTS).first();
// Extract data
const titleData = extractInfoFromTitle(body);
const tags = extractTags(body);
const prefixesData = parseGamePrefixes(body);
const src = extractPreviewSource(body);
const changelog = extractChangelog(mainPost);
const structuredData = extractStructuredData(body);
// Sometimes the JSON-LD are not set, especially in low-profile game
if(!structuredData) return null;
const parsedInfos = parseMainPostText(structuredData.description);
const overview = getOverview(structuredData.description, prefixesData.mod);
// Obtain the updated URL
const redirectUrl = await getUrlRedirect(url);
// Fill in the GameInfo element with the information obtained
const info = new GameInfo();
info.id = extractIDFromURL(url);
info.name = titleData.name;
info.author = titleData.author;
info.isMod = prefixesData.mod;
info.engine = prefixesData.engine;
info.status = prefixesData.status;
info.tags = tags;
info.url = redirectUrl;
info.language = parsedInfos.Language;
info.overview = overview;
info.supportedOS = parsedInfos.SupportedOS;
info.censored = parsedInfos.Censored;
info.lastUpdate = parsedInfos.LastUpdate;
info.previewSrc = src;
info.changelog = changelog;
info.version = titleData.version;
shared.logger.info(`Founded data for ${info.name}`);
return info;
};
//#region Private methods
/**
* @private
* Parse the game prefixes obtaining the engine used,
* the advancement status and if the game is actually a game or a mod.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, object>} Dictionary of values with keys `engine`, `status`, `mod`
*/
function parseGamePrefixes(body) {
shared.logger.trace("Parsing prefixes...");
// Local variables
let mod = false,
engine = null,
status = null;
// Obtain the title prefixes
const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES);
const $ = cheerio.load([].concat(body));
prefixeElements.each(function parseGamePrefix(idx, el) {
// Obtain the prefix text
let prefix = $(el).text().trim();
// Remove the square brackets
prefix = prefix.replace("[", "").replace("]", "");
// Check what the prefix indicates
if (isEngine(prefix)) engine = prefix;
else if (isStatus(prefix)) status = prefix;
else if (isMod(prefix)) mod = true;
});
// If the status is not set, then the game in in development (Ongoing)
status = !status ? "Ongoing" : status; // status ?? "Ongoing";
return {
engine,
status,
mod
};
}
/**
* @private
* Extracts all the possible informations from the title.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, string>} Dictionary of values with keys `name`, `author`, `version`
*/
function extractInfoFromTitle(body) {
shared.logger.trace("Extracting information from title...");
const title = body
.find(f95Selector.GT_TITLE)
.text()
.trim();
// From the title we can extract: Name, author and version
// [PREFIXES] TITLE [VERSION] [AUTHOR]
const matches = title.match(/\[(.*?)\]/g);
// Get the title name
let name = title;
matches.forEach(function replaceElementsInTitle(e) {
name = name.replace(e, "");
});
name = name.trim();
// The version is the penultimate element.
// If the matches are less than 2, than the title
// is malformes and only the author is fetched
// (usually the author is always present)
let version = null;
if (matches.length >= 2) {
// The regex [[\]]+ remove the square brackets
version = matches[matches.length - 2].replace(/[[\]]+/g, "").trim();
// Remove the trailing "v"
if (version[0] === "v") version = version.replace("v", "");
}
// Last element (the regex [[\]]+ remove the square brackets)
const author = matches[matches.length - 1].replace(/[[\]]+/g, "").trim();
return {
name,
version,
author,
};
}
/**
* @private
* Gets the tags used to classify the game.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String[]} List of tags
*/
function extractTags(body) {
shared.logger.trace("Extracting tags...");
// Get the game tags
const tagResults = body.find(f95Selector.GT_TAGS);
const $ = cheerio.load([].concat(body));
return tagResults.map(function parseGameTags(idx, el) {
return $(el).text().trim();
}).get();
}
/**
* @private
* Gets the URL of the image used as a preview.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String} URL of the image
*/
function extractPreviewSource(body) {
shared.logger.trace("Extracting image preview source...");
const image = body.find(f95Selector.GT_IMAGES);
// The "src" attribute is rendered only in a second moment,
// we need the "static" src value saved in the attribute "data-src"
const source = image ? image.attr("data-src") : null;
return source;
}
/**
* @private
* Gets the changelog of the latest version.
* @param {cheerio.Cheerio} mainPost main post selector
* @returns {String} Changelog of the last version or `null` if no changelog is fetched
*/
function extractChangelog(mainPost) {
shared.logger.trace("Extracting last changelog...");
// Obtain the changelog for ALL the versions
let changelog = mainPost.find(f95Selector.GT_LAST_CHANGELOG).text().trim();
// Parse the latest changelog
const endChangelog = changelog.indexOf("\nv"); // \n followed by version (v)
if (endChangelog !== -1) changelog = changelog.substring(0, endChangelog + 1);
// Clean changelog
changelog = changelog.replace("Spoiler", "");
changelog = changelog.replace(/\n+/g, "\n"); // Multiple /n
changelog = changelog.trim();
// Delete the version at the start of the changelog
const firstNewLine = changelog.indexOf("\n");
const supposedVersion = changelog.substring(0, firstNewLine);
if (supposedVersion[0] === "v") changelog = changelog.substring(firstNewLine).trim();
// Return changelog
return changelog ? changelog : null;
}
/**
* @private
* Process the main post text to get all the useful
* information in the format *DESCRIPTOR : VALUE*.
* Gets "standard" values such as: `Language`, `SupportedOS`, `Censored`, and `LastUpdate`.
* All non-canonical values are instead grouped together as a dictionary with the key `Various`.
* @param {String} text Structured text of the post
* @returns {Object.<string, object>} Dictionary of information
*/
function parseMainPostText(text) {
shared.logger.trace("Parsing main post raw text...");
const data = {};
// The information searched in the game post are one per line
const splittedText = text.split("\n");
for (const line of splittedText) {
if (!line.includes(":")) continue;
// Create pair key/value
const splitted = line.split(":");
const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch
const value = splitted[1].trim();
// Add pair to the dict if valid
if (value !== "") data[key] = value;
}
// Parse the standard pairs
const parsedDict = {};
// Check if the game is censored
if (data.CENSORED) {
const censored = data.CENSORED.toUpperCase() === "NO" ? false : true;
parsedDict["Censored"] = censored;
delete data.CENSORED;
}
// Last update of the main post
if (data.UPDATED && DateTime.fromISO(data.UPDATED).isValid) {
parsedDict["LastUpdate"] = new Date(data.UPDATED);
delete data.UPDATED;
}
else if (data.THREAD_UPDATED && DateTime.fromISO(data.THREAD_UPDATED).isValid) {
parsedDict["LastUpdate"] = new Date(data.THREAD_UPDATED);
delete data.THREAD_UPDATED;
}
else parsedDict["LastUpdate"] = null;
// Parse the supported OS
if (data.OS) {
const listOS = [];
// Usually the string is something like "Windows, Linux, Mac"
const splitted = data.OS.split(",");
splitted.forEach(function (os) {
listOS.push(os.trim());
});
parsedDict["SupportedOS"] = listOS;
delete data.OS;
}
// Rename the key for the language
if (data.LANGUAGE) {
parsedDict["Language"] = data.LANGUAGE;
delete data.LANGUAGE;
}
// What remains is added to a sub dictionary
parsedDict["Various"] = data;
return parsedDict;
}
/**
* @private
* Parse a JSON-LD element.
* @param {cheerio.Element} element
*/
function parseScriptTag(element) {
// Get the element HTML
const html = cheerio.load([].concat(element)).html().trim();
// Obtain the JSON-LD
const data = html
.replace("<script type=\"application/ld+json\">", "")
.replace("</script>", "");
// Convert the string to an object
const json = JSON.parse(data);
// Return only the data of the game
if (json["@type"] === "Book") return json;
}
/**
* @private
* Extracts and processes the JSON-LD values found at the bottom of the page.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
*/
function extractStructuredData(body) {
shared.logger.trace("Extracting JSON-LD data...");
// Fetch the JSON-LD data
const structuredDataElements = body.find(f95Selector.GT_JSONLD);
// Parse the data
const json = structuredDataElements.map((idx, el) => parseScriptTag(el)).get();
return json.lenght !== 0 ? json[0] : null;
}
/**
* @private
* Get the game description from its web page.
* Different processing depending on whether the game is a mod or not.
* @param {String} text Structured text extracted from the game's web page
* @param {Boolean} mod Specify if it is a game or a mod
* @returns {String} Game description
*/
function getOverview(text, mod) {
shared.logger.trace("Extracting game overview...");
// Get overview (different parsing for game and mod)
const overviewEndIndex = mod ? text.indexOf("Updated") : text.indexOf("Thread Updated");
return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim();
}
/**
* @private
* Check if the prefix is a game's engine.
* @param {String} prefix Prefix to check
* @return {Boolean}
*/
function isEngine(prefix) {
const engines = toUpperCaseArray(Object.values(shared.engines));
return engines.includes(prefix.toUpperCase());
}
/**
* @private
* Check if the prefix is a game's status.
* @param {String} prefix Prefix to check
* @return {Boolean}
*/
function isStatus(prefix) {
const statuses = toUpperCaseArray(Object.values(shared.statuses));
return statuses.includes(prefix.toUpperCase());
}
/**
* @private
* Check if the prefix indicates a mod.
* @param {String} prefix Prefix to check
* @return {Boolean}
*/
function isMod(prefix) {
const modPrefixes = ["MOD", "CHEAT MOD"];
return modPrefixes.includes(prefix.toUpperCase());
}
/**
* @private
* Extracts the game's unique ID from the game's URL.
* @param {String} url Game's URL
* @return {Number} Game's ID
*/
function extractIDFromURL(url) {
// URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/
// or https://f95zone.to/threads/ID/
const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/);
if(!match) return -1;
// Parse and return number
return parseInt(match[0], 10);
}
/**
* @private
* Makes an array of strings uppercase.
* @param {String[]} a
*/
function toUpperCaseArray(a) {
/**
* Makes a string uppercase.
* @param {String} s
* @returns {String}
*/
function toUpper(s) {
return s.toUpperCase();
}
return a.map(toUpper);
}
//#endregion Private methods

95
app/scripts/searcher.js Normal file
View File

@ -0,0 +1,95 @@
"use strict";
// Public modules from npm
const cheerio = require("cheerio");
// Modules from file
const { fetchHTML } = require("./network-helper.js");
const shared = require("./shared.js");
const f95Selector = require("./constants/css-selector.js");
const { F95_BASE_URL } = require("./constants/url.js");
//#region Public methods
/**
* @protected
* Search for a game on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Game name
* @returns {Promise<String[]>} URLs of results
*/
module.exports.searchGame = async function (name) {
shared.logger.info(`Searching games with name ${name}`);
// Replace the whitespaces with +
const searchName = encodeURIComponent(name.toUpperCase());
// Prepare the URL (only title, search in the "Games" section, order by relevance)
const url = `https://f95zone.to/search/83456043/?q="${searchName}"&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`;
// Fetch and parse the result URLs
return await fetchResultURLs(url);
};
/**
* @protected
* Search for a mod on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Mod name
* @returns {Promise<String[]>} URLs of results
*/
module.exports.searchMod = async function (name) {
shared.logger.info(`Searching mods with name ${name}`);
// Replace the whitespaces with +
const searchName = encodeURIComponent(name.toUpperCase());
// Prepare the URL (only title, search in the "Mods" section, order by relevance)
const url = `https://f95zone.to/search/83459796/?q="${searchName}"&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`;
// Fetch and parse the result URLs
return await fetchResultURLs(url);
};
//#endregion Public methods
//#region Private methods
/**
* @private
* Gets the URLs of the threads resulting from the F95Zone search.
* @param {String} url Search URL
* @return {Promise<String[]>} List of URLs
*/
async function fetchResultURLs(url) {
shared.logger.trace(`Fetching ${url}...`);
// Fetch HTML and prepare Cheerio
const html = await fetchHTML(url);
const $ = cheerio.load(html);
// Here we get all the DIV that are the body of the various query results
const results = $("body").find(f95Selector.GS_RESULT_BODY);
// Than we extract the URLs
const urls = results.map((idx, el) => {
const elementSelector = $(el);
return extractLinkFromResult(elementSelector);
}).get();
return urls;
}
/**
* @private
* Look for the URL to the thread referenced by the item.
* @param {cheerio.Cheerio} selector Element to search
* @returns {String} URL to thread
*/
function extractLinkFromResult(selector) {
shared.logger.trace("Extracting thread link from result...");
const partialLink = selector
.find(f95Selector.GS_RESULT_THREAD_TITLE)
.attr("href")
.trim();
// Compose and return the URL
return new URL(partialLink, F95_BASE_URL).toString();
}
//#endregion Private methods

135
app/scripts/shared.js Normal file
View File

@ -0,0 +1,135 @@
/* istanbul ignore file */
"use strict";
// Core modules
const {tmpdir} = require("os");
const {join} = require("path");
// Public modules from npm
const log4js = require("log4js");
// Modules from file
const Session = require("./classes/session");
/**
* Class containing variables shared between modules.
*/
class Shared {
//#region Properties
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
* @type Boolean
*/
static #_isLogged = false;
/**
* List of possible game engines used for development.
* @type Object<number,string>
*/
static #_engines = {};
/**
* List of possible development statuses that a game can assume.
* @type Object<number,string>
*/
static #_statuses = {};
/**
* List of other prefixes that a game can assume.
* @type Object<number,string>
*/
static #_others = {};
/**
* List of possible tags that a game can assume.
* @type Object<number,string>
*/
static #_tags = {};
/**
* Logger object used to write to both file and console.
* @type log4js.Logger
*/
static #_logger = log4js.getLogger();
/**
* Session on the F95Zone platform.
*/
static #_session = new Session(join(tmpdir(), "f95session.json"));
//#endregion Properties
//#region Getters
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
* @returns {Boolean}
*/
static get isLogged() {
return this.#_isLogged;
}
/**
* List of possible game engines used for development.
* @returns @returns {Object<number, string>}
*/
static get engines() {
return this.#_engines;
}
/**
* List of possible development states that a game can assume.
* @returns {Object<number, string>}
*/
static get statuses() {
return this.#_statuses;
}
/**
* List of other prefixes that a game can assume.
* @returns {Object<number, string>}
*/
static get others() {
return this.#_others;
}
/**
* List of possible tags that a game can assume.
* @returns {Object<number, string>}
*/
static get tags() {
return this.#_tags;
}
/**
* Logger object used to write to both file and console.
* @returns {log4js.Logger}
*/
static get logger() {
return this.#_logger;
}
/**
* Path to the cache used by this module wich contains engines, statuses, tags...
*/
static get cachePath() {
return join(tmpdir(), "f95cache.json");
}
/**
* Session on the F95Zone platform.
*/
static get session() {
return this.#_session;
}
//#endregion Getters
//#region Setters
static set engines(val) {
this.#_engines = val;
}
static set statuses(val) {
this.#_statuses = val;
}
static set tags(val) {
this.#_tags = val;
}
static set others(val) {
this.#_others = val;
}
static set isLogged(val) {
this.#_isLogged = val;
}
//#endregion Setters
}
module.exports = Shared;

131
app/scripts/user-scraper.js Normal file
View File

@ -0,0 +1,131 @@
"use strict";
// Public modules from npm
const cheerio = require("cheerio");
// Modules from file
const networkHelper = require("./network-helper.js");
const f95Selector = require("./constants/css-selector.js");
const f95url = require("./constants/url.js");
const UserData = require("./classes/user-data.js");
/**
* @protected
* Gets user data, such as username, url of watched threads, and profile picture url.
* @return {Promise<UserData>} User data
*/
module.exports.getUserData = async function() {
// Fetch data
const data = await fetchUsernameAndAvatar();
const urls = await fetchWatchedGameThreadURLs();
// Create object
const ud = new UserData();
ud.username = data.username;
ud.avatarSrc = data.source;
ud.watchedGameThreads = urls;
return ud;
};
//#region Private methods
/**
* @private
* It connects to the page and extracts the name
* of the currently logged in user and the URL
* of their profile picture.
* @return {Promise<Object.<string, string>>}
*/
async function fetchUsernameAndAvatar() {
// Fetch page
const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL);
// Load HTML response
const $ = cheerio.load(html);
const body = $("body");
// Fetch username
const username = body.find(f95Selector.UD_USERNAME_ELEMENT).first().text().trim();
// Fetch user avatar image source
const source = body.find(f95Selector.UD_AVATAR_PIC).first().attr("src");
return {
username,
source
};
}
/**
* @private
* Gets the list of URLs of game threads watched by the user.
* @returns {Promise<String[]>} List of URLs
*/
async function fetchWatchedGameThreadURLs() {
// Local variables
const watchedGameThreadURLs = [];
// Get the first page with the "unread" flag disabled
// and searching only the games forum
const firstPageURL = new URL(f95url.F95_WATCHED_THREADS);
firstPageURL.searchParams.append("unread", "0");
firstPageURL.searchParams.append("nodes[0]", "2"); // This is the forum filter
// Set the variable containing the current scraped page
let currentURL = firstPageURL.href;
do {
// Fetch page
const html = await networkHelper.fetchHTML(currentURL);
// Load HTML response
const $ = cheerio.load(html);
const body = $("body");
// Find the URLs
const urls = fetchPageURLs(body);
watchedGameThreadURLs.push(...urls);
// Find the next page (if any)
currentURL = fetchNextPageURL(body);
}
while (currentURL);
return watchedGameThreadURLs;
}
/**
* @private
* Gets the URLs of the watched threads on the page.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String[]}
*/
function fetchPageURLs(body) {
const elements = body.find(f95Selector.WT_URLS);
return elements.map(function extractURLs(idx, e) {
// Obtain the link (replace "unread" only for the unread threads)
const partialLink = e.attribs.href.replace("unread", "");
// Compose and return the URL
return new URL(partialLink, f95url.F95_BASE_URL).toString();
}).get();
}
/**
* @private
* Gets the URL of the next page containing the watched threads
* or `null` if that page does not exist.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {String}
*/
function fetchNextPageURL(body) {
const element = body.find(f95Selector.WT_NEXT_PAGE).first();
// No element found
if(element.length === 0) return null;
// Compose and return the URL
return new URL(element.attr("href"), f95url.F95_BASE_URL).toString();
}
//#endregion Private methods

930
coverage.lcov Normal file
View File

@ -0,0 +1,930 @@
TN:
SF:app\index.js
FN:39,isLogged
FN:57,(anonymous_1)
FN:86,(anonymous_2)
FN:115,(anonymous_3)
FN:144,(anonymous_4)
FN:166,(anonymous_5)
FN:196,(anonymous_6)
FN:209,(anonymous_7)
FN:225,(anonymous_8)
FNF:9
FNH:8
FNDA:1,isLogged
FNDA:1,(anonymous_1)
FNDA:3,(anonymous_2)
FNDA:1,(anonymous_3)
FNDA:4,(anonymous_4)
FNDA:1,(anonymous_5)
FNDA:1,(anonymous_6)
FNDA:0,(anonymous_7)
FNDA:3,(anonymous_8)
DA:4,1
DA:5,1
DA:6,1
DA:7,1
DA:8,1
DA:9,1
DA:12,1
DA:13,1
DA:14,1
DA:15,1
DA:16,1
DA:19,1
DA:20,1
DA:21,1
DA:22,1
DA:31,1
DA:32,1
DA:39,1
DA:40,1
DA:45,1
DA:57,1
DA:64,1
DA:65,1
DA:66,1
DA:68,1
DA:69,1
DA:70,1
DA:73,1
DA:74,0
DA:76,1
DA:86,1
DA:95,3
DA:96,3
DA:99,2
DA:100,2
DA:103,2
DA:115,1
DA:123,1
DA:128,1
DA:129,1
DA:131,1
DA:132,1
DA:134,1
DA:144,1
DA:152,4
DA:153,4
DA:154,4
DA:157,4
DA:166,1
DA:173,1
DA:196,1
DA:198,1
DA:201,1
DA:204,1
DA:205,1
DA:208,0
DA:209,0
DA:210,0
DA:212,0
DA:216,1
DA:222,1
DA:225,3
DA:226,1
LF:63
LH:58
BRDA:73,0,0,1
BRDA:73,0,1,0
BRDA:96,1,0,1
BRDA:96,1,1,2
BRDA:123,2,0,0
BRDA:123,2,1,1
BRDA:132,3,0,1
BRDA:132,3,1,0
BRDA:153,4,0,0
BRDA:153,4,1,4
BRDA:154,5,0,0
BRDA:154,5,1,4
BRDA:198,6,0,0
BRDA:198,6,1,1
BRDA:205,7,0,0
BRDA:205,7,1,1
BRDA:217,8,0,1
BRDA:217,8,1,0
BRDA:218,9,0,1
BRDA:218,9,1,0
BRDA:219,10,0,1
BRDA:219,10,1,0
BRF:22
BRH:12
end_of_record
TN:
SF:app\scripts\latest-fetch.js
FN:29,(anonymous_0)
FN:82,parseLatestURL
FNF:2
FNH:2
FNDA:1,(anonymous_0)
FNDA:1,parseLatestURL
DA:4,1
DA:5,1
DA:29,1
DA:31,1
DA:32,1
DA:33,1
DA:34,1
DA:35,1
DA:37,1
DA:39,1
DA:42,1
DA:45,1
DA:46,30
DA:47,3
DA:48,3
DA:49,3
DA:54,1
DA:55,1
DA:59,1
DA:84,1
DA:85,1
DA:86,1
DA:89,1
DA:90,1
DA:91,0
DA:93,1
DA:94,2
DA:98,1
DA:99,1
DA:100,2
DA:104,1
DA:105,1
DA:106,1
DA:107,0
DA:108,1
DA:111,1
DA:112,0
DA:113,0
DA:114,0
DA:115,0
DA:118,1
DA:120,1
LF:42
LH:36
BRDA:29,0,0,0
BRDA:57,1,0,1
BRDA:57,1,1,0
BRDA:46,2,0,3
BRDA:46,2,1,27
BRDA:55,3,0,0
BRDA:55,3,1,1
BRDA:82,4,0,0
BRDA:89,5,0,1
BRDA:89,5,1,0
BRDA:90,6,0,0
BRDA:90,6,1,1
BRDA:98,7,0,1
BRDA:98,7,1,0
BRDA:104,8,0,1
BRDA:104,8,1,0
BRDA:106,9,0,0
BRDA:106,9,1,1
BRDA:111,10,0,0
BRDA:111,10,1,1
BRDA:113,11,0,0
BRDA:113,11,1,0
BRDA:118,12,0,1
BRDA:118,12,1,0
BRF:24
BRH:11
end_of_record
TN:
SF:app\scripts\network-helper.js
FN:36,(anonymous_0)
FN:67,(anonymous_1)
FN:115,(anonymous_2)
FN:137,(anonymous_3)
FN:156,(anonymous_4)
FN:166,(anonymous_5)
FN:178,(anonymous_6)
FN:195,(anonymous_7)
FN:217,(anonymous_8)
FN:229,_axiosUrlExists
FNF:10
FNH:10
FNDA:28,(anonymous_0)
FNDA:7,(anonymous_1)
FNDA:8,(anonymous_2)
FNDA:39,(anonymous_3)
FNDA:49,(anonymous_4)
FNDA:6,(anonymous_5)
FNDA:61,(anonymous_6)
FNDA:12,(anonymous_7)
FNDA:16,(anonymous_8)
FNDA:11,_axiosUrlExists
DA:4,1
DA:5,1
DA:6,1
DA:7,1
DA:10,1
DA:11,1
DA:12,1
DA:13,1
DA:14,1
DA:17,1
DA:19,1
DA:21,1
DA:36,1
DA:38,28
DA:41,28
DA:45,28
DA:46,0
DA:49,28
DA:51,0
DA:54,28
DA:55,28
DA:67,1
DA:68,7
DA:69,7
DA:72,7
DA:75,7
DA:76,7
DA:77,7
DA:78,7
DA:79,7
DA:80,7
DA:81,7
DA:82,7
DA:83,7
DA:84,7
DA:86,7
DA:88,7
DA:89,7
DA:90,7
DA:93,6
DA:96,6
DA:99,6
DA:101,6
DA:102,6
DA:104,0
DA:106,1
DA:107,1
DA:115,1
DA:117,8
DA:119,8
DA:120,0
DA:121,0
DA:125,8
DA:126,8
DA:127,8
DA:137,1
DA:139,39
DA:141,39
DA:143,39
DA:145,1
DA:146,1
DA:156,1
DA:157,49
DA:166,1
DA:167,6
DA:168,1
DA:178,1
DA:180,61
DA:181,61
DA:182,61
DA:183,2
DA:195,1
DA:197,12
DA:199,12
DA:200,11
DA:202,11
DA:203,4
DA:204,4
DA:208,12
DA:217,1
DA:218,16
DA:219,16
DA:231,11
DA:232,11
DA:233,11
DA:234,10
DA:236,1
DA:237,0
DA:239,11
LF:89
LH:83
BRDA:45,0,0,0
BRDA:45,0,1,28
BRDA:49,1,0,0
BRDA:49,1,1,28
BRDA:69,2,0,0
BRDA:69,2,1,7
BRDA:89,3,0,1
BRDA:89,3,1,6
BRDA:99,4,0,6
BRDA:99,4,1,0
BRDA:119,5,0,0
BRDA:119,5,1,8
BRDA:157,6,0,48
BRDA:157,6,1,1
BRDA:167,7,0,5
BRDA:167,7,1,1
BRDA:182,8,0,59
BRDA:182,8,1,2
BRDA:195,9,0,8
BRDA:199,10,0,11
BRDA:199,10,1,1
BRDA:202,11,0,4
BRDA:202,11,1,7
BRDA:202,12,0,11
BRDA:202,12,1,10
BRDA:234,13,0,10
BRDA:234,13,1,10
BRDA:236,14,0,1
BRDA:236,14,1,0
BRF:29
BRH:23
end_of_record
TN:
SF:app\scripts\platform-data.js
FN:20,(anonymous_0)
FN:43,_readCache
FN:64,_saveCache
FN:82,_parseLatestPlatformHTML
FN:98,_assignLatestPlatformData
FN:107,(anonymous_5)
FNF:6
FNH:2
FNDA:6,(anonymous_0)
FNDA:6,_readCache
FNDA:0,_saveCache
FNDA:0,_parseLatestPlatformHTML
FNDA:0,_assignLatestPlatformData
FNDA:0,(anonymous_5)
DA:4,1
DA:7,1
DA:10,1
DA:11,1
DA:12,1
DA:13,1
DA:20,1
DA:22,6
DA:24,0
DA:27,0
DA:30,0
DA:33,0
DA:45,6
DA:47,6
DA:48,6
DA:49,6
DA:50,6
DA:51,6
DA:52,6
DA:53,6
DA:54,6
DA:56,6
DA:65,0
DA:71,0
DA:72,0
DA:83,0
DA:86,0
DA:87,0
DA:88,0
DA:89,0
DA:90,0
DA:100,0
DA:107,0
DA:108,0
DA:115,0
DA:117,0
DA:118,0
DA:121,0
DA:125,0
LF:39
LH:18
BRDA:22,0,0,0
BRDA:22,0,1,6
BRDA:47,1,0,6
BRDA:47,1,1,0
BRF:4
BRH:2
end_of_record
TN:
SF:app\scripts\scraper.js
FN:20,(anonymous_0)
FN:78,parseGamePrefixes
FN:89,parseGamePrefix
FN:118,extractInfoFromTitle
FN:131,replaceElementsInTitle
FN:165,extractTags
FN:170,parseGameTags
FN:181,extractPreviewSource
FN:197,extractChangelog
FN:230,parseMainPostText
FN:276,(anonymous_10)
FN:301,parseScriptTag
FN:323,extractStructuredData
FN:330,(anonymous_13)
FN:342,getOverview
FN:356,isEngine
FN:367,isStatus
FN:378,isMod
FN:389,extractIDFromURL
FN:404,toUpperCaseArray
FN:410,toUpper
FNF:21
FNH:21
FNDA:10,(anonymous_0)
FNDA:10,parseGamePrefixes
FNDA:19,parseGamePrefix
FNDA:10,extractInfoFromTitle
FNDA:39,replaceElementsInTitle
FNDA:10,extractTags
FNDA:212,parseGameTags
FNDA:10,extractPreviewSource
FNDA:10,extractChangelog
FNDA:10,parseMainPostText
FNDA:21,(anonymous_10)
FNDA:20,parseScriptTag
FNDA:10,extractStructuredData
FNDA:20,(anonymous_13)
FNDA:10,getOverview
FNDA:19,isEngine
FNDA:9,isStatus
FNDA:4,isMod
FNDA:10,extractIDFromURL
FNDA:28,toUpperCaseArray
FNDA:293,toUpper
DA:4,1
DA:5,1
DA:8,1
DA:9,1
DA:10,1
DA:11,1
DA:20,1
DA:21,10
DA:24,10
DA:25,10
DA:26,10
DA:27,10
DA:28,10
DA:31,10
DA:32,10
DA:33,10
DA:34,10
DA:35,10
DA:36,10
DA:39,10
DA:41,10
DA:42,10
DA:45,10
DA:48,10
DA:49,10
DA:50,10
DA:51,10
DA:52,10
DA:53,10
DA:54,10
DA:55,10
DA:56,10
DA:57,10
DA:58,10
DA:59,10
DA:60,10
DA:61,10
DA:62,10
DA:63,10
DA:64,10
DA:66,10
DA:67,10
DA:79,10
DA:82,10
DA:83,10
DA:84,10
DA:87,10
DA:89,10
DA:91,19
DA:94,19
DA:97,19
DA:98,9
DA:99,4
DA:103,10
DA:105,10
DA:119,10
DA:120,10
DA:127,10
DA:130,10
DA:131,10
DA:132,39
DA:134,10
DA:140,10
DA:141,10
DA:143,10
DA:146,10
DA:150,10
DA:152,10
DA:166,10
DA:169,10
DA:170,10
DA:171,212
DA:182,10
DA:183,10
DA:187,10
DA:188,10
DA:198,10
DA:201,10
DA:204,10
DA:205,10
DA:208,10
DA:209,10
DA:210,10
DA:213,10
DA:214,10
DA:215,10
DA:218,10
DA:231,10
DA:233,10
DA:236,10
DA:237,10
DA:238,1358
DA:241,248
DA:242,248
DA:243,248
DA:246,248
DA:250,10
DA:253,10
DA:254,8
DA:255,8
DA:256,8
DA:260,10
DA:261,0
DA:262,0
DA:264,10
DA:265,8
DA:266,8
DA:268,2
DA:271,10
DA:272,8
DA:275,8
DA:276,8
DA:277,21
DA:280,8
DA:281,8
DA:285,10
DA:286,10
DA:287,10
DA:291,10
DA:293,10
DA:303,20
DA:306,20
DA:311,20
DA:314,20
DA:324,10
DA:327,10
DA:330,20
DA:331,10
DA:343,10
DA:346,10
DA:347,10
DA:357,19
DA:358,19
DA:368,9
DA:369,9
DA:379,4
DA:380,4
DA:392,10
DA:393,10
DA:396,10
DA:411,293
DA:413,28
LF:142
LH:140
BRDA:25,0,0,0
BRDA:25,0,1,10
BRDA:39,1,0,0
BRDA:39,1,1,10
BRDA:97,2,0,10
BRDA:97,2,1,9
BRDA:98,3,0,5
BRDA:98,3,1,4
BRDA:99,4,0,1
BRDA:99,4,1,3
BRDA:103,5,0,5
BRDA:103,5,1,5
BRDA:141,6,0,10
BRDA:141,6,1,0
BRDA:146,7,0,10
BRDA:146,7,1,0
BRDA:187,8,0,10
BRDA:187,8,1,0
BRDA:205,9,0,7
BRDA:205,9,1,3
BRDA:215,10,0,7
BRDA:215,10,1,3
BRDA:218,11,0,8
BRDA:218,11,1,2
BRDA:238,12,0,1110
BRDA:238,12,1,248
BRDA:246,13,0,120
BRDA:246,13,1,128
BRDA:253,14,0,8
BRDA:253,14,1,2
BRDA:254,15,0,8
BRDA:254,15,1,0
BRDA:260,16,0,0
BRDA:260,16,1,10
BRDA:260,17,0,10
BRDA:260,17,1,0
BRDA:264,18,0,8
BRDA:264,18,1,2
BRDA:264,19,0,10
BRDA:264,19,1,8
BRDA:271,20,0,8
BRDA:271,20,1,2
BRDA:285,21,0,10
BRDA:285,21,1,0
BRDA:314,22,0,10
BRDA:314,22,1,10
BRDA:331,23,0,10
BRDA:331,23,1,0
BRDA:346,24,0,1
BRDA:346,24,1,9
BRDA:393,25,0,0
BRDA:393,25,1,10
BRF:52
BRH:41
end_of_record
TN:
SF:app\scripts\searcher.js
FN:19,(anonymous_0)
FN:38,(anonymous_1)
FN:59,fetchResultURLs
FN:70,(anonymous_3)
FN:84,extractLinkFromResult
FNF:5
FNH:5
FNDA:2,(anonymous_0)
FNDA:1,(anonymous_1)
FNDA:3,fetchResultURLs
FNDA:3,(anonymous_3)
FNDA:3,extractLinkFromResult
DA:4,1
DA:7,1
DA:8,1
DA:9,1
DA:10,1
DA:19,1
DA:20,2
DA:23,2
DA:26,2
DA:29,2
DA:38,1
DA:39,1
DA:42,1
DA:45,1
DA:48,1
DA:60,3
DA:63,3
DA:64,3
DA:67,3
DA:70,3
DA:71,3
DA:72,3
DA:75,3
DA:85,3
DA:87,3
DA:93,3
LF:26
LH:26
BRF:0
BRH:0
end_of_record
TN:
SF:app\scripts\user-scraper.js
FN:17,(anonymous_0)
FN:39,fetchUsernameAndAvatar
FN:64,fetchWatchedGameThreadURLs
FN:103,fetchPageURLs
FN:106,extractURLs
FN:122,fetchNextPageURL
FNF:6
FNH:6
FNDA:2,(anonymous_0)
FNDA:2,fetchUsernameAndAvatar
FNDA:2,fetchWatchedGameThreadURLs
FNDA:12,fetchPageURLs
FNDA:224,extractURLs
FNDA:12,fetchNextPageURL
DA:4,1
DA:7,1
DA:8,1
DA:9,1
DA:10,1
DA:17,1
DA:19,2
DA:20,2
DA:23,2
DA:24,2
DA:25,2
DA:26,2
DA:28,2
DA:41,2
DA:44,2
DA:45,2
DA:48,2
DA:51,2
DA:53,2
DA:66,2
DA:70,2
DA:71,2
DA:72,2
DA:75,2
DA:77,2
DA:79,12
DA:82,12
DA:83,12
DA:86,12
DA:87,12
DA:90,12
DA:94,2
DA:104,12
DA:106,12
DA:108,224
DA:111,224
DA:123,12
DA:126,12
DA:129,10
LF:39
LH:39
BRDA:126,0,0,2
BRDA:126,0,1,10
BRF:2
BRH:2
end_of_record
TN:
SF:app\scripts\classes\credentials.js
FN:7,(anonymous_0)
FN:17,(anonymous_1)
FNF:2
FNH:2
FNDA:8,(anonymous_0)
FNDA:8,(anonymous_1)
DA:4,1
DA:8,8
DA:9,8
DA:10,8
DA:18,8
DA:22,1
LF:6
LH:6
BRF:0
BRH:0
end_of_record
TN:
SF:app\scripts\classes\game-info.js
FN:4,(anonymous_0)
FN:120,(anonymous_1)
FNF:2
FNH:2
FNDA:14,(anonymous_0)
FNDA:1,(anonymous_1)
DA:10,14
DA:15,14
DA:20,14
DA:25,14
DA:30,14
DA:35,14
DA:40,14
DA:46,14
DA:51,14
DA:56,14
DA:61,14
DA:66,14
DA:71,14
DA:76,14
DA:81,14
DA:86,14
DA:122,1
DA:125,1
DA:126,1
DA:129,1
LF:20
LH:20
BRF:0
BRH:0
end_of_record
TN:
SF:app\scripts\classes\login-result.js
FN:7,(anonymous_0)
FNF:1
FNH:1
FNDA:7,(anonymous_0)
DA:12,7
DA:17,7
DA:20,1
LF:3
LH:3
BRF:0
BRH:0
end_of_record
TN:
SF:app\scripts\classes\prefix-parser.js
FN:10,(anonymous_0)
FN:21,(anonymous_1)
FN:22,(anonymous_2)
FN:31,(anonymous_3)
FN:37,toUpper
FN:49,(anonymous_5)
FN:62,(anonymous_6)
FN:85,(anonymous_7)
FNF:8
FNH:8
FNDA:2,(anonymous_0)
FNDA:11,(anonymous_1)
FNDA:65,(anonymous_2)
FNDA:27,(anonymous_3)
FNDA:984,toUpper
FNDA:27,(anonymous_5)
FNDA:3,(anonymous_6)
FNDA:1,(anonymous_7)
DA:4,1
DA:22,65
DA:38,984
DA:40,27
DA:50,27
DA:51,27
DA:52,27
DA:53,27
DA:63,3
DA:64,3
DA:66,11
DA:67,11
DA:68,9
DA:69,6
DA:70,1
DA:71,0
DA:74,11
DA:75,11
DA:77,3
DA:86,1
DA:87,1
DA:89,7
DA:90,7
DA:91,6
DA:92,4
DA:93,1
DA:94,0
DA:97,7
DA:99,1
DA:103,1
LF:30
LH:28
BRDA:67,0,0,2
BRDA:67,0,1,9
BRDA:68,1,0,3
BRDA:68,1,1,6
BRDA:69,2,0,5
BRDA:69,2,1,1
BRDA:70,3,0,1
BRDA:70,3,1,0
BRDA:75,4,0,11
BRDA:75,4,1,0
BRDA:90,5,0,1
BRDA:90,5,1,6
BRDA:91,6,0,2
BRDA:91,6,1,4
BRDA:92,7,0,3
BRDA:92,7,1,1
BRDA:93,8,0,1
BRDA:93,8,1,0
BRDA:97,9,0,7
BRDA:97,9,1,0
BRF:20
BRH:16
end_of_record
TN:
SF:app\scripts\classes\user-data.js
FN:7,(anonymous_0)
FNF:1
FNH:1
FNDA:2,(anonymous_0)
DA:12,2
DA:17,2
DA:22,2
DA:26,1
LF:4
LH:4
BRF:0
BRH:0
end_of_record
TN:
SF:app\scripts\constants\css-selector.js
FNF:0
FNH:0
DA:1,1
LF:1
LH:1
BRF:0
BRH:0
end_of_record
TN:
SF:app\scripts\constants\url.js
FNF:0
FNH:0
DA:1,1
LF:1
LH:1
BRF:0
BRH:0
end_of_record

7087
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -1,10 +1,9 @@
{
"main": "./src/index.ts",
"main": "./app/index.js",
"name": "f95api",
"version": "1.9.9",
"author": "Millennium Earl",
"description": "Unofficial Node JS module for scraping F95Zone platform",
"types": "dist/index.d.ts",
"repository": {
"type": "git",
"url": "git+https://github.com/MillenniumEarl/F95API.git"
@ -23,62 +22,31 @@
"user data"
],
"scripts": {
"lint": "eslint . --ext .ts",
"prettier-format": "prettier --config .prettierrc '{src,test}/**/*.ts' --write",
"compile": "tsc",
"test": "nyc --reporter=text mocha --require ts-node/register test/index.ts",
"coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov -t 38ad72bf-a29d-4c2e-9827-96cbe037afd2",
"publish": "npm publish",
"run-example": "npm run compile && node ./dist/example.js"
"test": "nyc --reporter=text mocha './test/index-test.js'",
"report-coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov -t 38ad72bf-a29d-4c2e-9827-96cbe037afd2",
"run-example": "node ./app/example.js",
"publish": "npm publish"
},
"engines": {
"node": ">=10.0"
},
"dependencies": {
"axios": "^0.21.1",
"axios": "^0.21.0",
"axios-cookiejar-support": "^1.0.1",
"cheerio": "^1.0.0-rc.5",
"class-validator": "^0.13.1",
"js-sha256": "^0.9.0",
"cheerio": "^1.0.0-rc.3",
"log4js": "^6.3.0",
"luxon": "^1.26.0",
"luxon": "^1.25.0",
"md5": "^2.3.0",
"tough-cookie": "^4.0.0"
},
"devDependencies": {
"@types/chai": "^4.2.15",
"@types/chai-as-promised": "^7.1.3",
"@types/inquirer": "^7.3.1",
"@types/luxon": "^1.25.2",
"@types/mocha": "^8.2.1",
"@types/node": "^14.14.27",
"@types/tough-cookie": "^4.0.0",
"@typescript-eslint/eslint-plugin": "^4.15.0",
"@typescript-eslint/parser": "^4.15.0",
"chai": "^4.3.3",
"chai-as-promised": "^7.1.1",
"babel-eslint": "^10.1.0",
"chai": "^4.2.0",
"dotenv": "^8.2.0",
"eslint": "^7.21.0",
"eslint-config-prettier": "^8.1.0",
"eslint-plugin-prettier": "^3.3.1",
"husky": "^5.1.3",
"inquirer": "^8.0.0",
"lint-staged": "^10.5.4",
"mocha": "^8.3.1",
"nyc": "^15.1.0",
"prettier": "^2.2.1",
"ts-node": "^9.1.1",
"typescript": "^4.2.3"
},
"husky": {
"hooks": {
"pre-commit": "lint-staged"
}
},
"lint-staged": {
"*.js": [
"prettier --write",
"git add"
]
"eslint": "^7.12.1",
"lodash": "^4.17.20",
"mocha": "^8.1.3",
"nyc": "^15.1.0"
},
"bugs": {
"url": "https://github.com/MillenniumEarl/F95API/issues"

View File

@ -1,136 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
/* eslint-disable no-console */
/* istanbul ignore file */
/*
to use this example, create an .env file
in the project root with the following values:
F95_USERNAME = YOUR_USERNAME
F95_PASSWORD = YOUR_PASSWORD
*/
"use strict";
// Public modules from npm
import inquirer from "inquirer";
import dotenv from "dotenv";
// Modules from file
import {
login,
getUserData,
getLatestUpdates,
LatestSearchQuery,
Game,
searchHandiwork,
HandiworkSearchQuery
} from "./index";
// Configure the .env reader
dotenv.config();
/**
* Ask the user to enter the OTP code
* necessary to authenticate on the server.
*/
async function insert2faCode(): Promise<number> {
const questions = [
{
type: "input",
name: "code",
message: "Insert 2FA code:"
}
];
// Prompt the user to insert the code
const answers = await inquirer.prompt(questions);
return answers.code as number;
}
/**
* Authenticate on the platform.
*/
async function authenticate(): Promise<boolean> {
// Log in the platform
console.log("Authenticating...");
const result = await login(process.env.F95_USERNAME, process.env.F95_PASSWORD, insert2faCode);
console.log(`Authentication result: ${result.message}\n`);
return result.success;
}
/**
* Fetch and show data of the current logger user.
*/
async function fetchUserData(): Promise<void> {
console.log("Fetching user data...");
const userdata = await getUserData();
const gameThreads = userdata.watched.filter((e) => e.forum === "Games");
const unread = gameThreads.filter((e) => e.unread).length;
console.log(`User: ${userdata.name}`);
console.log(`Threads followed: ${userdata.watched.length}`);
console.log(`Games followed: ${gameThreads.length}`);
console.log(`Unread game threads: ${unread}\n`);
}
/**
* Fetch the data of the latest `3D game` updated.
*/
async function fetchLatestGameInfo(): Promise<void> {
const latestQuery: LatestSearchQuery = new LatestSearchQuery();
latestQuery.category = "games";
latestQuery.includedTags = ["3d game"];
const latestUpdates = await getLatestUpdates<Game>(latestQuery, 1);
console.log(`"${latestUpdates.shift().name}" was the last "3d game" tagged game to be updated\n`);
}
/**
* Fetch data of the games given theirs names.
*/
async function fetchGameData(games: string[]): Promise<void> {
for (const gamename of games) {
console.log(`Searching '${gamename}'...`);
// Prepare the query
const query: HandiworkSearchQuery = new HandiworkSearchQuery();
query.category = "games";
query.keywords = gamename;
query.order = "likes"; // To find the most popular games
// Fetch the first result
const searchResult = await searchHandiwork<Game>(query, 1);
// No game found
if (searchResult.length !== 0) {
// Extract first game
const gamedata = searchResult.shift();
const authors = gamedata.authors.map((a) => a.name).join(", ");
console.log(`Found: ${gamedata.name} (${gamedata.version}) by ${authors}\n`);
} else console.log(`No data found for '${gamename}'\n`);
}
}
async function main() {
if (await authenticate()) {
// Fetch and log user data
await fetchUserData();
// Get latest `3D GAME` game updated
await fetchLatestGameInfo();
// Get game data
const gameList = ["City of broken dreamers", "Seeds of chaos", "MIST"];
await fetchGameData(gameList);
} else console.log("Failed authentication, impossible to continue");
}
main();

View File

@ -1,249 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from file
import shared from "./scripts/shared";
import search from "./scripts/search";
import { authenticate, urlExists, isF95URL, send2faCode } from "./scripts/network-helper";
import fetchLatestHandiworkURLs from "./scripts/fetch-data/fetch-latest";
import fetchPlatformData from "./scripts/fetch-data/fetch-platform-data";
import getHandiworkInformation from "./scripts/scrape-data/handiwork-parse";
import { IBasic } from "./scripts/interfaces";
// Classes from file
import Credentials from "./scripts/classes/credentials";
import LoginResult from "./scripts/classes/login-result";
import UserProfile from "./scripts/classes/mapping/user-profile";
import LatestSearchQuery from "./scripts/classes/query/latest-search-query";
import HandiworkSearchQuery from "./scripts/classes/query/handiwork-search-query";
import HandiWork from "./scripts/classes/handiwork/handiwork";
import { UserNotLogged, USER_NOT_LOGGED } from "./scripts/classes/errors";
//#region Re-export classes
export { default as PrefixParser } from "./scripts/classes/prefix-parser";
export { default as Animation } from "./scripts/classes/handiwork/animation";
export { default as Asset } from "./scripts/classes/handiwork/asset";
export { default as Comic } from "./scripts/classes/handiwork/comic";
export { default as Game } from "./scripts/classes/handiwork/game";
export { default as Handiwork } from "./scripts/classes/handiwork/handiwork";
export { default as PlatformUser } from "./scripts/classes/mapping/platform-user";
export { default as Post } from "./scripts/classes/mapping/post";
export { default as Thread } from "./scripts/classes/mapping/thread";
export { default as UserProfile } from "./scripts/classes/mapping/user-profile";
export { default as HandiworkSearchQuery } from "./scripts/classes/query/handiwork-search-query";
export { default as LatestSearchQuery } from "./scripts/classes/query/latest-search-query";
export { default as ThreadSearchQuery } from "./scripts/classes/query/thread-search-query";
//#endregion Re-export classes
//#region Export properties
/**
* Set the logger level for module debugging.
*/
// eslint-disable-next-line prefer-const
export let loggerLevel = shared.logger.level;
shared.logger.level = "warn"; // By default log only the warn messages
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
*/
export function isLogged(): boolean {
return shared.isLogged;
}
//#endregion Export properties
//#region Export methods
/**
* Log in to the F95Zone platform.
*
* This **must** be the first operation performed before accessing any other script functions.
*
* @param cb2fa
* Callback used if two-factor authentication is required for the profile.
* It must return he OTP code to use for the login.
*/
export async function login(
username: string,
password: string,
cb2fa?: () => Promise<number>
): Promise<LoginResult> {
// Try to load a previous session
await shared.session.load();
// If the session is valid, return
if (shared.session.isValid(username, password)) {
shared.logger.info(`Loading previous session for ${username}`);
// Load platform data
await fetchPlatformData();
shared.setIsLogged(true);
return new LoginResult(
true,
LoginResult.ALREADY_AUTHENTICATED,
`${username} already authenticated (session)`
);
}
// Creating credentials and fetch unique platform token
shared.logger.trace("Fetching token...");
const creds = new Credentials(username, password);
await creds.fetchToken();
shared.logger.trace(`Authentication for ${username}`);
let result = await authenticate(creds);
shared.setIsLogged(result.success);
// 2FA Authentication is required, fetch OTP
if (result.code === LoginResult.REQUIRE_2FA) {
const code = await cb2fa();
const response2fa = await send2faCode(code, creds.token);
if (response2fa.isSuccess()) result = response2fa.value;
else throw response2fa.value;
}
if (result.success) {
// Recreate the session, overwriting the old one
shared.session.create(username, password, creds.token);
await shared.session.save();
// Load platform data
await fetchPlatformData();
shared.logger.info("User logged in through the platform");
} else shared.logger.warn(`Error during authentication: ${result.message}`);
shared.setIsLogged(result.success);
return result;
}
/**
* Close the currently open session.
*
* You **must** be logged in to the portal before calling this method.
*/
export async function logout(): Promise<void> {
// Check if the user is logged
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
await shared.session.delete();
shared.setIsLogged(false);
}
/**
* Chek if exists a new version of the handiwork.
*
* You **must** be logged in to the portal before calling this method.
*/
export async function checkIfHandiworkHasUpdate(hw: HandiWork): Promise<boolean> {
// Local variables
let hasUpdate = false;
// Check if the user is logged
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// F95 change URL at every game update,
// so if the URL is different an update is available
if (await urlExists(hw.url, true)) {
// Fetch the online handiwork
const onlineHw = await getHandiworkFromURL<HandiWork>(hw.url);
// Compare the versions
hasUpdate = onlineHw.version?.toUpperCase() !== hw.version?.toUpperCase();
}
return hasUpdate;
}
/**
* Search for one or more handiworks identified by a specific query.
*
* You **must** be logged in to the portal before calling this method.
*
* @param {HandiworkSearchQuery} query Parameters used for the search.
* @param {Number} limit Maximum number of results. Default: 10
*/
export async function searchHandiwork<T extends IBasic>(
query: HandiworkSearchQuery,
limit: number = 10
): Promise<T[]> {
// Check if the user is logged
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
return search<T>(query, limit);
}
/**
* Given the url, it gets all the information about the handiwork requested.
*
* You **must** be logged in to the portal before calling this method.
*/
export async function getHandiworkFromURL<T extends IBasic>(url: string): Promise<T> {
// Check if the user is logged
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// Check URL validity
const exists = await urlExists(url);
if (!exists) throw new URIError(`${url} does not exists`);
if (!isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`);
// Get game data
return getHandiworkInformation<T>(url);
}
/**
* Gets the data of the currently logged in user.
*
* You **must** be logged in to the portal before calling this method.
*
* @returns {Promise<UserProfile>} Data of the user currently logged in
*/
export async function getUserData(): Promise<UserProfile> {
// Check if the user is logged
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// Create and fetch profile data
const profile = new UserProfile();
await profile.fetch();
return profile;
}
/**
* Gets the latest updated games that match the specified parameters.
*
* You **must** be logged in to the portal before calling this method.
*
* @param {LatestSearchQuery} query Parameters used for the search.
* @param {Number} limit Maximum number of results. Default: 10
*/
export async function getLatestUpdates<T extends IBasic>(
query: LatestSearchQuery,
limit: number = 10
): Promise<T[]> {
// Check limit value
if (limit <= 0) throw new Error("limit must be greater than 0");
// Check if the user is logged
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// Fetch the results
const urls = await fetchLatestHandiworkURLs(query, limit);
// Get the data from urls
const promiseList = urls.map((u: string) => getHandiworkInformation<T>(u));
return Promise.all(promiseList);
}
//#endregion

View File

@ -1,39 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from file
import { getF95Token } from "../network-helper";
/**
* Represents the credentials used to access the platform.
*/
export default class Credentials {
/**
* Username
*/
public username: string;
/**
* Password of the user.
*/
public password: string;
/**
* One time token used during login.
*/
public token: string = null;
constructor(username: string, password: string) {
this.username = username;
this.password = password;
}
/**
* Fetch and save the token used to log in to F95Zone.
*/
async fetchToken(): Promise<void> {
this.token = await getF95Token();
}
}

View File

@ -1,60 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
interface IBaseError {
/**
* Unique identifier of the error.
*/
id: number;
/**
* Error message.
*/
message: string;
/**
* Error to report.
*/
error: Error;
}
export const USER_NOT_LOGGED = "User not authenticated, unable to continue";
export const INVALID_USER_ID = "Invalid user ID";
export const INVALID_POST_ID = "Invalid post ID";
export const INVALID_THREAD_ID = "Invalid thread ID";
export class GenericAxiosError extends Error implements IBaseError {
id: number;
message: string;
error: Error;
constructor(args: IBaseError) {
super();
this.id = args.id;
this.message = args.message;
this.error = args.error;
}
}
export class UnexpectedResponseContentType extends Error implements IBaseError {
id: number;
message: string;
error: Error;
constructor(args: IBaseError) {
super();
this.id = args.id;
this.message = args.message;
this.error = args.error;
}
}
export class InvalidF95Token extends Error {}
export class UserNotLogged extends Error {}
export class InvalidID extends Error {}
export class ParameterError extends Error {}

View File

@ -1,34 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from files
import { TAuthor, IAnimation, TRating, TCategory, TChangelog } from "../../interfaces";
export default class Animation implements IAnimation {
//#region Properties
censored: boolean;
genre: string[];
installation: string;
language: string[];
lenght: string;
pages: string;
resolution: string[];
authors: TAuthor[];
category: TCategory;
changelog: TChangelog[];
cover: string;
id: number;
lastThreadUpdate: Date;
name: string;
overview: string;
prefixes: string[];
rating: TRating;
tags: string[];
threadPublishingDate: Date;
url: string;
//#endregion Properties
}

View File

@ -1,33 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from files
import { TAuthor, IAsset, TRating, TCategory, TChangelog } from "../../interfaces";
export default class Asset implements IAsset {
//#region Properties
assetLink: string;
associatedAssets: string[];
compatibleSoftware: string;
includedAssets: string[];
officialLinks: string[];
sku: string;
authors: TAuthor[];
category: TCategory;
changelog: TChangelog[];
cover: string;
id: number;
lastThreadUpdate: Date;
name: string;
overview: string;
prefixes: string[];
rating: TRating;
tags: string[];
threadPublishingDate: Date;
url: string;
//#endregion Properties
}

View File

@ -1,30 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from files
import { TAuthor, IComic, TRating, TCategory, TChangelog } from "../../interfaces";
export default class Comic implements IComic {
//#region Properties
genre: string[];
pages: string;
resolution: string[];
authors: TAuthor[];
category: TCategory;
changelog: TChangelog[];
cover: string;
id: number;
lastThreadUpdate: Date;
name: string;
overview: string;
prefixes: string[];
rating: TRating;
tags: string[];
threadPublishingDate: Date;
url: string;
//#endregion Properties
}

View File

@ -1,37 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from files
import { TAuthor, TEngine, IGame, TRating, TStatus, TCategory, TChangelog } from "../../interfaces";
export default class Game implements IGame {
//#region Properties
censored: boolean;
engine: TEngine;
genre: string[];
installation: string;
language: string[];
lastRelease: Date;
mod: boolean;
os: string[];
status: TStatus;
version: string;
authors: TAuthor[];
category: TCategory;
changelog: TChangelog[];
cover: string;
id: number;
lastThreadUpdate: Date;
name: string;
overview: string;
prefixes: string[];
rating: TRating;
tags: string[];
threadPublishingDate: Date;
url: string;
//#endregion Properties
}

View File

@ -1,57 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from files
import {
TAuthor,
TRating,
IHandiwork,
TEngine,
TCategory,
TStatus,
TChangelog
} from "../../interfaces";
/**
* It represents a generic work, be it a game, a comic, an animation or an asset.
*/
export default class HandiWork implements IHandiwork {
//#region Properties
censored: boolean;
engine: TEngine;
genre: string[];
installation: string;
language: string[];
lastRelease: Date;
mod: boolean;
os: string[];
status: TStatus;
version: string;
authors: TAuthor[];
category: TCategory;
changelog: TChangelog[];
cover: string;
id: number;
lastThreadUpdate: Date;
name: string;
overview: string;
prefixes: string[];
rating: TRating;
tags: string[];
threadPublishingDate: Date;
url: string;
pages: string;
resolution: string[];
lenght: string;
assetLink: string;
associatedAssets: string[];
compatibleSoftware: string;
includedAssets: string[];
officialLinks: string[];
sku: string;
//#endregion Properties
}

View File

@ -1,45 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
/**
* Object obtained in response to an attempt to login to the portal.
*/
export default class LoginResult {
//#region Login result codes
static REQUIRE_2FA = 100;
static AUTH_SUCCESSFUL = 200;
static AUTH_SUCCESSFUL_2FA = 201;
static ALREADY_AUTHENTICATED = 202;
static UNKNOWN_ERROR = 400;
static INCORRECT_CREDENTIALS = 401;
static INCORRECT_2FA_CODE = 402;
//#endregion Login result codes
//#region Properties
/**
* Result of the login operation
*/
readonly success: boolean;
/**
* Code associated with the result of the login operation.
*/
readonly code: number;
/**
* Login response message
*/
readonly message: string;
//#endregion Properties
constructor(success: boolean, code: number, message?: string) {
this.success = success;
this.code = code;
this.message = message;
}
}

View File

@ -1,215 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import cheerio from "cheerio";
import { DateTime } from "luxon";
// Modules from files
import { urls } from "../../constants/url";
import { fetchHTML } from "../../network-helper";
import { GENERIC, MEMBER } from "../../constants/css-selector";
import shared from "../../shared";
import { InvalidID, INVALID_USER_ID, UserNotLogged, USER_NOT_LOGGED } from "../errors";
import { ILazy } from "../../interfaces";
/**
* Represents a generic user registered on the platform.
*/
export default class PlatformUser implements ILazy {
//#region Fields
private _id: number;
private _name: string;
private _title: string;
private _banners: string[];
private _messages: number;
private _reactionScore: number;
private _points: number;
private _ratingsReceived: number;
private _joined: Date;
private _lastSeen: Date;
private _followed: boolean;
private _ignored: boolean;
private _private: boolean;
private _avatar: string;
private _amountDonated: number;
//#endregion Fields
//#region Getters
/**
* Unique user ID.
*/
public get id(): number {
return this._id;
}
/**
* Username.
*/
public get name(): string {
return this._name;
}
/**
* Title assigned to the user by the platform.
*/
public get title(): string {
return this._title;
}
/**
* List of banners assigned by the platform.
*/
public get banners(): string[] {
return this._banners;
}
/**
* Number of messages written by the user.
*/
public get messages(): number {
return this._messages;
}
/**
* @todo Reaction score.
*/
public get reactionScore(): number {
return this._reactionScore;
}
/**
* @todo Points.
*/
public get points(): number {
return this._points;
}
/**
* Number of ratings received.
*/
public get ratingsReceived(): number {
return this._ratingsReceived;
}
/**
* Date of joining the platform.
*/
public get joined(): Date {
return this._joined;
}
/**
* Date of the last connection to the platform.
*/
public get lastSeen(): Date {
return this._lastSeen;
}
/**
* Indicates whether the user is followed by the currently logged in user.
*/
public get followed(): boolean {
return this._followed;
}
/**
* Indicates whether the user is ignored by the currently logged on user.
*/
public get ignored(): boolean {
return this._ignored;
}
/**
* Indicates that the profile is private and not viewable by the user.
*/
public get private(): boolean {
return this._private;
}
/**
* URL of the image used as the user's avatar.
*/
public get avatar(): string {
return this._avatar;
}
/**
* Value of donations made.
*/
public get donation(): number {
return this._amountDonated;
}
//#endregion Getters
constructor(id?: number) {
this._id = id;
}
//#region Public methods
public setID(id: number): void {
// Check ID
if (!id || id < 1) throw new InvalidID(INVALID_USER_ID);
this._id = id;
}
public async fetch(): Promise<void> {
// Check login
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// Check ID
if (!this.id || this.id < 1) throw new InvalidID(INVALID_USER_ID);
// Prepare the URL
const url = new URL(this.id.toString(), `${urls.MEMBERS}/`).toString();
// Fetch the page
const response = await fetchHTML(url);
const result = response.applyOnSuccess((html) => this.elaborateResponse(html));
if (result.isFailure()) throw response.value;
}
//#endregion Public methods
//#region Private methods
/**
* Process the HTML code received as
* an answer and gets the data contained in it.
*/
private elaborateResponse(html: string): void {
// Prepare cheerio
const $ = cheerio.load(html);
// Check if the profile is private
this._private =
$(GENERIC.ERROR_BANNER)?.text().trim() ===
"This member limits who may view their full profile.";
if (!this._private) {
// Parse the elements
this._name = $(MEMBER.NAME).text();
this._title = $(MEMBER.TITLE).text();
this._banners = $(MEMBER.BANNERS)
.toArray()
.map((el, idx) => $(el).text().trim())
.filter((el) => el);
this._avatar = $(MEMBER.AVATAR).attr("src");
this._followed = $(MEMBER.FOLLOWED).text() === "Unfollow";
this._ignored = $(MEMBER.IGNORED).text() === "Unignore";
this._messages = parseInt($(MEMBER.MESSAGES).text(), 10);
this._reactionScore = parseInt($(MEMBER.REACTION_SCORE).text(), 10);
this._points = parseInt($(MEMBER.POINTS).text(), 10);
this._ratingsReceived = parseInt($(MEMBER.RATINGS_RECEIVED).text(), 10);
// Parse date
const joined = $(MEMBER.JOINED)?.attr("datetime");
if (DateTime.fromISO(joined).isValid) this._joined = new Date(joined);
const lastSeen = $(MEMBER.LAST_SEEN)?.attr("datetime");
if (DateTime.fromISO(lastSeen).isValid) this._joined = new Date(lastSeen);
// Parse donation
const donation = $(MEMBER.AMOUNT_DONATED)?.text().replace("$", "");
this._amountDonated = donation ? parseInt(donation, 10) : 0;
}
}
//#endregion Private methods
}

View File

@ -1,175 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import cheerio from "cheerio";
// Modules from file
import PlatformUser from "./platform-user";
import { IPostElement, parseF95ThreadPost } from "../../scrape-data/post-parse";
import { POST, THREAD } from "../../constants/css-selector";
import { urls } from "../../constants/url";
import { fetchHTML } from "../../network-helper";
import shared from "../../shared";
import { InvalidID, INVALID_POST_ID, UserNotLogged, USER_NOT_LOGGED } from "../errors";
import { ILazy } from "../../interfaces";
/**
* Represents a post published by a user on the F95Zone platform.
*/
export default class Post implements ILazy {
//#region Fields
private _id: number;
private _number: number;
private _published: Date;
private _lastEdit: Date;
private _owner: PlatformUser;
private _bookmarked: boolean;
private _message: string;
private _body: IPostElement[];
//#endregion Fields
//#region Getters
/**
* Represents a post published by a user on the F95Zone platform.
*/
public get id(): number {
return this._id;
}
/**
* Unique ID of the post within the thread in which it is present.
*/
public get number(): number {
return this._number;
}
/**
* Date the post was first published.
*/
public get published(): Date {
return this._published;
}
/**
* Date the post was last modified.
*/
public get lastEdit(): Date {
return this._lastEdit;
}
/**
* User who owns the post.
*/
public get owner(): PlatformUser {
return this._owner;
}
/**
* Indicates whether the post has been bookmarked.
*/
public get bookmarked(): boolean {
return this._bookmarked;
}
/**
* Post message text.
*/
public get message(): string {
return this._message;
}
/**
* Set of the elements that make up the body of the post.
*/
public get body(): IPostElement[] {
return this._body;
}
//#endregion Getters
constructor(id: number) {
this._id = id;
}
//#region Public methods
/**
* Gets the post data starting from its unique ID for the entire platform.
*/
public async fetch(): Promise<void> {
// Check login
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// Check ID
if (!this.id || this.id < 1) throw new InvalidID(INVALID_POST_ID);
// Fetch HTML page containing the post
const url = new URL(this.id.toString(), urls.POSTS).toString();
const response = await fetchHTML(url);
if (response.isSuccess()) await this.elaborateResponse(response.value);
else throw response.value;
}
//#endregion Public methods
//#region Private methods
/**
* Process the HTML code received as
* an answer and gets the data contained in it.
*/
private async elaborateResponse(html: string) {
// Load cheerio and find post
const $ = cheerio.load(html);
const post = $(THREAD.POSTS_IN_PAGE)
.toArray()
.find((el, idx) => {
// Fetch the ID and check if it is what we are searching
const sid: string = $(el).find(POST.ID).attr("id").replace("post-", "");
const id = parseInt(sid, 10);
if (id === this.id) return el;
});
// Finally parse the post
await this.parsePost($, $(post));
}
private async parsePost($: cheerio.Root, post: cheerio.Cheerio): Promise<void> {
// Find post's ID
const sid: string = post.find(POST.ID).attr("id").replace("post-", "");
this._id = parseInt(sid, 10);
// Find post's number
const sNumber: string = post.find(POST.NUMBER).text().replace("#", "");
this._number = parseInt(sNumber, 10);
// Find post's publishing date
const sPublishing: string = post.find(POST.PUBLISH_DATE).attr("datetime");
this._published = new Date(sPublishing);
// Find post's last edit date
const sLastEdit: string = post.find(POST.LAST_EDIT).attr("datetime");
this._lastEdit = new Date(sLastEdit);
// Find post's owner
const sOwnerID: string = post.find(POST.OWNER_ID).attr("data-user-id").trim();
this._owner = new PlatformUser(parseInt(sOwnerID, 10));
await this._owner.fetch();
// Find if the post is bookmarked
this._bookmarked = post.find(POST.BOOKMARKED).length !== 0;
// Find post's message
this._message = post.find(POST.BODY).text();
// Parse post's body
const body = post.find(POST.BODY);
this._body = parseF95ThreadPost($, body);
}
//#endregion
}

View File

@ -1,290 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import cheerio from "cheerio";
import { DateTime } from "luxon";
// Modules from files
import Post from "./post";
import PlatformUser from "./platform-user";
import { ILazy, TCategory, TRating } from "../../interfaces";
import { urls } from "../../constants/url";
import { POST, THREAD } from "../../constants/css-selector";
import { fetchHTML, fetchPOSTResponse } from "../../network-helper";
import Shared from "../../shared";
import {
InvalidID,
INVALID_THREAD_ID,
ParameterError,
UserNotLogged,
USER_NOT_LOGGED
} from "../errors";
import { getJSONLD, TJsonLD } from "../../scrape-data/json-ld";
import shared from "../../shared";
/**
* Represents a generic F95Zone platform thread.
*/
export default class Thread implements ILazy {
//#region Fields
private POST_FOR_PAGE = 20;
private _id: number;
private _url: string;
private _title: string;
private _tags: string[];
private _prefixes: string[];
private _rating: TRating;
private _owner: PlatformUser;
private _publication: Date;
private _modified: Date;
private _category: TCategory;
//#endregion Fields
//#region Getters
/**
* Unique ID of the thread on the platform.
*/
public get id(): number {
return this._id;
}
/**
* URL of the thread.
*
* It may vary depending on any versions of the contained product.
*/
public get url(): string {
return this._url;
}
/**
* Thread title.
*/
public get title(): string {
return this._title;
}
/**
* Tags associated with the thread.
*/
public get tags(): string[] {
return this._tags;
}
/**
* Prefixes associated with the thread
*/
public get prefixes(): string[] {
return this._prefixes;
}
/**
* Rating assigned to the thread.
*/
public get rating(): TRating {
return this._rating;
}
/**
* Owner of the thread.
*/
public get owner(): PlatformUser {
return this._owner;
}
/**
* Date the thread was first published.
*/
public get publication(): Date {
return this._publication;
}
/**
* Date the thread was last modified.
*/
public get modified(): Date {
return this._modified;
}
/**
* Category to which the content of the thread belongs.
*/
public get category(): TCategory {
return this._category;
}
//#endregion Getters
/**
* Initializes an object for mapping a thread.
*
* The unique ID of the thread must be specified.
*/
constructor(id: number) {
this._id = id;
}
//#region Private methods
/**
* Set the number of posts to display for the current thread.
*/
private async setMaximumPostsForPage(n: 20 | 40 | 60 | 100): Promise<void> {
// Prepare the parameters to send via POST request
const params = {
_xfResponseType: "json",
_xfRequestUri: `/account/dpp-update?content_type=thread&content_id=${this.id}`,
_xfToken: Shared.session.token,
_xfWithData: "1",
content_id: this.id.toString(),
content_type: "thread",
"dpp_custom_config[posts]": n.toString()
};
// Send POST request
const response = await fetchPOSTResponse(urls.POSTS_NUMBER, params);
if (response.isFailure()) throw response.value;
}
/**
* Gets all posts on a page.
*/
private parsePostsInPage(html: string): Post[] {
// Load the HTML
const $ = cheerio.load(html);
// Start parsing the posts
const posts = $(THREAD.POSTS_IN_PAGE)
.toArray()
.map((el, idx) => {
const id = $(el).find(POST.ID).attr("id").replace("post-", "");
return new Post(parseInt(id, 10));
});
// Wait for the post to be fetched
return posts;
}
/**
* It processes the rating of the thread
* starting from the data contained in the JSON+LD tag.
*/
private parseRating(data: TJsonLD): TRating {
const ratingTree = data["aggregateRating"] as TJsonLD;
const rating: TRating = {
average: ratingTree ? parseFloat(ratingTree["ratingValue"] as string) : 0,
best: ratingTree ? parseInt(ratingTree["bestRating"] as string, 10) : 0,
count: ratingTree ? parseInt(ratingTree["ratingCount"] as string, 10) : 0
};
return rating;
}
/**
* Clean the title of a thread, removing prefixes
* and generic elements between square brackets, and
* returns the clean title of the work.
*/
private cleanHeadline(headline: string): string {
// From the title we can extract: Name, author and version
// [PREFIXES] TITLE [VERSION] [AUTHOR]
const matches = headline.match(/\[(.*?)\]/g);
// Get the title name
let name = headline;
if (matches) matches.forEach((e) => (name = name.replace(e, "")));
return name.trim();
}
/**
* Process the HTML code received as
* an answer and gets the data contained in it.
*/
private async elaborateResponse(html: string) {
// Load the HTML
const $ = cheerio.load(html);
// Fetch data from selectors
const ownerID = $(THREAD.OWNER_ID).attr("data-user-id");
const tagArray = $(THREAD.TAGS).toArray();
const prefixArray = $(THREAD.PREFIXES).toArray();
const JSONLD = getJSONLD($("body"));
const published = JSONLD["datePublished"] as string;
const modified = JSONLD["dateModified"] as string;
// Parse the thread's data
this._title = this.cleanHeadline(JSONLD["headline"] as string);
this._tags = tagArray.map((el) => $(el).text().trim());
this._prefixes = prefixArray.map((el) => $(el).text().trim());
this._owner = new PlatformUser(parseInt(ownerID, 10));
await this._owner.fetch();
this._rating = this.parseRating(JSONLD);
this._category = JSONLD["articleSection"] as TCategory;
// Validate the dates
if (DateTime.fromISO(modified).isValid) this._modified = new Date(modified);
if (DateTime.fromISO(published).isValid) this._publication = new Date(published);
}
//#endregion Private methods
//#region Public methods
/**
* Gets information about this thread.
*/
public async fetch(): Promise<void> {
// Check login
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// Check ID
if (!this.id || this.id < 1) throw new InvalidID(INVALID_THREAD_ID);
// Prepare the url
this._url = new URL(this.id.toString(), urls.THREADS).toString();
// Fetch the HTML source
const response = await fetchHTML(this.url);
if (response.isSuccess()) await this.elaborateResponse(response.value);
else throw response.value;
}
/**
* Gets the post in the `index` position with respect to the posts in the thread.
*
* `index` must be greater or equal to 1.
* If the post is not found, `null` is returned.
*/
public async getPost(index: number): Promise<Post | null> {
// Validate parameters
if (index < 1) throw new ParameterError("Index must be greater or equal than 1");
// Local variables
let returnValue = null;
// Get the page number of the post
const page = Math.ceil(index / this.POST_FOR_PAGE);
// Fetch the page
const url = new URL(`page-${page}`, `${this.url}/`).toString();
const htmlResponse = await fetchHTML(url);
if (htmlResponse.isSuccess()) {
// Parse the post
const posts = this.parsePostsInPage(htmlResponse.value);
// Find the searched post
for (const p of posts) {
await p.fetch();
if (p.number === index) {
returnValue = p;
break;
}
}
return returnValue;
} else throw htmlResponse.value;
}
//#endregion Public methods
}

View File

@ -1,212 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import cheerio from "cheerio";
// Modules from files
import Post from "./post";
import PlatformUser from "./platform-user";
import { urls } from "../../constants/url";
import { GENERIC, WATCHED_THREAD } from "../../constants/css-selector";
import { fetchHTML } from "../../network-helper";
import {
GenericAxiosError,
UnexpectedResponseContentType,
UserNotLogged,
USER_NOT_LOGGED
} from "../errors";
import { Result } from "../result";
import shared from "../../shared";
// Interfaces
interface IWatchedThread {
/**
* URL of the thread
*/
url: string;
/**
* Indicates whether the thread has any unread posts.
*/
unread: boolean;
/**
* Specifies the forum to which the thread belongs.
*/
forum: string;
}
// Types
type TFetchResult = Result<GenericAxiosError | UnexpectedResponseContentType, string>;
/**
* Class containing the data of the user currently connected to the F95Zone platform.
*/
export default class UserProfile extends PlatformUser {
//#region Fields
private _watched: IWatchedThread[] = [];
private _bookmarks: Post[] = [];
private _alerts: string[] = [];
private _conversations: string[];
//#endregion Fields
//#region Getters
/**
* List of followed thread data.
*/
public get watched(): IWatchedThread[] {
return this._watched;
}
/**
* List of bookmarked posts.
* @todo
*/
public get bookmarks(): Post[] {
return this._bookmarks;
}
/**
* List of alerts.
* @todo
*/
public get alerts(): string[] {
return this._alerts;
}
/**
* List of conversations.
* @todo
*/
public get conversation(): string[] {
return this._conversations;
}
//#endregion Getters
constructor() {
super();
}
//#region Public methods
public async fetch(): Promise<void> {
// Check login
if (!shared.isLogged) throw new UserNotLogged(USER_NOT_LOGGED);
// First get the user ID and set it
const id = await this.fetchUserID();
super.setID(id);
// Than fetch the basic data
await super.fetch();
// Now fetch the watched threads
this._watched = await this.fetchWatchedThread();
}
//#endregion Public methods
//#region Private methods
/**
* Gets the ID of the user currently logged.
*/
private async fetchUserID(): Promise<number> {
// Local variables
const url = new URL(urls.BASE).toString();
// Fetch and parse page
const response = await fetchHTML(url);
const result = response.applyOnSuccess((html) => {
// Load page with cheerio
const $ = cheerio.load(html);
const sid = $(GENERIC.CURRENT_USER_ID).attr("data-user-id").trim();
return parseInt(sid, 10);
});
if (result.isFailure()) throw result.value;
else return result.value;
}
/**
* Gets the list of threads followed by the user currently logged.
*/
private async fetchWatchedThread(): Promise<IWatchedThread[]> {
// Prepare and fetch URL
const url = new URL(urls.WATCHED_THREADS);
url.searchParams.set("unread", "0");
const response = await fetchHTML(url.toString());
const result = response.applyOnSuccess(async (html) => {
// Load page in cheerio
const $ = cheerio.load(html);
// Fetch the pages
const lastPage = parseInt($(WATCHED_THREAD.LAST_PAGE).text().trim(), 10);
const pages = await this.fetchPages(url, lastPage);
const watchedThreads = pages.map((r, idx) => {
const elements = r.applyOnSuccess(this.fetchPageThreadElements);
if (elements.isSuccess()) return elements.value;
});
return [].concat(...watchedThreads);
});
if (result.isFailure()) throw result.value;
else return result.value as Promise<IWatchedThread[]>;
}
/**
* Gets the pages containing the thread data.
* @param url Base URL to use for scraping a page
* @param n Total number of pages
* @param s Page to start from
*/
private async fetchPages(url: URL, n: number, s = 1): Promise<TFetchResult[]> {
// Local variables
const responsePromiseList: Promise<TFetchResult>[] = [];
// Fetch the page' HTML
for (let page = s; page <= n; page++) {
// Set the page URL
url.searchParams.set("page", page.toString());
// Fetch HTML but not wait for it
const promise = fetchHTML(url.toString());
responsePromiseList.push(promise);
}
// Wait for the promises to resolve
return Promise.all(responsePromiseList);
}
/**
* Gets thread data starting from the source code of the page passed by parameter.
*/
private fetchPageThreadElements(html: string): IWatchedThread[] {
// Local variables
const $ = cheerio.load(html);
return $(WATCHED_THREAD.BODIES)
.map((idx, el) => {
// Parse the URL
const partialURL = $(el).find(WATCHED_THREAD.URL).attr("href");
const url = new URL(partialURL.replace("unread", ""), `${urls.BASE}`).toString();
return {
url: url.toString(),
unread: partialURL.endsWith("unread"),
forum: $(el).find(WATCHED_THREAD.FORUM).text().trim()
} as IWatchedThread;
})
.get();
}
//#endregion Private methods
}

View File

@ -1,114 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from file
import shared, { TPrefixDict } from "../shared";
/**
* Convert prefixes and platform tags from string to ID and vice versa.
*/
export default class PrefixParser {
//#region Private methods
/**
* Gets the key associated with a given value from a dictionary.
* @param {Object} object Dictionary to search
* @param {Any} value Value associated with the key
* @returns {String|undefined} Key found or undefined
*/
private getKeyByValue(object: TPrefixDict, value: string): string | undefined {
return Object.keys(object).find((key) => object[key] === value);
}
/**
* Makes an array of strings uppercase.
*/
private toUpperCaseArray(a: string[]): string[] {
/**
* Makes a string uppercase.
*/
function toUpper(s: string): string {
return s.toUpperCase();
}
return a.map(toUpper);
}
/**
* Check if `dict` contains `value` as a value.
*/
private valueInDict(dict: TPrefixDict, value: string): boolean {
const array = Object.values(dict);
const upperArr = this.toUpperCaseArray(array);
const element = value.toUpperCase();
return upperArr.includes(element);
}
/**
* Search within the platform prefixes for the
* desired element and return the dictionary that contains it.
* @param element Element to search in the prefixes as a key or as a value
*/
private searchElementInPrefixes(element: string | number): TPrefixDict | null {
// Local variables
let dictName = null;
// Iterate the key/value pairs in order to find the element
for (const [key, subdict] of Object.entries(shared.prefixes)) {
// Check if the element is a value in the sub-dict
const valueInDict =
typeof element === "string" && this.valueInDict(subdict, element as string);
// Check if the element is a key in the subdict
const keyInDict =
typeof element === "number" && Object.keys(subdict).includes(element.toString());
if (valueInDict || keyInDict) {
dictName = key;
break;
}
}
return shared.prefixes[dictName] ?? null;
}
//#endregion Private methods
/**
* Convert a list of prefixes to their respective IDs.
*/
public prefixesToIDs(prefixes: string[]): number[] {
const ids: number[] = [];
for (const p of prefixes) {
// Check what dict contains the value
const dict = this.searchElementInPrefixes(p);
if (dict) {
// Extract the key from the dict
const key = this.getKeyByValue(dict, p);
ids.push(parseInt(key, 10));
}
}
return ids;
}
/**
* It converts a list of IDs into their respective prefixes.
*/
public idsToPrefixes(ids: number[]): string[] {
const prefixes: string[] = [];
for (const id of ids) {
// Check what dict contains the key
const dict = this.searchElementInPrefixes(id);
// Add the key to the list
if (dict) {
prefixes.push(dict[id]);
}
}
return prefixes;
}
}

View File

@ -1,185 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import { IsInt, Min, validateSync } from "class-validator";
import { AxiosResponse } from "axios";
// Module from files
import { IQuery, TCategory, TQueryInterface } from "../../interfaces";
import { GenericAxiosError } from "../errors";
import { Result } from "../result";
import LatestSearchQuery, { TLatestOrder } from "./latest-search-query";
import ThreadSearchQuery, { TThreadOrder } from "./thread-search-query";
// Type definitions
/**
* Method of sorting results. Try to unify the two types of
* sorts in the "Latest" section and in the "Thread search"
* section. Being dynamic research, if a sorting type is not
* available, the replacement sort is chosen.
*
* `date`: Order based on the latest update
*
* `likes`: Order based on the number of likes received. Replacement: `replies`.
*
* `relevance`: Order based on the relevance of the result (or rating).
*
* `replies`: Order based on the number of answers to the thread. Replacement: `views`.
*
* `title`: Order based on the growing alphabetical order of the titles.
*
* `views`: Order based on the number of visits. Replacement: `replies`.
*/
type THandiworkOrder = "date" | "likes" | "relevance" | "replies" | "title" | "views";
type TExecuteResult = Result<GenericAxiosError, AxiosResponse<any>>;
export default class HandiworkSearchQuery implements IQuery {
//#region Private fields
static MIN_PAGE = 1;
//#endregion Private fields
//#region Properties
/**
* Keywords to use in the search.
*/
public keywords = "";
/**
* The results must be more recent than the date indicated.
*/
public newerThan: Date = null;
/**
* The results must be older than the date indicated.
*/
public olderThan: Date = null;
public includedTags: string[] = [];
/**
* Tags to exclude from the search.
*/
public excludedTags: string[] = [];
public includedPrefixes: string[] = [];
public category: TCategory = null;
/**
* Results presentation order.
*/
public order: THandiworkOrder = "relevance";
@IsInt({
message: "$property expect an integer, received $value"
})
@Min(HandiworkSearchQuery.MIN_PAGE, {
message: "The minimum $property value must be $constraint1, received $value"
})
public page = 1;
itype: TQueryInterface = "HandiworkSearchQuery";
//#endregion Properties
//#region Public methods
/**
* Select what kind of search should be
* performed based on the properties of
* the query.
*/
public selectSearchType(): "latest" | "thread" {
// Local variables
const MAX_TAGS_LATEST_SEARCH = 5;
const DEFAULT_SEARCH_TYPE = "latest";
// If the keywords are set or the number
// of included tags is greather than 5,
// we must perform a thread search
if (this.keywords || this.includedTags.length > MAX_TAGS_LATEST_SEARCH) return "thread";
return DEFAULT_SEARCH_TYPE;
}
public validate(): boolean {
return validateSync(this).length === 0;
}
public async execute(): Promise<TExecuteResult> {
// Local variables
let response: TExecuteResult = null;
// Check if the query is valid
if (!this.validate()) {
throw new Error(`Invalid query: ${validateSync(this).join("\n")}`);
}
// Convert the query
if (this.selectSearchType() === "latest")
response = await this.cast<LatestSearchQuery>("LatestSearchQuery").execute();
else response = await this.cast<ThreadSearchQuery>("ThreadSearchQuery").execute();
return response;
}
public cast<T extends IQuery>(type: TQueryInterface): T {
// Local variables
let returnValue = null;
// Convert the query
if (type === "LatestSearchQuery") returnValue = this.castToLatest();
else if (type === "ThreadSearchQuery") returnValue = this.castToThread();
else returnValue = this as HandiworkSearchQuery;
// Cast the result to T
return returnValue as T;
}
//#endregion Public methods
//#region Private methods
private castToLatest(): LatestSearchQuery {
// Cast the basic query object and copy common values
const query: LatestSearchQuery = new LatestSearchQuery();
Object.keys(this).forEach((key) => {
if (Object.prototype.hasOwnProperty.call(query, key)) {
query[key] = this[key];
}
});
// Adapt order filter
let orderFilter = this.order as string;
if (orderFilter === "relevance") orderFilter = "rating";
else if (orderFilter === "replies") orderFilter = "views";
query.order = orderFilter as TLatestOrder;
// Adapt date
if (this.newerThan) query.date = query.findNearestDate(this.newerThan);
return query;
}
private castToThread(): ThreadSearchQuery {
// Cast the basic query object and copy common values
const query: ThreadSearchQuery = new ThreadSearchQuery();
Object.keys(this).forEach((key) => {
if (Object.prototype.hasOwnProperty.call(query, key)) {
query[key] = this[key];
}
});
// Set uncommon values
query.onlyTitles = true;
// Adapt order filter
let orderFilter = this.order as string;
if (orderFilter === "title") orderFilter = "relevance";
else if (orderFilter === "likes") orderFilter = "replies";
query.order = orderFilter as TThreadOrder;
return query;
}
//#endregion
}

View File

@ -1,152 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import { ArrayMaxSize, IsInt, Min, validateSync } from "class-validator";
// Modules from file
import { urls } from "../../constants/url";
import PrefixParser from "../prefix-parser";
import { IQuery, TCategory, TQueryInterface } from "../../interfaces";
import { fetchGETResponse } from "../../network-helper";
import { AxiosResponse } from "axios";
import { GenericAxiosError } from "../errors";
import { Result } from "../result";
// Type definitions
export type TLatestOrder = "date" | "likes" | "views" | "title" | "rating";
type TDate = 365 | 180 | 90 | 30 | 14 | 7 | 3 | 1;
/**
* Query used to search handiwork in the "Latest" tab.
*/
export default class LatestSearchQuery implements IQuery {
//#region Private fields
private static MAX_TAGS = 5;
private static MIN_PAGE = 1;
//#endregion Private fields
//#region Properties
public category: TCategory = "games";
/**
* Ordering type.
*
* Default: `date`.
*/
public order: TLatestOrder = "date";
/**
* Date limit in days, to be understood as "less than".
* Use `1` to indicate "today" or `null` to indicate "anytime".
*
* Default: `null`
*/
public date: TDate = null;
@ArrayMaxSize(LatestSearchQuery.MAX_TAGS, {
message: "Too many tags: $value instead of $constraint1"
})
public includedTags: string[] = [];
public includedPrefixes: string[] = [];
@IsInt({
message: "$property expect an integer, received $value"
})
@Min(LatestSearchQuery.MIN_PAGE, {
message: "The minimum $property value must be $constraint1, received $value"
})
public page = LatestSearchQuery.MIN_PAGE;
itype: TQueryInterface = "LatestSearchQuery";
//#endregion Properties
//#region Public methods
public validate(): boolean {
return validateSync(this).length === 0;
}
public async execute(): Promise<Result<GenericAxiosError, AxiosResponse<any>>> {
// Check if the query is valid
if (!this.validate()) {
throw new Error(`Invalid query: ${validateSync(this).join("\n")}`);
}
// Prepare the URL
const url = this.prepareGETurl();
const decoded = decodeURIComponent(url.toString());
// Fetch the result
return fetchGETResponse(decoded);
}
/**
* Gets the value (in days) acceptable in the query starting from a generic date.
*/
public findNearestDate(d: Date): TDate {
// Find the difference between today and the passed date
const diff = this.dateDiffInDays(new Date(), d);
// Find the closest valid value in the array
const closest = [365, 180, 90, 30, 14, 7, 3, 1].reduce(function (prev, curr) {
return Math.abs(curr - diff) < Math.abs(prev - diff) ? curr : prev;
});
return closest as TDate;
}
//#endregion Public methods
//#region Private methods
/**
* Prepare the URL by filling out the GET parameters with the data in the query.
*/
private prepareGETurl(): URL {
// Create the URL
const url = new URL(urls.LATEST_PHP);
url.searchParams.set("cmd", "list");
// Set the category
const cat: TCategory = this.category === "mods" ? "games" : this.category;
url.searchParams.set("cat", cat);
// Add tags and prefixes
const parser = new PrefixParser();
for (const tag of parser.prefixesToIDs(this.includedTags)) {
url.searchParams.append("tags[]", tag.toString());
}
for (const p of parser.prefixesToIDs(this.includedPrefixes)) {
url.searchParams.append("prefixes[]", p.toString());
}
// Set the other values
url.searchParams.set("sort", this.order.toString());
url.searchParams.set("page", this.page.toString());
if (this.date) url.searchParams.set("date", this.date.toString());
return url;
}
/**
*
*/
private dateDiffInDays(a: Date, b: Date) {
const MS_PER_DAY = 1000 * 60 * 60 * 24;
// Discard the time and time-zone information.
const utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate());
const utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate());
return Math.floor((utc2 - utc1) / MS_PER_DAY);
}
//#endregion Private methodss
}

View File

@ -1,182 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import { IsInt, Min, validateSync } from "class-validator";
// Module from files
import { IQuery, TCategory, TQueryInterface } from "../../interfaces";
import { urls } from "../../constants/url";
import PrefixParser from "./../prefix-parser";
import { fetchPOSTResponse } from "../../network-helper";
import { AxiosResponse } from "axios";
import { GenericAxiosError } from "../errors";
import { Result } from "../result";
import Shared from "../../shared";
// Type definitions
export type TThreadOrder = "relevance" | "date" | "last_update" | "replies";
export default class ThreadSearchQuery implements IQuery {
//#region Private fields
static MIN_PAGE = 1;
//#endregion Private fields
//#region Properties
/**
* Keywords to use in the search.
*/
public keywords = "";
/**
* Indicates to search by checking only the thread titles and not the content.
*/
public onlyTitles = false;
/**
* The results must be more recent than the date indicated.
*/
public newerThan: Date = null;
/**
* The results must be older than the date indicated.
*/
public olderThan: Date = null;
public includedTags: string[] = [];
/**
* Tags to exclude from the search.
*/
public excludedTags: string[] = [];
/**
* Minimum number of answers that the thread must possess.
*/
public minimumReplies = 0;
public includedPrefixes: string[] = [];
public category: TCategory = null;
/**
* Results presentation order.
*/
public order: TThreadOrder = "relevance";
@IsInt({
message: "$property expect an integer, received $value"
})
@Min(ThreadSearchQuery.MIN_PAGE, {
message: "The minimum $property value must be $constraint1, received $value"
})
public page = 1;
itype: TQueryInterface = "ThreadSearchQuery";
//#endregion Properties
//#region Public methods
public validate(): boolean {
return validateSync(this).length === 0;
}
public async execute(): Promise<Result<GenericAxiosError, AxiosResponse<any>>> {
// Check if the query is valid
if (!this.validate()) {
throw new Error(`Invalid query: ${validateSync(this).join("\n")}`);
}
// Define the POST parameters
const params = this.preparePOSTParameters();
// Return the POST response
return fetchPOSTResponse(urls.SEARCH, params);
}
//#endregion Public methods
//#region Private methods
/**
* Prepare the parameters for post request with the data in the query.
*/
private preparePOSTParameters(): { [s: string]: string } {
// Local variables
const params = {};
// Ad the session token
params["_xfToken"] = Shared.session.token;
// Specify if only the title should be searched
if (this.onlyTitles) params["c[title_only]"] = "1";
// Add keywords
params["keywords"] = this.keywords ?? "*";
// Specify the scope of the search (only "threads/post")
params["search_type"] = "post";
// Set the dates
if (this.newerThan) {
const date = this.convertShortDate(this.newerThan);
params["c[newer_than]"] = date;
}
if (this.olderThan) {
const date = this.convertShortDate(this.olderThan);
params["c[older_than]"] = date;
}
// Set included and excluded tags (joined with a comma)
if (this.includedTags) params["c[tags]"] = this.includedTags.join(",");
if (this.excludedTags) params["c[excludeTags]"] = this.excludedTags.join(",");
// Set minimum reply number
if (this.minimumReplies > 0) params["c[min_reply_count]"] = this.minimumReplies.toString();
// Add prefixes
const parser = new PrefixParser();
const ids = parser.prefixesToIDs(this.includedPrefixes);
for (let i = 0; i < ids.length; i++) {
const name = `c[prefixes][${i}]`;
params[name] = ids[i].toString();
}
// Set the category
params["c[child_nodes]"] = "1"; // Always set
if (this.category) {
const catID = this.categoryToID(this.category).toString();
params["c[nodes][0]"] = catID;
}
// Set the other values
params["order"] = this.order.toString();
params["page"] = this.page.toString();
return params;
}
/**
* Convert a date in the YYYY-MM-DD format taking into account the time zone.
*/
private convertShortDate(d: Date): string {
const offset = d.getTimezoneOffset();
d = new Date(d.getTime() - offset * 60 * 1000);
return d.toISOString().split("T")[0];
}
/**
* Gets the unique ID of the selected category.
*/
private categoryToID(category: TCategory): number {
const catMap = {
games: 2,
mods: 41,
comics: 40,
animations: 94,
assets: 95
};
return catMap[category as string];
}
//#endregion Private methods
}

View File

@ -1,56 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
/* istanbul ignore file */
export type Result<L, A> = Failure<L, A> | Success<L, A>;
export class Failure<L, A> {
readonly value: L;
constructor(value: L) {
this.value = value;
}
isFailure(): this is Failure<L, A> {
return true;
}
isSuccess(): this is Success<L, A> {
return false;
}
applyOnSuccess<B>(_: (a: A) => B): Result<L, B> {
return this as any;
}
}
export class Success<L, A> {
readonly value: A;
constructor(value: A) {
this.value = value;
}
isFailure(): this is Failure<L, A> {
return false;
}
isSuccess(): this is Success<L, A> {
return true;
}
applyOnSuccess<B>(func: (a: A) => B): Result<L, B> {
return new Success(func(this.value));
}
}
export const failure = <L, A>(l: L): Result<L, A> => {
return new Failure(l);
};
export const success = <L, A>(a: A): Result<L, A> => {
return new Success<L, A>(a);
};

View File

@ -1,220 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Core modules
import * as fs from "fs";
import { promisify } from "util";
import path from "path";
// Public modules from npm
import { sha256 } from "js-sha256";
import tough, { CookieJar } from "tough-cookie";
// Promisifed functions
const areadfile = promisify(fs.readFile);
const awritefile = promisify(fs.writeFile);
const aunlinkfile = promisify(fs.unlink);
export default class Session {
//#region Fields
/**
* Max number of days the session is valid.
*/
private readonly SESSION_TIME: number = 1;
private readonly COOKIEJAR_FILENAME: string = "f95cookiejar.json";
private _path: string;
private _isMapped: boolean;
private _created: Date;
private _hash: string;
private _token: string;
private _cookieJar: CookieJar;
private _cookieJarPath: string;
//#endregion Fields
//#region Getters
/**
* Path of the session map file on disk.
*/
public get path(): string {
return this._path;
}
/**
* Indicates if the session is mapped on disk.
*/
public get isMapped(): boolean {
return this._isMapped;
}
/**
* Date of creation of the session.
*/
public get created(): Date {
return this._created;
}
/**
* MD5 hash of the username and the password.
*/
public get hash(): string {
return this._hash;
}
/**
* Token used to login to F95Zone.
*/
public get token(): string {
return this._token;
}
/**
* Cookie holder.
*/
public get cookieJar(): tough.CookieJar {
return this._cookieJar;
}
//#endregion Getters
/**
* Initializes the session by setting the path for saving information to disk.
*/
constructor(p: string) {
this._path = p;
this._isMapped = fs.existsSync(this.path);
this._created = new Date(Date.now());
this._hash = null;
this._token = null;
this._cookieJar = new tough.CookieJar();
// Define the path for the cookiejar
const basedir = path.dirname(p);
this._cookieJarPath = path.join(basedir, this.COOKIEJAR_FILENAME);
}
//#region Private Methods
/**
* Get the difference in days between two dates.
*/
private dateDiffInDays(a: Date, b: Date) {
const MS_PER_DAY = 1000 * 60 * 60 * 24;
// Discard the time and time-zone information.
const utc1 = Date.UTC(a.getFullYear(), a.getMonth(), a.getDate());
const utc2 = Date.UTC(b.getFullYear(), b.getMonth(), b.getDate());
return Math.abs(Math.floor((utc2 - utc1) / MS_PER_DAY));
}
/**
* Convert the object to a dictionary serializable in JSON.
*/
private toJSON(): Record<string, unknown> {
return {
_created: this._created,
_hash: this._hash,
_token: this._token
};
}
//#endregion Private Methods
//#region Public Methods
/**
* Create a new session.
*/
create(username: string, password: string, token: string): void {
// First, create the _hash of the credentials
const value = `${username}%%%${password}`;
this._hash = sha256(value);
// Set the token
this._token = token;
// Update the creation date
this._created = new Date(Date.now());
}
/**
* Save the session to disk.
*/
async save(): Promise<void> {
// Update the creation date
this._created = new Date(Date.now());
// Convert data
const json = this.toJSON();
const data = JSON.stringify(json);
// Write data
await awritefile(this.path, data);
// Write cookiejar
const serializedJar = await this._cookieJar.serialize();
await awritefile(this._cookieJarPath, JSON.stringify(serializedJar));
}
/**
* Load the session from disk.
*/
async load(): Promise<void> {
if (this.isMapped) {
// Read data
const data = await areadfile(this.path, { encoding: "utf-8", flag: "r" });
const json = JSON.parse(data);
// Assign values
this._created = new Date(json._created);
this._hash = json._hash;
this._token = json._token;
// Load cookiejar
const serializedJar = await areadfile(this._cookieJarPath, {
encoding: "utf-8",
flag: "r"
});
this._cookieJar = await CookieJar.deserialize(JSON.parse(serializedJar));
}
}
/**
* Delete the session from disk.
*/
async delete(): Promise<void> {
if (this.isMapped) {
// Delete the session data
await aunlinkfile(this.path);
// Delete the cookiejar
await aunlinkfile(this._cookieJarPath);
}
}
/**
* Check if the session is valid.
*/
isValid(username: string, password: string): boolean {
// Get the number of days from the file creation
const diff = this.dateDiffInDays(new Date(Date.now()), this.created);
// The session is valid if the number of days is minor than SESSION_TIME
const dateValid = diff < this.SESSION_TIME;
// Check the hash
const value = `${username}%%%${password}`;
const hashValid = sha256(value) === this._hash;
// Search for expired cookies
const jarValid =
this._cookieJar.getCookiesSync("https://f95zone.to").filter((el) => el.TTL() === 0).length ===
0;
return dateValid && hashValid && jarValid;
}
//#endregion Public Methods
}

View File

@ -1,224 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
export const GENERIC = {
/**
* The ID of the user currently logged into
* the platform in the attribute `data-user-id`.
*/
CURRENT_USER_ID: "span.avatar[data-user-id]",
/**
* Banner containing any error messages as text.
*/
ERROR_BANNER: "div.p-body-pageContent > div.blockMessage",
/**
* Provider that the platform expects to use to verify the code for two-factor authentication.
*/
EXPECTED_2FA_PROVIDER: 'input[name="provider"]',
/**
* Locate the token used for the session.
*/
GET_REQUEST_TOKEN: 'input[name="_xfToken"]',
/**
* Block containing the text of any errors that occurred during the login.
*/
LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic",
/**
* Locate the script containing the tags and prefixes of the platform content in JSON format.
*/
LATEST_UPDATES_TAGS_SCRIPT: "script:contains('latestUpdates')"
};
export const WATCHED_THREAD = {
/**
* List of elements containing the data of the watched threads.
*/
BODIES: "div.structItem-cell--main",
/**
* Link element containing the partial URL
* of the thread in the `href` attribute.
*
* It may be followed by the `/unread` segment.
*
* For use within a `WATCHED_THREAD.BODIES` selector.
*/
URL: "div > a[data-tp-primary]",
/**
* Name of the forum to which the thread belongs as text.
*
* For use within a `WATCHED_THREAD.BODIES` selector.
*/
FORUM:
"div.structItem-cell--main > div.structItem-minor > ul.structItem-parts > li:last-of-type > a",
/**
* Index of the last page available as text.
*/
LAST_PAGE: "ul.pageNav-main > li:last-child > a"
};
export const THREAD = {
/**
* Number of pages in the thread (as text of the element).
*
* Two identical elements are identified.
*/
LAST_PAGE: "ul.pageNav-main > li:last-child > a",
/**
* Identify the creator of the thread.
*
* The ID is contained in the `data-user-id` attribute.
*/
OWNER_ID: "div.uix_headerInner > * a.username[data-user-id]",
/**
* Contains the creation date of the thread.
*
* The date is contained in the `datetime` attribute as an ISO string.
*/
CREATION: "div.uix_headerInner > * time",
/**
* List of tags assigned to the thread.
*/
TAGS: "a.tagItem",
/**
* List of prefixes assigned to the thread.
*/
PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]',
/**
* Thread title.
*/
TITLE: "h1.p-title-value",
/**
* JSON containing thread information.
*
* Two different elements are found.
*/
JSONLD: 'script[type="application/ld+json"]',
/**
* Posts on the current page.
*/
POSTS_IN_PAGE: "article.message"
};
export const THREAD_SEARCH = {
/**
* Thread title resulting from research.
*/
THREAD_TITLE: "h3.contentRow-title > a",
/**
*Thread body resulting from research.
*/
BODY: "div.contentRow-main"
};
export const POST = {
/**
* Unique post number for the current thread.
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
NUMBER: '* ul.message-attribution-opposite > li > a:not([id])[rel="nofollow"]',
/**
* Unique ID of the post in the F95Zone platform in the `id` attribute.
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
ID: 'span[id^="post"]',
/**
* Unique ID of the post author in the `data-user-id` attribute.
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
OWNER_ID: "* div.message-cell--user > * a[data-user-id]",
/**
* Main body of the post where the message written by the user is contained.
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
BODY: "* article.message-body > div.bbWrapper",
/**
* Publication date of the post contained in the `datetime` attribute as an ISO date.
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
PUBLISH_DATE: "* div.message-attribution-main > a > time",
/**
* Last modified date of the post contained in the `datetime` attribute as the ISO date.
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
LAST_EDIT: "* div.message-lastEdit > time",
/**
* Gets the element only if the post has been bookmarked.
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
BOOKMARKED: '* ul.message-attribution-opposite >li > a[title="Bookmark"].is-bookmarked',
/**
* Name visualized on the button used to hide/show a spoiler element of a post.
*/
SPOILER_NAME: "button.bbCodeSpoiler-button > * span.bbCodeSpoiler-button-title",
/**
* Contents of a spoiler element in a post.
*/
SPOILER_CONTENT: "div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content"
};
export const MEMBER = {
/**
* Name of the user.
*
* It also contains the unique ID of the user in the `data-user-id` attribute.
*/
NAME: 'span[class^="username"]',
/**
* Title of the user in the platform.
*
* i.e.: Member
*/
TITLE: "span.userTitle",
/**
* Avatar used by the user.
*
* Source in the attribute `src`.
*/
AVATAR: "span.avatarWrapper > a.avatar > img",
/**
* User assigned banners.
*
* The last element is always empty and can be ignored.
*/
BANNERS: "em.userBanner > strong",
/**
* Date the user joined the platform.
*
* The date is contained in the `datetime` attribute as an ISO string.
*/
JOINED: "div.uix_memberHeader__extra > div.memberHeader-blurb:nth-child(1) > * time",
/**
* Last time the user connected to the platform.
*
* The date is contained in the `datetime` attribute as an ISO string.
*/
LAST_SEEN: "div.uix_memberHeader__extra > div.memberHeader-blurb:nth-child(2) > * time",
MESSAGES: "div.pairJustifier > dl:nth-child(1) > * a",
REACTION_SCORE: "div.pairJustifier > dl:nth-child(2) > dd",
POINTS: "div.pairJustifier > dl:nth-child(3) > * a",
RATINGS_RECEIVED: "div.pairJustifier > dl:nth-child(4) > dd",
AMOUNT_DONATED: "div.pairJustifier > dl:nth-child(5) > dd",
/**
* Button used to follow/unfollow the user.
*
* If the text is `Unfollow` then the user is followed.
* If the text is `Follow` then the user is not followed.
*/
FOLLOWED: "div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-follow] > span",
/**
* Button used to ignore/unignore the user.
*
* If the text is `Unignore` then the user is ignored.
* If the text is `Ignore` then the user is not ignored.
*/
IGNORED: "div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-ignore]"
};

View File

@ -1,69 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
export const urls = {
/**
* Page with the list of alerts for the user currently logged.
*/
ALERTS: "https://f95zone.to/account/alerts",
/**
* Basic URL of the platform.
*/
BASE: "https://f95zone.to",
/**
* Page with the list of favorite posts of the user currently logged.
*/
BOOKMARKS: "https://f95zone.to/account/bookmarks",
/**
* Page with the list of conversations of the currently logged user.
*/
CONVERSATIONS: "https://f95zone.to/conversations/",
/**
* URL of the script used for searching for content
* in the "Latest Updates" section of the platform.
*/
LATEST_PHP: "https://f95zone.to/new_latest.php",
/**
* Page with the latest updated platform content.
*/
LATEST_UPDATES: "https://f95zone.to/latest",
/**
* Page used for user login.
*/
LOGIN: "https://f95zone.to/login/login",
/**
* Page used for entering the OTP code in the case of two-factor authentication.
*/
LOGIN_2FA: "https://f95zone.to/login/two-step",
/**
* Summary page of users registered on the platform.
* Used for the search for a specific member through ID.
*/
MEMBERS: "https://f95zone.to/members",
/**
* Add the unique ID of the post to
* get the thread page where the post
* is present.
*/
POSTS: "https://f95zone.to/posts/",
/**
* URL used to send a POST request and change
* the number of posts that can be viewed per
* page of a specific thread.
*/
POSTS_NUMBER: "https://f95zone.to/account/dpp-update",
/**
* URL used to search the platform by POST request.
*/
SEARCH: "https://f95zone.to/search/search/",
/**
* Add the unique ID of the thread to get it's page.
*/
THREADS: "https://f95zone.to/threads/",
/**
* Page with the list of watched threads of the currently logged user.
*/
WATCHED_THREADS: "https://f95zone.to/watched/threads"
};

View File

@ -1,48 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from file
import HandiworkSearchQuery from "../classes/query/handiwork-search-query";
import LatestSearchQuery from "../classes/query/latest-search-query";
import ThreadSearchQuery from "../classes/query/thread-search-query";
import fetchLatestHandiworkURLs from "./fetch-latest";
import fetchThreadHandiworkURLs from "./fetch-thread";
/**
* Gets the URLs of the handiworks that match the passed parameters.
* You *must* be logged.
* @param {LatestSearchQuery} query
* Query used for the search
* @param {Number} limit
* Maximum number of items to get. Default: 30
* @returns {Promise<String[]>} URLs of the handiworks
*/
export default async function fetchHandiworkURLs(
query: HandiworkSearchQuery,
limit = 30
): Promise<string[]> {
// Local variables
let urls: string[] = null;
const searchType = query.selectSearchType();
// Convert the query
if (searchType === "latest") {
// Cast the query
const castedQuery = query.cast<LatestSearchQuery>("LatestSearchQuery");
// Fetch the urls
urls = await fetchLatestHandiworkURLs(castedQuery, limit);
} else {
// Cast the query
const castedQuery = query.cast<ThreadSearchQuery>("ThreadSearchQuery");
// Fetch the urls
urls = await fetchThreadHandiworkURLs(castedQuery, limit);
}
return urls;
}

View File

@ -1,58 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from file
import LatestSearchQuery from "../classes/query/latest-search-query";
import { urls } from "../constants/url";
/**
* Gets the URLs of the latest handiworks that match the passed parameters.
*
* You *must* be logged.
* @param {LatestSearchQuery} query
* Query used for the search
* @param {Number} limit
* Maximum number of items to get. Default: 30
* @returns {Promise<String[]>} URLs of the handiworks
*/
export default async function fetchLatestHandiworkURLs(
query: LatestSearchQuery,
limit: number = 30
): Promise<string[]> {
// Local variables
const shallowQuery: LatestSearchQuery = Object.assign(new LatestSearchQuery(), query);
const resultURLs = [];
let fetchedResults = 0;
let noMorePages = false;
do {
// Fetch the response (application/json)
const response = await shallowQuery.execute();
// Save the URLs
if (response.isSuccess()) {
// In-loop variables
const data: [{ thread_id: number }] = response.value.data.msg.data;
const totalPages: number = response.value.data.msg.pagination.total;
data.map((e, idx) => {
if (fetchedResults < limit) {
const gameURL = new URL(e.thread_id.toString(), urls.THREADS).href;
resultURLs.push(gameURL);
fetchedResults += 1;
}
});
// Increment page and check for it's existence
shallowQuery.page += 1;
noMorePages = shallowQuery.page > totalPages;
} else throw response.value;
} while (fetchedResults < limit && !noMorePages);
return resultURLs;
}

View File

@ -1,168 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Core modules
import { readFileSync, writeFileSync, existsSync } from "fs";
// Public modules from npm
import cheerio from "cheerio";
// Modules from file
import shared, { TPrefixDict } from "../shared";
import { urls as f95url } from "../constants/url";
import { GENERIC } from "../constants/css-selector";
import { fetchHTML } from "../network-helper";
//#region Interface definitions
/**
* Represents the single element contained in the data categories.
*/
interface ISingleOption {
id: number;
name: string;
class: string;
}
/**
* Represents the set of values associated with a specific category of data.
*/
interface ICategoryResource {
id: number;
name: string;
prefixes: ISingleOption[];
}
/**
* Represents the set of tags present on the platform.
*/
interface ILatestResource {
prefixes: { [s: string]: ICategoryResource[] };
tags: TPrefixDict;
options: string;
}
//#endregion Interface definitions
//#region Public methods
/**
* Gets the basic data used for game data processing
* (such as graphics engines and progress statuses)
*/
export default async function fetchPlatformData(): Promise<void> {
// Check if the data are cached
if (!readCache(shared.cachePath)) {
// Load the HTML
const response = await fetchHTML(f95url.LATEST_UPDATES);
// Parse data
const result = response.applyOnSuccess((html) => {
const data = parseLatestPlatformHTML(html);
// Assign data
assignLatestPlatformData(data);
// Cache data
saveCache(shared.cachePath);
});
if (result.isFailure()) throw result.value;
}
}
//#endregion Public methods
//#region Private methods
/**
* Read the platform cache (if available).
*/
function readCache(path: string) {
// Local variables
let returnValue = false;
if (existsSync(path)) {
const data = readFileSync(path, { encoding: "utf-8", flag: "r" });
const json: { [s: string]: TPrefixDict } = JSON.parse(data);
shared.setPrefixPair("engines", json.engines);
shared.setPrefixPair("statuses", json.statuses);
shared.setPrefixPair("tags", json.tags);
shared.setPrefixPair("others", json.others);
returnValue = true;
}
return returnValue;
}
/**
* Save the current platform variables to disk.
*/
function saveCache(path: string): void {
const saveDict = {
engines: shared.prefixes["engines"],
statuses: shared.prefixes["statuses"],
tags: shared.prefixes["tags"],
others: shared.prefixes["others"]
};
const json = JSON.stringify(saveDict);
writeFileSync(path, json);
}
/**
* Given the HTML code of the response from the F95Zone,
* parse it and return the result.
*/
function parseLatestPlatformHTML(html: string): ILatestResource {
const $ = cheerio.load(html);
// Clean the JSON string
const unparsedText = $(GENERIC.LATEST_UPDATES_TAGS_SCRIPT).html().trim();
const startIndex = unparsedText.indexOf("{");
const endIndex = unparsedText.lastIndexOf("}");
const parsedText = unparsedText.substring(startIndex, endIndex + 1);
return JSON.parse(parsedText);
}
/**
* Assign to the local variables the values from the F95Zone.
*/
function assignLatestPlatformData(data: ILatestResource): void {
// Local variables
const scrapedData = {};
// Parse and assign the values that are NOT tags
for (const [key, value] of Object.entries(data.prefixes)) {
for (const res of value) {
// Prepare the dict
const dict: TPrefixDict = {};
// Assign values
res.prefixes.map((e) => (dict[e.id] = e.name.replace("&#039;", "'")));
// Merge the dicts ("Other"/"Status" field)
if (scrapedData[res.name]) {
const newKeys = Object.keys(dict)
.map((k) => parseInt(k, 10))
.filter((k) => !scrapedData[res.name][k]);
newKeys.map((k) => (scrapedData[res.name][k] = dict[k]));
}
// Assign the property
else scrapedData[res.name] = dict;
}
}
// Save the values
shared.setPrefixPair("engines", Object.assign({}, scrapedData["Engine"]));
shared.setPrefixPair("statuses", Object.assign({}, scrapedData["Status"]));
shared.setPrefixPair("others", Object.assign({}, scrapedData["Other"]));
shared.setPrefixPair("tags", data.tags);
}
//#endregion

View File

@ -1,83 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import cheerio from "cheerio";
// Modules from file
import shared from "../shared";
import { THREAD_SEARCH } from "../constants/css-selector";
import { urls as f95urls } from "../constants/url";
import ThreadSearchQuery from "../classes/query/thread-search-query";
//#region Public methods
/**
* Gets the URLs of the handiwork' threads that match the passed parameters.
*
* You *must* be logged.
* @param {ThreadSearchQuery} query
* Query used for the search
* @param {number} limit
* Maximum number of items to get. Default: 30
* @returns {Promise<String[]>} URLs of the handiworks
*/
export default async function fetchThreadHandiworkURLs(
query: ThreadSearchQuery,
limit: number = 30
): Promise<string[]> {
// Execute the query
const response = await query.execute();
// Fetch the results from F95 and return the handiwork urls
if (response.isSuccess()) return fetchResultURLs(response.value.data as string, limit);
else throw response.value;
}
//#endregion Public methods
//#region Private methods
/**
* Gets the URLs of the threads resulting from the F95Zone search.
* @param {number} limit
* Maximum number of items to get. Default: 30
*/
async function fetchResultURLs(html: string, limit: number = 30): Promise<string[]> {
// Prepare cheerio
const $ = cheerio.load(html);
// Here we get all the DIV that are the body of the various query results
const results = $("body").find(THREAD_SEARCH.BODY);
// Than we extract the URLs
const urls = results
.slice(0, limit)
.map((idx, el) => {
const elementSelector = $(el);
return extractLinkFromResult(elementSelector);
})
.get();
return urls;
}
/**
* Look for the URL to the thread referenced by the item.
* @param {cheerio.Cheerio} selector Element to search
* @returns {String} URL to thread
*/
function extractLinkFromResult(selector: cheerio.Cheerio): string {
shared.logger.trace("Extracting thread link from result...");
const partialLink = selector.find(THREAD_SEARCH.THREAD_TITLE).attr("href").trim();
// Compose and return the URL
return new URL(partialLink, f95urls.BASE).toString();
}
//#endregion Private methods

View File

@ -1,343 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
/* istanbul ignore file */
/**
* Data relating to an external platform (i.e. Patreon).
*/
export type TExternalPlatform = {
/**
* name of the platform.
*/
name: string;
/**
* link to the platform.
*/
link: string;
};
/**
* Information about the author of a work.
*/
export type TAuthor = {
/**
* Plain name or username of the author.
*/
name: string;
/**
*
*/
platforms: TExternalPlatform[];
};
/**
* Information on the evaluation of a work.
*/
export type TRating = {
/**
* average value of evaluations.
*/
average: number;
/**
* Best rating received.
*/
best: number;
/**
* Number of ratings made by users.
*/
count: number;
};
/**
* Information about a single version of the product.
*/
export type TChangelog = {
/**
* Product version.
*/
version: string;
/**
* Version information.
*/
information: string[];
};
/**
* List of possible graphics engines used for game development.
*/
export type TEngine =
| "QSP"
| "RPGM"
| "Unity"
| "HTML"
| "RAGS"
| "Java"
| "Ren'Py"
| "Flash"
| "ADRIFT"
| "Others"
| "Tads"
| "Wolf RPG"
| "Unreal Engine"
| "WebGL";
/**
* List of possible progress states associated with a game.
*/
export type TStatus = "Completed" | "Ongoing" | "Abandoned" | "Onhold";
/**
* List of possible categories of a particular work.
*/
export type TCategory = "games" | "mods" | "comics" | "animations" | "assets";
/**
* Valid names of classes that implement the IQuery interface.
*/
export type TQueryInterface = "LatestSearchQuery" | "ThreadSearchQuery" | "HandiworkSearchQuery";
/**
* Collection of values defined for each
* handiwork on the F95Zone platform.
*/
export interface IBasic {
/**
* Authors of the work.
*/
authors: TAuthor[];
/**
* Category of the work..
*/
category: TCategory;
/**
* List of changes of the work for each version.
*/
changelog: TChangelog[];
/**
* link to the cover image of the work.
*/
cover: string;
/**
* Unique ID of the work on the platform.
*/
id: number;
/**
* Last update of the opera thread.
*/
lastThreadUpdate: Date;
/**
* Plain name of the work (without tags and/or prefixes)
*/
name: string;
/**
* Work description
*/
overview: string;
/**
* List of prefixes associated with the work.
*/
prefixes: string[];
/**
* Evaluation of the work by the users of the platform.
*/
rating: TRating;
/**
* List of tags associated with the work.
*/
tags: string[];
/**
* Date of publication of the thread associated with the work.
*/
threadPublishingDate: Date;
/**
* URL to the work's official conversation on the F95Zone portal.
*/
url: string;
}
/**
* Collection of values representing a game present on the F95Zone platform.
*/
export interface IGame extends IBasic {
/**
* Specify whether the work has censorship
* measures regarding NSFW scenes
*/
censored: boolean;
/**
* Graphics engine used for game development.
*/
engine: TEngine;
/**
* List of genres associated with the work.
*/
genre: string[];
/**
* Author's Guide to Installation.
*/
installation: string;
/**
* List of available languages.
*/
language: string[];
/**
* Last time the work underwent updates.
*/
lastRelease: Date;
/**
* Indicates that this item represents a mod.
*/
mod: boolean;
/**
* List of OS for which the work is compatible.
*/
os: string[];
/**
* Indicates the progress of a game.
*/
status: TStatus;
/**
* Version of the work.
*/
version: string;
}
/**
* Collection of values representing a comic present on the F95Zone platform.
*/
export interface IComic extends IBasic {
/**
* List of genres associated with the work.
*/
genre: string[];
/**
* Number of pages or elements that make up the work.
*/
pages: string;
/**
* List of resolutions available for the work.
*/
resolution: string[];
}
/**
* Collection of values representing an animation present on the F95Zone platform.
*/
export interface IAnimation extends IBasic {
/**
* Specify whether the work has censorship
* measures regarding NSFW scenes
*/
censored: boolean;
/**
* List of genres associated with the work.
*/
genre: string[];
/**
* Author's Guide to Installation.
*/
installation: string;
/**
* List of available languages.
*/
language: string[];
/**
* Length of the animation.
*/
lenght: string;
/**
* Number of pages or elements that make up the work.
*/
pages: string;
/**
* List of resolutions available for the work.
*/
resolution: string[];
}
/**
* Collection of values representing an asset present on the F95Zone platform.
*/
export interface IAsset extends IBasic {
/**
* External URL of the asset.
*/
assetLink: string;
/**
* List of URLs of assets associated with the work
* (for example same collection).
*/
associatedAssets: string[];
/**
* Software compatible with the work.
*/
compatibleSoftware: string;
/**
* List of assets url included in the work or used to develop it.
*/
includedAssets: string[];
/**
* List of official links of the work, external to the platform.
*/
officialLinks: string[];
/**
* Unique SKU value of the work.
*/
sku: string;
}
/**
* Collection of values extrapolated from the
* F95 platform representing a particular work.
*/
export interface IHandiwork extends IGame, IComic, IAnimation, IAsset {}
export interface IQuery {
/**
* Name of the implemented interface.
*/
itype: TQueryInterface;
/**
* Category of items to search among.
*/
category: TCategory;
/**
* Tags to be include in the search.
* Max. 5 tags
*/
includedTags: string[];
/**
* Prefixes to include in the search.
*/
includedPrefixes: string[];
/**
* Index of the page to be obtained.
* Between 1 and infinity.
*/
page: number;
/**
* Verify that the query values are valid.
*/
validate(): boolean;
/**
* Search with the data in the query and returns the result.
*
* If the query is invalid it throws an exception.
*/
execute(): any;
}
/**
* It represents an object that obtains the data
* only on the explicit request of the user and
* only after its establishment.
*/
export interface ILazy {
/**
* Gets the data relating to the object.
*/
fetch(): Promise<void>;
}

View File

@ -1,426 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import axios, { AxiosResponse } from "axios";
import cheerio from "cheerio";
import axiosCookieJarSupport from "axios-cookiejar-support";
// Modules from file
import shared from "./shared";
import { urls } from "./constants/url";
import { GENERIC } from "./constants/css-selector";
import LoginResult from "./classes/login-result";
import { failure, Result, success } from "./classes/result";
import {
GenericAxiosError,
InvalidF95Token,
UnexpectedResponseContentType
} from "./classes/errors";
import Credentials from "./classes/credentials";
// Configure axios to use the cookie jar
axiosCookieJarSupport(axios);
// Types
type LookupMapCodeT = {
code: number;
message: string;
};
type ProviderT = "auto" | "totp" | "email";
// Global variables
const USER_AGENT =
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " +
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15";
const AUTH_SUCCESSFUL_MESSAGE = "Authentication successful";
const INVALID_2FA_CODE_MESSAGE =
"The two-step verification value could not be confirmed. Please try again";
const INCORRECT_CREDENTIALS_MESSAGE = "Incorrect password. Please try again.";
/**
* Common configuration used to send request via Axios.
*/
const commonConfig = {
/**
* Headers to add to the request.
*/
headers: {
"User-Agent": USER_AGENT,
Connection: "keep-alive"
},
/**
* Specify if send credentials along the request.
*/
withCredentials: true,
/**
* Jar of cookies to send along the request.
*/
jar: shared.session.cookieJar,
validateStatus: function (status: number) {
return status < 500; // Resolve only if the status code is less than 500
},
timeout: 5000
};
/**
* Gets the HTML code of a page.
*/
export async function fetchHTML(
url: string
): Promise<Result<GenericAxiosError | UnexpectedResponseContentType, string>> {
// Fetch the response of the platform
const response = await fetchGETResponse(url);
if (response.isSuccess()) {
// Check if the response is a HTML source code
const isHTML = response.value.headers["content-type"].includes("text/html");
const unexpectedResponseError = new UnexpectedResponseContentType({
id: 2,
message: `Expected HTML but received ${response.value["content-type"]}`,
error: null
});
return isHTML ? success(response.value.data as string) : failure(unexpectedResponseError);
} else return failure(response.value as GenericAxiosError);
}
/**
* It authenticates to the platform using the credentials
* and token obtained previously. Save cookies on your
* device after authentication.
* @param {Credentials} credentials Platform access credentials
* @param {Boolean} force Specifies whether the request should be forced, ignoring any saved cookies
* @returns {Promise<LoginResult>} Result of the operation
*/
export async function authenticate(
credentials: Credentials,
force: boolean = false
): Promise<LoginResult> {
shared.logger.info(`Authenticating with user ${credentials.username}`);
if (!credentials.token) throw new InvalidF95Token(`Invalid token for auth: ${credentials.token}`);
// Secure the URL
const secureURL = enforceHttpsUrl(urls.LOGIN);
// Prepare the parameters to send to the platform to authenticate
const params = {
login: credentials.username,
url: "",
password: credentials.password,
password_confirm: "",
additional_security: "",
remember: "1",
_xfRedirect: "https://f95zone.to/",
website_code: "",
_xfToken: credentials.token
};
// Try to log-in
let authResult: LoginResult = null;
// Fetch the response to the login request
const response = await fetchPOSTResponse(secureURL, params, force);
// Parse the response
const result = response.applyOnSuccess((r) => manageLoginPOSTResponse(r));
// Manage result
if (result.isFailure()) {
const message = `Error ${result.value.message} occurred while authenticating`;
shared.logger.error(message);
authResult = new LoginResult(false, LoginResult.UNKNOWN_ERROR, message);
} else authResult = result.value;
return authResult;
}
/**
* Send an OTP code if the login procedure requires it.
* @param code OTP code.
* @param token Unique token for the session associated with the credentials in use.
* @param provider Provider used to generate the access code.
* @param trustedDevice If the device in use is trusted, 2FA authentication is not required for 30 days.
*/
export async function send2faCode(
code: number,
token: string,
provider: ProviderT = "auto",
trustedDevice: boolean = false
): Promise<Result<GenericAxiosError, LoginResult>> {
// Prepare the parameters to send via POST request
const params = {
_xfRedirect: urls.BASE,
_xfRequestUri: "/login/two-step?_xfRedirect=https%3A%2F%2Ff95zone.to%2F&remember=1",
_xfResponseType: "json",
_xfToken: token,
_xfWithData: "1",
code: code.toString(),
confirm: "1",
provider: provider,
remember: "1",
trust: trustedDevice ? "1" : "0"
};
// Send 2FA params
const response = await fetchPOSTResponse(urls.LOGIN_2FA, params);
// Check if the authentication is valid
const validAuth = response.applyOnSuccess((r) => manage2faResponse(r));
if (validAuth.isSuccess() && validAuth.value.isSuccess()) {
// Valid login
return success(validAuth.value.value);
} else if (validAuth.isSuccess() && validAuth.value.isFailure()) {
// Wrong provider, try with another
const expectedProvider = validAuth.value.value;
return await send2faCode(code, token, expectedProvider, trustedDevice);
} else failure(validAuth.value);
}
/**
* Obtain the token used to authenticate the user to the platform.
*/
export async function getF95Token(): Promise<string> {
// Fetch the response of the platform
const response = await fetchGETResponse(urls.LOGIN);
if (response.isSuccess()) {
// The response is a HTML page, we need to find the <input> with name "_xfToken"
const $ = cheerio.load(response.value.data as string);
return $("body").find(GENERIC.GET_REQUEST_TOKEN).attr("value");
} else throw response.value;
}
//#region Utility methods
/**
* Performs a GET request to a specific URL and returns the response.
*/
export async function fetchGETResponse(
url: string
): Promise<Result<GenericAxiosError, AxiosResponse<any>>> {
// Secure the URL
const secureURL = enforceHttpsUrl(url);
try {
// Fetch and return the response
commonConfig.jar = shared.session.cookieJar;
const response = await axios.get(secureURL, commonConfig);
return success(response);
} catch (e) {
shared.logger.error(`(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`);
const genericError = new GenericAxiosError({
id: 1,
message: `(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`,
error: e
});
return failure(genericError);
}
}
/**
* Performs a POST request through axios.
* @param url URL to request
* @param params List of value pairs to send with the request
* @param force If `true`, the request ignores the sending of cookies already present on the device.
*/
export async function fetchPOSTResponse(
url: string,
params: { [s: string]: string },
force = false
): Promise<Result<GenericAxiosError, AxiosResponse<any>>> {
// Secure the URL
const secureURL = enforceHttpsUrl(url);
// Prepare the parameters for the POST request
const urlParams = new URLSearchParams();
for (const [key, value] of Object.entries(params)) urlParams.append(key, value);
// Shallow copy of the common configuration object
commonConfig.jar = shared.session.cookieJar;
const config = Object.assign({}, commonConfig);
// Remove the cookies if forced
if (force) delete config.jar;
// Send the POST request and await the response
try {
const response = await axios.post(secureURL, urlParams, config);
return success(response);
} catch (e) {
const message = `(POST) Error ${e.message} occurred while trying to fetch ${secureURL}`;
shared.logger.error(message);
const genericError = new GenericAxiosError({
id: 3,
message: message,
error: e
});
return failure(genericError);
}
}
/**
* Enforces the scheme of the URL is https and returns the new URL.
*/
export function enforceHttpsUrl(url: string): string {
if (isStringAValidURL(url)) return url.replace(/^(https?:)?\/\//, "https://");
else throw new Error(`${url} is not a valid URL`);
}
/**
* Check if the url belongs to the domain of the F95 platform.
*/
export function isF95URL(url: string): boolean {
return url.startsWith(urls.BASE);
}
/**
* Checks if the string passed by parameter has a
* properly formatted and valid path to a URL (HTTP/HTTPS).
*
* @author Daveo
* @see https://preview.tinyurl.com/y2f2e2pc
*/
export function isStringAValidURL(url: string): boolean {
// Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc)
const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/;
const regex = new RegExp(expression);
return url.match(regex).length > 0;
}
/**
* Check if a particular URL is valid and reachable on the web.
* @param {string} url URL to check
* @param {boolean} [checkRedirect]
* If `true`, the function will consider redirects a violation and return `false`.
* Default: `false`
*/
export async function urlExists(url: string, checkRedirect: boolean = false): Promise<boolean> {
// Local variables
let valid = false;
if (isStringAValidURL(url)) {
valid = await axiosUrlExists(url);
if (valid && checkRedirect) {
const redirectUrl = await getUrlRedirect(url);
valid = redirectUrl === url;
}
}
return valid;
}
/**
* Check if the URL has a redirect to another page.
* @param {String} url URL to check for redirect
* @returns {Promise<String>} Redirect URL or the passed URL
*/
export async function getUrlRedirect(url: string): Promise<string> {
commonConfig.jar = shared.session.cookieJar;
const response = await axios.head(url, commonConfig);
return response.config.url;
}
//#endregion Utility methods
//#region Private methods
/**
* Check with Axios if a URL exists.
*/
async function axiosUrlExists(url: string): Promise<boolean> {
// Local variables
const ERROR_CODES = ["ENOTFOUND", "ETIMEDOUT"];
let valid = false;
try {
commonConfig.jar = shared.session.cookieJar;
const response = await axios.head(url, commonConfig);
valid = response && !/4\d\d/.test(response.status.toString());
} catch (error) {
// Throw error only if the error is unknown
if (!ERROR_CODES.includes(error.code)) throw error;
}
return valid;
}
/**
* Manages the response obtained from the server after requesting authentication.
*/
function manageLoginPOSTResponse(response: AxiosResponse<any>) {
// Parse the response HTML
const $ = cheerio.load(response.data as string);
// Check if 2 factor authentication is required
if (response.config.url.startsWith(urls.LOGIN_2FA)) {
return new LoginResult(
false,
LoginResult.REQUIRE_2FA,
"Two-factor authentication is needed to continue"
);
}
// Get the error message (if any) and remove the new line chars
const errorMessage = $("body").find(GENERIC.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, "");
// Return the result of the authentication
const result = errorMessage.trim() === "";
const message = result ? AUTH_SUCCESSFUL_MESSAGE : errorMessage;
const code = messageToCode(message);
return new LoginResult(result, code, message);
}
/**
* Given the login message response of the
* platform, return the login result code.
*/
function messageToCode(message: string): number {
// Prepare the lookup dict
const mapDict: LookupMapCodeT[] = [];
mapDict.push({
code: LoginResult.AUTH_SUCCESSFUL,
message: AUTH_SUCCESSFUL_MESSAGE
});
mapDict.push({
code: LoginResult.INCORRECT_CREDENTIALS,
message: INCORRECT_CREDENTIALS_MESSAGE
});
mapDict.push({
code: LoginResult.INCORRECT_2FA_CODE,
message: INVALID_2FA_CODE_MESSAGE
});
const result = mapDict.find((e) => e.message === message);
return result ? result.code : LoginResult.UNKNOWN_ERROR;
}
/**
* Manage the response given by the platform when the 2FA is required.
*/
function manage2faResponse(r: AxiosResponse<any>): Result<ProviderT, LoginResult> {
// The html property exists only if the provider is wrong
const rightProvider = !("html" in r.data);
// Wrong provider!
if (!rightProvider) {
const $ = cheerio.load(r.data.html.content);
const expectedProvider = $(GENERIC.EXPECTED_2FA_PROVIDER).attr("value");
return failure(expectedProvider as ProviderT);
}
// r.data.status is 'ok' if the authentication is successful
const result = r.data.status === "ok";
const message: string = result ? AUTH_SUCCESSFUL_MESSAGE : r.data.errors.join(",");
const loginCode = messageToCode(message);
return success(new LoginResult(result, loginCode, message));
}
//#endregion

View File

@ -1,317 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import { DateTime } from "luxon";
// Modules from files
import HandiWork from "../classes/handiwork/handiwork";
import Thread from "../classes/mapping/thread";
import { IBasic, TAuthor, TChangelog, TEngine, TExternalPlatform, TStatus } from "../interfaces";
import shared, { TPrefixDict } from "../shared";
import { ILink, IPostElement } from "./post-parse";
/**
* Gets information of a particular handiwork from its thread.
*
* If you don't want to specify the object type, use `HandiWork`.
*
* @todo It does not currently support assets.
*/
export default async function getHandiworkInformation<T extends IBasic>(
arg: string | Thread
): Promise<T> {
// Local variables
let thread: Thread = null;
if (typeof arg === "string") {
// Fetch thread data
const id = extractIDFromURL(arg);
thread = new Thread(id);
await thread.fetch();
} else thread = arg;
shared.logger.info(`Obtaining handiwork from ${thread.url}`);
// Convert the info from thread to handiwork
const hw: HandiWork = {} as HandiWork;
hw.id = thread.id;
hw.url = thread.url;
hw.name = thread.title;
hw.category = thread.category;
hw.threadPublishingDate = thread.publication;
hw.lastThreadUpdate = thread.modified;
hw.tags = thread.tags;
hw.rating = thread.rating;
fillWithPrefixes(hw, thread.prefixes);
// Fetch info from first post
const post = await thread.getPost(1);
fillWithPostData(hw, post.body);
return <T>(<unknown>hw);
}
//#region Private methods
//#region Utilities
/**
* Extracts the work's unique ID from its URL.
*/
function extractIDFromURL(url: string): number {
shared.logger.trace("Extracting ID from URL...");
// URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/
// or https://f95zone.to/threads/ID/
const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/);
if (!match) return -1;
// Parse and return number
return parseInt(match[0], 10);
}
/**
* Makes an array of strings uppercase.
*/
function toUpperCaseArray(a: string[]): string[] {
/**
* Makes a string uppercase.
*/
function toUpper(s: string): string {
return s.toUpperCase();
}
return a.map(toUpper);
}
/**
* Check if the string `s` is in the dict `a`.
*
* Case insensitive.
*/
function stringInDict(s: string, a: TPrefixDict): boolean {
// Make uppercase all the strings in the array
const values = toUpperCaseArray(Object.values(a));
return values.includes(s.toUpperCase());
}
/**
* Convert a string to a boolean.
*
* Check also for `yes`/`no` and `1`/`0`.
*/
function stringToBoolean(s: string): boolean {
// Local variables
const positiveTerms = ["true", "yes", "1"];
const negativeTerms = ["false", "no", "0"];
const cleanString = s.toLowerCase().trim();
let result = Boolean(s);
if (positiveTerms.includes(cleanString)) result = true;
else if (negativeTerms.includes(cleanString)) result = false;
return result;
}
/**
* Gets the element with the given name or `undefined`.
*
* Case-insensitive.
*/
function getPostElementByName(elements: IPostElement[], name: string): IPostElement | undefined {
return elements.find((el) => el.name.toUpperCase() === name.toUpperCase());
}
//#endregion Utilities
/**
* Parse the post prefixes.
*
* In particular, it elaborates the following prefixes for games:
* `Engine`, `Status`, `Mod`.
*/
function fillWithPrefixes(hw: HandiWork, prefixes: string[]) {
shared.logger.trace("Parsing prefixes...");
// Local variables
let mod = false;
let engine: TEngine = null;
let status: TStatus = null;
/**
* Emulated dictionary of mod prefixes.
*/
const fakeModDict: TPrefixDict = {
0: "MOD",
1: "CHEAT MOD"
};
// Initialize the array
hw.prefixes = [];
prefixes.map((item, idx) => {
// Remove the square brackets
const prefix = item.replace("[", "").replace("]", "");
// Check what the prefix indicates
if (stringInDict(prefix, shared.prefixes["engines"])) engine = prefix as TEngine;
else if (stringInDict(prefix, shared.prefixes["statuses"])) status = prefix as TStatus;
else if (stringInDict(prefix, fakeModDict)) mod = true;
// Anyway add the prefix to list
hw.prefixes.push(prefix);
});
// If the status is not set, then the game is in development (Ongoing)
status = !status && hw.category === "games" ? status : "Ongoing";
hw.engine = engine;
hw.status = status;
hw.mod = mod;
}
/**
* Compiles a HandiWork object with the data extracted
* from the main post of the HandiWork page.
*
* The values that will be added are:
* `Overview`, `OS`, `Language`, `Version`, `Installation`,
* `Pages`, `Resolution`, `Lenght`, `Genre`, `Censored`,
* `LastRelease`, `Authors`, `Changelog`, `Cover`.
*/
function fillWithPostData(hw: HandiWork, elements: IPostElement[]) {
// First fill the "simple" elements
hw.overview = getPostElementByName(elements, "overview")?.text;
hw.os = getPostElementByName(elements, "os")
?.text?.split(",")
.map((s) => s.trim());
hw.language = getPostElementByName(elements, "language")
?.text?.split(",")
.map((s) => s.trim());
hw.version = getPostElementByName(elements, "version")?.text;
hw.installation = getPostElementByName(elements, "installation")?.text;
hw.pages = getPostElementByName(elements, "pages")?.text;
hw.resolution = getPostElementByName(elements, "resolution")
?.text?.split(",")
.map((s) => s.trim());
hw.lenght = getPostElementByName(elements, "lenght")?.text;
// Parse the censorship
const censored =
getPostElementByName(elements, "censored") || getPostElementByName(elements, "censorship");
if (censored) hw.censored = stringToBoolean(censored.text);
// Get the genres
const genre = getPostElementByName(elements, "genre")?.text;
hw.genre = genre
?.split(",")
.map((s) => s.trim())
.filter((s) => s !== "");
// Get the cover
const cover = elements.find((e) => e.type === "Image") as ILink;
hw.cover = cover?.href;
// Fill the dates
const releaseDate = getPostElementByName(elements, "release date")?.text;
if (DateTime.fromISO(releaseDate).isValid) hw.lastRelease = new Date(releaseDate);
// Get the author
hw.authors = parseAuthor(elements);
// Get the changelog
hw.changelog = parseChangelog(elements);
}
/**
* Parse the author from the post's data.
*/
function parseAuthor(elements: IPostElement[]): TAuthor[] {
// Local variables
const author: TAuthor = {
name: "",
platforms: []
};
// Fetch the authors from the post data
const authorElement =
getPostElementByName(elements, "developer") ||
getPostElementByName(elements, "developer/publisher") ||
getPostElementByName(elements, "artist");
if (authorElement) {
// Set the author name
author.name = authorElement.text;
// Add the found platforms
authorElement.content.forEach((e: ILink) => {
// Ignore invalid links
if (e.href) {
// Create and push the new platform
const platform: TExternalPlatform = {
name: e.text,
link: e.href
};
author.platforms.push(platform);
}
});
}
return [author];
}
/**
* Parse the changelog from the post's data.
*/
function parseChangelog(elements: IPostElement[]): TChangelog[] {
// Local variables
const changelog = [];
const changelogElement =
getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log");
if (changelogElement) {
// regex used to match version tags
const versionRegex = /^v[0-9]+\.[0-9]+.*/;
// Get the indexes of the version tags
const indexesVersion = changelogElement.content
.filter((e) => e.type === "Text" && versionRegex.test(e.text))
.map((e) => changelogElement.content.indexOf(e));
const results = indexesVersion.map((i, j) => {
// In-loop variable
const versionChangelog: TChangelog = {
version: "",
information: []
};
// Get the difference in indexes between this and the next version tag
const diff = indexesVersion[j + 1] ?? changelogElement.content.length;
// fetch the group of data of this version tag
const group = changelogElement.content.slice(i, diff);
versionChangelog.version = group.shift().text.replace("v", "").trim();
// parse the data
group.forEach((e) => {
if (e.type === "Generic" || e.type === "Spoiler") {
const textes = e.content.map((c) => c.text);
versionChangelog.information.push(...textes);
} else versionChangelog.information.push(e.text);
});
return versionChangelog;
});
changelog.push(...results);
}
return changelog;
}
//#endregion Private methods

View File

@ -1,67 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Public modules from npm
import cheerio from "cheerio";
// Modules from file
import shared from "../shared";
import { THREAD } from "../constants/css-selector";
/**
* Represents information contained in a JSON+LD tag.
*/
export type TJsonLD = { [s: string]: string | TJsonLD };
/**
* Extracts and processes the JSON-LD values of the page.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {TJsonLD[]} List of data obtained from the page
*/
export function getJSONLD(body: cheerio.Cheerio): TJsonLD {
shared.logger.trace("Extracting JSON-LD data...");
// Fetch the JSON-LD data
const structuredDataElements = body.find(THREAD.JSONLD);
// Parse the data
const values = structuredDataElements.map((idx, el) => parseJSONLD(el)).get();
// Merge the data and return a single value
return mergeJSONLD(values);
}
//#region Private methods
/**
* Merges multiple JSON+LD tags into one object.
* @param data List of JSON+LD tags
*/
function mergeJSONLD(data: TJsonLD[]): TJsonLD {
// Local variables
let merged: TJsonLD = {};
for (const value of data) {
merged = Object.assign(merged, value);
}
return merged;
}
/**
* Parse a JSON-LD element source code.
*/
function parseJSONLD(element: cheerio.Element): TJsonLD {
// Get the element HTML
const html = cheerio(element).html().trim();
// Obtain the JSON-LD
const data = html.replace('<script type="application/ld+json">', "").replace("</script>", "");
// Convert the string to an object
return JSON.parse(data);
}
//#endregion Private methods

View File

@ -1,520 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Import from files
import { POST } from "../constants/css-selector";
// Types
type TNodeType = "Text" | "Formatted" | "Spoiler" | "Link" | "List" | "Noscript" | "Unknown";
//#region Interfaces
/**
* Represents an element contained in the post.
*/
export interface IPostElement {
/**
* Type of element.
*/
type: "Generic" | "Text" | "Link" | "Image" | "Spoiler";
/**
* Name associated with the element.
*/
name: string;
/**
* Text of the content of the element excluding any children.
*/
text: string;
/**
* Children elements contained in this element.
*/
content: IPostElement[];
}
/**
* Represents a link type link in the post.
*/
export interface ILink extends IPostElement {
type: "Image" | "Link";
/**
* Link to the resource.
*/
href: string;
}
//#endregion Interfaces
//#region Public methods
/**
* Given a post of a thread page it extracts the information contained in the body.
*/
export function parseF95ThreadPost($: cheerio.Root, post: cheerio.Cheerio): IPostElement[] {
// The data is divided between "tag" and "text" elements.
// Simple data is composed of a "tag" element followed
// by a "text" element, while more complex data (contained
// in spoilers) is composed of a "tag" element, followed
// by a text containing only ":" and then by an additional
// "tag" element having as the first term "Spoiler"
// First fetch all the elements in the post
const elements = post
.contents()
.toArray()
.map((e) => parseCheerioNode($, e)); // Parse the nodes
// Create a supernode
let supernode = createGenericElement();
supernode.content = elements;
// Reduce the nodes
supernode = reducePostElement(supernode);
// Remove the empty nodes
supernode = removeEmptyContentFromElement(supernode);
// Finally parse the elements to create the pairs of title/data
return pairUpElements(supernode.content);
}
//#endregion Public methods
//#region Private methods
//#region Node type
/**
* Check if the node passed as a parameter is a formatting one (i.e. `<b>`).
*/
function isFormattingNode(node: cheerio.Element): boolean {
const formattedTags = ["b", "i"];
return node.type === "tag" && formattedTags.includes(node.name);
}
/**
* Check if the node passed as a parameter is of text type.
*/
function isTextNode(node: cheerio.Element): boolean {
return node.type === "text";
}
/**
* Check if the node is a spoiler.
*/
function isSpoilerNode(node: cheerio.Cheerio): boolean {
return node.attr("class") === "bbCodeSpoiler";
}
/**
* Check if the node is a link or a image.
*/
function isLinkNode(node: cheerio.Element): boolean {
// Local variables
let valid = false;
// The node is a valid DOM element
if (node.type === "tag") {
const e = node as cheerio.TagElement;
valid = e.name === "a" || e.name === "img";
}
return valid;
}
/**
* Check if the node is a `noscript` tag.
*/
function isNoScriptNode(node: cheerio.Element): boolean {
return node.type === "tag" && node.name === "noscript";
}
/**
* Check if the node is a list element, i.e. `<li>` or `<ul>` tag.
*/
function isListNode(node: cheerio.Element): boolean {
return node.type === "tag" && (node.name === "ul" || node.name === "li");
}
/**
* Idetnify the type of node passed by parameter.
*/
function nodeType($: cheerio.Root, node: cheerio.Element): TNodeType {
// Function map
const functionMap = {
Text: (node: cheerio.Element) => isTextNode(node) && !isFormattingNode(node),
Formatted: (node: cheerio.Element) => isFormattingNode(node),
Spoiler: (node: cheerio.Element) => isSpoilerNode($(node)),
Link: (node: cheerio.Element) => isLinkNode(node),
List: (node: cheerio.Element) => isListNode(node),
Noscript: (node: cheerio.Element) => isNoScriptNode(node)
};
// Parse and return the type of the node
const result = Object.keys(functionMap).find((e) => functionMap[e](node));
return result ? (result as TNodeType) : "Unknown";
}
//#endregion Node Type
//#region Parse Cheerio node
/**
* Process a spoiler element by getting its text broken
* down by any other spoiler elements present.
*/
function parseCheerioSpoilerNode($: cheerio.Root, node: cheerio.Cheerio): IPostElement {
// A spoiler block is composed of a div with class "bbCodeSpoiler",
// containing a div "bbCodeSpoiler-content" containing, in cascade,
// a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content".
// This last tag contains the required data.
// Local variables
const spoiler: IPostElement = {
type: "Spoiler",
name: "",
text: "",
content: []
};
// Find the title of the spoiler (contained in the button)
const name = node.find(POST.SPOILER_NAME)?.first();
spoiler.name = name ? name.text().trim() : "";
// Parse the content of the spoiler
spoiler.content = node
.find(POST.SPOILER_CONTENT)
.contents()
.toArray()
.map((e) => parseCheerioNode($, e));
// Clean text (Spoiler has no text) @todo
// spoiler.text = spoiler.text.replace(/\s\s+/g, " ").trim();
return spoiler;
}
/**
* Process a node that contains a link or image.
*/
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink {
// Local variable
const link: ILink = {
type: "Link",
name: "",
text: "",
href: "",
content: []
};
if (element.is("img")) {
link.type = "Image";
link.text = element.attr("alt") ?? "";
link.href = element.attr("data-src");
} else if (element.is("a")) {
link.type = "Link";
link.text = element.text().replace(/\s\s+/g, " ").trim();
link.href = element.attr("href");
}
return link;
}
/**
* Process a text only node.
*/
function parseCheerioTextNode(node: cheerio.Cheerio): IPostElement {
const content: IPostElement = {
type: "Text",
name: "",
text: getCheerioNonChildrenText(node),
content: []
};
return content;
}
/**
* Gets the text of the node only, excluding child nodes.
* Also includes formatted text elements (i.e. `<b>`).
*/
function getCheerioNonChildrenText(node: cheerio.Cheerio): string {
// Local variable
let text = "";
// If the node has no children, return the node's text
if (node.contents().length === 1) {
// @todo Remove IF after cheerio RC6
text = node.text();
} else {
// Find all the text nodes in the node
text = node
.first()
.contents() // @todo Change to children() after cheerio RC6
.filter((idx, e) => isTextNode(e))
.text();
}
// Clean and return the text
return text.replace(/\s\s+/g, " ").trim();
}
//#endregion Parse Cheerio node
//#region IPostElement utility
/**
* Check if the node has non empty `name` and `text`.
*/
function isPostElementUnknown(node: IPostElement): boolean {
// @todo For some strange reason, if the node IS empty but
// node.type === "Text" the 2nd statement return false.
return node.name.trim() === "" && node.text.trim() === "";
}
/**
* Check if the node has a non empty property
* between `name`, `text` and `content`.
*/
function isPostElementEmpty(node: IPostElement): boolean {
return node.content.length === 0 && isPostElementUnknown(node);
}
/**
* Create a `IPostElement` without name, text or content.
*/
function createGenericElement(): IPostElement {
return {
type: "Generic",
name: "",
text: "",
content: []
};
}
/**
* Clean the element `name` and `text` removing initial and final special characters.
*/
function cleanElement(element: IPostElement): IPostElement {
// Local variables
const shallow = Object.assign({}, element);
const specialCharSet = /[-!$%^&*()_+|~=`{}[\]:";'<>?,./]/;
const startsWithSpecialCharsRegex = new RegExp("^" + specialCharSet.source);
const endsWithSpecialCharsRegex = new RegExp(specialCharSet.source + "$");
shallow.name = shallow.name
.replace(startsWithSpecialCharsRegex, "")
.replace(endsWithSpecialCharsRegex, "")
.trim();
shallow.text = shallow.text
.replace(startsWithSpecialCharsRegex, "")
.replace(endsWithSpecialCharsRegex, "")
.trim();
return shallow;
}
//#endregion IPostElement utility
/**
* Collapse an `IPostElement` element with a single subnode
* in the `Content` field in case it has no information.
*/
function reducePostElement(element: IPostElement): IPostElement {
// Local variables
const shallowCopy = Object.assign({}, element);
// If the node has only one child, reduce and return it
if (isPostElementUnknown(shallowCopy) && shallowCopy.content.length === 1) {
return reducePostElement(shallowCopy.content[0]);
}
// Reduce element's childs
shallowCopy.content = shallowCopy.content.map((e) => reducePostElement(e));
return shallowCopy;
}
/**
* Remove all empty children elements of the elements for parameter.
*/
function removeEmptyContentFromElement(element: IPostElement, recursive = true): IPostElement {
// Create a copy of the element
const copy = Object.assign({}, element);
// Reduce nested contents if recursive
const recursiveResult = recursive
? element.content.map((e) => removeEmptyContentFromElement(e))
: copy.content;
// Find the non-empty nodes
const validNodes = recursiveResult
.filter((e) => !isPostElementEmpty(e)) // Remove the empty nodes
.filter((e) => !isPostElementEmpty(cleanElement(e))); // Remove the useless nodes
// Assign the nodes
copy.content = validNodes;
return copy;
}
/**
* Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes.
*/
function parseCheerioNode($: cheerio.Root, node: cheerio.Element): IPostElement {
// Local variables
const cheerioNode = $(node);
// Function mapping
const functionMap = {
Text: (node: cheerio.Cheerio) => parseCheerioTextNode(node),
Spoiler: (node: cheerio.Cheerio) => parseCheerioSpoilerNode($, node),
Link: (node: cheerio.Cheerio) => parseCheerioLinkNode(node)
};
// Get the type of node
const type = nodeType($, node);
// Get the post based on the type of node
const post = Object.keys(functionMap).includes(type)
? functionMap[type]($(node))
: createGenericElement();
// Parse the childrens only if the node is a <b>/<i> element, a list
// or a unknown element. For the link in unnecessary while for the
// spoilers is already done in parseCheerioSpoilerNode
const includeTypes: TNodeType[] = ["Formatted", "List", "Unknown"];
if (includeTypes.includes(type)) {
const childPosts = cheerioNode
.contents() // @todo Change to children() after cheerio RC6
.toArray()
.filter((e) => e) // Ignore undefined elements
.map((e) => parseCheerioNode($, e))
.filter((e) => !isPostElementEmpty(e));
post.content.push(...childPosts);
}
return post;
}
/**
* It simplifies the `IPostElement` elements by associating
* the corresponding value to each characterizing element (i.e. author).
*/
function pairUpElements(elements: IPostElement[]): IPostElement[] {
// Local variables
const shallow = [...elements];
// Parse all the generic elements that
// act as "container" for other information
shallow
.filter((e) => e.type === "Generic")
.map((e) => ({
element: e,
pairs: pairUpElements(e.content)
}))
.forEach((e) => {
// Find the index of the elements
const index = shallow.indexOf(e.element);
// Remove that elements
shallow.splice(index, 1);
// Add the pairs at the index of the deleted element
e.pairs.forEach((e, i) => shallow.splice(index + i, 0, e));
});
// Than we find all the IDs of the elements that are "titles".
const indexes = shallow
.filter((e, i) => isValidTitleElement(e, i, shallow))
.map((e) => shallow.indexOf(e));
// Now we find all the elements between indexes and
// associate them with the previous "title" element
return indexes.map((i, j) => parseGroupData(i, j, indexes, shallow));
}
/**
* Verify if the `element` is a valid title.
* @param element Element to check
* @param index Index of the element in `array`
* @param array Array of elements to check
*/
function isValidTitleElement(element: IPostElement, index: number, array: IPostElement[]): boolean {
// Check if this element is a "title" checking also the next element
const isPostfixDoublePoints = element.text.endsWith(":") && element.text !== ":";
const nextElementIsValue = array[index + 1]?.text.startsWith(":");
const elementIsTextTitle =
element.type === "Text" && (isPostfixDoublePoints || nextElementIsValue);
// Special values tha must be set has "title"
const specialValues = ["DOWNLOAD", "CHANGELOG", "CHANGE-LOG", "GENRE"];
const specialTypes = ["Image"];
// Used to ignore already merged elements with name (ignore spoilers)
// because they have as name the content of the spoiler button
const hasName = element.name !== "" && element.type !== "Spoiler";
return (
elementIsTextTitle ||
specialTypes.includes(element.type) ||
specialValues.includes(element.text.toUpperCase()) ||
hasName
);
}
/**
* Associate the relative values to a title.
* @param start Title index in the `elements` array
* @param index `start` index in `indexes`
* @param indexes List of titles indices in the `elements` array
* @param elements Array of elements to group
*/
function parseGroupData(
start: number,
index: number,
indexes: number[],
elements: IPostElement[]
): IPostElement {
// Local variables
const endsWithSpecialCharsRegex = /[-:]$/;
const startsWithDoublePointsRegex = /^[:]/;
// Find all the elements (title + data) of the same data group
const nextIndex = indexes[index + 1] ?? elements.length;
const group = elements.slice(start, nextIndex);
// Extract the title
const title = group.shift();
// If the title is already named (beacuse it was
// previously elaborated) return it witout
if (title.name !== "" && title.type !== "Spoiler") return title;
// Assign name and text of the title
title.name = title.text.replace(endsWithSpecialCharsRegex, "").trim();
title.text = group
.filter((e) => e.type === "Text")
.map((e) =>
e.text
.replace(startsWithDoublePointsRegex, "") // Remove the starting ":" from the element's text
.replace(endsWithSpecialCharsRegex, "") // Remove any special chars at the end
.trim()
)
.join(" ") // Join with space
.trim();
// Append all the content of the elements.
group.forEach(
(e) =>
e.type === "Spoiler"
? title.content.push(...e.content) // Add all the content fo the spoiler
: title.content.push(e) // Add the element itself
);
return title;
}
//#endregion Private methods

View File

@ -1,56 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
"use strict";
// Modules from file
import { IBasic, IQuery } from "./interfaces";
import getHandiworkInformation from "./scrape-data/handiwork-parse";
import { HandiworkSearchQuery, LatestSearchQuery, ThreadSearchQuery } from "..";
import fetchHandiworkURLs from "./fetch-data/fetch-handiwork";
import fetchLatestHandiworkURLs from "./fetch-data/fetch-latest";
import fetchThreadHandiworkURLs from "./fetch-data/fetch-thread";
/**
* Gets the handiworks that match the passed parameters.
*
* You *must* be logged.
* @param {Number} limit
* Maximum number of items to get. Default: 30
*/
export default async function search<T extends IBasic>(
query: IQuery,
limit: number = 30
): Promise<T[]> {
// Fetch the URLs
const urls: string[] = await getURLsFromQuery(query, limit);
// Fetch the data
const results = urls.map((url) => getHandiworkInformation<T>(url));
return Promise.all(results);
}
//#region Private methods
/**
* @param query Query used for the search
* @param limit Maximum number of items to get. Default: 30
* @returns URLs of the fetched games
*/
async function getURLsFromQuery(query: IQuery, limit = 30): Promise<string[]> {
switch (query.itype) {
case "HandiworkSearchQuery":
return fetchHandiworkURLs(query as HandiworkSearchQuery, limit);
case "LatestSearchQuery":
return fetchLatestHandiworkURLs(query as LatestSearchQuery, limit);
case "ThreadSearchQuery":
return fetchThreadHandiworkURLs(query as ThreadSearchQuery, limit);
default:
throw Error(`Invalid query type: ${query.itype}`);
}
}
//#endregion Private methods

View File

@ -1,83 +0,0 @@
// Copyright (c) 2021 MillenniumEarl
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
/* istanbul ignore file */
"use strict";
// Core modules
import { tmpdir } from "os";
import { join } from "path";
// Public modules from npm
import log4js from "log4js";
// Modules from file
import Session from "./classes/session";
// Types declaration
export type TPrefixDict = { [n: number]: string };
type TPrefixKey = "engines" | "statuses" | "tags" | "others";
type TPrefixes = { [key in TPrefixKey]: TPrefixDict };
/**
* Class containing variables shared between modules.
*/
export default abstract class Shared {
//#region Fields
private static _isLogged = false;
private static _prefixes: TPrefixes = {} as TPrefixes;
private static _logger: log4js.Logger = log4js.getLogger();
private static _session = new Session(join(tmpdir(), "f95session.json"));
//#endregion Fields
//#region Getters
/**
* Indicates whether a user is logged in to the F95Zone platform or not.
*/
static get isLogged(): boolean {
return this._isLogged;
}
/**
* List of platform prefixes and tags.
*/
static get prefixes(): { [s: string]: TPrefixDict } {
return this._prefixes;
}
/**
* Logger object used to write to both file and console.
*/
static get logger(): log4js.Logger {
return this._logger;
}
/**
* Path to the cache used by this module wich contains engines, statuses, tags...
*/
static get cachePath(): string {
return join(tmpdir(), "f95cache.json");
}
/**
* Session on the F95Zone platform.
*/
static get session(): Session {
return this._session;
}
//#endregion Getters
//#region Setters
static setPrefixPair(key: TPrefixKey, val: TPrefixDict): void {
this._prefixes[key] = val;
}
static setIsLogged(val: boolean): void {
this._isLogged = val;
}
//#endregion Setters
}

View File

@ -1,50 +0,0 @@
"use strict";
// Public module from npm
import { expect } from "chai";
// Modules from file
import Credentials from "../../src/scripts/classes/credentials";
export function suite(): void {
it("Check token formatting", async function testValidToken() {
// Token example:
// 1604309951,0338213c00fcbd894fd9415e6ba08403
// 1604309986,ebdb75502337699381f0f55c86353555
// 1604310008,2d50d55808e5ec3a157ec01953da9d26
// Fetch token (is a GET request, we don't need the credentials)
const cred = new Credentials(null, null);
await cred.fetchToken();
// Parse token for assert
const splitted = cred.token.split(",");
const unique = splitted[0];
const hash = splitted[1];
expect(splitted.length).to.be.equal(2, "The token consists of two parts");
// Check type of parts
expect(isNumeric(unique)).to.be.true;
expect(isNumeric(hash)).to.be.false;
// The second part is most probably the MD5 hash of something
expect(hash.length).to.be.equal(32, "Hash should have 32 hex chars");
});
}
//#region Private methods
/**
* Check if a string is a number.
* @author Jeremy
* @see https://preview.tinyurl.com/y46jqwkt
*/
function isNumeric(num: any): boolean {
const isNan = isNaN(num as number);
const isNum = typeof num === "number";
const isValidString = typeof num === "string" && num.trim() !== "";
return (isNum || isValidString) && !isNan;
}
//#endregion

View File

@ -1,45 +0,0 @@
"use strict";
// Public module from npm
import chai from "chai";
import chaiAsPromised from "chai-as-promised";
import { INVALID_USER_ID, USER_NOT_LOGGED } from "../../../src/scripts/classes/errors";
// Module from files
import { PlatformUser } from "../../../src";
import Shared from "../../../src/scripts/shared";
chai.use(chaiAsPromised);
const { expect } = chai;
export function suite(): void {
it("Set invalid ID", function setInvalidID() {
const user = new PlatformUser();
expect(user.setID(-1)).to.be.rejectedWith(INVALID_USER_ID);
expect(user.setID(null)).to.be.rejectedWith(INVALID_USER_ID);
});
it("Fetch platform user without ID", async function fetchWithoutID() {
Shared.setIsLogged(true);
const user = new PlatformUser();
await expect(user.fetch()).to.be.rejectedWith(INVALID_USER_ID);
});
it("Fetch platform user with null ID", async function fetchWithNullID() {
Shared.setIsLogged(true);
const user = new PlatformUser(null);
await expect(user.fetch()).to.be.rejectedWith(INVALID_USER_ID);
});
it("Fetch platform user with invalid ID", async function fetchWithInvalidID() {
Shared.setIsLogged(true);
const user = new PlatformUser(-1);
await expect(user.fetch()).to.be.rejectedWith(INVALID_USER_ID);
});
it("Fetch platform user without authentication", async function fetchWithoutAuth() {
Shared.setIsLogged(false);
const user = new PlatformUser(1234);
await expect(user.fetch()).to.be.rejectedWith(USER_NOT_LOGGED);
});
}

View File

@ -1,33 +0,0 @@
"use strict";
// Public module from npm
import chai from "chai";
import chaiAsPromised from "chai-as-promised";
import { INVALID_POST_ID, USER_NOT_LOGGED } from "../../../src/scripts/classes/errors";
// Module from files
import { Post } from "../../../src";
import Shared from "../../../src/scripts/shared";
chai.use(chaiAsPromised);
const { expect } = chai;
export function suite(): void {
it("Fetch post with null ID", async function fetchWithNullID() {
Shared.setIsLogged(true);
const post = new Post(null);
await expect(post.fetch()).to.be.rejectedWith(INVALID_POST_ID);
});
it("Fetch post with invalid ID", async function fetchWithInvalidID() {
Shared.setIsLogged(true);
const post = new Post(-1);
await expect(post.fetch()).to.be.rejectedWith(INVALID_POST_ID);
});
it("Fetch post without authentication", async function fetchWithoutAuth() {
Shared.setIsLogged(false);
const post = new Post(1234);
await expect(post.fetch()).to.be.rejectedWith(USER_NOT_LOGGED);
});
}

View File

@ -1,39 +0,0 @@
"use strict";
// Public module from npm
import chai from "chai";
import chaiAsPromised from "chai-as-promised";
import { INVALID_THREAD_ID, USER_NOT_LOGGED } from "../../../src/scripts/classes/errors";
// Module from files
import { Thread } from "../../../src";
import Shared from "../../../src/scripts/shared";
chai.use(chaiAsPromised);
const { expect } = chai;
export function suite(): void {
it("Fetch thread with invalid ID", async function fetchWithInvalidID() {
Shared.setIsLogged(true);
const thread = new Thread(-1);
await expect(thread.fetch()).to.be.rejectedWith(INVALID_THREAD_ID);
});
it("Fetch thread with null ID", async function fetchWithNullID() {
Shared.setIsLogged(true);
const thread = new Thread(null);
await expect(thread.fetch()).to.be.rejectedWith(INVALID_THREAD_ID);
});
it("Fetch thread without authentication", async function fetchWithoutAuth() {
Shared.setIsLogged(false);
const thread = new Thread(1234);
await expect(thread.fetch()).to.be.rejectedWith(USER_NOT_LOGGED);
});
it("Fetch post with invalid ID", async function fetchWithInvalidID() {
Shared.setIsLogged(true);
const thread = new Thread(-1);
await expect(thread.getPost(0)).to.be.rejectedWith("Index must be greater or equal than 1");
});
}

View File

@ -1,21 +0,0 @@
"use strict";
// Public module from npm
import chai from "chai";
import chaiAsPromised from "chai-as-promised";
import { INVALID_USER_ID, USER_NOT_LOGGED } from "../../../src/scripts/classes/errors";
// Module from files
import { UserProfile } from "../../../src";
import Shared from "../../../src/scripts/shared";
chai.use(chaiAsPromised);
const { expect } = chai;
export function suite(): void {
it("Fetch profile without authentication", async function fetchWithoutAuth() {
Shared.setIsLogged(false);
const up = new UserProfile();
await expect(up.fetch()).to.be.rejectedWith(USER_NOT_LOGGED);
});
}

View File

@ -1,42 +0,0 @@
"use strict";
// Public module from npm
import { expect } from "chai";
import dotenv from "dotenv";
// Modules from file
import { login, PrefixParser } from "../../src/index";
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
export function suite(): void {
//#region Setup
before(async function beforeAll() {
await login(USERNAME, PASSWORD);
});
//#endregion Setup
it("Parse prefixes", async function testPrefixParser() {
// Create a new parser
const parser = new PrefixParser();
// Test values
const testIDs = [103, 225, 44, 13, 2, 7, 22];
const testPrefixes = ["corruption", "pregnancy", "slave", "VN", "RPGM", "Ren'Py", "Abandoned"];
// Parse values
const ids = parser.prefixesToIDs(testPrefixes);
const tags = parser.idsToPrefixes(ids);
// Assert equality
expect(testPrefixes).to.be.deep.equal(tags, "The tags must be the same");
expect(testIDs).to.be.deep.equal(ids, "The IDs must be the same");
});
}

View File

@ -1,38 +0,0 @@
"use strict";
// Public modules from npm
import dotenv from "dotenv";
import inquirer from "inquirer";
// Modulee from files
import { login } from "../src/index";
import LoginResult from "../src/scripts/classes/login-result";
// Configure the .env reader
dotenv.config();
export async function auth(): Promise<LoginResult> {
return login(process.env.F95_USERNAME, process.env.F95_PASSWORD, insert2faCode);
}
//#region Private methods
/**
* Ask the user to enter the OTP code
* necessary to authenticate on the server.
*/
async function insert2faCode(): Promise<number> {
const questions = [
{
type: "input",
name: "code",
message: "Insert 2FA code:"
}
];
// Prompt the user to insert the code
const answers = await inquirer.prompt(questions);
return answers.code as number;
}
//#endregion Private methods

40
test/index-test.js Normal file
View File

@ -0,0 +1,40 @@
"use strict";
// Test suite
const api = require("./suites/api-test.js").suite;
const credentials = require("./suites/credentials-test.js").suite;
const network = require("./suites/network-helper-test.js").suite;
const platform = require("./suites/platform-data-test.js").suite;
const scraper = require("./suites/scraper-test.js").suite;
const searcher = require("./suites/searcher-test.js").suite;
const uScraper = require("./suites/user-scraper-test.js").suite;
const prefixParser = require("./suites/prefix-parser-test.js").suite;
describe("Test basic function", function testBasic() {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
describe("Test credentials class", credentials.bind(this));
describe("Test network helper", network.bind(this));
describe("Test prefix parser", prefixParser.bind(this));
});
describe("Test F95 modules", function testF95Modules() {
//#region Set-up
this.timeout(15000); // All tests in this suite get 15 seconds before timeout
//#endregion Set-up
describe("Test platform data fetch", platform.bind(this));
describe("Test scraper methods", scraper.bind(this));
describe("Test searcher methods", searcher.bind(this));
describe("Test user scraper methods", uScraper.bind(this));
});
describe("Test complete API", function testAPI() {
//#region Set-up
this.timeout(15000); // All tests in this suite get 15 seconds before timeout
//#endregion Set-up
describe("Test API", api.bind(this));
});

View File

@ -1,25 +0,0 @@
"use strict";
// Import suites
import { suite as credentials } from "./classes/credentials";
import { suite as prefixParser } from "./classes/prefix-parser";
import { suite as platformUser } from "./classes/mapping/platform-user";
import { suite as post } from "./classes/mapping/post";
import { suite as thread } from "./classes/mapping/thread";
import { suite as userProfile } from "./classes/mapping/user-profile";
describe("Test basic function", function testBasic() {
//#region Set-up
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
//#endregion Set-up
// describe("Test network helper", network.bind(this));
describe("Test Credentials", credentials.bind(this));
describe("Test PrefixParser", prefixParser.bind(this));
describe("Test PlatformUser", platformUser.bind(this));
describe("Test Post", post.bind(this));
describe("Test Thread", thread.bind(this));
describe("Test UserProfile", userProfile.bind(this));
});

120
test/suites/api-test.js Normal file
View File

@ -0,0 +1,120 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
const {
isEqual
} = require("lodash");
// Modules from file
const F95API = require("../../app/index.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
// Global suite variables
const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/";
const updatedGameURL = "https://f95zone.to/threads/noxian-nights-v1-2-4-hreinn-games.2/";
it("Test login", async function testLogin() {
const result = await F95API.login(USERNAME, PASSWORD);
expect(result.success).to.be.true;
expect(F95API.isLogged()).to.be.true;
});
it("Test user data fetching", async function testUserDataFetch() {
const userdata = await F95API.getUserData();
expect(userdata.username).to.be.equal(USERNAME);
});
it("Test game for existing update", async function checkUpdateByURL() {
// We force the creation of a GameInfo object,
// knowing that the checkIfGameHasUpdate() function
// only needs the game URL
const info = new F95API.GameInfo();
// The gameURL identifies a game for which we know there is an update
info.url = gameURL;
// Check for updates
const update = await F95API.checkIfGameHasUpdate(info);
expect(update).to.be.true;
});
it("Test game for non existing update", async function checkUpdateByVersion() {
// We force the creation of a GameInfo object,
// knowing that the checkIfGameHasUpdate() function
// only needs the game URL
const info = new F95API.GameInfo();
// The updatedGameURL identifies a game for which
// we know there is **not** an update
info.url = updatedGameURL;
info.version = "1.2.4"; // The hame is marked as "Completed" so it shouldn't change it's version
// Check for updates
const update = await F95API.checkIfGameHasUpdate(info);
expect(update).to.be.false;
});
it("Test game for fake update", async function checkFakeUpdateByVersion() {
// We force the creation of a GameInfo object,
// knowing that the checkIfGameHasUpdate() function
// only needs the game URL
const info = new F95API.GameInfo();
// The updatedGameURL identifies a game for which
// we know there is **not** an update
info.url = updatedGameURL;
info.version = "ThisIsAFakeVersion"; // The real version is "1.2.4"
// Check for updates
const update = await F95API.checkIfGameHasUpdate(info);
expect(update).to.be.true;
});
it("Test game data fetching", async function testGameDataFetch() {
// Search a game by name
const gameList = await F95API.getGameData("perverted education", false);
// We know that there is only one game with the selected name
expect(gameList.length).to.be.equal(1, `There should be only one game, not ${gameList.length}`);
const game = gameList[0];
// Than we fetch a game from URL
const gameFromURL = await F95API.getGameDataFromURL(game.url);
// The two games must be equal
const equal = isEqual(game, gameFromURL);
expect(equal).to.be.true;
});
it("Test latest games fetching", async function testLatestFetch() {
// Prepare a search query
const query = {
datelimit: 0,
tags: ["male protagonist", "3dcg"],
prefixes: ["Completed", "Unity"],
sorting: "views",
};
// TODO
// First test the parameters validation
// assert.throws(() => { F95API.getLatestUpdates(query, 0); },
// Error,
// "Error thrown if limit is <= 0");
// Now we fetch certain games that are "stables" as per 2020
const LIMIT = 3;
const result = await F95API.getLatestUpdates(query, LIMIT);
expect(result[0].id).to.be.equal(3691, "The game should be: 'Man of the house'");
expect(result[1].id).to.be.equal(5483, "The game should be: 'Lucky mark'");
expect(result[2].id).to.be.equal(5949, "The game should be: 'Timestamps, Unconditional Love'");
});
};

View File

@ -0,0 +1,52 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
module.exports.suite = function suite() {
it("Check token formatting", async function testValidToken() {
// Token example:
// 1604309951,0338213c00fcbd894fd9415e6ba08403
// 1604309986,ebdb75502337699381f0f55c86353555
// 1604310008,2d50d55808e5ec3a157ec01953da9d26
// Fetch token (is a GET request, we don't need the credentials)
const cred = new Credentials(null, null);
await cred.fetchToken();
// Parse token for assert
const splitted = cred.token.split(",");
const unique = splitted[0];
const hash = splitted[1];
expect(splitted.length).to.be.equal(2, "The token consists of two parts");
// Check type of parts
expect(isNumeric(unique)).to.be.true;
expect(isNumeric(hash)).to.be.false;
// The second part is most probably the MD5 hash of something
expect(hash.length).to.be.equal(32, "Hash should have 32 hex chars");
});
};
//#region Private methods
/**
* @private
* Check if a string is a number
* @param {String} str
* @author Dan, Ben Aston
* @see https://preview.tinyurl.com/y46jqwkt
*/
function isNumeric(str) {
// We only process strings!
if (typeof str != "string") return false;
// Use type coercion to parse the _entirety_ of the string
// (`parseFloat` alone does not do this) and ensure strings
// of whitespace fail
return !isNaN(str) && !isNaN(parseFloat(str));
}
//#endregion

View File

@ -0,0 +1,107 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
const networkHelper = require("../../app/scripts/network-helper.js");
const {
F95_SEARCH_URL
} = require("../../app/scripts/constants/url.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
const FAKE_USERNAME = "Fake_Username091276";
const FAKE_PASSWORD = "fake_password";
module.exports.suite = function suite() {
// Global suite variables
const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/";
it("Check if URL exists", async function checkURLExistence() {
// Check generic URLs...
let exists = await networkHelper.urlExists("https://www.google.com/");
expect(exists, "Complete valid URL").to.be.true;
exists = await networkHelper.urlExists("www.google.com");
expect(exists, "URl without protocol prefix").to.be.false;
exists = await networkHelper.urlExists("https://www.google/");
expect(exists, "URL without third level domain").to.be.false;
// Now check for more specific URLs (with redirect)...
exists = await networkHelper.urlExists(gameURL);
expect(exists, "URL with redirect without check").to.be.true;
exists = await networkHelper.urlExists(gameURL, true);
expect(exists, "URL with redirect with check").to.be.false;
});
it("Check if URL belong to the platform", function checkIfURLIsF95() {
let belong = networkHelper.isF95URL(gameURL);
expect(belong).to.be.true;
belong = networkHelper.isF95URL("https://www.google/");
expect(belong).to.be.false;
});
it("Enforce secure URLs", function testSecureURLEnforcement() {
// This URL is already secure, should remain the same
let enforced = networkHelper.enforceHttpsUrl(gameURL);
expect(enforced).to.be.equal(gameURL, "The game URL is already secure");
// This URL is not secure
enforced = networkHelper.enforceHttpsUrl("http://www.google.com");
expect(enforced).to.be.equal("https://www.google.com", "The URL was without SSL/TLS (HTTPs)");
// Finally, we check when we pass a invalid URL
enforced = networkHelper.enforceHttpsUrl("http://invalidurl");
expect(enforced).to.be.null;
});
it("Check URL redirect", async function checkURLRedirect() {
// gameURL is an old URL it has been verified that it generates a redirect
const redirectURL = await networkHelper.getUrlRedirect(gameURL);
expect(redirectURL).to.not.be.equal(gameURL, "The original URL has redirect");
// If we recheck the new URL, we find that no redirect happens
const secondRedirectURL = await networkHelper.getUrlRedirect(redirectURL);
expect(secondRedirectURL).to.be.equal(redirectURL, "The URL has no redirect");
});
it("Check response to GET request", async function testGETResponse() {
// We should be able to fetch a game page
let response = await networkHelper.fetchGETResponse(gameURL);
expect(response.status).to.be.equal(200, "The operation must be successful");
// We should NOT be able to fetch the search page (we must be logged)
response = await networkHelper.fetchGETResponse(F95_SEARCH_URL);
expect(response).to.be.null;
});
it("Test for authentication to platform", async function testAuthentication() {
// Try to authenticate with valid credentials
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
const validResult = await networkHelper.authenticate(creds);
expect(validResult.success).to.be.true;
// Now we use fake credentials
const fakeCreds = new Credentials(FAKE_USERNAME, FAKE_PASSWORD);
await fakeCreds.fetchToken();
const invalidResult = await networkHelper.authenticate(fakeCreds, true);
expect(invalidResult.success).to.be.false;
});
it("Test fetching HTML", async function testFetchHTML() {
// This should return the HTML code of the page
const html = await networkHelper.fetchHTML(gameURL);
expect(html.startsWith("<!DOCTYPE html>")).to.be.true;
});
};

View File

@ -0,0 +1,65 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
const { isEqual } = require("lodash");
// Core modules
const fs = require("fs");
// Modules from file
const shared = require("../../app/scripts/shared.js");
const platform = require("../../app/scripts/platform-data.js");
const Credentials = require("../../app/scripts/classes/credentials.js");
const { authenticate } = require("../../app/scripts/network-helper.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
//#region Setup
before(async function beforeAll() {
// Authenticate
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
await authenticate(creds);
});
//#endregion Setup
it("Fetch new platform data", async function fetchNewPlatformData() {
// Delete the current platform data (if exists)
if(fs.existsSync(shared.cachePath)) fs.unlinkSync(shared.cachePath);
// Fetch data
await platform.fetchPlatformData();
// Check data
const enginesEquality = isEqual({}, shared.engines);
const statusEquality = isEqual({}, shared.statuses);
const tagsEquality = isEqual({}, shared.tags);
expect(enginesEquality, "Should not be empty").to.be.false;
expect(statusEquality, "Should not be empty").to.be.false;
expect(tagsEquality, "Should not be empty").to.be.false;
// Check if the file exists
expect(fs.existsSync(shared.cachePath)).to.be.true;
});
it("Fetch cached platform data", async function fetchCachedPlatformData() {
// Fetch data
await platform.fetchPlatformData();
// Check data
const enginesEquality = isEqual({}, shared.engines);
const statusEquality = isEqual({}, shared.statuses);
const tagsEquality = isEqual({}, shared.tags);
expect(enginesEquality, "Should not be empty").to.be.false;
expect(statusEquality, "Should not be empty").to.be.false;
expect(tagsEquality, "Should not be empty").to.be.false;
});
};

View File

@ -0,0 +1,45 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
const { isEqual } = require("lodash");
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
const PrefixParser = require("../../app/scripts/classes/prefix-parser.js");
const { authenticate } = require("../../app/scripts/network-helper.js");
const { fetchPlatformData } = require("../../app/scripts/platform-data.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
//#region Setup
before(async function beforeAll() {
// Authenticate
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
await authenticate(creds);
await fetchPlatformData();
});
//#endregion Setup
it("Parse prefixes", async function testPrefixParser() {
// Create a new parser
const parser = new PrefixParser();
const testPrefixes = ["corruption", "pregnancy", "slave", "VN", "RPGM", "Ren'Py", "Abandoned"];
const ids = parser.prefixesToIDs(testPrefixes);
const tags = parser.idsToPrefixes(ids);
const tagsEquality = isEqual(testPrefixes, tags);
expect(tagsEquality, "The tags must be the same").to.be.true;
const idsEquality = isEqual([103, 225, 44, 13, 2, 7, 22], ids);
expect(idsEquality, "The IDs must be the same").to.be.true;
});
};

View File

@ -0,0 +1,56 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const { isEqual } = require("lodash");
const GameInfo = require("../../app/scripts/classes/game-info.js");
// Modules from file
const scraper = require("../../app/scripts/scraper.js");
module.exports.suite = function suite() {
// Global suite variables
const gameURL = "https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/";
const modURL = "https://f95zone.to/threads/witch-trainer-silver-mod-v1-39-silver-studio-games.1697/";
const previewSrc = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png";
const modPreviewSrc = "https://attachments.f95zone.to/2018/10/178357_banner.png";
it("Search game", async function () {
// This test depend on the data on F95Zone at gameURL
const result = await scraper.getGameInfo(gameURL);
// Test only the main information
expect(result.name).to.equal("Kingdom of Deception");
expect(result.author).to.equal("Hreinn Games");
expect(result.isMod, "Should be false").to.be.false;
expect(result.engine).to.equal("Ren'Py");
expect(result.previewSrc).to.equal(previewSrc, "Preview not equals");
});
it("Search mod", async function () {
// This test depend on the data on F95Zone at modURL
const result = await scraper.getGameInfo(modURL);
// Test only the main information
expect(result.name).to.equal("Witch Trainer: Silver Mod");
expect(result.author).to.equal("Silver Studio Games");
expect(result.isMod, "Should be true").to.be.true;
expect(result.engine).to.equal("Ren'Py");
expect(result.previewSrc).to.equal(modPreviewSrc, "Preview not equals");
});
it("Test game serialization", async function testGameSerialization() {
// This test depend on the data on F95Zone at gameURL
const testGame = await scraper.getGameInfo(gameURL);
// Serialize...
const json = JSON.stringify(testGame);
// Deserialize...
const parsedGameInfo = GameInfo.fromJSON(json);
// Compare with lodash
const result = isEqual(parsedGameInfo, testGame);
expect(result).to.be.true;
});
};

View File

@ -0,0 +1,65 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
const searcher = require("../../app/scripts/searcher.js");
const { authenticate } = require("../../app/scripts/network-helper.js");
const { fetchPlatformData } = require("../../app/scripts/platform-data.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
// TODO:
// This method should delete the store F95Zone cookies,
// but what if the other tests require them?
// it("Search game when not logged", async function searchGameWhenNotLogged() {
// // Search for a game that we know has only one result
// // but without logging in first
// const urls = await searcher.searchGame("kingdom of deception");
// expect(urls.lenght).to.be.equal(0, "There should not be any URL");
// });
it("Search game", async function searchGame() {
// Authenticate
const result = await auth();
expect(result.success, "Authentication should be successful").to.be.true;
// Search for a game that we know has only one result
const urls = await searcher.searchGame("kingdom of deception");
expect(urls.length).to.be.equal(1, `There should be only one game result instead of ${urls.length}`);
});
it("Search mod", async function searchMod() {
// Authenticate
const result = await auth();
expect(result.success, "Authentication should be successful").to.be.true;
// Search for a mod that we know has only one result
const urls = await searcher.searchMod("kingdom of deception jdmod");
expect(urls.length).to.be.equal(1, `There should be only one mod result instead of ${urls.length}`);
});
};
//#region Private methods
/**
* @private
* Simple wrapper for authentication.
*/
async function auth() {
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
const result = await authenticate(creds);
if (result.success) await fetchPlatformData();
return result;
}
//#endregion Private methods

View File

@ -0,0 +1,55 @@
"use strict";
// Public module from npm
const expect = require("chai").expect;
const dotenv = require("dotenv");
// Modules from file
const Credentials = require("../../app/scripts/classes/credentials.js");
const uScraper = require("../../app/scripts/user-scraper.js");
const { authenticate } = require("../../app/scripts/network-helper.js");
const { fetchPlatformData } = require("../../app/scripts/platform-data.js");
// Configure the .env reader
dotenv.config();
// Global variables
const USERNAME = process.env.F95_USERNAME;
const PASSWORD = process.env.F95_PASSWORD;
module.exports.suite = function suite() {
// TODO:
// This method should delete the store F95Zone cookies,
// but what if the other tests require them?
// it("Fetch data when not logged", async function fetchUserDataWhenLogged() {
// const data = await uScraper.getUserData();
// expect(data.username).to.be.equal("");
// expect(data.avatarSrc).to.be.equal("");
// expect(data.watchedThreads.length).to.be.equal(0);
// });
it("Fetch data when logged", async function fetchUserDataWhenNotLogged() {
// Authenticate
const result = await auth();
expect(result.success, "Authentication should be successful").to.be.true;
// We test only for the username, the other test data depends on the user logged
const data = await uScraper.getUserData();
expect(data.username).to.be.equal(USERNAME);
});
};
//#region Private methods
/**
* @private
* Simple wrapper for authentication.
*/
async function auth() {
const creds = new Credentials(USERNAME, PASSWORD);
await creds.fetchToken();
const result = await authenticate(creds);
if (result.success) await fetchPlatformData();
return result;
}
//#endregion Private methods

View File

@ -1,25 +0,0 @@
{
"compileOnSave": true,
"compilerOptions": {
"experimentalDecorators": true,
"outDir": "./dist",
"allowJs": true,
"module": "commonjs",
"target": "es5",
"moduleResolution": "node",
"esModuleInterop": true,
"allowSyntheticDefaultImports": true,
"incremental": true,
"sourceMap": true,
"alwaysStrict": true,
"declaration": true,
},
"include": [
"./src/**/*.ts"
],
"exclude": [
"./tests/",
"./node_modules/",
"./dist/"
]
}