Resolved merge conflicts
MillenniumEarl
commit
42f8de6c6b
96
app/index.js
96
app/index.js
|
|
@ -1,28 +1,28 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Core modules
|
||||
const fs = require('fs');
|
||||
const fs = require("fs");
|
||||
|
||||
// Modules from file
|
||||
const shared = require('./scripts/shared.js');
|
||||
const constURLs = require('./scripts/constants/urls.js');
|
||||
const selectors = require('./scripts/constants/css-selectors.js');
|
||||
const shared = require("./scripts/shared.js");
|
||||
const constURLs = require("./scripts/constants/urls.js");
|
||||
const selectors = require("./scripts/constants/css-selectors.js");
|
||||
const {
|
||||
isStringAValidURL,
|
||||
urlExists,
|
||||
isF95URL
|
||||
} = require('./scripts/urls-helper.js');
|
||||
const scraper = require('./scripts/game-scraper.js');
|
||||
isF95URL,
|
||||
} = require("./scripts/urls-helper.js");
|
||||
const scraper = require("./scripts/game-scraper.js");
|
||||
const {
|
||||
prepareBrowser,
|
||||
preparePage
|
||||
} = require('./scripts/puppeteer-helper.js');
|
||||
const searcher = require('./scripts/game-searcher.js');
|
||||
preparePage,
|
||||
} = require("./scripts/puppeteer-helper.js");
|
||||
const searcher = require("./scripts/game-searcher.js");
|
||||
|
||||
// Classes from file
|
||||
const GameInfo = require('./scripts/classes/game-info.js');
|
||||
const LoginResult = require('./scripts/classes/login-result.js');
|
||||
const UserData = require('./scripts/classes/user-data.js');
|
||||
const GameInfo = require("./scripts/classes/game-info.js");
|
||||
const LoginResult = require("./scripts/classes/login-result.js");
|
||||
const UserData = require("./scripts/classes/user-data.js");
|
||||
|
||||
//#region Export classes
|
||||
module.exports.GameInfo = GameInfo;
|
||||
|
|
@ -99,27 +99,27 @@ var _browser = null;
|
|||
*/
|
||||
module.exports.login = async function (username, password) {
|
||||
if (shared.isLogged) {
|
||||
if (shared.debug) console.log('Already logged in');
|
||||
if (shared.debug) console.log("Already logged in");
|
||||
const result = new LoginResult();
|
||||
result.success = true;
|
||||
result.message = 'Already logged in';
|
||||
result.message = "Already logged in";
|
||||
return result;
|
||||
}
|
||||
|
||||
// If cookies are loaded, use them to authenticate
|
||||
shared.cookies = loadCookies();
|
||||
if (shared.cookies !== null) {
|
||||
if (shared.debug) console.log('Valid session, no need to re-authenticate');
|
||||
if (shared.debug) console.log("Valid session, no need to re-authenticate");
|
||||
shared.isLogged = true;
|
||||
const result = new LoginResult();
|
||||
result.success = true;
|
||||
result.message = 'Logged with cookies';
|
||||
result.message = "Logged with cookies";
|
||||
return result;
|
||||
}
|
||||
|
||||
// Else, log in throught browser
|
||||
if (shared.debug)
|
||||
console.log('No saved sessions or expired session, login on the platform');
|
||||
console.log("No saved sessions or expired session, login on the platform");
|
||||
|
||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
||||
|
|
@ -130,9 +130,9 @@ module.exports.login = async function (username, password) {
|
|||
if (result.success) {
|
||||
// Reload cookies
|
||||
shared.cookies = loadCookies();
|
||||
if (shared.debug) console.log('User logged in through the platform');
|
||||
if (shared.debug) console.log("User logged in through the platform");
|
||||
} else {
|
||||
console.warn('Error during authentication: ' + result.message);
|
||||
console.warn("Error during authentication: " + result.message);
|
||||
}
|
||||
if (shared.isolation) await browser.close();
|
||||
return result;
|
||||
|
|
@ -146,11 +146,11 @@ module.exports.login = async function (username, password) {
|
|||
*/
|
||||
module.exports.loadF95BaseData = async function () {
|
||||
if (!shared.isLogged || !shared.cookies) {
|
||||
console.warn('User not authenticated, unable to continue');
|
||||
console.warn("User not authenticated, unable to continue");
|
||||
return false;
|
||||
}
|
||||
|
||||
if (shared.debug) console.log('Loading base data...');
|
||||
if (shared.debug) console.log("Loading base data...");
|
||||
|
||||
// Prepare a new web page
|
||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
||||
|
|
@ -161,7 +161,7 @@ module.exports.loadF95BaseData = async function () {
|
|||
|
||||
// Go to latest update page and wait for it to load
|
||||
await page.goto(constURLs.F95_LATEST_UPDATES, {
|
||||
waitUntil: shared.WAIT_STATEMENT
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
});
|
||||
|
||||
// Obtain engines (disc/online)
|
||||
|
|
@ -170,7 +170,7 @@ module.exports.loadF95BaseData = async function () {
|
|||
page,
|
||||
shared.enginesCachePath,
|
||||
selectors.ENGINE_ID_SELECTOR,
|
||||
'engines'
|
||||
"engines"
|
||||
);
|
||||
|
||||
// Obtain statuses (disc/online)
|
||||
|
|
@ -179,11 +179,11 @@ module.exports.loadF95BaseData = async function () {
|
|||
page,
|
||||
shared.statusesCachePath,
|
||||
selectors.STATUS_ID_SELECTOR,
|
||||
'statuses'
|
||||
"statuses"
|
||||
);
|
||||
|
||||
if (shared.isolation) await browser.close();
|
||||
if (shared.debug) console.log('Base data loaded');
|
||||
if (shared.debug) console.log("Base data loaded");
|
||||
return true;
|
||||
};
|
||||
/**
|
||||
|
|
@ -195,7 +195,7 @@ module.exports.loadF95BaseData = async function () {
|
|||
*/
|
||||
module.exports.chekIfGameHasUpdate = async function (info) {
|
||||
if (!shared.isLogged || !shared.cookies) {
|
||||
console.warn('user not authenticated, unable to continue');
|
||||
console.warn("user not authenticated, unable to continue");
|
||||
return info.version;
|
||||
}
|
||||
|
||||
|
|
@ -225,7 +225,7 @@ module.exports.chekIfGameHasUpdate = async function (info) {
|
|||
*/
|
||||
module.exports.getGameData = async function (name, includeMods) {
|
||||
if (!shared.isLogged || !shared.cookies) {
|
||||
console.warn('user not authenticated, unable to continue');
|
||||
console.warn("user not authenticated, unable to continue");
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -262,13 +262,13 @@ module.exports.getGameData = async function (name, includeMods) {
|
|||
*/
|
||||
module.exports.getGameDataFromURL = async function (url) {
|
||||
if (!shared.isLogged || !shared.cookies) {
|
||||
console.warn('user not authenticated, unable to continue');
|
||||
console.warn("user not authenticated, unable to continue");
|
||||
return null;
|
||||
}
|
||||
|
||||
// Check URL
|
||||
if (!urlExists(url)) return null;
|
||||
if (!isF95URL(url)) throw new Error(url + ' is not a valid F95Zone URL');
|
||||
if (!isF95URL(url)) throw new Error(url + " is not a valid F95Zone URL");
|
||||
|
||||
// Gets the search results of the game being searched for
|
||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
||||
|
|
@ -288,7 +288,7 @@ module.exports.getGameDataFromURL = async function (url) {
|
|||
*/
|
||||
module.exports.getUserData = async function () {
|
||||
if (!shared.isLogged || !shared.cookies) {
|
||||
console.warn('user not authenticated, unable to continue');
|
||||
console.warn("user not authenticated, unable to continue");
|
||||
return null;
|
||||
}
|
||||
|
||||
|
|
@ -313,7 +313,7 @@ module.exports.getUserData = async function () {
|
|||
|
||||
const avatarSrc = await page.evaluate(
|
||||
/* istanbul ignore next */ (selector) =>
|
||||
document.querySelector(selector).getAttribute('src'),
|
||||
document.querySelector(selector).getAttribute("src"),
|
||||
selectors.AVATAR_PIC
|
||||
);
|
||||
|
||||
|
|
@ -334,7 +334,7 @@ module.exports.getUserData = async function () {
|
|||
*/
|
||||
module.exports.logout = async function () {
|
||||
if (!shared.isLogged || !shared.cookies) {
|
||||
console.warn('user not authenticated, unable to continue');
|
||||
console.warn("user not authenticated, unable to continue");
|
||||
return;
|
||||
}
|
||||
shared.isLogged = false;
|
||||
|
|
@ -385,14 +385,14 @@ function isCookieExpired(cookie) {
|
|||
// Ignore cookies that never expire
|
||||
const expirationUnixTimestamp = cookie.expire;
|
||||
|
||||
if (expirationUnixTimestamp !== '-1') {
|
||||
if (expirationUnixTimestamp !== "-1") {
|
||||
// Convert UNIX epoch timestamp to normal Date
|
||||
const expirationDate = new Date(expirationUnixTimestamp * 1000);
|
||||
|
||||
if (expirationDate < Date.now()) {
|
||||
if (shared.debug)
|
||||
console.log(
|
||||
'Cookie ' + cookie.name + ' expired, you need to re-authenticate'
|
||||
"Cookie " + cookie.name + " expired, you need to re-authenticate"
|
||||
);
|
||||
expiredCookies = true;
|
||||
}
|
||||
|
|
@ -421,7 +421,7 @@ async function loadValuesFromLatestPage(
|
|||
elementRequested
|
||||
) {
|
||||
// If the values already exist they are loaded from disk without having to connect to F95
|
||||
if (shared.debug) console.log('Load ' + elementRequested + ' from disk...');
|
||||
if (shared.debug) console.log("Load " + elementRequested + " from disk...");
|
||||
if (fs.existsSync(path)) {
|
||||
const valueJSON = fs.readFileSync(path);
|
||||
return JSON.parse(valueJSON);
|
||||
|
|
@ -429,11 +429,11 @@ async function loadValuesFromLatestPage(
|
|||
|
||||
// Otherwise, connect and download the data from the portal
|
||||
if (shared.debug)
|
||||
console.log('No ' + elementRequested + ' cached, downloading...');
|
||||
console.log("No " + elementRequested + " cached, downloading...");
|
||||
const values = await getValuesFromLatestPage(
|
||||
page,
|
||||
selector,
|
||||
'Getting ' + elementRequested + ' from page'
|
||||
"Getting " + elementRequested + " from page"
|
||||
);
|
||||
fs.writeFileSync(path, JSON.stringify(values));
|
||||
return values;
|
||||
|
|
@ -489,8 +489,8 @@ async function loginF95(browser, username, password) {
|
|||
await Promise.all([
|
||||
page.click(selectors.LOGIN_BUTTON), // Click on the login button
|
||||
page.waitForNavigation({
|
||||
waitUntil: shared.WAIT_STATEMENT
|
||||
}) // Wait for page to load
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}), // Wait for page to load
|
||||
]);
|
||||
|
||||
// Prepare result
|
||||
|
|
@ -507,7 +507,7 @@ async function loginF95(browser, username, password) {
|
|||
if (result.success) {
|
||||
const c = await page.cookies();
|
||||
fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c));
|
||||
result.message = 'Authentication successful';
|
||||
result.message = "Authentication successful";
|
||||
} else if (
|
||||
// Obtain the error message
|
||||
await page.evaluate(
|
||||
|
|
@ -522,15 +522,15 @@ async function loginF95(browser, username, password) {
|
|||
selectors.LOGIN_MESSAGE_ERROR
|
||||
);
|
||||
|
||||
if (errorMessage === 'Incorrect password. Please try again.') {
|
||||
result.message = 'Incorrect password';
|
||||
if (errorMessage === "Incorrect password. Please try again.") {
|
||||
result.message = "Incorrect password";
|
||||
} else if (
|
||||
errorMessage ===
|
||||
'The requested user "' + username + '" could not be found.'
|
||||
) {
|
||||
result.message = 'Incorrect username';
|
||||
result.message = "Incorrect username";
|
||||
} else result.message = errorMessage;
|
||||
} else result.message = 'Unknown error';
|
||||
} else result.message = "Unknown error";
|
||||
|
||||
await page.close(); // Close the page
|
||||
return result;
|
||||
|
|
@ -557,7 +557,7 @@ async function getUserWatchedGameThreads(browser) {
|
|||
// Set the filters
|
||||
await page.evaluate(
|
||||
/* istanbul ignore next */ (selector) =>
|
||||
document.querySelector(selector).removeAttribute('checked'),
|
||||
document.querySelector(selector).removeAttribute("checked"),
|
||||
selectors.UNREAD_THREAD_CHECKBOX
|
||||
); // Also read the threads already read
|
||||
|
||||
|
|
@ -578,7 +578,7 @@ async function getUserWatchedGameThreads(browser) {
|
|||
handle
|
||||
);
|
||||
// If 'unread' is left, it will redirect to the last unread post
|
||||
const url = src.replace('/unread', '');
|
||||
const url = src.replace("/unread", "");
|
||||
urls.push(url);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
/* istanbul ignore file */
|
||||
|
||||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Core modules
|
||||
const fs = require('fs');
|
||||
const fs = require("fs");
|
||||
|
||||
// Public modules from npm
|
||||
// const { File } = require('megajs');
|
||||
|
||||
// Modules from file
|
||||
const { prepareBrowser, preparePage } = require('../puppeteer-helper.js');
|
||||
const shared = require('../shared.js');
|
||||
const { prepareBrowser, preparePage } = require("../puppeteer-helper.js");
|
||||
const shared = require("../shared.js");
|
||||
|
||||
class GameDownload {
|
||||
constructor() {
|
||||
|
|
@ -19,7 +19,7 @@ class GameDownload {
|
|||
* Platform that hosts game files
|
||||
* @type String
|
||||
*/
|
||||
this.hosting = '';
|
||||
this.hosting = "";
|
||||
/**
|
||||
* @public
|
||||
* Link to game files
|
||||
|
|
@ -43,9 +43,9 @@ class GameDownload {
|
|||
* @return {Promise<Boolean>} Result of the operation
|
||||
*/
|
||||
async download(path) {
|
||||
if (this.link.includes('mega.nz'))
|
||||
if (this.link.includes("mega.nz"))
|
||||
return await downloadMEGA(this.link, path);
|
||||
else if (this.link.includes('nopy.to'))
|
||||
else if (this.link.includes("nopy.to"))
|
||||
return await downloadNOPY(this.link, path);
|
||||
}
|
||||
}
|
||||
|
|
@ -57,13 +57,13 @@ async function downloadMEGA(url, savepath) {
|
|||
const page = await preparePage(browser);
|
||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||
await page.goto(url);
|
||||
await page.waitForSelector('a.host_link');
|
||||
await page.waitForSelector("a.host_link");
|
||||
|
||||
// Obtain the link for the unmasked page and click it
|
||||
const link = await page.$('a.host_link');
|
||||
const link = await page.$("a.host_link");
|
||||
await link.click();
|
||||
await page.goto(url, {
|
||||
waitUntil: shared.WAIT_STATEMENT
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Go to the game page and wait until it loads
|
||||
|
||||
// Obtain the URL after the redirect
|
||||
|
|
@ -84,22 +84,22 @@ async function downloadNOPY(url, savepath) {
|
|||
const browser = await prepareBrowser();
|
||||
const page = await preparePage(browser);
|
||||
await page.goto(url);
|
||||
await page.waitForSelector('#download');
|
||||
await page.waitForSelector("#download");
|
||||
|
||||
// Set the save path
|
||||
await page._client.send('Page.setDownloadBehavior', {
|
||||
behavior: 'allow',
|
||||
downloadPath: path.basename(path.dirname(savepath)) // It's a directory
|
||||
await page._client.send("Page.setDownloadBehavior", {
|
||||
behavior: "allow",
|
||||
downloadPath: path.basename(path.dirname(savepath)), // It's a directory
|
||||
});
|
||||
|
||||
// Obtain the download button and click it
|
||||
const downloadButton = await page.$('#download');
|
||||
const downloadButton = await page.$("#download");
|
||||
await downloadButton.click();
|
||||
|
||||
// Await for all the connections to close
|
||||
await page.waitForNavigation({
|
||||
waitUntil: 'networkidle0',
|
||||
timeout: 0 // Disable timeout
|
||||
waitUntil: "networkidle0",
|
||||
timeout: 0, // Disable timeout
|
||||
});
|
||||
|
||||
// Close browser and page
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
const UNKNOWN = 'Unknown';
|
||||
const UNKNOWN = "Unknown";
|
||||
|
||||
class GameInfo {
|
||||
constructor() {
|
||||
|
|
@ -103,7 +103,7 @@ class GameInfo {
|
|||
isMod: this.isMod,
|
||||
changelog: this.changelog,
|
||||
gameDir: this.gameDir,
|
||||
downloadInfo: this.downloadInfo
|
||||
downloadInfo: this.downloadInfo,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
/**
|
||||
* Object obtained in response to an attempt to login to the portal.
|
||||
|
|
@ -14,7 +14,7 @@ class LoginResult {
|
|||
* Login response message
|
||||
* @type String
|
||||
*/
|
||||
this.message = '';
|
||||
this.message = "";
|
||||
}
|
||||
}
|
||||
module.exports = LoginResult;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
/**
|
||||
* Class containing the data of the user currently connected to the F95Zone platform.
|
||||
|
|
@ -9,7 +9,7 @@ class UserData {
|
|||
* User username.
|
||||
* @type String
|
||||
*/
|
||||
this.username = '';
|
||||
this.username = "";
|
||||
/**
|
||||
* Path to the user's profile picture.
|
||||
* @type String
|
||||
|
|
|
|||
|
|
@ -1,32 +1,32 @@
|
|||
module.exports = Object.freeze({
|
||||
AVATAR_INFO: 'span.avatar',
|
||||
AVATAR_INFO: "span.avatar",
|
||||
AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]',
|
||||
ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span',
|
||||
FILTER_THREADS_BUTTON: 'button[class="button--primary button"]',
|
||||
GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]',
|
||||
GAME_TAGS: 'a.tagItem',
|
||||
GAME_TITLE: 'h1.p-title-value',
|
||||
GAME_TAGS: "a.tagItem",
|
||||
GAME_TITLE: "h1.p-title-value",
|
||||
GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]',
|
||||
LOGIN_BUTTON: 'button.button--icon--login',
|
||||
LOGIN_BUTTON: "button.button--icon--login",
|
||||
LOGIN_MESSAGE_ERROR:
|
||||
'div.blockMessage.blockMessage--error.blockMessage--iconic',
|
||||
"div.blockMessage.blockMessage--error.blockMessage--iconic",
|
||||
ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]',
|
||||
PASSWORD_INPUT: 'input[name="password"]',
|
||||
SEARCH_BUTTON: 'form.block > * button.button--icon--search',
|
||||
SEARCH_BUTTON: "form.block > * button.button--icon--search",
|
||||
SEARCH_FORM_TEXTBOX: 'input[name="keywords"]',
|
||||
STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span',
|
||||
THREAD_POSTS:
|
||||
'article.message-body:first-child > div.bbWrapper:first-of-type',
|
||||
THREAD_TITLE: 'h3.contentRow-title',
|
||||
"article.message-body:first-child > div.bbWrapper:first-of-type",
|
||||
THREAD_TITLE: "h3.contentRow-title",
|
||||
TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]',
|
||||
UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]',
|
||||
USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText',
|
||||
USERNAME_INPUT: 'input[name="login"]',
|
||||
WATCHED_THREAD_FILTER_POPUP_BUTTON: 'a.filterBar-menuTrigger',
|
||||
WATCHED_THREAD_NEXT_PAGE: 'a.pageNav-jump--next',
|
||||
WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
|
||||
WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next",
|
||||
WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]',
|
||||
DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]',
|
||||
SEARCH_THREADS_RESULTS_BODY: 'div.contentRow-main',
|
||||
SEARCH_THREADS_MEMBERSHIP: 'li > a:not(.username)',
|
||||
THREAD_LAST_CHANGELOG: 'div.bbCodeBlock-content > div:first-of-type'
|
||||
SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main",
|
||||
SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)",
|
||||
THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type",
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
module.exports = Object.freeze({
|
||||
F95_BASE_URL: 'https://f95zone.to',
|
||||
F95_SEARCH_URL: 'https://f95zone.to/search',
|
||||
F95_LATEST_UPDATES: 'https://f95zone.to/latest',
|
||||
F95_LOGIN_URL: 'https://f95zone.to/login',
|
||||
F95_WATCHED_THREADS: 'https://f95zone.to/watched/threads'
|
||||
F95_BASE_URL: "https://f95zone.to",
|
||||
F95_SEARCH_URL: "https://f95zone.to/search",
|
||||
F95_LATEST_UPDATES: "https://f95zone.to/latest",
|
||||
F95_LOGIN_URL: "https://f95zone.to/login",
|
||||
F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,16 +1,16 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
const HTMLParser = require('node-html-parser');
|
||||
const puppeteer = require('puppeteer'); // skipcq: JS-0128
|
||||
const HTMLParser = require("node-html-parser");
|
||||
const puppeteer = require("puppeteer"); // skipcq: JS-0128
|
||||
|
||||
// Modules from file
|
||||
const shared = require('./shared.js');
|
||||
const selectors = require('./constants/css-selectors.js');
|
||||
const { preparePage } = require('./puppeteer-helper.js');
|
||||
const GameDownload = require('./classes/game-download.js');
|
||||
const GameInfo = require('./classes/game-info.js');
|
||||
const { isStringAValidURL, isF95URL, urlExists } = require('./urls-helper.js');
|
||||
const shared = require("./shared.js");
|
||||
const selectors = require("./constants/css-selectors.js");
|
||||
const { preparePage } = require("./puppeteer-helper.js");
|
||||
const GameDownload = require("./classes/game-download.js");
|
||||
const GameInfo = require("./classes/game-info.js");
|
||||
const { isStringAValidURL, isF95URL, urlExists } = require("./urls-helper.js");
|
||||
|
||||
/**
|
||||
* @protected
|
||||
|
|
@ -21,17 +21,17 @@ const { isStringAValidURL, isF95URL, urlExists } = require('./urls-helper.js');
|
|||
* looking for or null if the URL doesn't exists
|
||||
*/
|
||||
module.exports.getGameInfo = async function (browser, url) {
|
||||
if (shared.debug) console.log('Obtaining game info');
|
||||
if (shared.debug) console.log("Obtaining game info");
|
||||
|
||||
// Verify the correctness of the URL
|
||||
if (!isF95URL(url)) throw new Error(url + ' is not a valid F95Zone URL');
|
||||
if (!isF95URL(url)) throw new Error(url + " is not a valid F95Zone URL");
|
||||
const exists = await urlExists(url);
|
||||
if (!exists) return null;
|
||||
|
||||
const page = await preparePage(browser); // Set new isolated page
|
||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||
await page.goto(url, {
|
||||
waitUntil: shared.WAIT_STATEMENT
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Go to the game page and wait until it loads
|
||||
|
||||
// It asynchronously searches for the elements and
|
||||
|
|
@ -59,7 +59,7 @@ module.exports.getGameInfo = async function (browser, url) {
|
|||
? parsedInfos.UPDATED
|
||||
: parsedInfos.THREAD_UPDATED;
|
||||
info.previewSource = await previewSource;
|
||||
info.changelog = (await changelog) || 'Unknown changelog';
|
||||
info.changelog = (await changelog) || "Unknown changelog";
|
||||
//info.downloadInfo = await downloadData;
|
||||
/* Downloading games without going directly to
|
||||
* the platform appears to be prohibited by
|
||||
|
|
@ -67,7 +67,7 @@ module.exports.getGameInfo = async function (browser, url) {
|
|||
* keep the links for downloading the games. */
|
||||
|
||||
await page.close(); // Close the page
|
||||
if (shared.debug) console.log('Founded data for ' + info.name);
|
||||
if (shared.debug) console.log("Founded data for " + info.name);
|
||||
return info;
|
||||
};
|
||||
|
||||
|
|
@ -81,7 +81,7 @@ module.exports.getGameVersionFromTitle = async function (browser, info) {
|
|||
const page = await preparePage(browser); // Set new isolated page
|
||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||
await page.goto(info.f95url, {
|
||||
waitUntil: shared.WAIT_STATEMENT
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Go to the game page and wait until it loads
|
||||
|
||||
// Get the title
|
||||
|
|
@ -93,10 +93,10 @@ module.exports.getGameVersionFromTitle = async function (browser, info) {
|
|||
const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText;
|
||||
|
||||
// The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR]
|
||||
const startIndex = title.indexOf('[') + 1;
|
||||
const endIndex = title.indexOf(']', startIndex);
|
||||
const startIndex = title.indexOf("[") + 1;
|
||||
const endIndex = title.indexOf("]", startIndex);
|
||||
let version = title.substring(startIndex, endIndex).trim().toUpperCase();
|
||||
if (version.startsWith('V')) version = version.replace('V', ''); // Replace only the first occurrence
|
||||
if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence
|
||||
return version;
|
||||
};
|
||||
|
||||
|
|
@ -112,9 +112,9 @@ module.exports.getGameVersionFromTitle = async function (browser, info) {
|
|||
function getOverview(text, isMod) {
|
||||
// Get overview (different parsing for game and mod)
|
||||
let overviewEndIndex;
|
||||
if (isMod) overviewEndIndex = text.indexOf('Updated');
|
||||
else overviewEndIndex = text.indexOf('Thread Updated');
|
||||
return text.substring(0, overviewEndIndex).replace('Overview:\n', '').trim();
|
||||
if (isMod) overviewEndIndex = text.indexOf("Updated");
|
||||
else overviewEndIndex = text.indexOf("Thread Updated");
|
||||
return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -156,7 +156,7 @@ async function getGameAuthor(page) {
|
|||
const gameTitle = structuredTitle.childNodes.pop().rawText;
|
||||
|
||||
// The last square brackets contain the author
|
||||
const startTitleIndex = gameTitle.lastIndexOf('[') + 1;
|
||||
const startTitleIndex = gameTitle.lastIndexOf("[") + 1;
|
||||
return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim();
|
||||
}
|
||||
|
||||
|
|
@ -171,17 +171,17 @@ function parseConversationPage(text) {
|
|||
const dataPairs = {};
|
||||
|
||||
// The information searched in the game post are one per line
|
||||
const splittedText = text.split('\n');
|
||||
const splittedText = text.split("\n");
|
||||
for (const line of splittedText) {
|
||||
if (!line.includes(':')) continue;
|
||||
if (!line.includes(":")) continue;
|
||||
|
||||
// Create pair key/value
|
||||
const splitted = line.split(':');
|
||||
const key = splitted[0].trim().toUpperCase().replaceAll(' ', '_'); // Uppercase to avoid mismatch
|
||||
const splitted = line.split(":");
|
||||
const key = splitted[0].trim().toUpperCase().replaceAll(" ", "_"); // Uppercase to avoid mismatch
|
||||
const value = splitted[1].trim();
|
||||
|
||||
// Add pair to the dict if valid
|
||||
if (value !== '') dataPairs[key] = value;
|
||||
if (value !== "") dataPairs[key] = value;
|
||||
}
|
||||
|
||||
return dataPairs;
|
||||
|
|
@ -200,7 +200,7 @@ async function getGamePreviewSource(page) {
|
|||
// Get the firs image available
|
||||
const img = document.querySelector(selector);
|
||||
|
||||
if (img) return img.getAttribute('src');
|
||||
if (img) return img.getAttribute("src");
|
||||
else return null;
|
||||
},
|
||||
selectors.GAME_IMAGES
|
||||
|
|
@ -227,7 +227,7 @@ async function getGameTitle(page) {
|
|||
|
||||
// The last element **shoud be** the title without prefixes (engines, status, other...)
|
||||
const gameTitle = structuredTitle.childNodes.pop().rawText;
|
||||
const endTitleIndex = gameTitle.indexOf('[');
|
||||
const endTitleIndex = gameTitle.indexOf("[");
|
||||
return gameTitle.substring(0, endTitleIndex).trim();
|
||||
}
|
||||
|
||||
|
|
@ -261,10 +261,10 @@ async function getGameTags(page) {
|
|||
* @returns {Promise<GameInfo>} GameInfo object passed in to which the identified information has been added
|
||||
*/
|
||||
async function parsePrefixes(page, info) {
|
||||
const MOD_PREFIX = 'MOD';
|
||||
const MOD_PREFIX = "MOD";
|
||||
|
||||
// The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete'
|
||||
info.status = 'Ongoing';
|
||||
info.status = "Ongoing";
|
||||
for (const handle of await page.$$(selectors.GAME_TITLE_PREFIXES)) {
|
||||
const value = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
|
|
@ -273,7 +273,7 @@ async function parsePrefixes(page, info) {
|
|||
);
|
||||
|
||||
// Clean the prefix
|
||||
const prefix = value.toUpperCase().replace('[', '').replace(']', '').trim();
|
||||
const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim();
|
||||
|
||||
// Getting infos...
|
||||
if (shared.statuses.includes(prefix)) info.status = prefix;
|
||||
|
|
@ -303,7 +303,7 @@ async function getLastChangelog(page) {
|
|||
spoiler
|
||||
);
|
||||
const parsedText = HTMLParser.parse(changelogHTML).structuredText;
|
||||
return parsedText.replace('Spoiler', '').trim();
|
||||
return parsedText.replace("Spoiler", "").trim();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -316,17 +316,17 @@ async function getLastChangelog(page) {
|
|||
async function getGameDownloadLink(page) {
|
||||
// Most used hosting platforms
|
||||
const hostingPlatforms = [
|
||||
'MEGA',
|
||||
'NOPY',
|
||||
'FILESUPLOAD',
|
||||
'MIXDROP',
|
||||
'UPLOADHAVEN',
|
||||
'PIXELDRAIN',
|
||||
'FILESFM'
|
||||
"MEGA",
|
||||
"NOPY",
|
||||
"FILESUPLOAD",
|
||||
"MIXDROP",
|
||||
"UPLOADHAVEN",
|
||||
"PIXELDRAIN",
|
||||
"FILESFM",
|
||||
];
|
||||
|
||||
// Supported OS platforms
|
||||
const platformOS = ['WIN', 'LINUX', 'MAC', 'ALL'];
|
||||
const platformOS = ["WIN", "LINUX", "MAC", "ALL"];
|
||||
|
||||
// Gets the <span> which contains the download links
|
||||
const temp = await page.$$(selectors.DOWNLOAD_LINKS_CONTAINER);
|
||||
|
|
@ -383,13 +383,13 @@ async function getGameDownloadLink(page) {
|
|||
* @returns {GameDownload[]} List of game download links for the selected platform
|
||||
*/
|
||||
function extractGameHostingData(platform, text) {
|
||||
const PLATFORM_BOLD_OPEN = '<b>';
|
||||
const CONTAINER_SPAN_CLOSE = '</span>';
|
||||
const LINK_OPEN = '<a';
|
||||
const LINK_CLOSE = '</a>';
|
||||
const HREF_START = 'href=\'';
|
||||
const HREF_END = '\'';
|
||||
const TAG_CLOSE = '>';
|
||||
const PLATFORM_BOLD_OPEN = "<b>";
|
||||
const CONTAINER_SPAN_CLOSE = "</span>";
|
||||
const LINK_OPEN = "<a";
|
||||
const LINK_CLOSE = "</a>";
|
||||
const HREF_START = "href='";
|
||||
const HREF_END = "'";
|
||||
const TAG_CLOSE = ">";
|
||||
|
||||
// Identify the individual platforms
|
||||
let startIndex = text.indexOf(platform.toLowerCase());
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
const puppeteer = require('puppeteer'); // skipcq: JS-0128
|
||||
const puppeteer = require("puppeteer"); // skipcq: JS-0128
|
||||
|
||||
// Modules from file
|
||||
const shared = require('./shared.js');
|
||||
const constURLs = require('./constants/urls.js');
|
||||
const selectors = require('./constants/css-selectors.js');
|
||||
const { preparePage } = require('./puppeteer-helper.js');
|
||||
const { isF95URL } = require('./urls-helper.js');
|
||||
const shared = require("./shared.js");
|
||||
const constURLs = require("./constants/urls.js");
|
||||
const selectors = require("./constants/css-selectors.js");
|
||||
const { preparePage } = require("./puppeteer-helper.js");
|
||||
const { isF95URL } = require("./urls-helper.js");
|
||||
|
||||
/**
|
||||
* @protected
|
||||
|
|
@ -18,12 +18,12 @@ const { isF95URL } = require('./urls-helper.js');
|
|||
* @returns {Promise<String[]>} List of URL of possible games obtained from the preliminary research on the F95 portal
|
||||
*/
|
||||
module.exports.getSearchGameResults = async function (browser, gamename) {
|
||||
if (shared.debug) console.log('Searching ' + gamename + ' on F95Zone');
|
||||
if (shared.debug) console.log("Searching " + gamename + " on F95Zone");
|
||||
|
||||
const page = await preparePage(browser); // Set new isolated page
|
||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||
await page.goto(constURLs.F95_SEARCH_URL, {
|
||||
waitUntil: shared.WAIT_STATEMENT
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Go to the search form and wait for it
|
||||
|
||||
// Explicitly wait for the required items to load
|
||||
|
|
@ -36,21 +36,21 @@ module.exports.getSearchGameResults = async function (browser, gamename) {
|
|||
await Promise.all([
|
||||
page.click(selectors.SEARCH_BUTTON), // Execute search
|
||||
page.waitForNavigation({
|
||||
waitUntil: shared.WAIT_STATEMENT
|
||||
}) // Wait for page to load
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}), // Wait for page to load
|
||||
]);
|
||||
|
||||
// Select all conversation titles
|
||||
const resultsThread = await page.$$(selectors.SEARCH_THREADS_RESULTS_BODY);
|
||||
|
||||
// For each element found extract the info about the conversation
|
||||
if (shared.debug) console.log('Extracting info from conversations');
|
||||
if (shared.debug) console.log("Extracting info from conversations");
|
||||
const results = [];
|
||||
for (const element of resultsThread) {
|
||||
const gameUrl = await getOnlyGameThreads(page, element);
|
||||
if (gameUrl !== null) results.push(gameUrl);
|
||||
}
|
||||
if (shared.debug) console.log('Find ' + results.length + ' conversations');
|
||||
if (shared.debug) console.log("Find " + results.length + " conversations");
|
||||
await page.close(); // Close the page
|
||||
|
||||
return results;
|
||||
|
|
@ -71,7 +71,7 @@ async function getOnlyGameThreads(page, divHandle) {
|
|||
|
||||
// Get the forum where the thread was posted
|
||||
const forum = await getMembershipForum(page, forumHandle);
|
||||
if (forum !== 'GAMES' && forum !== 'MODS') return null;
|
||||
if (forum !== "GAMES" && forum !== "MODS") return null;
|
||||
|
||||
// Get the URL of the thread from the title
|
||||
return await getThreadURL(page, titleHandle);
|
||||
|
|
@ -92,13 +92,13 @@ async function getMembershipForum(page, handle) {
|
|||
|
||||
let link = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
(e) => e.getAttribute('href'),
|
||||
(e) => e.getAttribute("href"),
|
||||
handle
|
||||
);
|
||||
|
||||
// Parse link
|
||||
link = link.replace('/forums/', '');
|
||||
const endIndex = link.indexOf('.');
|
||||
link = link.replace("/forums/", "");
|
||||
const endIndex = link.indexOf(".");
|
||||
const forum = link.substring(0, endIndex);
|
||||
|
||||
return forum.toUpperCase();
|
||||
|
|
@ -114,7 +114,7 @@ async function getMembershipForum(page, handle) {
|
|||
async function getThreadURL(page, handle) {
|
||||
const relativeURLThread = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
(e) => e.querySelector('a').href,
|
||||
(e) => e.querySelector("a").href,
|
||||
handle
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
const puppeteer = require('puppeteer');
|
||||
const puppeteer = require("puppeteer");
|
||||
|
||||
// Modules from file
|
||||
const shared = require('./shared.js');
|
||||
const shared = require("./shared.js");
|
||||
|
||||
/**
|
||||
* @protected
|
||||
|
|
@ -18,11 +18,11 @@ module.exports.prepareBrowser = async function () {
|
|||
if (shared.chromiumLocalPath) {
|
||||
browser = await puppeteer.launch({
|
||||
executablePath: shared.chromiumLocalPath,
|
||||
headless: !shared.debug // Use GUI when debug = true
|
||||
headless: !shared.debug, // Use GUI when debug = true
|
||||
});
|
||||
} else {
|
||||
browser = await puppeteer.launch({
|
||||
headless: !shared.debug // Use GUI when debug = true
|
||||
headless: !shared.debug, // Use GUI when debug = true
|
||||
});
|
||||
}
|
||||
|
||||
|
|
@ -42,9 +42,9 @@ module.exports.preparePage = async function (browser) {
|
|||
|
||||
// Block image download
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => {
|
||||
if (request.resourceType() === 'image') request.abort();
|
||||
else if (request.resourceType === 'font') request.abort();
|
||||
page.on("request", (request) => {
|
||||
if (request.resourceType() === "image") request.abort();
|
||||
else if (request.resourceType === "font") request.abort();
|
||||
// else if (request.resourceType() == 'stylesheet') request.abort();
|
||||
// else if(request.resourceType == 'media') request.abort();
|
||||
else request.continue();
|
||||
|
|
@ -52,8 +52,8 @@ module.exports.preparePage = async function (browser) {
|
|||
|
||||
// Set custom user-agent
|
||||
const userAgent =
|
||||
'Mozilla/5.0 (X11; Linux x86_64)' +
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36';
|
||||
"Mozilla/5.0 (X11; Linux x86_64)" +
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
|
||||
await page.setUserAgent(userAgent);
|
||||
|
||||
return page;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Core modules
|
||||
const { join } = require('path');
|
||||
const { join } = require("path");
|
||||
|
||||
/**
|
||||
* Class containing variables shared between modules.
|
||||
|
|
@ -37,12 +37,12 @@ class Shared {
|
|||
* Wait instruction for the browser created by puppeteer.
|
||||
* @type String
|
||||
*/
|
||||
static WAIT_STATEMENT = 'domcontentloaded';
|
||||
static WAIT_STATEMENT = "domcontentloaded";
|
||||
/**
|
||||
* Path to the directory to save the cache generated by the API.
|
||||
* @type String
|
||||
*/
|
||||
static _cacheDir = './f95cache';
|
||||
static _cacheDir = "./f95cache";
|
||||
/**
|
||||
* If true, it opens a new browser for each request to
|
||||
* the F95Zone platform, otherwise it reuses the same.
|
||||
|
|
@ -99,21 +99,21 @@ class Shared {
|
|||
* @returns {String}
|
||||
*/
|
||||
static get cookiesCachePath() {
|
||||
return join(this._cacheDir, 'cookies.json');
|
||||
return join(this._cacheDir, "cookies.json");
|
||||
}
|
||||
/**
|
||||
* Path to the game engine cache.
|
||||
* @returns {String}
|
||||
*/
|
||||
static get enginesCachePath() {
|
||||
return join(this._cacheDir, 'engines.json');
|
||||
return join(this._cacheDir, "engines.json");
|
||||
}
|
||||
/**
|
||||
* Path to the cache of possible game states.
|
||||
* @returns {String}
|
||||
*/
|
||||
static get statusesCachePath() {
|
||||
return join(this._cacheDir, 'statuses.json');
|
||||
return join(this._cacheDir, "statuses.json");
|
||||
}
|
||||
/**
|
||||
* If true, it opens a new browser for each request
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
const ky = require('ky-universal').create({
|
||||
throwHttpErrors: false
|
||||
const ky = require("ky-universal").create({
|
||||
throwHttpErrors: false,
|
||||
});
|
||||
|
||||
// Modules from file
|
||||
const { F95_BASE_URL } = require('./constants/urls.js');
|
||||
const { F95_BASE_URL } = require("./constants/urls.js");
|
||||
|
||||
/**
|
||||
* @protected
|
||||
|
|
|
|||
|
|
@ -5,9 +5,9 @@ const {
|
|||
getGameData,
|
||||
loadF95BaseData,
|
||||
getUserData,
|
||||
logout
|
||||
} = require('../app/index.js');
|
||||
const GameDownload = require('../app/scripts/classes/game-download.js');
|
||||
logout,
|
||||
} = require("../app/index.js");
|
||||
const GameDownload = require("../app/scripts/classes/game-download.js");
|
||||
|
||||
debug(true);
|
||||
main();
|
||||
|
|
|
|||
Loading…
Reference in New Issue