'use strict'; const path = require('path'); const fs = require('fs'); const LoginResult = require('./scripts/login-result').LoginResult; const GameInfo = require('./scripts/game-info').GameInfo; const GameDownload = require('./scripts/game-download').GameDownload; const HTMLParser = require('node-html-parser'); const puppeteer = require('puppeteer'); const urlExist = require('url-exist'); //#region URL const F95_BASE_URL = 'https://f95zone.to'; const F95_SEARCH_URL = 'https://f95zone.to/search'; const F95_LATEST_UPDATES = 'https://f95zone.to/latest'; const F95_LOGIN_URL = 'https://f95zone.to/login' //#endregion //#region CSS Selectors const SEARCH_FORM_TEXTBOX = 'input[name="keywords"]'; const PASSWORD_INPUT = 'input[name="password"]'; const USERNAME_INPUT = 'input[name="login"]'; const LOGIN_BUTTON = 'button.button--icon--login'; const AVATAR_INFO = 'span.avatar'; const TITLE_ONLY_CHECKBOX = 'form.block > * input[name="c[title_only]"]'; const SEARCH_BUTTON = 'form.block > * button.button--icon--search'; const ENGINE_ID_SELECTOR = 'div[id^="btn-prefix_1_"]>span'; const STATUS_ID_SELECTOR = 'div[id^="btn-prefix_4_"]>span'; const THREAD_TITLE = 'h3.contentRow-title'; const THREAD_POSTS = 'article.message-body:first-child > div.bbWrapper:first-of-type'; const GAME_TITLE = 'h1.p-title-value'; const GAME_IMAGES = 'img[src^="https://attachments.f95zone.to"]'; const LOGIN_MESSAGE_ERROR = 'div.blockMessage.blockMessage--error.blockMessage--iconic'; //#endregion CSS Selectors //#region Game prefixes const MOD_PREFIX = 'MOD'; const GAME_RECOMMENDATION_PREFIX = 'RECOMMENDATION'; //#endregion Game prefixes //#region Directories const CACHE_PATH = './f95cache'; const COOKIES_SAVE_PATH = path.join(CACHE_PATH, 'cookies.json'); const ENGINES_SAVE_PATH = path.join(CACHE_PATH, 'engines.json'); const STATUSES_SAVE_PATH = path.join(CACHE_PATH, 'statuses.json'); // Create directory if it doesn't exist if (!fs.existsSync(CACHE_PATH)) fs.mkdirSync(CACHE_PATH); //#endregion Directories //#region Various const WAIT_STATEMENT = 'domcontentloaded'; //#endregion Various //#region Fields /** * @private * @type Object[] */ let _cookies = loadCookies(); /** * @private * @type String[] */ let _engines = null; /** * @private * @type String[] */ let _statuses = null; /** @private * @type Boolean */ let _isLogged = false; /** * @private * @type Boolean */ let _debug = false; //#endregion Fields //#region Properties /** * * @param {Boolean} value */ module.exports.debug = function(value){ _debug = value; } module.exports.isLogged = function() { return _isLogged; }; //#endregion Properties //#region Export methods /** * @public * Log in to the F95Zone platform. * This **must** be the first operation performed before accessing any other script functions. * @param {String} username Username used for login * @param {String} password Password used for login * @returns {Promise} Result of the operation */ module.exports.login = async function (username, password) { if(_isLogged){ if(_debug) console.log("Already logged in"); let result = new LoginResult(); result.success = true; result.message = 'Already logged in'; return result; } // If cookies are loaded, use them to authenticate if (_cookies !== null) { if (_debug) console.log('Valid session, no need to re-authenticate'); _isLogged = true; let result = new LoginResult(); result.success = true; result.message = 'Logged with cookies'; return result; } // Else, log in throught browser if (_debug) console.log('No saved sessions or expired session, login on the platform'); let browser = await prepareBrowser(); let result = await loginF95(browser, username, password); _isLogged = result.success; if (result.success) { // Reload cookies _cookies = loadCookies(); if (_debug) console.log('User logged in through the platform'); } else { console.warn('Error during authentication: ' + result.message); } await browser.close(); return result; } /** * @public * This method loads the main data from the F95 portal * used to provide game information. You **must** be logged * in to the portal before calling this method. * @returns {Promise} Result of the operation */ module.exports.loadF95BaseData = async function () { if (!_isLogged) { console.warn('User not authenticated, unable to continue'); return false; } if(_debug) console.log('Loading base data...'); // Prepare a new web page let browser = await prepareBrowser(); let page = await preparePage(browser); // Set new isolated page await page.setCookie(..._cookies); // Set cookies to avoid login // Go to latest update page and wait for it to load await page.goto(F95_LATEST_UPDATES, { waitUntil: WAIT_STATEMENT }); // Obtain engines (disc/online) await page.waitForSelector(ENGINE_ID_SELECTOR); _engines = await loadValuesFromLatestPage(page, ENGINES_SAVE_PATH, ENGINE_ID_SELECTOR, 'engines'); // Obtain statuses (disc/online) await page.waitForSelector(STATUS_ID_SELECTOR); _statuses = await loadValuesFromLatestPage(page, STATUSES_SAVE_PATH, STATUS_ID_SELECTOR, 'statuses'); await browser.close(); if (_debug) console.log('Base data loaded'); return true; } /** * @public * Returns the currently online version of the specified game. * You **must** be logged in to the portal before calling this method. * @param {GameInfo} info Information about the game to get the version for * @returns {Promise} Currently online version of the specified game */ module.exports.getGameVersion = async function (info) { if (!_isLogged) { console.warn('user not authenticated, unable to continue'); return info.version; } let urlExists = await urlExist(info.f95url.toString()); // F95 change URL at every game update, so if the URL is the same no update is available if (urlExists) return info.version; else return await module.exports.getGameData(info.name, info.isMod).version; } /** * @public * Starting from the name, it gets all the information about the game you are looking for. * You **must** be logged in to the portal before calling this method. * @param {String} name Name of the game searched * @param {Boolean} includeMods Indicates whether to also take mods into account when searching * @returns {Promise} Information about the game searched or null if no game were found */ module.exports.getGameData = async function (name, includeMods) { if (!_isLogged) { console.warn('user not authenticated, unable to continue'); return null; } // Gets the search results of the game being searched for let browser = await prepareBrowser(); let infoList = await getSearchGameResults(browser, name, _cookies); // Process previous partial results let result = null; for (let info of infoList) { // Skip mods if not required if (info.isMod && !includeMods) continue; // TODO // What if there are more games with the same name? // For the moment, return the first result = await getGameInfo(browser, info); break; } await browser.close(); return result; } /** * @deprecated * @public * @param {*} platform * @param {*} url */ module.exports.getDownloadLink = async function(platform, url){ if (!_isLogged) { console.warn('user not authenticated, unable to continue'); return null; } // Gets the search results of the game being searched for let browser = await prepareBrowser(); getGameDownloadLink(browser, url); await browser.close(); } /** * @public * Properly closes all instances opened by the API. * @returns {Promise} */ module.exports.close = async function() { if(_debug) console.log("Closing F95API") } //#endregion //#region Private methods //#region Puppeteer helpers /** * @private * Create a Chromium instance used to navigate with Puppeteer. * By default the browser is headless. * @returns {Promise} Created browser */ async function prepareBrowser() { // Create a headless browser let browser = await puppeteer.launch({ headless: true, }); return browser; } /** * @private * Prepare a page used to navigate the browser. * The page is set up to reject image download requests. The user agent is also changed. * @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created * @returns {Promise} New page */ async function preparePage(browser) { // Create new page in the browser argument let page = await browser.newPage(); // Block image download await page.setRequestInterception(true); page.on('request', (request) => { if (request.resourceType() === 'image') request.abort(); // else if(request.resourceType == 'font') request.abort(); // else if(request.resourceType == 'media') request.abort(); else request.continue(); }); // Set custom user-agent let userAgent = 'Mozilla/5.0 (X11; Linux x86_64)' + 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36'; await page.setUserAgent(userAgent); return page; } //#endregion //#region Cookies functions /** * @private * Loads and verifies the expiration of previously stored cookies from disk * if they exist, otherwise it returns null. * @return {object[]} List of dictionaries or null if cookies don't exist */ function loadCookies() { // Check the existence of the cookie file if (fs.existsSync(COOKIES_SAVE_PATH)) { // Read cookies let cookiesJSON = fs.readFileSync(COOKIES_SAVE_PATH); let cookies = JSON.parse(cookiesJSON); // Check if the cookies have expired for (let cookie of cookies) { if (isCookieExpired(cookie)) return null; } // Cookies loaded and verified return cookies; } else return null; } /** * @private * Check the validity of a cookie. * @param {object} cookie Cookies to verify the validity. It's a dictionary * @returns {Boolean} true if the cookie has expired, false otherwise */ function isCookieExpired(cookie) { // Local variables let expiredCookies = false; // Ignore cookies that never expire let expirationUnixTimestamp = cookie['expire']; if (expirationUnixTimestamp !== '-1') { // Convert UNIX epoch timestamp to normal Date let expirationDate = new Date(expirationUnixTimestamp * 1000); if (expirationDate < Date.now()) { if (_debug) console.log('Cookie ' + cookie['name'] + ' expired, you need to re-authenticate'); expiredCookies = true; } } return expiredCookies; } //#endregion Cookies functions //#region Latest Updates page parserer /** * @private * If present, it reads the file containing the searched values (engines or states) * from the disk, otherwise it connects to the F95 portal (at the page * https://f95zone.to/latest) and downloads them. * @param {puppeteer.Page} page Page used to locate the required elements * @param {String} path Path to disk of the JSON file containing the data to read / write * @param {String} selector CSS selector of the required elements * @param {String} elementRequested Required element (engines or states) used to detail log messages * @returns {Promise} List of required values in uppercase */ async function loadValuesFromLatestPage(page, path, selector, elementRequested) { // If the values already exist they are loaded from disk without having to connect to F95 if (_debug) console.log('Load ' + elementRequested + ' from disk...'); if (fs.existsSync(path)) { let valueJSON = fs.readFileSync(path); return JSON.parse(valueJSON); } // Otherwise, connect and download the data from the portal if (_debug) console.log('No ' + elementRequested + ' cached, downloading...'); let values = await getValuesFromLatestPage(page, selector, 'Getting ' + elementRequested + ' from page'); fs.writeFileSync(path, JSON.stringify(values)); return values; } /** * @private * Gets all the textual values of the elements present * in the F95 portal page and identified by the selector * passed by parameter * @param {puppeteer.Page} page Page used to locate items specified by the selector * @param {String} selector CSS selector * @param {String} logMessage Log message indicating which items the selector is requesting * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector */ async function getValuesFromLatestPage(page, selector, logMessage) { if (_debug) console.log(logMessage); let result = []; let elements = await page.$$(selector); for (let element of elements) { let text = await element.evaluate(e => e.innerText); // Save as upper text for better match if used in query result.push(text.toUpperCase()); } return result; } //#endregion //#region URL methods /** * @private * Check if the url belongs to the domain of the F95 platform. * @param {URL} url URL to check * @returns {Boolean} true if the url belongs to the domain, false otherwise */ function isF95URL(url) { if (url.toString().startsWith(F95_BASE_URL)) return true; else return false; } /** * @private * Checks if the string passed by parameter has a properly formatted and valid path to a URL. * @param {String} url String to check for correctness * @returns {Boolean} true if the string is a valid URL, false otherwise */ function isStringAValidURL(url){ try { new URL(url); return true; } catch (err) { return false; } } //#endregion //#region User /** * @private * Log in to the F95Zone portal and, if successful, save the cookies. * @param {puppeteer.Browser} browser Browser object used for navigation * @param {String} username Username to use during login * @param {String} password Password to use during login * @returns {Promise} Result of the operation */ async function loginF95(browser, username, password) { let page = await preparePage(browser); // Set new isolated page await page.goto(F95_LOGIN_URL); // Go to login page // Explicitly wait for the required items to load await page.waitForSelector(USERNAME_INPUT); await page.waitForSelector(PASSWORD_INPUT); await page.waitForSelector(LOGIN_BUTTON); await page.type(USERNAME_INPUT, username); // Insert username await page.type(PASSWORD_INPUT, password); // Insert password await Promise.all([ page.click(LOGIN_BUTTON), // Click on the login button page.waitForNavigation({ waitUntil: WAIT_STATEMENT }), // Wait for page to load ]); await page.waitForSelector(AVATAR_INFO); // Prepare result let result = new LoginResult(); // Check if the user is logged in result.success = await page.evaluate((selector) => document.querySelector(selector) !== null, AVATAR_INFO); // Save cookies to avoid re-auth if (result.success) { const cookies = await page.cookies(); fs.writeFileSync(COOKIES_SAVE_PATH, JSON.stringify(cookies)); result.message = 'Authentication successful'; } // Obtain the error message else if (await page.evaluate((selector) => document.querySelector(selector) !== null, LOGIN_MESSAGE_ERROR)) { let errorMessage = await page.evaluate((selector) => document.querySelector(selector).innerText, LOGIN_MESSAGE_ERROR); if (errorMessage === 'Incorrect password. Please try again.') result.message = 'Incorrect password'; else if (errorMessage === "The requested user '" + username + "' could not be found.") result.message = 'Incorrect username'; else result.message = errorMessage; } else result.message = "Unknown error"; await page.close(); // Close the page return result; } /** * @deprecated */ async function getUserData(){ } /** * @deprecated */ async function getUserWatchedGameThreads(){ } //#endregion User //#region Game data parser /** * @private * Get information from the game's main page. * @param {puppeteer.Browser} browser Browser object used for navigation * @param {GameInfo} info Partial game information * @return {Promise} Complete information about the game you are looking for */ async function getGameInfo(browser, info) { if(_debug) console.log('Obtaining game info'); // Verify the correctness of the URL if (!isF95URL(info.f95url)) throw info.f95url + ' is not a valid F95Zone URL'; let exists = await urlExist(info.f95url.toString()); if (!exists) return new GameInfo(); let page = await preparePage(browser); // Set new isolated page await page.setCookie(..._cookies); // Set cookies to avoid login await page.goto(info.f95url.toString(), { waitUntil: WAIT_STATEMENT }); // Go to the game page and wait until it loads // Get the game/mod name (without square brackets) let title = getGameTitle(page); // Get the game/mod author (without square brackets) let author = getGameAuthor(page); // Get the game title image (the first is what we are searching) let previewSource = await getGamePreviewSource(page); if (previewSource === null) console.warn('Cannot find game preview image for ' + await title); // Gets the first post, where are listed all the game's informations let post = (await page.$$(THREAD_POSTS))[0]; // The info are plain text so we need to parse the HTML code let bodyHTML = await page.evaluate((mainPost) => mainPost.innerHTML, post); let structuredText = HTMLParser.parse(bodyHTML).structuredText; // Get overview (different parsing for game and mod) let overviewEndIndex; if (info.isMod) overviewEndIndex = structuredText.indexOf('Updated'); else overviewEndIndex = structuredText.indexOf('Thread Updated'); let overview = structuredText.substring(0, overviewEndIndex).replace('Overview:\n', '').trim(); // Parse all the information in the format DESCRIPTION : VALUE let parsedInfos = parseConversationPage(structuredText); // Fill in the GameInfo element with the information obtained info.name = await title; info.author = await author; info.overview = overview; info.version = info.isMod ? parsedInfos['MOD VERSION'] : parsedInfos['VERSION']; info.lastUpdate = info.isMod ? parsedInfos['UPDATED'] : parsedInfos['THREAD UPDATED']; info.previewSource = previewSource; await page.close(); // Close the page if(_debug) console.log('Founded data for ' + await title); return info; } /** * @private * Extrapolates and cleans the author from the page passed by parameter. * @param {puppeteer.Page} page Page containing the author to be extrapolated * @returns {Promise} Game author */ async function getGameAuthor(page) { // Get the game/mod name (without square brackets) let titleHTML = await page.evaluate((selector) => document.querySelector(selector).innerHTML, GAME_TITLE); let structuredTitle = HTMLParser.parse(titleHTML); // The last element **shoud be** the title without prefixes (engines, status, other...) let gameTitle = structuredTitle.childNodes.pop().rawText; // The last square brackets contain the author let startTitleIndex = gameTitle.lastIndexOf('[') + 1; return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); } /** * @private * Process the post text to get all the useful * information in the format *DESCRIPTOR : VALUE*. * @param {String} text Structured text of the post * @returns {Object} Dictionary of information */ function parseConversationPage(text) { let dataPairs = {}; // The information searched in the game post are one per line let splittedText = text.split('\n'); for (let line of splittedText) { if (!line.includes(':')) continue; // Create pair key/value let splitted = line.split(':'); let key = splitted[0].trim().toUpperCase(); // Uppercase to avoid mismatch let value = splitted[1].trim(); // Add pair to the dict if valid if (value != '') dataPairs[key] = value; } return dataPairs; } /** * @private * Gets the URL of the image used as a preview for the game in the conversation. * @param {puppeteer.Page} page Page containing the URL to be extrapolated * @returns {Promise} URL of the image or null if failed to get it */ async function getGamePreviewSource(page) { let src = await page.evaluate((selector) => { // Get the firs image available let img = document.querySelector(selector); if (img === null || img === undefined) return null; else return img.getAttribute('src'); }, GAME_IMAGES); // Check if the URL is valid return isStringAValidURL(src) ? new URL(src) : null; } /** * @private * Extrapolates and cleans the title from the page passed by parameter. * @param {puppeteer.Page} page Page containing the title to be extrapolated * @returns {Promise} Game title */ async function getGameTitle(page) { // Get the game/mod name (without square brackets) const titleHTML = await page.evaluate((selector) => document.querySelector(selector).innerHTML, GAME_TITLE); const structuredTitle = HTMLParser.parse(titleHTML); // The last element **shoud be** the title without prefixes (engines, status, other...) var gameTitle = structuredTitle.childNodes.pop().rawText; const endTitleIndex = gameTitle.indexOf('['); return gameTitle.substring(0, endTitleIndex).trim(); } /** * @deprecated * @param {puppeteer.Browser} browser * @param {URL} url */ async function getGameDownloadLink(browser, url) { // Verify the correctness of the URL if (!isF95URL(url)) throw url + ' is not a valid F95Zone URL'; let exists = await urlExist(url.toString()); if (!exists) return new GameDownload(); let page = await preparePage(browser); // Set new isolated page await page.setCookie(..._cookies); // Set cookies to avoid login await page.goto(url.toString(), { waitUntil: WAIT_STATEMENT }); // Go to the game page and wait until it loads // Gets the first post, where are listed all the game's informations let post = (await page.$$(THREAD_POSTS))[0]; // Get the HTML text let postHTML = await page.evaluate((mainPost) => mainPost.innerHTML, post); let startIndex = postHTML.indexOf('DOWNLOAD'); let endIndex = postHTML.indexOf('class="js-lbImage"'); postHTML = postHTML.substring(startIndex, endIndex - startIndex); console.log(postHTML); } //#endregion Game data parser //#region Game search /** * @private * Search the F95Zone portal to find possible conversations regarding the game you are looking for. * @param {puppeteer.Browser} browser Browser object used for navigation * @param {String} gamename Name of the game to search for * @returns {Promise} List of information obtained from the preliminary research on the F95 portal */ async function getSearchGameResults(browser, gamename) { if (_debug) console.log('Searching ' + gamename + ' on F95Zone'); let page = await preparePage(browser); // Set new isolated page await page.setCookie(..._cookies); // Set cookies to avoid login await page.goto(F95_SEARCH_URL, { waitUntil: WAIT_STATEMENT }); // Go to the search form and wait for it // Explicitly wait for the required items to load await page.waitForSelector(SEARCH_FORM_TEXTBOX); await page.waitForSelector(TITLE_ONLY_CHECKBOX); await page.waitForSelector(SEARCH_BUTTON); await page.type(SEARCH_FORM_TEXTBOX, gamename) // Type the game we desire await page.click(TITLE_ONLY_CHECKBOX) // Select only the thread with the game in the titles await Promise.all([ page.click(SEARCH_BUTTON), // Execute search page.waitForNavigation({ waitUntil: WAIT_STATEMENT }), // Wait for page to load ]); // Select all conversation titles let threadTitleList = await page.$$(THREAD_TITLE); // For each title extract the info about the conversation if(_debug) console.log('Extracting info from conversation titles'); let results = []; for (const title of threadTitleList) { var info = await getSearchThreadInfo(page, title); // Append the game's informations if (info !== null) results.push(info); } if (_debug) console.log('Find ' + results.length + ' conversations'); await page.close(); // Close the page return results; } /** * @private * Starting from the title of a conversation on the F95 portal, * he obtains basic information about the element (in case it is a game) * @param {puppeteer.Page} page Page containing the conversation to be analyzed * @param {puppeteer.ElementHandle} titleHandle Title of the conversation to be analyzed * @return {Promise} Object containing the information obtained from the analysis or null if it is not a game conversation */ async function getSearchThreadInfo(page, titleHandle) { let info = new GameInfo(); // Get the URL of the thread from the title let relativeURLThread = await page.evaluate((element) => element.querySelector('a').href, titleHandle); info.f95url = new URL(relativeURLThread, F95_BASE_URL); // Select infos like the engine used for the game or it's status // parsing the title let elements = await titleHandle.$$('span[dir="auto"]'); // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' are info.status = 'Ongoing'; for (let element of elements) { // Elaborate the prefixes let prefix = await page.evaluate(element => element.textContent.toUpperCase(), element); prefix = prefix.replace('[', '').replace(']', ''); // This is not a game nor a mod, we can exit if (prefix === GAME_RECOMMENDATION_PREFIX) return null; // Getting infos... else if (_statuses.includes(prefix)) info.status = prefix; else if (_engines.includes(prefix)) info.engine = prefix; // This is probably a mod for the game we are searching else if (prefix === MOD_PREFIX) info.isMod = true; } return info; } //#endregion Game search //#endregion Private methods