diff --git a/app/index.js b/app/index.js index 9d5478d..5bf9af0 100644 --- a/app/index.js +++ b/app/index.js @@ -482,7 +482,7 @@ async function loginF95(browser, username, password) { await page.waitForNavigation({ waitUntil: shared.WAIT_STATEMENT, }); // Wait for page to load - + // Prepare result let result = new LoginResult(); diff --git a/app/scripts/constants/css-selectors.js b/app/scripts/constants/css-selectors.js index d0ca9cd..f36ecff 100644 --- a/app/scripts/constants/css-selectors.js +++ b/app/scripts/constants/css-selectors.js @@ -1,29 +1,31 @@ module.exports = Object.freeze({ - AVATAR_INFO: 'span.avatar', - AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]', - ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span', - FILTER_THREADS_BUTTON: 'button[class="button--primary button"]', - GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]', - GAME_TAGS: 'a.tagItem', - GAME_TITLE: 'h1.p-title-value', - GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]', - LOGIN_BUTTON: 'button.button--icon--login', - LOGIN_MESSAGE_ERROR: 'div.blockMessage.blockMessage--error.blockMessage--iconic', - ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]', - PASSWORD_INPUT: 'input[name="password"]', - SEARCH_BUTTON: 'form.block > * button.button--icon--search', - SEARCH_FORM_TEXTBOX: 'input[name="keywords"]', - STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span', - THREAD_POSTS: 'article.message-body:first-child > div.bbWrapper:first-of-type', - THREAD_TITLE: 'h3.contentRow-title', - TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]', - UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]', - USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText', - USERNAME_INPUT: 'input[name="login"]', - WATCHED_THREAD_FILTER_POPUP_BUTTON: 'a.filterBar-menuTrigger', - WATCHED_THREAD_NEXT_PAGE: 'a.pageNav-jump--next', - WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]', - DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]', - SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", - SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)" -}); \ No newline at end of file + AVATAR_INFO: "span.avatar", + AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]', + ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span', + FILTER_THREADS_BUTTON: 'button[class="button--primary button"]', + GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]', + GAME_TAGS: "a.tagItem", + GAME_TITLE: "h1.p-title-value", + GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]', + LOGIN_BUTTON: "button.button--icon--login", + LOGIN_MESSAGE_ERROR: + "div.blockMessage.blockMessage--error.blockMessage--iconic", + ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]', + PASSWORD_INPUT: 'input[name="password"]', + SEARCH_BUTTON: "form.block > * button.button--icon--search", + SEARCH_FORM_TEXTBOX: 'input[name="keywords"]', + STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span', + THREAD_POSTS: + "article.message-body:first-child > div.bbWrapper:first-of-type", + THREAD_TITLE: "h3.contentRow-title", + TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]', + UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]', + USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText', + USERNAME_INPUT: 'input[name="login"]', + WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", + WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]', + DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]', + SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", + SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", +}); diff --git a/app/scripts/game-scraper.js b/app/scripts/game-scraper.js index 50bb80d..92a553b 100644 --- a/app/scripts/game-scraper.js +++ b/app/scripts/game-scraper.js @@ -77,7 +77,7 @@ module.exports.getGameInfo = async function (browser, url) { * @param {GameInfo} info Information about the game * @returns {Promise} Online version of the game */ -module.exports.getGameVersionFromTitle = async function(browser, info) { +module.exports.getGameVersionFromTitle = async function (browser, info) { let page = await preparePage(browser); // Set new isolated page await page.setCookie(...shared.cookies); // Set cookies to avoid login await page.goto(info.f95url, { @@ -87,8 +87,7 @@ module.exports.getGameVersionFromTitle = async function(browser, info) { // Get the title let titleHTML = await page.evaluate( /* istanbul ignore next */ - (selector) => - document.querySelector(selector).innerHTML, + (selector) => document.querySelector(selector).innerHTML, selectors.GAME_TITLE ); let title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; @@ -97,9 +96,9 @@ module.exports.getGameVersionFromTitle = async function(browser, info) { let startIndex = title.indexOf("[") + 1; let endIndex = title.indexOf("]", startIndex); let version = title.substring(startIndex, endIndex).trim().toUpperCase(); - if(version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence + if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence return version; -} +}; //#region Private methods /** diff --git a/app/scripts/game-searcher.js b/app/scripts/game-searcher.js index 57d0f95..9c7221a 100644 --- a/app/scripts/game-searcher.js +++ b/app/scripts/game-searcher.js @@ -1,15 +1,13 @@ "use strict"; // Public modules from npm -const puppeteer = require('puppeteer'); +const puppeteer = require("puppeteer"); // Modules from file const shared = require("./shared.js"); const constURLs = require("./constants/urls.js"); const selectors = require("./constants/css-selectors.js"); -const { - preparePage, -} = require("./puppeteer-helper.js"); +const { preparePage } = require("./puppeteer-helper.js"); /** * @protected @@ -18,42 +16,42 @@ const { * @param {String} gamename Name of the game to search for * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal */ -module.exports.getSearchGameResults = async function(browser, gamename) { - if (shared.debug) console.log("Searching " + gamename + " on F95Zone"); +module.exports.getSearchGameResults = async function (browser, gamename) { + if (shared.debug) console.log("Searching " + gamename + " on F95Zone"); - let page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(constURLs.F95_SEARCH_URL, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the search form and wait for it + let page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(constURLs.F95_SEARCH_URL, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the search form and wait for it - // Explicitly wait for the required items to load - await page.waitForSelector(selectors.SEARCH_FORM_TEXTBOX); - await page.waitForSelector(selectors.TITLE_ONLY_CHECKBOX); - await page.waitForSelector(selectors.SEARCH_BUTTON); + // Explicitly wait for the required items to load + await page.waitForSelector(selectors.SEARCH_FORM_TEXTBOX); + await page.waitForSelector(selectors.TITLE_ONLY_CHECKBOX); + await page.waitForSelector(selectors.SEARCH_BUTTON); - await page.type(selectors.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire - await page.click(selectors.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles - await page.click(selectors.SEARCH_BUTTON); // Execute search - await page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }); // Wait for page to load + await page.type(selectors.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire + await page.click(selectors.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles + await page.click(selectors.SEARCH_BUTTON); // Execute search + await page.waitForNavigation({ + waitUntil: shared.WAIT_STATEMENT, + }); // Wait for page to load - // Select all conversation titles - let resultsThread = await page.$$(selectors.SEARCH_THREADS_RESULTS_BODY); + // Select all conversation titles + let resultsThread = await page.$$(selectors.SEARCH_THREADS_RESULTS_BODY); - // For each element found extract the info about the conversation - if (shared.debug) console.log("Extracting info from conversations"); - let results = []; - for (let element of resultsThread) { - let gameUrl = await getOnlyGameThreads(page, element); - if (gameUrl !== null) results.push(gameUrl); - } - if (shared.debug) console.log("Find " + results.length + " conversations"); - await page.close(); // Close the page + // For each element found extract the info about the conversation + if (shared.debug) console.log("Extracting info from conversations"); + let results = []; + for (let element of resultsThread) { + let gameUrl = await getOnlyGameThreads(page, element); + if (gameUrl !== null) results.push(gameUrl); + } + if (shared.debug) console.log("Find " + results.length + " conversations"); + await page.close(); // Close the page - return results; -} + return results; +}; //#region Private methods /** @@ -64,16 +62,16 @@ module.exports.getSearchGameResults = async function(browser, gamename) { * @return {Promise} URL of the game/mod or null if the URL is not of a game */ async function getOnlyGameThreads(page, divHandle) { - // Obtain the elements containing the basic information - let titleHandle = await divHandle.$(selectors.THREAD_TITLE); - let forumHandle = await divHandle.$(selectors.SEARCH_THREADS_MEMBERSHIP); - - // Get the forum where the thread was posted - let forum = await getMembershipForum(page, forumHandle); - if(forum !== "GAMES" && forum != "MODS") return null; + // Obtain the elements containing the basic information + let titleHandle = await divHandle.$(selectors.THREAD_TITLE); + let forumHandle = await divHandle.$(selectors.SEARCH_THREADS_MEMBERSHIP); - // Get the URL of the thread from the title - return await getThreadURL(page, titleHandle); + // Get the forum where the thread was posted + let forum = await getMembershipForum(page, forumHandle); + if (forum !== "GAMES" && forum != "MODS") return null; + + // Get the URL of the thread from the title + return await getThreadURL(page, titleHandle); } /** @@ -81,26 +79,26 @@ async function getOnlyGameThreads(page, divHandle) { * Obtain the membership forum of the thread passed throught "handle". * @param {puppeteer.Page} page Page containing the conversation to be analyzed * @param {puppeteer.ElementHandle} handle Handle containing the forum membership - * @returns {Promise} Uppercase membership category + * @returns {Promise} Uppercase membership category */ async function getMembershipForum(page, handle) { - // The link can be something like: - // + /forums/request.NUMBER/ - // + /forums/game-recommendations-identification.NUMBER/ - // + /forums/games.NUMBER/ <-- We need this + // The link can be something like: + // + /forums/request.NUMBER/ + // + /forums/game-recommendations-identification.NUMBER/ + // + /forums/games.NUMBER/ <-- We need this - let link = await page.evaluate( - /* istanbul ignore next */ - (e) => e.getAttribute('href'), - handle - ); + let link = await page.evaluate( + /* istanbul ignore next */ + (e) => e.getAttribute("href"), + handle + ); - // Parse link - link = link.replace("/forums/", ""); - let endIndex = link.indexOf("."); - let forum = link.substring(0, endIndex); + // Parse link + link = link.replace("/forums/", ""); + let endIndex = link.indexOf("."); + let forum = link.substring(0, endIndex); - return forum.toUpperCase(); + return forum.toUpperCase(); } /** @@ -111,12 +109,12 @@ async function getMembershipForum(page, handle) { * @returns {Promise} URL of the thread */ async function getThreadURL(page, handle) { - let relativeURLThread = await page.evaluate( - /* istanbul ignore next */ - (e) => e.querySelector("a").href, - handle - ); - let urlThread = new URL(relativeURLThread, constURLs.F95_BASE_URL).toString(); - return urlThread; + let relativeURLThread = await page.evaluate( + /* istanbul ignore next */ + (e) => e.querySelector("a").href, + handle + ); + let urlThread = new URL(relativeURLThread, constURLs.F95_BASE_URL).toString(); + return urlThread; } //#endregion Private methods diff --git a/app/scripts/puppeteer-helper.js b/app/scripts/puppeteer-helper.js index 5d14222..ebb3eb4 100644 --- a/app/scripts/puppeteer-helper.js +++ b/app/scripts/puppeteer-helper.js @@ -1,25 +1,25 @@ -'use strict'; +"use strict"; // Public modules from npm -const puppeteer = require('puppeteer'); +const puppeteer = require("puppeteer"); // Modules from file -const shared = require('./shared.js'); +const shared = require("./shared.js"); /** * @protected - * Create a Chromium instance used to navigate with Puppeteer. + * Create a Chromium instance used to navigate with Puppeteer. * By default the browser is headless. * @returns {Promise} Created browser */ -module.exports.prepareBrowser = async function() { - // Create a headless browser - let browser = await puppeteer.launch({ - headless: !shared.debug, // Use GUI when debug = true - }); +module.exports.prepareBrowser = async function () { + // Create a headless browser + let browser = await puppeteer.launch({ + headless: !shared.debug, // Use GUI when debug = true + }); - return browser; -} + return browser; +}; /** * @protected @@ -28,24 +28,25 @@ module.exports.prepareBrowser = async function() { * @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created * @returns {Promise} New page */ -module.exports.preparePage = async function(browser) { - // Create new page in the browser argument - let page = await browser.newPage(); +module.exports.preparePage = async function (browser) { + // Create new page in the browser argument + let page = await browser.newPage(); - // Block image download - await page.setRequestInterception(true); - page.on('request', (request) => { - if (request.resourceType() === 'image') request.abort(); - else if(request.resourceType == 'font') request.abort(); - // else if (request.resourceType() == 'stylesheet') request.abort(); - // else if(request.resourceType == 'media') request.abort(); - else request.continue(); - }); + // Block image download + await page.setRequestInterception(true); + page.on("request", (request) => { + if (request.resourceType() === "image") request.abort(); + else if (request.resourceType == "font") request.abort(); + // else if (request.resourceType() == 'stylesheet') request.abort(); + // else if(request.resourceType == 'media') request.abort(); + else request.continue(); + }); - // Set custom user-agent - let userAgent = 'Mozilla/5.0 (X11; Linux x86_64)' + - 'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36'; - await page.setUserAgent(userAgent); + // Set custom user-agent + let userAgent = + "Mozilla/5.0 (X11; Linux x86_64)" + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; + await page.setUserAgent(userAgent); - return page; -} \ No newline at end of file + return page; +}; diff --git a/test/index-test.js b/test/index-test.js index e27a36b..bae1753 100644 --- a/test/index-test.js +++ b/test/index-test.js @@ -3,8 +3,8 @@ const expect = require("chai").expect; const F95API = require("../app/index"); const fs = require("fs"); -const sleep = require('sleep'); -const dotenv = require('dotenv'); +const sleep = require("sleep"); +const dotenv = require("dotenv"); dotenv.config(); const COOKIES_SAVE_PATH = "./f95cache/cookies.json"; @@ -26,9 +26,9 @@ describe("Login without cookies", function () { //#region Set-up this.timeout(30000); // All tests in this suite get 30 seconds before timeout - before("Set isolation", function() { + before("Set isolation", function () { F95API.setIsolation(true); - }) + }); beforeEach("Remove all cookies", function () { // Runs before each test in this block @@ -42,7 +42,7 @@ describe("Login without cookies", function () { it("Test with valid credentials", async function () { // Gain exclusive use of the cookies while (testOrder !== 0) randomSleep(); - + const result = await F95API.login(USERNAME, PASSWORD); expect(result.success).to.be.true; expect(result.message).equal("Authentication successful"); @@ -221,7 +221,8 @@ describe("Check game update", function () { // This test depend on the data on F95Zone at // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ - let url = "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; + let url = + "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; const result = await F95API.getGameDataFromURL(url); result.version = "0.9600";