From d24593a04a4d902ff62a02092276e784d354959e Mon Sep 17 00:00:00 2001 From: "samuele.berlusconi" Date: Sat, 24 Oct 2020 20:52:29 +0200 Subject: [PATCH] Better and cleaner game search --- app/scripts/constants/css-selector.js | 3 ++- app/scripts/constants/url.js | 2 +- app/scripts/game-scraper.js | 22 ++++++---------------- app/scripts/game-searcher.js | 6 ++++-- test/user-test.js | 22 ++++++---------------- 5 files changed, 19 insertions(+), 36 deletions(-) diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index 8b268c3..6ef347e 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -13,7 +13,8 @@ module.exports = Object.freeze({ ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]', PASSWORD_INPUT: 'input[name="password"]', SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: 'input[name="keywords"]', + SEARCH_FORM_TEXTBOX: 'input[name="keywords"][type="search"]', + SEARCH_ONLY_GAMES_OPTION: 'select[name="c[nodes][]"] > option[value="1"]', STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span', THREAD_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", diff --git a/app/scripts/constants/url.js b/app/scripts/constants/url.js index 816a00a..87a8847 100644 --- a/app/scripts/constants/url.js +++ b/app/scripts/constants/url.js @@ -1,6 +1,6 @@ module.exports = Object.freeze({ F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search", + F95_SEARCH_URL: "https://f95zone.to/search/?type=post", F95_LATEST_UPDATES: "https://f95zone.to/latest", F95_LOGIN_URL: "https://f95zone.to/login", F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", diff --git a/app/scripts/game-scraper.js b/app/scripts/game-scraper.js index 3265704..dd52d52 100644 --- a/app/scripts/game-scraper.js +++ b/app/scripts/game-scraper.js @@ -25,9 +25,9 @@ module.exports.getGameInfo = async function (browser, url) { // Verify the correctness of the URL const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); + if (!exists) throw new URIError(`${url} is not a valid URL`); if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); + throw new Error(`${url} is not a valid F95Zone URL`); const page = await preparePage(browser); // Set new isolated page await page.setCookie(...shared.cookies); // Set cookies to avoid login @@ -45,7 +45,6 @@ module.exports.getGameInfo = async function (browser, url) { info = await parsePrefixes(page, info); // Fill status/engines/isMod const structuredText = await getMainPostStructuredText(page); const overview = getOverview(structuredText, info.isMod); - const parsedInfos = parseConversationPage(structuredText); const previewSource = getGamePreviewSource(page); const changelog = getLastChangelog(page); @@ -283,10 +282,8 @@ async function getGameTags(page) { * @returns {Promise} GameInfo object passed in to which the identified information has been added */ async function parsePrefixes(page, info) { - const MOD_PREFIX = "MOD"; - // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' - info.status = "Ongoing"; + info.status = "ONGOING"; for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { const value = await page.evaluate( /* istanbul ignore next */ @@ -298,10 +295,10 @@ async function parsePrefixes(page, info) { const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); // Getting infos... - if (shared.statuses.includes(prefix)) info.status = capitalize(prefix); - else if (shared.engines.includes(prefix)) info.engine = capitalize(prefix); + if (shared.statuses.includes(prefix)) info.status = prefix; + else if (shared.engines.includes(prefix)) info.engine = prefix; // This is not a game but a mod - else if (prefix === MOD_PREFIX) info.isMod = true; + else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; } return info; } @@ -467,11 +464,4 @@ function extractGameHostingData(platform, text) { return downloadData; } -/** - * Capitalize a string - * @param {String} string - */ -function capitalize(string) { - return string.toLowerCase().charAt(0).toUpperCase() + string.slice(1); -} //#endregion Private methods diff --git a/app/scripts/game-searcher.js b/app/scripts/game-searcher.js index 4cd6ea6..592b490 100644 --- a/app/scripts/game-searcher.js +++ b/app/scripts/game-searcher.js @@ -18,7 +18,7 @@ const { isF95URL } = require("./url-helper.js"); * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal */ module.exports.getSearchGameResults = async function (browser, gamename) { - shared.logger.info("Searching " + gamename + " on F95Zone"); + shared.logger.info(`Searching ${gamename} on F95Zone`); const page = await preparePage(browser); // Set new isolated page await page.setCookie(...shared.cookies); // Set cookies to avoid login @@ -30,11 +30,13 @@ module.exports.getSearchGameResults = async function (browser, gamename) { await Promise.all([ page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), + page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), page.waitForSelector(selectorK.SEARCH_BUTTON), ]); await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles + await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod await Promise.all([ page.click(selectorK.SEARCH_BUTTON), // Execute search page.waitForNavigation({ @@ -52,7 +54,7 @@ module.exports.getSearchGameResults = async function (browser, gamename) { const gameUrl = await getOnlyGameThreads(page, element); if (gameUrl !== null) results.push(gameUrl); } - shared.logger.info("Find " + results.length + " conversations"); + shared.logger.info(`Find ${results.length} conversations`); await page.close(); // Close the page return results; diff --git a/test/user-test.js b/test/user-test.js index 60152ae..c7a1442 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -1,25 +1,15 @@ -const { - debug, - login, - getGameData, - loadF95BaseData, - getUserData, - logout, -} = require("../app/index.js"); +const F95API = require("../app/index.js"); -debug(true); +F95API.debug(true); main(); async function main() { - const loginResult = await login("MillenniumEarl", "f9vTcRNuvxj4YpK"); + const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); if (loginResult.success) { - await loadF95BaseData(); - const gameData = await getGameData("kingdom of deception", false); + await F95API.loadF95BaseData(); + const gameData = await F95API.getGameData("a struggle with sin", false); console.log(gameData); - - // let userData = await getUserData(); - // console.log(userData); } - logout(); + F95API.logout(); }