From d24593a04a4d902ff62a02092276e784d354959e Mon Sep 17 00:00:00 2001 From: "samuele.berlusconi" Date: Sat, 24 Oct 2020 20:52:29 +0200 Subject: [PATCH 1/3] Better and cleaner game search --- app/scripts/constants/css-selector.js | 3 ++- app/scripts/constants/url.js | 2 +- app/scripts/game-scraper.js | 22 ++++++---------------- app/scripts/game-searcher.js | 6 ++++-- test/user-test.js | 22 ++++++---------------- 5 files changed, 19 insertions(+), 36 deletions(-) diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index 8b268c3..6ef347e 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -13,7 +13,8 @@ module.exports = Object.freeze({ ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]', PASSWORD_INPUT: 'input[name="password"]', SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: 'input[name="keywords"]', + SEARCH_FORM_TEXTBOX: 'input[name="keywords"][type="search"]', + SEARCH_ONLY_GAMES_OPTION: 'select[name="c[nodes][]"] > option[value="1"]', STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span', THREAD_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", diff --git a/app/scripts/constants/url.js b/app/scripts/constants/url.js index 816a00a..87a8847 100644 --- a/app/scripts/constants/url.js +++ b/app/scripts/constants/url.js @@ -1,6 +1,6 @@ module.exports = Object.freeze({ F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search", + F95_SEARCH_URL: "https://f95zone.to/search/?type=post", F95_LATEST_UPDATES: "https://f95zone.to/latest", F95_LOGIN_URL: "https://f95zone.to/login", F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", diff --git a/app/scripts/game-scraper.js b/app/scripts/game-scraper.js index 3265704..dd52d52 100644 --- a/app/scripts/game-scraper.js +++ b/app/scripts/game-scraper.js @@ -25,9 +25,9 @@ module.exports.getGameInfo = async function (browser, url) { // Verify the correctness of the URL const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); + if (!exists) throw new URIError(`${url} is not a valid URL`); if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); + throw new Error(`${url} is not a valid F95Zone URL`); const page = await preparePage(browser); // Set new isolated page await page.setCookie(...shared.cookies); // Set cookies to avoid login @@ -45,7 +45,6 @@ module.exports.getGameInfo = async function (browser, url) { info = await parsePrefixes(page, info); // Fill status/engines/isMod const structuredText = await getMainPostStructuredText(page); const overview = getOverview(structuredText, info.isMod); - const parsedInfos = parseConversationPage(structuredText); const previewSource = getGamePreviewSource(page); const changelog = getLastChangelog(page); @@ -283,10 +282,8 @@ async function getGameTags(page) { * @returns {Promise} GameInfo object passed in to which the identified information has been added */ async function parsePrefixes(page, info) { - const MOD_PREFIX = "MOD"; - // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' - info.status = "Ongoing"; + info.status = "ONGOING"; for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { const value = await page.evaluate( /* istanbul ignore next */ @@ -298,10 +295,10 @@ async function parsePrefixes(page, info) { const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); // Getting infos... - if (shared.statuses.includes(prefix)) info.status = capitalize(prefix); - else if (shared.engines.includes(prefix)) info.engine = capitalize(prefix); + if (shared.statuses.includes(prefix)) info.status = prefix; + else if (shared.engines.includes(prefix)) info.engine = prefix; // This is not a game but a mod - else if (prefix === MOD_PREFIX) info.isMod = true; + else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; } return info; } @@ -467,11 +464,4 @@ function extractGameHostingData(platform, text) { return downloadData; } -/** - * Capitalize a string - * @param {String} string - */ -function capitalize(string) { - return string.toLowerCase().charAt(0).toUpperCase() + string.slice(1); -} //#endregion Private methods diff --git a/app/scripts/game-searcher.js b/app/scripts/game-searcher.js index 4cd6ea6..592b490 100644 --- a/app/scripts/game-searcher.js +++ b/app/scripts/game-searcher.js @@ -18,7 +18,7 @@ const { isF95URL } = require("./url-helper.js"); * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal */ module.exports.getSearchGameResults = async function (browser, gamename) { - shared.logger.info("Searching " + gamename + " on F95Zone"); + shared.logger.info(`Searching ${gamename} on F95Zone`); const page = await preparePage(browser); // Set new isolated page await page.setCookie(...shared.cookies); // Set cookies to avoid login @@ -30,11 +30,13 @@ module.exports.getSearchGameResults = async function (browser, gamename) { await Promise.all([ page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), + page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), page.waitForSelector(selectorK.SEARCH_BUTTON), ]); await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles + await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod await Promise.all([ page.click(selectorK.SEARCH_BUTTON), // Execute search page.waitForNavigation({ @@ -52,7 +54,7 @@ module.exports.getSearchGameResults = async function (browser, gamename) { const gameUrl = await getOnlyGameThreads(page, element); if (gameUrl !== null) results.push(gameUrl); } - shared.logger.info("Find " + results.length + " conversations"); + shared.logger.info(`Find ${results.length} conversations`); await page.close(); // Close the page return results; diff --git a/test/user-test.js b/test/user-test.js index 60152ae..c7a1442 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -1,25 +1,15 @@ -const { - debug, - login, - getGameData, - loadF95BaseData, - getUserData, - logout, -} = require("../app/index.js"); +const F95API = require("../app/index.js"); -debug(true); +F95API.debug(true); main(); async function main() { - const loginResult = await login("MillenniumEarl", "f9vTcRNuvxj4YpK"); + const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); if (loginResult.success) { - await loadF95BaseData(); - const gameData = await getGameData("kingdom of deception", false); + await F95API.loadF95BaseData(); + const gameData = await F95API.getGameData("a struggle with sin", false); console.log(gameData); - - // let userData = await getUserData(); - // console.log(userData); } - logout(); + F95API.logout(); } From 681844688e50ffd0b22c0f50cd70514344928035 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Sat, 24 Oct 2020 18:53:04 +0000 Subject: [PATCH 2/3] Format code with prettier This commit fixes the style issues introduced in d24593a according to the output from prettier. Details: https://deepsource.io/gh/MillenniumEarl/F95API/transform/ae7afd9f-e945-4a6d-b27b-f910046a2311/ --- test/user-test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/user-test.js b/test/user-test.js index c7a1442..c6f6b29 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -1,4 +1,4 @@ -const F95API = require("../app/index.js"); +const F95API = require("../app/index.js"); F95API.debug(true); main(); From 9a37342363694fe1e70ee00423540483b5c26ca0 Mon Sep 17 00:00:00 2001 From: "samuele.berlusconi" Date: Sat, 24 Oct 2020 20:53:47 +0200 Subject: [PATCH 3/3] Reported coverage --- coverage.lcov | 304 +++++++++++++++++++++++++------------------------- 1 file changed, 153 insertions(+), 151 deletions(-) diff --git a/coverage.lcov b/coverage.lcov index 8522b42..ee63729 100644 --- a/coverage.lcov +++ b/coverage.lcov @@ -382,20 +382,21 @@ TN: SF:app\scripts\game-scraper.js FN:23,(anonymous_0) FN:83,(anonymous_1) -FN:115,getOverview -FN:130,getMainPostStructuredText -FN:149,getGameAuthor -FN:173,parseConversationPage -FN:199,getGamePreviewSource -FN:229,getGameTitle -FN:250,getGameTags -FN:273,parsePrefixes -FN:303,getLastChangelog -FN:462,capitalize +FN:113,cleanFSString +FN:126,getOverview +FN:141,getMainPostStructuredText +FN:160,getGameAuthor +FN:184,parseConversationPage +FN:210,getGamePreviewSource +FN:240,getGameTitle +FN:261,getGameTags +FN:284,parsePrefixes +FN:312,getLastChangelog FNF:12 FNH:12 FNDA:5,(anonymous_0) -FNDA:2,(anonymous_1) +FNDA:7,(anonymous_1) +FNDA:7,cleanFSString FNDA:5,getOverview FNDA:5,getMainPostStructuredText FNDA:5,getGameAuthor @@ -405,7 +406,6 @@ FNDA:5,getGameTitle FNDA:5,getGameTags FNDA:5,parsePrefixes FNDA:5,getLastChangelog -FNDA:5,capitalize DA:4,1 DA:5,1 DA:8,1 @@ -440,123 +440,124 @@ DA:55,5 DA:56,5 DA:57,5 DA:58,5 -DA:59,5 +DA:61,5 DA:62,5 DA:63,5 DA:72,5 DA:73,5 DA:74,5 DA:83,1 -DA:84,2 -DA:85,2 -DA:86,2 -DA:91,2 -DA:96,2 -DA:99,2 -DA:100,2 -DA:101,2 -DA:102,2 -DA:103,2 -DA:118,5 -DA:119,3 -DA:120,5 -DA:132,5 -DA:135,5 -DA:140,5 +DA:84,7 +DA:85,7 +DA:86,7 +DA:91,7 +DA:96,7 +DA:99,7 +DA:100,7 +DA:101,7 +DA:102,7 +DA:103,7 +DA:104,7 +DA:114,7 +DA:115,7 +DA:129,5 +DA:130,3 +DA:131,5 +DA:143,5 +DA:146,5 DA:151,5 -DA:156,5 -DA:159,5 DA:162,5 -DA:163,5 +DA:167,5 +DA:170,5 +DA:173,5 DA:174,5 -DA:177,5 -DA:178,5 -DA:179,1161 -DA:182,310 -DA:183,310 -DA:184,310 -DA:187,310 -DA:190,5 +DA:185,5 +DA:188,5 +DA:189,5 +DA:190,1161 +DA:193,310 +DA:194,310 +DA:195,310 +DA:198,310 DA:201,5 -DA:202,5 -DA:204,0 -DA:207,5 -DA:220,5 +DA:212,5 +DA:213,5 +DA:215,0 +DA:218,5 DA:231,5 -DA:236,5 -DA:239,5 -DA:240,5 -DA:241,5 +DA:242,5 +DA:247,5 +DA:250,5 DA:251,5 -DA:254,5 -DA:255,46 -DA:260,46 +DA:252,5 DA:262,5 -DA:274,5 -DA:277,5 -DA:278,5 -DA:279,9 -DA:286,9 -DA:289,9 -DA:290,9 -DA:292,4 -DA:294,5 -DA:305,5 -DA:307,5 -DA:308,5 -DA:310,5 -DA:315,5 -DA:318,5 -DA:319,3 -DA:320,5 -DA:321,5 -DA:463,5 -LF:105 -LH:103 +DA:265,5 +DA:266,46 +DA:271,46 +DA:273,5 +DA:286,5 +DA:287,5 +DA:288,9 +DA:295,9 +DA:298,9 +DA:299,9 +DA:301,4 +DA:303,5 +DA:314,5 +DA:316,5 +DA:317,5 +DA:319,5 +DA:324,5 +DA:327,5 +DA:328,3 +DA:329,5 +DA:330,5 +LF:106 +LH:104 BRDA:28,0,0,0 BRDA:28,0,1,5 BRDA:29,1,0,0 BRDA:29,1,1,5 BRDA:58,2,0,2 BRDA:58,2,1,3 -BRDA:59,3,0,2 -BRDA:59,3,1,3 -BRDA:102,4,0,2 -BRDA:102,4,1,0 -BRDA:118,5,0,2 -BRDA:118,5,1,3 -BRDA:179,6,0,851 -BRDA:179,6,1,310 -BRDA:187,7,0,151 -BRDA:187,7,1,159 -BRDA:220,8,0,5 -BRDA:220,8,1,0 -BRDA:289,9,0,0 -BRDA:289,9,1,9 -BRDA:290,10,0,5 -BRDA:290,10,1,4 -BRDA:292,11,0,2 -BRDA:292,11,1,2 -BRDA:308,12,0,0 -BRDA:308,12,1,5 -BRDA:318,13,0,3 -BRDA:318,13,1,2 -BRDA:320,14,0,1 -BRDA:320,14,1,4 +BRDA:102,3,0,5 +BRDA:102,3,1,2 +BRDA:129,4,0,2 +BRDA:129,4,1,3 +BRDA:190,5,0,851 +BRDA:190,5,1,310 +BRDA:198,6,0,151 +BRDA:198,6,1,159 +BRDA:231,7,0,5 +BRDA:231,7,1,0 +BRDA:298,8,0,0 +BRDA:298,8,1,9 +BRDA:299,9,0,5 +BRDA:299,9,1,4 +BRDA:301,10,0,2 +BRDA:301,10,1,2 +BRDA:301,11,0,4 +BRDA:301,11,1,2 +BRDA:317,12,0,0 +BRDA:317,12,1,5 +BRDA:327,13,0,3 +BRDA:327,13,1,2 +BRDA:329,14,0,1 +BRDA:329,14,1,4 BRF:30 -BRH:24 +BRH:25 end_of_record TN: SF:app\scripts\game-searcher.js FN:20,(anonymous_0) -FN:69,getOnlyGameThreads -FN:89,getMembershipForum -FN:116,getThreadURL +FN:71,getOnlyGameThreads +FN:91,getMembershipForum +FN:118,getThreadURL FNF:4 FNH:4 FNDA:2,(anonymous_0) -FNDA:12,getOnlyGameThreads -FNDA:12,getMembershipForum +FNDA:4,getOnlyGameThreads +FNDA:4,getMembershipForum FNDA:4,getThreadURL DA:4,1 DA:7,1 @@ -570,44 +571,45 @@ DA:23,2 DA:24,2 DA:25,2 DA:30,2 -DA:36,2 DA:37,2 DA:38,2 -DA:46,2 -DA:49,2 -DA:50,2 +DA:39,2 +DA:40,2 +DA:48,2 DA:51,2 -DA:52,12 -DA:53,12 -DA:55,2 -DA:56,2 +DA:52,2 +DA:53,2 +DA:54,4 +DA:55,4 +DA:57,2 DA:58,2 -DA:71,12 -DA:72,12 -DA:75,12 -DA:76,12 -DA:79,4 -DA:95,12 -DA:102,12 -DA:103,12 -DA:104,12 -DA:106,12 -DA:117,4 -DA:124,4 -DA:127,0 -DA:128,0 -LF:38 -LH:36 -BRDA:53,0,0,4 -BRDA:53,0,1,8 -BRDA:76,1,0,8 -BRDA:76,1,1,4 -BRDA:76,2,0,12 -BRDA:76,2,1,10 -BRDA:124,3,0,4 -BRDA:124,3,1,0 +DA:60,2 +DA:73,4 +DA:74,4 +DA:77,4 +DA:78,4 +DA:81,4 +DA:97,4 +DA:104,4 +DA:105,4 +DA:106,4 +DA:108,4 +DA:119,4 +DA:126,4 +DA:129,0 +DA:130,0 +LF:39 +LH:37 +BRDA:55,0,0,4 +BRDA:55,0,1,0 +BRDA:78,1,0,0 +BRDA:78,1,1,4 +BRDA:78,2,0,4 +BRDA:78,2,1,2 +BRDA:126,3,0,4 +BRDA:126,3,1,0 BRF:8 -BRH:7 +BRH:5 end_of_record TN: SF:app\scripts\puppeteer-helper.js @@ -617,8 +619,8 @@ FN:45,(anonymous_2) FNF:3 FNH:3 FNDA:14,(anonymous_0) -FNDA:19,(anonymous_1) -FNDA:2266,(anonymous_2) +FNDA:24,(anonymous_1) +FNDA:2770,(anonymous_2) DA:4,1 DA:7,1 DA:15,1 @@ -628,23 +630,23 @@ DA:19,0 DA:24,14 DA:29,14 DA:39,1 -DA:41,19 -DA:44,19 -DA:45,19 -DA:46,2266 -DA:47,1391 -DA:50,1391 -DA:55,19 -DA:57,19 -DA:59,19 +DA:41,24 +DA:44,24 +DA:45,24 +DA:46,2770 +DA:47,1657 +DA:50,1657 +DA:55,24 +DA:57,24 +DA:59,24 LF:18 LH:17 BRDA:18,0,0,0 BRDA:18,0,1,14 -BRDA:46,1,0,875 -BRDA:46,1,1,1391 +BRDA:46,1,0,1113 +BRDA:46,1,1,1657 BRDA:47,2,0,0 -BRDA:47,2,1,1391 +BRDA:47,2,1,1657 BRF:6 BRH:4 end_of_record @@ -672,7 +674,7 @@ FNF:18 FNH:15 FNDA:14,(anonymous_0) FNDA:38,(anonymous_1) -FNDA:36,(anonymous_2) +FNDA:41,(anonymous_2) FNDA:9,(anonymous_3) FNDA:9,(anonymous_4) FNDA:0,(anonymous_5) @@ -701,7 +703,7 @@ DA:53,1 DA:58,1 DA:67,14 DA:74,38 -DA:81,36 +DA:81,41 DA:88,9 DA:95,9 DA:102,0