From 20fea5c3152d00f18e2a1872b3f3c410fb032332 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Fri, 30 Oct 2020 20:41:56 +0100 Subject: [PATCH] Implemented search for games and mods --- app/scriptsV2/classes/game-info.js | 111 ++++++++++++++++++++++ app/scriptsV2/classes/login-result.js | 20 ++++ app/scriptsV2/classes/user-data.js | 26 ++++++ app/scriptsV2/constants/css-selector.js | 31 +++++++ app/scriptsV2/constants/url.js | 7 ++ app/scriptsV2/network-helper.js | 35 +++++++ app/scriptsV2/search.js | 88 ++++++++++++++++++ package-lock.json | 118 +++++++++++++++++++++++- package.json | 2 + 9 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 app/scriptsV2/classes/game-info.js create mode 100644 app/scriptsV2/classes/login-result.js create mode 100644 app/scriptsV2/classes/user-data.js create mode 100644 app/scriptsV2/constants/css-selector.js create mode 100644 app/scriptsV2/constants/url.js create mode 100644 app/scriptsV2/network-helper.js create mode 100644 app/scriptsV2/search.js diff --git a/app/scriptsV2/classes/game-info.js b/app/scriptsV2/classes/game-info.js new file mode 100644 index 0000000..44e7137 --- /dev/null +++ b/app/scriptsV2/classes/game-info.js @@ -0,0 +1,111 @@ +"use strict"; + +class GameInfo { + constructor() { + //#region Properties + /** + * Game name + * @type String + */ + this.name = null; + /** + * Game author + * @type String + */ + this.author = null; + /** + * URL to the game's official conversation on the F95Zone portal + * @type String + */ + this.url = null; + /** + * Game description + * @type String + */ + this.overview = null; + /** + * List of tags associated with the game + * @type String[] + */ + this.tags = []; + /** + * Graphics engine used for game development + * @type String + */ + this.engine = null; + /** + * Progress of the game + * @type String + */ + this.status = null; + /** + * Game description image URL + * @type String + */ + this.previewSrc = null; + /** + * Game version + * @type String + */ + this.version = null; + /** + * Last time the game underwent updates + * @type String + */ + this.lastUpdate = null; + /** + * Last time the local copy of the game was run + * @type String + */ + this.lastPlayed = null; + /** + * Specifies if the game is original or a mod + * @type Boolean + */ + this.isMod = false; + /** + * Changelog for the last version. + * @type String + */ + this.changelog = null; + /** + * Directory containing the local copy of the game + * @type String + */ + this.gameDir = null; + //#endregion Properties + } + + /** + * Converts the object to a dictionary used for JSON serialization. + */ + /* istanbul ignore next */ + toJSON() { + return { + name: this.name, + author: this.author, + url: this.url, + overview: this.overview, + engine: this.engine, + status: this.status, + previewSrc: this.previewSrc, + version: this.version, + lastUpdate: this.lastUpdate, + lastPlayed: this.lastPlayed, + isMod: this.isMod, + changelog: this.changelog, + gameDir: this.gameDir, + }; + } + + /** + * Return a new GameInfo from a JSON string. + * @param {String} json JSON string used to create the new object + * @returns {GameInfo} + */ + /* istanbul ignore next */ + static fromJSON(json) { + return Object.assign(new GameInfo(), json); + } +} +module.exports = GameInfo; diff --git a/app/scriptsV2/classes/login-result.js b/app/scriptsV2/classes/login-result.js new file mode 100644 index 0000000..51fd9bc --- /dev/null +++ b/app/scriptsV2/classes/login-result.js @@ -0,0 +1,20 @@ +"use strict"; + +/** + * Object obtained in response to an attempt to login to the portal. + */ +class LoginResult { + constructor(success, message) { + /** + * Result of the login operation + * @type Boolean + */ + this.success = success; + /** + * Login response message + * @type String + */ + this.message = message; + } +} +module.exports = LoginResult; diff --git a/app/scriptsV2/classes/user-data.js b/app/scriptsV2/classes/user-data.js new file mode 100644 index 0000000..7edb904 --- /dev/null +++ b/app/scriptsV2/classes/user-data.js @@ -0,0 +1,26 @@ +"use strict"; + +/** + * Class containing the data of the user currently connected to the F95Zone platform. + */ +class UserData { + constructor() { + /** + * User username. + * @type String + */ + this.username = ""; + /** + * Path to the user's profile picture. + * @type String + */ + this.avatarSrc = null; + /** + * List of followed thread URLs. + * @type URL[] + */ + this.watchedThreads = []; + } +} + +module.exports = UserData; diff --git a/app/scriptsV2/constants/css-selector.js b/app/scriptsV2/constants/css-selector.js new file mode 100644 index 0000000..032e6c9 --- /dev/null +++ b/app/scriptsV2/constants/css-selector.js @@ -0,0 +1,31 @@ +module.exports = Object.freeze({ + AVATAR_INFO: "span.avatar", + AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", + ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", + FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", + GT_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", + GT_TAGS: "a.tagItem", + GT_TITLE: "h1.p-title-value", + GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", + LOGIN_BUTTON: "button.button--icon--login", + LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", + ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", + PASSWORD_INPUT: "input[name=\"password\"]", + SEARCH_BUTTON: "form.block > * button.button--icon--search", + SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", + SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", + STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", + THREAD_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", + GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", + TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", + WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", + USERNAME_INPUT: "input[name=\"login\"]", + WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WT_NEXT_PAGE: "a.pageNav-jump--next", + WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]", + DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", + GS_RESULT_BODY: "div.contentRow-main", + GS_MEMBERSHIP: "li > a:not(.username)", + THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", +}); diff --git a/app/scriptsV2/constants/url.js b/app/scriptsV2/constants/url.js new file mode 100644 index 0000000..fac63ba --- /dev/null +++ b/app/scriptsV2/constants/url.js @@ -0,0 +1,7 @@ +module.exports = Object.freeze({ + F95_BASE_URL: "https://f95zone.to", + F95_SEARCH_URL: "https://f95zone.to/search/?type=post", + F95_LATEST_UPDATES: "https://f95zone.to/latest", + F95_LOGIN_URL: "https://f95zone.to/login", + F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", +}); diff --git a/app/scriptsV2/network-helper.js b/app/scriptsV2/network-helper.js new file mode 100644 index 0000000..6bf95b9 --- /dev/null +++ b/app/scriptsV2/network-helper.js @@ -0,0 +1,35 @@ +"use strict"; + +// Public modules from npm +const axios = require("axios").default; +const _ = require("lodash"); + +// Modules from file +const shared = require("./scripts/shared.js"); + +/** + * @protected + * Gets the HTML code of a page. + * @param {String} url URL to fetch + * @returns {Promise} HTML code or `null` if an error arise + */ +module.exports = async function fetchHTML(url) { + try { + const response = await axios.get(url); + return response.data; + } catch { + shared.logger.error(`An error occurred while trying to fetch the URL: ${url}`); + return null; + } +}; + +/** + * @protected + * Enforces the scheme of the URL is https and returns the new URL. + * @param {String} url + * @returns {String} + */ +module.exports = function enforceHttpsUrl(url) { + const value = _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; + return value; +}; \ No newline at end of file diff --git a/app/scriptsV2/search.js b/app/scriptsV2/search.js new file mode 100644 index 0000000..d348169 --- /dev/null +++ b/app/scriptsV2/search.js @@ -0,0 +1,88 @@ +"use strict"; + +// Public modules from npm +const cheerio = require("cheerio"); + +// Modules from file +const { fetchHTML } = require("./network-helper.js"); +const shared = require("./scripts/shared.js"); +const f95Selector = require("./constants/css-selector.js"); + +/** + * @protected + * Search for a game on F95Zone and return a list of URLs, one for each search result. + * @param {String} name Game name + * @returns {Promise} URLs of results + */ +module.exports = async function searchGame(name) { + shared.logger.info(`Searching games with name ${name}`); + + // Replace the whitespaces with + + const searchName = name.replaceAll(" ", "+").toUpperCase(); + + // Prepare the URL (only title, search in the "Games" section, order by relevance) + const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; + + // Fetch and parse the result URLs + return await fetchResultURLs(url); +}; + +/** + * @protected + * Search for a mod on F95Zone and return a list of URLs, one for each search result. + * @param {String} name Mod name + * @returns {Promise} URLs of results + */ +module.exports = async function searchMod(name) { + shared.logger.info(`Searching mods with name ${name}`); + // Replace the whitespaces with + + const searchName = name.replaceAll(" ", "+").toUpperCase(); + + // Prepare the URL (only title, search in the "Mods" section, order by relevance) + const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; + + // Fetch and parse the result URLs + return await fetchResultURLs(url); +}; + +//#region Private methods +/** + * @private + * Gets the URLs of the threads resulting from the F95Zone search. + * @param {String} url Search URL + * @return {Promise} List of URLs + */ +async function fetchResultURLs(url) { + shared.logger.info(`Fetching ${url}...`); + + // Fetch HTML and prepare Cheerio + const html = await fetchHTML(url); + const $ = cheerio.load(html); + + // Here we get all the DIV that are the body of the various query results + const results = $("body").find(f95Selector.GS_RESULT_BODY); + + // Than we extract the URLs + const urls = results.map((idx, el) => { + const elementSelector = $(el); + return extractLinkFromResult(elementSelector); + }).get(); + + return urls; +} + +/** + * @private + * Look for the URL to the thread referenced by the item. + * @param {cheerio.Cheerio} selector Element to search + * @returns {String} URL to thread + */ +function extractLinkFromResult(selector) { + const link = selector + .find(f95Selector.GS_RESULT_THREAD_TITLE) + .attr("href") + .trim(); + + return link; +} +//#endregion Private methods diff --git a/package-lock.json b/package-lock.json index 3018479..d65e78b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -285,8 +285,7 @@ "@types/node": { "version": "14.11.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", - "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==", - "optional": true + "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==" }, "@types/yauzl": { "version": "2.9.1", @@ -423,6 +422,14 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "axios": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz", + "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, "babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -471,6 +478,11 @@ "readable-stream": "^3.4.0" } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -564,6 +576,19 @@ "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "chokidar": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz", @@ -648,6 +673,22 @@ "which": "^2.0.1" } }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==" + }, "data-uri-to-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", @@ -725,6 +766,37 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dotenv": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", @@ -754,6 +826,11 @@ "ansi-colors": "^4.1.1" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, "es-abstract": { "version": "1.17.6", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", @@ -1159,6 +1236,11 @@ "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" + }, "foreground-child": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", @@ -1327,6 +1409,19 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + } + }, "https-proxy-agent": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", @@ -1724,8 +1819,7 @@ "lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", - "dev": true + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, "lodash.flattendeep": { "version": "4.4.0", @@ -2127,6 +2221,14 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, "nyc": { "version": "15.1.0", "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", @@ -2259,6 +2361,14 @@ "callsites": "^3.0.0" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", diff --git a/package.json b/package.json index 2836d66..c178177 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "node": ">=10.0" }, "dependencies": { + "axios": "^0.21.0", + "cheerio": "^1.0.0-rc.3", "ky": "^0.24.0", "ky-universal": "^0.8.2", "log4js": "^6.3.0",