From 8d0a30649e19045c38b158bc4645e7cba9eafbdf Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 30 Nov 2020 10:40:23 +0100 Subject: [PATCH] Added support for fetch latest updates --- app/index.js | 60 ++++++++++++++++++ app/scripts/latest-fetch.js | 119 ++++++++++++++++++++++++++++++++++++ 2 files changed, 179 insertions(+) create mode 100644 app/scripts/latest-fetch.js diff --git a/app/index.js b/app/index.js index 4862d08..c5a4ac1 100644 --- a/app/index.js +++ b/app/index.js @@ -6,6 +6,7 @@ const networkHelper = require("./scripts/network-helper.js"); const scraper = require("./scripts/scraper.js"); const searcher = require("./scripts/searcher.js"); const uScraper = require("./scripts/user-scraper.js"); +const latestFetch = require("./scripts/latest-fetch.js"); // Classes from file const Credentials = require("./scripts/classes/credentials.js"); @@ -171,4 +172,63 @@ module.exports.getUserData = async function () { return await uScraper.getUserData(); }; + +/** + * @public + * Gets the latest updated games that match the specified parameters. + * You **must** be logged in to the portal before calling this method. + * @param {Object} args + * Parameters used for the search. + * @param {String[]} [args.tags] + * List of tags to be included in the search (max 5). + * @param {Number} [args.datelimit] + * Number of days since the game was last updated. + * The entered value will be approximated to the nearest valid one. + * Use `0` to select no time limit. + * @param {String[]} [args.prefixes] + * Prefixes to be included in the search. + * @param {String} [args.sorting] + * Method of sorting the results between (default: `date`): + * `date`, `likes`, `views`, `name`, `weighted` + * @param {Number} limit Maximum number of results + * @returns {Promise} List of games + */ +module.exports.getLatestUpdates = async function(args, limit) { + // Check limit value + if(limit <= 0) throw new Error("limit must be greater than 0"); + + // Prepare the parser + const tp = new TagParser(); + tp.fetch(); + + // Get the closest date limit + let filterDate = 0; + if(args.datelimit) { + // Script taken from: + // https://www.gavsblog.com/blog/find-closest-number-in-array-javascript + const validDate = [365, 180, 90, 30, 14, 7, 3, 1, 0]; + validDate.sort((a, b) => { + return Math.abs(args.datelimit - a) - Math.abs(args.datelimit - b); + }); + filterDate = validDate[0]; + } + + // Fetch the games + const query = { + tags: args.tags ? tp.tagsToIDs(args.tags) : [], + prefixes: [], // TODO: Add prefix parser + sort: args.sorting ? args.sorting : "date", + date: filterDate, + }; + const urls = await latestFetch.fetchLatest(query, limit); + + // Get the gamedata from urls + const gameInfoList = []; + for(const url of urls) { + const gameinfo = await exports.getGameDataFromURL(url); + gameInfoList.push(gameinfo); + } + + return gameInfoList; +}; //#endregion diff --git a/app/scripts/latest-fetch.js b/app/scripts/latest-fetch.js new file mode 100644 index 0000000..e94959f --- /dev/null +++ b/app/scripts/latest-fetch.js @@ -0,0 +1,119 @@ +"use strict"; + +// Modules from file +const { fetchGETResponse } = require("./network-helper.js"); +const f95url = require("./constants/url.js"); + +/** + * @public + * Gets the URLs of the latest updated games that match the passed parameters. + * You *must* be logged. + * @param {Object} query + * Query used for the search + * @param {Number[]} [query.tags] + * List of tags to be included in the search. Max. 5 tags + * @param {Number[]} [query.prefixes] + * List of prefixes to be included in the search. + * @param {String} [query.sort] + * Sorting type between (default: `date`): + * `date`, `likes`, `views`, `name`, `weighted` + * @param {Number} [query.date] + * Date limit in days, to be understood as "less than". + * Possible values: + * `365`, `180`, `90`, `30`, `14`, `7`, `3`, `1`. + * Use `1` to indicate "today" or set no value to indicate "anytime" + * @param {Number} limit + * Maximum number of items to get. Default: 30 + * @returns {Promise} URLs of the fetched games + */ +module.exports.fetchLatest = async function(query, limit = 30) { + // Local variables + const threadURL = new URL("threads/", f95url.F95_BASE_URL).href; + const resultURLs = []; + let fetchedResults = 0; + let page = 1; + let noMorePages = false; + + do { + // Prepare the URL + const url = parseLatestURL(query, page); + + // Fetch the response (application/json) + const response = await fetchGETResponse(url); + + // Save the URLs + for(const result of response.data.msg.data) { + if(fetchedResults >= limit) continue; + const gameURL = new URL(result.thread_id, threadURL).href; + resultURLs.push(gameURL); + fetchedResults += 1; + } + + // Increment page and check for it's existence + page += 1; + if (page > response.data.msg.pagination.total) noMorePages = true; + } + while (fetchedResults < limit && !noMorePages); + + return resultURLs; +}; + +/** + * @private + * Parse the URL with the passed parameters. + * @param {Object} query + * Query used for the search + * @param {Number[]} [query.tags] + * List of tags to be included in the search. Max. 5 tags + * @param {Number[]} [query.prefixes] + * List of prefixes to be included in the search. + * @param {String} [query.sort] + * Sorting type between (default: `date`): + * `date`, `likes`, `views`, `title`, `rating` + * @param {Number} [query.date] + * Date limit in days, to be understood as "less than". + * Possible values: + * `365`, `180`, `90`, `30`, `14`, `7`, `3`, `1`. + * Use `1` to indicate "today" or set no value to indicate "anytime" + * @param {Number} [page] + * Index of the page to be obtained. Default: 1. + */ +function parseLatestURL(query, page = 1) { + // Create the URL + const url = new URL("https://f95zone.to/new_latest.php"); + url.searchParams.set("cmd", "list"); + url.searchParams.set("cat", "games"); + + // Add the parameters + if (query.tags) { + if (query.tags.length > 5) + throw new Error(`Too many tags: ${query.tags.length} instead of 5`); + + for(const tag of query.tags) { + url.searchParams.append("tags[]", tag); + } + } + if (query.prefixes) { + for (const p of query.prefixes) { + url.searchParams.append("prefixes[]", p); + } + } + + if(query.sort) { + const validSort = ["date", "likes", "views", "title", "rating"]; + if (!validSort.includes(query.sort)) + throw new Error(`Invalid sort parameter: ${query.sort}`); + url.searchParams.set("sort", query.sort); + } + + if (query.date) { + const validDate = [365, 180, 90, 30, 14, 7, 3, 1]; + if (!validDate.includes(query.date)) + throw new Error(`Invalid date parameter: ${query.date}`); + url.searchParams.set("date", query.date); + } + + if (page) url.searchParams.set("page", page); + + return url.toString(); +} \ No newline at end of file