Added support for fetch latest updates

pull/57/head
MillenniumEarl 2020-11-30 10:40:23 +01:00
parent 9ff4d6b12f
commit 8d0a30649e
2 changed files with 179 additions and 0 deletions

View File

@ -6,6 +6,7 @@ const networkHelper = require("./scripts/network-helper.js");
const scraper = require("./scripts/scraper.js");
const searcher = require("./scripts/searcher.js");
const uScraper = require("./scripts/user-scraper.js");
const latestFetch = require("./scripts/latest-fetch.js");
// Classes from file
const Credentials = require("./scripts/classes/credentials.js");
@ -171,4 +172,63 @@ module.exports.getUserData = async function () {
return await uScraper.getUserData();
};
/**
* @public
* Gets the latest updated games that match the specified parameters.
* You **must** be logged in to the portal before calling this method.
* @param {Object} args
* Parameters used for the search.
* @param {String[]} [args.tags]
* List of tags to be included in the search (max 5).
* @param {Number} [args.datelimit]
* Number of days since the game was last updated.
* The entered value will be approximated to the nearest valid one.
* Use `0` to select no time limit.
* @param {String[]} [args.prefixes]
* Prefixes to be included in the search.
* @param {String} [args.sorting]
* Method of sorting the results between (default: `date`):
* `date`, `likes`, `views`, `name`, `weighted`
* @param {Number} limit Maximum number of results
* @returns {Promise<GameInfo[]>} List of games
*/
module.exports.getLatestUpdates = async function(args, limit) {
// Check limit value
if(limit <= 0) throw new Error("limit must be greater than 0");
// Prepare the parser
const tp = new TagParser();
tp.fetch();
// Get the closest date limit
let filterDate = 0;
if(args.datelimit) {
// Script taken from:
// https://www.gavsblog.com/blog/find-closest-number-in-array-javascript
const validDate = [365, 180, 90, 30, 14, 7, 3, 1, 0];
validDate.sort((a, b) => {
return Math.abs(args.datelimit - a) - Math.abs(args.datelimit - b);
});
filterDate = validDate[0];
}
// Fetch the games
const query = {
tags: args.tags ? tp.tagsToIDs(args.tags) : [],
prefixes: [], // TODO: Add prefix parser
sort: args.sorting ? args.sorting : "date",
date: filterDate,
};
const urls = await latestFetch.fetchLatest(query, limit);
// Get the gamedata from urls
const gameInfoList = [];
for(const url of urls) {
const gameinfo = await exports.getGameDataFromURL(url);
gameInfoList.push(gameinfo);
}
return gameInfoList;
};
//#endregion

119
app/scripts/latest-fetch.js Normal file
View File

@ -0,0 +1,119 @@
"use strict";
// Modules from file
const { fetchGETResponse } = require("./network-helper.js");
const f95url = require("./constants/url.js");
/**
* @public
* Gets the URLs of the latest updated games that match the passed parameters.
* You *must* be logged.
* @param {Object} query
* Query used for the search
* @param {Number[]} [query.tags]
* List of tags to be included in the search. Max. 5 tags
* @param {Number[]} [query.prefixes]
* List of prefixes to be included in the search.
* @param {String} [query.sort]
* Sorting type between (default: `date`):
* `date`, `likes`, `views`, `name`, `weighted`
* @param {Number} [query.date]
* Date limit in days, to be understood as "less than".
* Possible values:
* `365`, `180`, `90`, `30`, `14`, `7`, `3`, `1`.
* Use `1` to indicate "today" or set no value to indicate "anytime"
* @param {Number} limit
* Maximum number of items to get. Default: 30
* @returns {Promise<String[]>} URLs of the fetched games
*/
module.exports.fetchLatest = async function(query, limit = 30) {
// Local variables
const threadURL = new URL("threads/", f95url.F95_BASE_URL).href;
const resultURLs = [];
let fetchedResults = 0;
let page = 1;
let noMorePages = false;
do {
// Prepare the URL
const url = parseLatestURL(query, page);
// Fetch the response (application/json)
const response = await fetchGETResponse(url);
// Save the URLs
for(const result of response.data.msg.data) {
if(fetchedResults >= limit) continue;
const gameURL = new URL(result.thread_id, threadURL).href;
resultURLs.push(gameURL);
fetchedResults += 1;
}
// Increment page and check for it's existence
page += 1;
if (page > response.data.msg.pagination.total) noMorePages = true;
}
while (fetchedResults < limit && !noMorePages);
return resultURLs;
};
/**
* @private
* Parse the URL with the passed parameters.
* @param {Object} query
* Query used for the search
* @param {Number[]} [query.tags]
* List of tags to be included in the search. Max. 5 tags
* @param {Number[]} [query.prefixes]
* List of prefixes to be included in the search.
* @param {String} [query.sort]
* Sorting type between (default: `date`):
* `date`, `likes`, `views`, `title`, `rating`
* @param {Number} [query.date]
* Date limit in days, to be understood as "less than".
* Possible values:
* `365`, `180`, `90`, `30`, `14`, `7`, `3`, `1`.
* Use `1` to indicate "today" or set no value to indicate "anytime"
* @param {Number} [page]
* Index of the page to be obtained. Default: 1.
*/
function parseLatestURL(query, page = 1) {
// Create the URL
const url = new URL("https://f95zone.to/new_latest.php");
url.searchParams.set("cmd", "list");
url.searchParams.set("cat", "games");
// Add the parameters
if (query.tags) {
if (query.tags.length > 5)
throw new Error(`Too many tags: ${query.tags.length} instead of 5`);
for(const tag of query.tags) {
url.searchParams.append("tags[]", tag);
}
}
if (query.prefixes) {
for (const p of query.prefixes) {
url.searchParams.append("prefixes[]", p);
}
}
if(query.sort) {
const validSort = ["date", "likes", "views", "title", "rating"];
if (!validSort.includes(query.sort))
throw new Error(`Invalid sort parameter: ${query.sort}`);
url.searchParams.set("sort", query.sort);
}
if (query.date) {
const validDate = [365, 180, 90, 30, 14, 7, 3, 1];
if (!validDate.includes(query.date))
throw new Error(`Invalid date parameter: ${query.date}`);
url.searchParams.set("date", query.date);
}
if (page) url.searchParams.set("page", page);
return url.toString();
}