diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 16b1503..8c34bc9 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -21,7 +21,6 @@ const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15"; axiosCookieJarSupport(axios); const cookieJar = new tough.CookieJar(); - const commonConfig = { headers: { "User-Agent": userAgent, diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js index bfa823a..e5cd538 100644 --- a/app/scripts/scraper.js +++ b/app/scripts/scraper.js @@ -66,7 +66,7 @@ module.exports.getGameInfo = async function (url) { * Parse the game prefixes obtaining the engine used, * the advancement status and if the game is actually a game or a mod. * @param {cheerio.Cheerio} body Page `body` selector - * @returns {Object} Dictionary of values + * @returns {Object.} Dictionary of values with keys `engine`, `status`, `mod` */ function parseGamePrefixes(body) { shared.logger.trace("Parsing prefixes..."); @@ -106,7 +106,7 @@ function parseGamePrefixes(body) { * @private * Extracts all the possible informations from the title. * @param {cheerio.Cheerio} body Page `body` selector - * @returns {Object} Dictionary of values + * @returns {Object.} Dictionary of values with keys `name`, `author`, `version` */ function extractInfoFromTitle(body) { shared.logger.trace("Extracting information from title..."); @@ -202,8 +202,10 @@ function extractChangelog(mainPost) { * @private * Process the main post text to get all the useful * information in the format *DESCRIPTOR : VALUE*. + * Gets "standard" values such as: `Language`, `SupportedOS`, `Censored`, and `LastUpdate`. + * All non-canonical values are instead grouped together as a dictionary with the key `Various`. * @param {String} text Structured text of the post - * @returns {Object} Dictionary of information + * @returns {Object.} Dictionary of information */ function parseMainPostText(text) { shared.logger.trace("Parsing main post raw text..."); @@ -274,7 +276,7 @@ function parseMainPostText(text) { * @private * Extracts and processes the JSON-LD values found at the bottom of the page. * @param {cheerio.Cheerio} body Page `body` selector - * @returns {Object} JSON-LD or `null` if no valid JSON is found + * @returns {Object.} JSON-LD or `null` if no valid JSON is found */ function extractStructuredData(body) { shared.logger.trace("Extracting JSON-LD data..."); diff --git a/app/scripts/user-scraper.js b/app/scripts/user-scraper.js index 0d577de..05cff21 100644 --- a/app/scripts/user-scraper.js +++ b/app/scripts/user-scraper.js @@ -9,6 +9,11 @@ const f95Selector = require("./constants/css-selector.js"); const f95url = require("./constants/url.js"); const UserData = require("./classes/user-data.js"); +/** + * @protected + * Gets user data, such as username, url of watched threads, and profile picture url. + * @return {UserData} User data + */ module.exports.getUserData = async function() { // Fetch data const data = await fetchUsernameAndAvatar(); @@ -24,6 +29,13 @@ module.exports.getUserData = async function() { }; //#region Private methods +/** + * @private + * It connects to the page and extracts the name + * of the currently logged in user and the URL + * of their profile picture. + * @return {Object.} + */ async function fetchUsernameAndAvatar() { // Fetch page const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL); @@ -44,6 +56,11 @@ async function fetchUsernameAndAvatar() { }; } +/** + * @private + * Gets the list of URLs of threads watched by the user. + * @returns {String[]} List of URLs + */ async function fetchWatchedThreadURLs() { // Local variables let currentURL = f95url.F95_WATCHED_THREADS; @@ -69,6 +86,12 @@ async function fetchWatchedThreadURLs() { return wathcedThreadURLs; } +/** + * @private + * Gets the URLs of the watched threads on the page. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String[]} + */ function fetchPageURLs(body) { const elements = body.find(f95Selector.WT_URLS); @@ -82,8 +105,11 @@ function fetchPageURLs(body) { } /** - * - * @param {cheerio.Cheerio} body + * @private + * Gets the URL of the next page containing the watched threads + * or `null` if that page does not exist. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String} */ function fetchNextPageURL(body) { const element = body.find(f95Selector.WT_NEXT_PAGE).first();