Addes user data fetching
MillenniumEarl
parent
49712b2337
commit
90100709ba
94
app/index.js
94
app/index.js
|
|
@ -2,11 +2,10 @@
|
|||
|
||||
// Modules from file
|
||||
const shared = require("./scripts/shared.js");
|
||||
const f95url = require("./scripts/constants/url.js");
|
||||
const f95selector = require("./scripts/constants/css-selector.js");
|
||||
const networkHelper = require("./scripts/network-helper.js");
|
||||
const scraper = require("./scripts/scraper.js");
|
||||
const searcher = require("./scripts/searcher.js");
|
||||
const uScraper = require("./scripts/user-scraper.js");
|
||||
|
||||
// Classes from file
|
||||
const Credentials = require("./scripts/classes/credentials.js");
|
||||
|
|
@ -163,95 +162,6 @@ module.exports.getUserData = async function () {
|
|||
return null;
|
||||
}
|
||||
|
||||
const threads = await getUserWatchedGameThreads(null);
|
||||
|
||||
const username = await page.evaluate(
|
||||
/* istanbul ignore next */ (selector) =>
|
||||
document.querySelector(selector).innerText,
|
||||
f95selector.USERNAME_ELEMENT
|
||||
);
|
||||
|
||||
const avatarSrc = await page.evaluate(
|
||||
/* istanbul ignore next */ (selector) =>
|
||||
document.querySelector(selector).getAttribute("src"),
|
||||
f95selector.AVATAR_PIC
|
||||
);
|
||||
|
||||
const ud = new UserData();
|
||||
ud.username = username;
|
||||
ud.avatarSrc = networkHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null;
|
||||
ud.watchedThreads = threads;
|
||||
|
||||
return ud;
|
||||
return await uScraper.getUserData();
|
||||
};
|
||||
//#endregion
|
||||
|
||||
//#region Private methods
|
||||
|
||||
//#region User
|
||||
/**
|
||||
* @private
|
||||
* Gets the list of URLs of threads the user follows.
|
||||
* @param {puppeteer.Browser} browser Browser object used for navigation
|
||||
* @returns {Promise<String[]>} URL list
|
||||
*/
|
||||
async function getUserWatchedGameThreads() {
|
||||
const page = null;
|
||||
await page.goto(f95url.F95_WATCHED_THREADS); // Go to the thread page
|
||||
|
||||
// Explicitly wait for the required items to load
|
||||
await page.waitForSelector(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON);
|
||||
|
||||
// Show the popup
|
||||
await Promise.all([
|
||||
page.click(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON),
|
||||
page.waitForSelector(f95selector.UNREAD_THREAD_CHECKBOX),
|
||||
page.waitForSelector(f95selector.ONLY_GAMES_THREAD_OPTION),
|
||||
page.waitForSelector(f95selector.FILTER_THREADS_BUTTON),
|
||||
]);
|
||||
|
||||
// Set the filters
|
||||
await page.evaluate(
|
||||
/* istanbul ignore next */ (selector) =>
|
||||
document.querySelector(selector).removeAttribute("checked"),
|
||||
f95selector.UNREAD_THREAD_CHECKBOX
|
||||
); // Also read the threads already read
|
||||
|
||||
// Filter the threads
|
||||
await page.click(f95selector.ONLY_GAMES_THREAD_OPTION);
|
||||
await page.click(f95selector.FILTER_THREADS_BUTTON);
|
||||
await page.waitForSelector(f95selector.WATCHED_THREAD_URLS);
|
||||
|
||||
// Get the threads urls
|
||||
const urls = [];
|
||||
let nextPageExists = false;
|
||||
do {
|
||||
// Get all the URLs
|
||||
for (const handle of await page.$$(f95selector.WATCHED_THREAD_URLS)) {
|
||||
const src = await page.evaluate(
|
||||
/* istanbul ignore next */ (element) => element.href,
|
||||
handle
|
||||
);
|
||||
// If 'unread' is left, it will redirect to the last unread post
|
||||
const url = src.replace("/unread", "");
|
||||
urls.push(url);
|
||||
}
|
||||
|
||||
nextPageExists = await page.evaluate(
|
||||
/* istanbul ignore next */ (selector) => document.querySelector(selector),
|
||||
f95selector.WATCHED_THREAD_NEXT_PAGE
|
||||
);
|
||||
|
||||
// Click to next page
|
||||
if (nextPageExists) {
|
||||
await page.click(f95selector.WATCHED_THREAD_NEXT_PAGE);
|
||||
await page.waitForSelector(f95selector.WATCHED_THREAD_URLS);
|
||||
}
|
||||
} while (nextPageExists);
|
||||
|
||||
await page.close();
|
||||
return urls;
|
||||
}
|
||||
//#endregion User
|
||||
|
||||
//#endregion Private methods
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
module.exports = Object.freeze({
|
||||
BD_ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span",
|
||||
BD_STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span",
|
||||
|
||||
GT_IMAGES: "img:not([title])[data-src^=\"https://attachments.f95zone.to\"][data-url=\"\"]",
|
||||
GT_TAGS: "a.tagItem",
|
||||
GT_TITLE: "h1.p-title-value",
|
||||
|
|
@ -16,13 +17,7 @@ module.exports = Object.freeze({
|
|||
GS_RESULT_BODY: "div.contentRow-main",
|
||||
GS_MEMBERSHIP: "li > a:not(.username)",
|
||||
GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]",
|
||||
|
||||
LOGIN_BUTTON: "button.button--icon--login",
|
||||
UD_USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText",
|
||||
UD_AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]",
|
||||
LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic",
|
||||
PASSWORD_INPUT: "input[name=\"password\"]",
|
||||
USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText",
|
||||
USERNAME_INPUT: "input[name=\"login\"]",
|
||||
AVATAR_INFO: "span.avatar",
|
||||
AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]",
|
||||
FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]",
|
||||
});
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
// Public modules from npm
|
||||
const axios = require("axios").default;
|
||||
const _ = require("lodash");
|
||||
const { isString } = require("lodash");
|
||||
const ky = require("ky-universal").create({
|
||||
throwHttpErrors: false,
|
||||
});
|
||||
|
|
@ -166,7 +166,7 @@ module.exports.fetchGETResponse = async function(url) {
|
|||
* @returns {String}
|
||||
*/
|
||||
module.exports.enforceHttpsUrl = function (url) {
|
||||
return _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
|
||||
return isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1 +1,97 @@
|
|||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
const cheerio = require("cheerio");
|
||||
|
||||
// Modules from file
|
||||
const networkHelper = require("./network-helper.js");
|
||||
const f95Selector = require("./constants/css-selector.js");
|
||||
const f95url = require("./constants/url.js");
|
||||
const UserData = require("./classes/user-data.js");
|
||||
|
||||
module.exports.getUserData = async function() {
|
||||
// Fetch data
|
||||
const data = await fetchUsernameAndAvatar();
|
||||
const urls = await fetchWatchedThreadURLs();
|
||||
|
||||
// Create object
|
||||
const ud = new UserData();
|
||||
ud.username = data.username;
|
||||
ud.avatarSrc = data.source;
|
||||
ud.watchedThreads = urls;
|
||||
|
||||
return ud;
|
||||
};
|
||||
|
||||
//#region Private methods
|
||||
async function fetchUsernameAndAvatar() {
|
||||
// Fetch page
|
||||
const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL);
|
||||
|
||||
// Load HTML response
|
||||
const $ = cheerio.load(html);
|
||||
const body = $("body");
|
||||
|
||||
// Fetch username
|
||||
const username = body.find(f95Selector.UD_USERNAME_ELEMENT).first().text().trim();
|
||||
|
||||
// Fetch user avatar image source
|
||||
const source = body.find(f95Selector.UD_AVATAR_PIC).first().attr("src");
|
||||
|
||||
return {
|
||||
username,
|
||||
source
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchWatchedThreadURLs() {
|
||||
// Local variables
|
||||
let currentURL = f95url.F95_WATCHED_THREADS;
|
||||
const wathcedThreadURLs = [];
|
||||
|
||||
do {
|
||||
// Fetch page
|
||||
const html = await networkHelper.fetchHTML(currentURL);
|
||||
|
||||
// Load HTML response
|
||||
const $ = cheerio.load(html);
|
||||
const body = $("body");
|
||||
|
||||
// Find the URLs
|
||||
const urls = fetchPageURLs(body);
|
||||
wathcedThreadURLs.push(...urls);
|
||||
|
||||
// Find the next page (if any)
|
||||
currentURL = fetchNextPageURL(body);
|
||||
}
|
||||
while (currentURL);
|
||||
|
||||
return wathcedThreadURLs;
|
||||
}
|
||||
|
||||
function fetchPageURLs(body) {
|
||||
const elements = body.find(f95Selector.WT_URLS);
|
||||
|
||||
return elements.map(function extractURLs(idx, e) {
|
||||
// Obtain the link (replace "unread" only for the unread threads)
|
||||
const partialLink = e.attribs.href.replace("unread", "");
|
||||
|
||||
// Compose and return the URL
|
||||
return new URL(partialLink, f95url.F95_BASE_URL).toString();
|
||||
}).get();
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param {cheerio.Cheerio} body
|
||||
*/
|
||||
function fetchNextPageURL(body) {
|
||||
const element = body.find(f95Selector.WT_NEXT_PAGE).first();
|
||||
|
||||
// No element found
|
||||
if(element.length === 0) return null;
|
||||
|
||||
// Compose and return the URL
|
||||
return new URL(element.attr("href"), f95url.F95_BASE_URL).toString();
|
||||
}
|
||||
//#endregion Private methods
|
||||
|
|
@ -8,6 +8,7 @@ const searcher = require("../app/scripts/searcher.js");
|
|||
const scraper = require("../app/scripts/scraper.js");
|
||||
const Credentials = require("../app/scripts/classes/credentials.js");
|
||||
const networkHelper = require("../app/scripts/network-helper.js");
|
||||
const uScraper = require("../app/scripts/user-scraper.js");
|
||||
|
||||
// Configure the .env reader
|
||||
dotenv.config();
|
||||
|
|
@ -16,6 +17,10 @@ dotenv.config();
|
|||
auth().then(async function searchGames(result) {
|
||||
if(!result) return;
|
||||
|
||||
console.log("Fetching user data...");
|
||||
const userdata = await uScraper.getUserData();
|
||||
console.log(`${userdata.username} follows ${userdata.watchedThreads.length} threads`);
|
||||
|
||||
// Search for Kingdom Of Deception data
|
||||
await search("kingdom of deception");
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue