From ecfd1784b12eb47c4b01d98306a01a3b4aee3832 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Sun, 1 Nov 2020 14:56:07 +0100 Subject: [PATCH] Added authentication to platform, game searcher tested --- app/scripts/classes/credentials.js | 2 +- app/scripts/constants/url.js | 2 +- app/scripts/network-helper.js | 80 ++++++++++++++++-------------- app/scripts/scraper.js | 2 +- app/scripts/searcher.js | 23 ++++----- package-lock.json | 37 +++++++++++++- package.json | 4 +- test/user-test.js | 5 +- 8 files changed, 97 insertions(+), 58 deletions(-) diff --git a/app/scripts/classes/credentials.js b/app/scripts/classes/credentials.js index 6eb7ab2..3041f03 100644 --- a/app/scripts/classes/credentials.js +++ b/app/scripts/classes/credentials.js @@ -7,7 +7,7 @@ class Credentials { constructor(username, password) { this.username = username; this.password = password; - this.token = ""; + this.token = null; } async fetchToken() { diff --git a/app/scripts/constants/url.js b/app/scripts/constants/url.js index fac63ba..6931f82 100644 --- a/app/scripts/constants/url.js +++ b/app/scripts/constants/url.js @@ -2,6 +2,6 @@ module.exports = Object.freeze({ F95_BASE_URL: "https://f95zone.to", F95_SEARCH_URL: "https://f95zone.to/search/?type=post", F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_LOGIN_URL: "https://f95zone.to/login", + F95_LOGIN_URL: "https://f95zone.to/login/login", F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", }); diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index d2a160b..3ed132b 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -7,7 +7,8 @@ const ky = require("ky-universal").create({ throwHttpErrors: false, }); const cheerio = require("cheerio"); -const qs = require("querystring"); +const axiosCookieJarSupport = require("axios-cookiejar-support").default; +const tough = require("tough-cookie"); // Modules from file const shared = require("./shared.js"); @@ -17,6 +18,8 @@ const f95url = require("./constants/url.js"); const userAgent = "Mozilla/5.0 (X11; Linux x86_64)" + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; +axiosCookieJarSupport(axios); +const cookieJar = new tough.CookieJar(); /** * @protected @@ -30,6 +33,8 @@ module.exports.fetchHTML = async function (url) { headers: { "User-Agent": userAgent }, + withCredentials: true, + jar: cookieJar }); return response.data; } catch (e) { @@ -40,50 +45,44 @@ module.exports.fetchHTML = async function (url) { /** * @protected - * Gets the HTML code of a login-protected page. - * @param {String} url URL to fetch + * It authenticates to the platform using the credentials + * and token obtained previously. Save cookies on your + * device after authentication. * @param {Credentials} credentials Platform access credentials - * @returns {Promise} HTML code or `null` if an error arise + * @returns {Promise} Result of the operation */ -module.exports.fetchHTMLWithAuth = async function (url, credentials) { - shared.logger.trace(`Fetching ${url} with user ${credentials.username}`); +module.exports.autenticate = async function (credentials) { + shared.logger.info(`Authenticating with user ${credentials.username}`); + if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); - const data = { - "login": credentials.username, - "url": "", - "password": credentials.password, - "password_confirm": "", - "additional_security": "", - "remember": "1", - "_xfRedirect": "https://f95zone.to/", - "website_code": "", - "_xfToken": credentials.token, - }; + // Prepare the parameters to send to the platform to authenticate + const params = new URLSearchParams(); + params.append("login", credentials.username); + params.append("url", ""); + params.append("password", credentials.password); + params.append("password_confirm", ""); + params.append("additional_security", ""); + params.append("remember", "1"); + params.append("_xfRedirect", "https://f95zone.to/"); + params.append("website_code", ""); + params.append("_xfToken", credentials.token); const config = { headers: { "User-Agent": userAgent, - "Content-Type": "application/x-www-form-urlencoded" - } + "Content-Type": "application/x-www-form-urlencoded", + "Connection": "keep-alive" + }, + withCredentials: true, + jar: cookieJar // Retrieve the stored cookies! What a pain to understand that this is a MUST! }; try { - console.log(qs.stringify(data)); - const response = await axios({ - method: "post", - url: url, - data: qs.stringify(data), - headers: { - "user-agent": userAgent, - "content-type": "application/x-www-form-urlencoded;charset=utf-8" - }, - withCredentials: true - }); - //const response = await axios.post(url, qs.stringify(data), config); - return response.data; + await axios.post(f95url.F95_LOGIN_URL, params, config); + return true; } catch (e) { - shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`); - return null; + shared.logger.error(`Error ${e.message} occurred while authenticating to ${f95url.F95_LOGIN_URL}`); + return false; } }; @@ -93,12 +92,17 @@ module.exports.fetchHTMLWithAuth = async function (url, credentials) { */ module.exports.getF95Token = async function() { try { - // Fetch the response of the platform - const response = await axios.get(f95url.F95_LOGIN_URL, { + const config = { headers: { - "User-Agent": userAgent + "User-Agent": userAgent, + "Connection": "keep-alive" }, - }); + withCredentials: true, + jar: cookieJar // Used to store the token in the PC + }; + + // Fetch the response of the platform + const response = await axios.get(f95url.F95_LOGIN_URL, config); // The response is a HTML page, we need to find the with name "_xfToken" const $ = cheerio.load(response.data); diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js index a59eb7c..e0fd443 100644 --- a/app/scripts/scraper.js +++ b/app/scripts/scraper.js @@ -75,7 +75,7 @@ function extractInfoFromTitle(body) { // From the title we can extract: Name, author and version // TITLE [VERSION] [AUTHOR] - const matches = title.match(/\[(.*?)\]/); + const matches = title.match(/\[(.*?)\]/g); const endIndex = title.indexOf("["); // The open bracket of the version const name = title.substring(0, endIndex).trim(); const version = matches[0].trim(); diff --git a/app/scripts/searcher.js b/app/scripts/searcher.js index a7ba734..3285ccd 100644 --- a/app/scripts/searcher.js +++ b/app/scripts/searcher.js @@ -4,19 +4,19 @@ const cheerio = require("cheerio"); // Modules from file -const { fetchHTMLWithAuth } = require("./network-helper.js"); +const { fetchHTML } = require("./network-helper.js"); const shared = require("./shared.js"); const f95Selector = require("./constants/css-selector.js"); +const { F95_BASE_URL } = require("./constants/url.js"); //#region Public methods /** * @protected * Search for a game on F95Zone and return a list of URLs, one for each search result. * @param {String} name Game name - * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchGame = async function (name, credentials) { +module.exports.searchGame = async function (name) { shared.logger.info(`Searching games with name ${name}`); // Replace the whitespaces with + @@ -26,17 +26,16 @@ module.exports.searchGame = async function (name, credentials) { const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url, credentials); + return await fetchResultURLs(url); }; /** * @protected * Search for a mod on F95Zone and return a list of URLs, one for each search result. * @param {String} name Mod name - * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchMod = async function (name, credentials) { +module.exports.searchMod = async function (name) { shared.logger.info(`Searching mods with name ${name}`); // Replace the whitespaces with + @@ -46,7 +45,7 @@ module.exports.searchMod = async function (name, credentials) { const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url, credentials); + return await fetchResultURLs(url); }; //#endregion Public methods @@ -55,14 +54,13 @@ module.exports.searchMod = async function (name, credentials) { * @private * Gets the URLs of the threads resulting from the F95Zone search. * @param {String} url Search URL - * @param {Credentials} credentials Platform access credentials * @return {Promise} List of URLs */ -async function fetchResultURLs(url, credentials) { +async function fetchResultURLs(url) { shared.logger.info(`Fetching ${url}...`); // Fetch HTML and prepare Cheerio - const html = await fetchHTMLWithAuth(url, credentials); + const html = await fetchHTML(url); const $ = cheerio.load(html); // Here we get all the DIV that are the body of the various query results @@ -84,11 +82,12 @@ async function fetchResultURLs(url, credentials) { * @returns {String} URL to thread */ function extractLinkFromResult(selector) { - const link = selector + const partialLink = selector .find(f95Selector.GS_RESULT_THREAD_TITLE) .attr("href") .trim(); - return link; + // Compose and return the URL + return new URL(partialLink, F95_BASE_URL).toString(); } //#endregion Private methods diff --git a/package-lock.json b/package-lock.json index 2029248..26492f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -416,6 +416,15 @@ "follow-redirects": "^1.10.0" } }, + "axios-cookiejar-support": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz", + "integrity": "sha512-IZJxnAJ99XxiLqNeMOqrPbfR7fRyIfaoSLdPUf4AMQEGkH8URs0ghJK/xtqBsD+KsSr3pKl4DEQjCn834pHMig==", + "requires": { + "is-redirect": "^1.0.0", + "pify": "^5.0.0" + } + }, "babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -1459,6 +1468,11 @@ "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", "dev": true }, + "is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=" + }, "is-regex": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", @@ -2299,6 +2313,11 @@ "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", "dev": true }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" + }, "pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -2342,11 +2361,15 @@ "iterate-value": "^1.0.0" } }, + "psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" + }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "randombytes": { "version": "2.1.0", @@ -2711,6 +2734,16 @@ "is-number": "^7.0.0" } }, + "tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.1.2" + } + }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index c57c96c..2fb3213 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,12 @@ }, "dependencies": { "axios": "^0.21.0", + "axios-cookiejar-support": "^1.0.1", "cheerio": "^1.0.0-rc.3", "ky": "^0.24.0", "ky-universal": "^0.8.2", - "log4js": "^6.3.0" + "log4js": "^6.3.0", + "tough-cookie": "^4.0.0" }, "devDependencies": { "babel-eslint": "^10.1.0", diff --git a/test/user-test.js b/test/user-test.js index afbdc94..71ebf4a 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -21,8 +21,9 @@ async function searchKOD() { await creds.fetchToken(); console.log(`Token obtained: ${creds.token}`); - const html = await networkHelper.fetchHTMLWithAuth("https://f95zone.to/login/login", creds); - console.log(html); + console.log("Authenticating..."); + const authenticated = await networkHelper.autenticate(creds); + console.log(`Authentication result: ${authenticated}`); console.log("Searching KOD..."); const urls = await searcher.searchGame("kingdom of deception", creds);