Added authentication to platform, game searcher tested

app/scripts/classes/credentials.js

@@ -7,7 +7,7 @@ class Credentials {
     constructor(username, password) {
         this.username = username;
         this.password = password;
-        this.token = "";
+        this.token = null;
     }
 
     async fetchToken() {

app/scripts/constants/url.js

@@ -2,6 +2,6 @@ module.exports = Object.freeze({
     F95_BASE_URL: "https://f95zone.to",
     F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
     F95_LATEST_UPDATES: "https://f95zone.to/latest",
-    F95_LOGIN_URL: "https://f95zone.to/login",
+    F95_LOGIN_URL: "https://f95zone.to/login/login",
     F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
 });

app/scripts/network-helper.js

@@ -7,7 +7,8 @@ const ky = require("ky-universal").create({
     throwHttpErrors: false,
 });
 const cheerio = require("cheerio");
-const qs = require("querystring");
+const axiosCookieJarSupport = require("axios-cookiejar-support").default;
+const tough = require("tough-cookie");
 
 // Modules from file
 const shared = require("./shared.js");
@@ -17,6 +18,8 @@ const f95url = require("./constants/url.js");
 const userAgent =
     "Mozilla/5.0 (X11; Linux x86_64)" +
     "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
+axiosCookieJarSupport(axios);
+const cookieJar = new tough.CookieJar();
 
 /**
  * @protected
@@ -30,6 +33,8 @@ module.exports.fetchHTML = async function (url) {
             headers: {
                 "User-Agent": userAgent
             },
+            withCredentials: true,
+            jar: cookieJar
         });
         return response.data;
     } catch (e) {
@@ -40,50 +45,44 @@ module.exports.fetchHTML = async function (url) {
 
 /**
  * @protected
- * Gets the HTML code of a login-protected page.
+ * It authenticates to the platform using the credentials 
- * @param {String} url URL to fetch
+ * and token obtained previously. Save cookies on your 
+ * device after authentication.
  * @param {Credentials} credentials Platform access credentials
- * @returns {Promise<String>} HTML code or `null` if an error arise
+ * @returns {Promise<Boolean>} Result of the operation
  */
-module.exports.fetchHTMLWithAuth = async function (url, credentials) {
+module.exports.autenticate = async function (credentials) {
-    shared.logger.trace(`Fetching ${url} with user ${credentials.username}`);
+    shared.logger.info(`Authenticating with user ${credentials.username}`);
+    if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
 
-    const data = {
+    // Prepare the parameters to send to the platform to authenticate
-        "login": credentials.username,
+    const params = new URLSearchParams();
-        "url": "",
+    params.append("login", credentials.username);
-        "password": credentials.password,
+    params.append("url", "");
-        "password_confirm": "",
+    params.append("password", credentials.password);
-        "additional_security": "",
+    params.append("password_confirm", "");
-        "remember": "1",
+    params.append("additional_security", "");
-        "_xfRedirect": "https://f95zone.to/",
+    params.append("remember", "1");
-        "website_code": "",
+    params.append("_xfRedirect", "https://f95zone.to/");
-        "_xfToken": credentials.token,
+    params.append("website_code", "");
-    };
+    params.append("_xfToken", credentials.token);
 
     const config = {
         headers: {
             "User-Agent": userAgent,
-            "Content-Type": "application/x-www-form-urlencoded"
+            "Content-Type": "application/x-www-form-urlencoded",
-        }
+            "Connection": "keep-alive"
+        },
+        withCredentials: true,
+        jar: cookieJar // Retrieve the stored cookies! What a pain to understand that this is a MUST!
     };
 
     try {
-        console.log(qs.stringify(data));
+        await axios.post(f95url.F95_LOGIN_URL, params, config);
-        const response = await axios({
+        return true;
-            method: "post",
-            url: url,
-            data: qs.stringify(data),
-            headers: {
-                "user-agent": userAgent,
-                "content-type": "application/x-www-form-urlencoded;charset=utf-8"
-            },
-            withCredentials: true
-        });
-        //const response = await axios.post(url, qs.stringify(data), config);
-        return response.data;
     } catch (e) {
-        shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`);
+        shared.logger.error(`Error ${e.message} occurred while authenticating to ${f95url.F95_LOGIN_URL}`);
-        return null;
+        return false;
     }
 };
 
@@ -93,12 +92,17 @@ module.exports.fetchHTMLWithAuth = async function (url, credentials) {
  */
 module.exports.getF95Token = async function() {
     try {
-        // Fetch the response of the platform
+        const config = {
-        const response = await axios.get(f95url.F95_LOGIN_URL, {
             headers: {
-                "User-Agent": userAgent
+                "User-Agent": userAgent,
+                "Connection": "keep-alive"
             },
-        });
+            withCredentials: true,
+            jar: cookieJar // Used to store the token in the PC
+        };
+
+        // Fetch the response of the platform
+        const response = await axios.get(f95url.F95_LOGIN_URL, config);
 
         // The response is a HTML page, we need to find the <input> with name "_xfToken"
         const $ = cheerio.load(response.data);

app/scripts/scraper.js

@@ -75,7 +75,7 @@ function extractInfoFromTitle(body) {
 
     // From the title we can extract: Name, author and version
     // TITLE [VERSION] [AUTHOR]
-    const matches = title.match(/\[(.*?)\]/);
+    const matches = title.match(/\[(.*?)\]/g);
     const endIndex = title.indexOf("["); // The open bracket of the version
     const name = title.substring(0, endIndex).trim();
     const version = matches[0].trim();

app/scripts/searcher.js

@@ -4,19 +4,19 @@
 const cheerio = require("cheerio");
 
 // Modules from file
-const { fetchHTMLWithAuth } = require("./network-helper.js");
+const { fetchHTML } = require("./network-helper.js");
 const shared = require("./shared.js");
 const f95Selector = require("./constants/css-selector.js");
+const { F95_BASE_URL } = require("./constants/url.js");
 
 //#region Public methods
 /**
  * @protected
  * Search for a game on F95Zone and return a list of URLs, one for each search result.
  * @param {String} name Game name
- * @param {Credentials} credentials Platform access credentials
  * @returns {Promise<String[]>} URLs of results
  */
-module.exports.searchGame = async function (name, credentials) {
+module.exports.searchGame = async function (name) {
     shared.logger.info(`Searching games with name ${name}`);
 
     // Replace the whitespaces with +
@@ -26,17 +26,16 @@ module.exports.searchGame = async function (name, credentials) {
     const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`;
 
     // Fetch and parse the result URLs
-    return await fetchResultURLs(url, credentials);
+    return await fetchResultURLs(url);
 };
 
 /**
  * @protected
  * Search for a mod on F95Zone and return a list of URLs, one for each search result.
  * @param {String} name Mod name
- * @param {Credentials} credentials Platform access credentials
  * @returns {Promise<String[]>} URLs of results
  */
-module.exports.searchMod = async function (name, credentials) {
+module.exports.searchMod = async function (name) {
     shared.logger.info(`Searching mods with name ${name}`);
     
     // Replace the whitespaces with +
@@ -46,7 +45,7 @@ module.exports.searchMod = async function (name, credentials) {
     const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`;
 
     // Fetch and parse the result URLs
-    return await fetchResultURLs(url, credentials);
+    return await fetchResultURLs(url);
 };
 //#endregion Public methods
 
@@ -55,14 +54,13 @@ module.exports.searchMod = async function (name, credentials) {
  * @private
  * Gets the URLs of the threads resulting from the F95Zone search.
  * @param {String} url Search URL
- * @param {Credentials} credentials Platform access credentials
  * @return {Promise<String[]>} List of URLs
  */
-async function fetchResultURLs(url, credentials) {
+async function fetchResultURLs(url) {
     shared.logger.info(`Fetching ${url}...`);
 
     // Fetch HTML and prepare Cheerio
-    const html = await fetchHTMLWithAuth(url, credentials);
+    const html = await fetchHTML(url);
     const $ = cheerio.load(html);
 
     // Here we get all the DIV that are the body of the various query results
@@ -84,11 +82,12 @@ async function fetchResultURLs(url, credentials) {
  * @returns {String} URL to thread
  */
 function extractLinkFromResult(selector) {
-    const link = selector
+    const partialLink = selector
         .find(f95Selector.GS_RESULT_THREAD_TITLE)
         .attr("href")
         .trim();
 
-    return link;
+    // Compose and return the URL
+    return new URL(partialLink, F95_BASE_URL).toString();
 }
 //#endregion Private methods

package-lock.json

@@ -416,6 +416,15 @@
         "follow-redirects": "^1.10.0"
       }
     },
+    "axios-cookiejar-support": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
+      "integrity": "sha512-IZJxnAJ99XxiLqNeMOqrPbfR7fRyIfaoSLdPUf4AMQEGkH8URs0ghJK/xtqBsD+KsSr3pKl4DEQjCn834pHMig==",
+      "requires": {
+        "is-redirect": "^1.0.0",
+        "pify": "^5.0.0"
+      }
+    },
     "babel-eslint": {
       "version": "10.1.0",
       "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz",
@@ -1459,6 +1468,11 @@
       "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
       "dev": true
     },
+    "is-redirect": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz",
+      "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ="
+    },
     "is-regex": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz",
@@ -2299,6 +2313,11 @@
       "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
       "dev": true
     },
+    "pify": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz",
+      "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA=="
+    },
     "pkg-dir": {
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
@@ -2342,11 +2361,15 @@
         "iterate-value": "^1.0.0"
       }
     },
+    "psl": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
+      "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
+    },
     "punycode": {
       "version": "2.1.1",
       "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
-      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
+      "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
-      "dev": true
     },
     "randombytes": {
       "version": "2.1.0",
@@ -2711,6 +2734,16 @@
         "is-number": "^7.0.0"
       }
     },
+    "tough-cookie": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
+      "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
+      "requires": {
+        "psl": "^1.1.33",
+        "punycode": "^2.1.1",
+        "universalify": "^0.1.2"
+      }
+    },
     "type-check": {
       "version": "0.4.0",
       "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",

package.json

@@ -32,10 +32,12 @@
   },
   "dependencies": {
     "axios": "^0.21.0",
+    "axios-cookiejar-support": "^1.0.1",
     "cheerio": "^1.0.0-rc.3",
     "ky": "^0.24.0",
     "ky-universal": "^0.8.2",
-    "log4js": "^6.3.0"
+    "log4js": "^6.3.0",
+    "tough-cookie": "^4.0.0"
   },
   "devDependencies": {
     "babel-eslint": "^10.1.0",

test/user-test.js

@@ -21,8 +21,9 @@ async function searchKOD() {
     await creds.fetchToken();
     console.log(`Token obtained: ${creds.token}`);
 
-    const html = await networkHelper.fetchHTMLWithAuth("https://f95zone.to/login/login", creds);
+    console.log("Authenticating...");
-    console.log(html);
+    const authenticated = await networkHelper.autenticate(creds);
+    console.log(`Authentication result: ${authenticated}`);
 
     console.log("Searching KOD...");
     const urls = await searcher.searchGame("kingdom of deception", creds);