From 4c273def446495dc98cef66de2a32537b4b256b4 Mon Sep 17 00:00:00 2001 From: "samuele.berlusconi" Date: Sun, 1 Nov 2020 09:54:59 +0100 Subject: [PATCH] Try to auth in the platform via POST request --- app/scripts/classes/credentials.js | 18 +++ app/scripts/network-helper.js | 91 +++++++++-- app/scripts/searcher.js | 17 +- package-lock.json | 246 ++++------------------------- test/user-test.js | 22 ++- 5 files changed, 158 insertions(+), 236 deletions(-) create mode 100644 app/scripts/classes/credentials.js diff --git a/app/scripts/classes/credentials.js b/app/scripts/classes/credentials.js new file mode 100644 index 0000000..6eb7ab2 --- /dev/null +++ b/app/scripts/classes/credentials.js @@ -0,0 +1,18 @@ +"use strict"; + +// Modules from file +const { getF95Token } = require("../network-helper.js"); + +class Credentials { + constructor(username, password) { + this.username = username; + this.password = password; + this.token = ""; + } + + async fetchToken() { + this.token = await getF95Token(); + } +} + +module.exports = Credentials; \ No newline at end of file diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 682e4bd..d2a160b 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -6,12 +6,17 @@ const _ = require("lodash"); const ky = require("ky-universal").create({ throwHttpErrors: false, }); +const cheerio = require("cheerio"); +const qs = require("querystring"); // Modules from file const shared = require("./shared.js"); -const { - F95_BASE_URL -} = require("./constants/url.js"); +const f95url = require("./constants/url.js"); + +// Global variables +const userAgent = + "Mozilla/5.0 (X11; Linux x86_64)" + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; /** * @protected @@ -20,15 +25,11 @@ const { * @returns {Promise} HTML code or `null` if an error arise */ module.exports.fetchHTML = async function (url) { - const userAgent = - "Mozilla/5.0 (X11; Linux x86_64)" + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; - try { const response = await axios.get(url, { headers: { "User-Agent": userAgent - } + }, }); return response.data; } catch (e) { @@ -37,6 +38,78 @@ module.exports.fetchHTML = async function (url) { } }; +/** + * @protected + * Gets the HTML code of a login-protected page. + * @param {String} url URL to fetch + * @param {Credentials} credentials Platform access credentials + * @returns {Promise} HTML code or `null` if an error arise + */ +module.exports.fetchHTMLWithAuth = async function (url, credentials) { + shared.logger.trace(`Fetching ${url} with user ${credentials.username}`); + + const data = { + "login": credentials.username, + "url": "", + "password": credentials.password, + "password_confirm": "", + "additional_security": "", + "remember": "1", + "_xfRedirect": "https://f95zone.to/", + "website_code": "", + "_xfToken": credentials.token, + }; + + const config = { + headers: { + "User-Agent": userAgent, + "Content-Type": "application/x-www-form-urlencoded" + } + }; + + try { + console.log(qs.stringify(data)); + const response = await axios({ + method: "post", + url: url, + data: qs.stringify(data), + headers: { + "user-agent": userAgent, + "content-type": "application/x-www-form-urlencoded;charset=utf-8" + }, + withCredentials: true + }); + //const response = await axios.post(url, qs.stringify(data), config); + return response.data; + } catch (e) { + shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`); + return null; + } +}; + +/** + * Obtain the token used to authenticate the user to the platform. + * @returns {Promise} Token or `null` if an error arise + */ +module.exports.getF95Token = async function() { + try { + // Fetch the response of the platform + const response = await axios.get(f95url.F95_LOGIN_URL, { + headers: { + "User-Agent": userAgent + }, + }); + + // The response is a HTML page, we need to find the with name "_xfToken" + const $ = cheerio.load(response.data); + const token = $("body").find("input[name='_xfToken']").attr("value"); + return token; + } catch (e) { + shared.logger.error(`Error ${e.message} occurred while trying to fetch F95 token`); + return null; + } +}; + /** * @protected * Enforces the scheme of the URL is https and returns the new URL. @@ -55,7 +128,7 @@ module.exports.enforceHttpsUrl = function (url) { * @returns {Boolean} true if the url belongs to the domain, false otherwise */ module.exports.isF95URL = function (url) { - if (url.toString().startsWith(F95_BASE_URL)) return true; + if (url.toString().startsWith(f95url.F95_BASE_URL)) return true; else return false; }; diff --git a/app/scripts/searcher.js b/app/scripts/searcher.js index eac6f55..a7ba734 100644 --- a/app/scripts/searcher.js +++ b/app/scripts/searcher.js @@ -4,7 +4,7 @@ const cheerio = require("cheerio"); // Modules from file -const { fetchHTML } = require("./network-helper.js"); +const { fetchHTMLWithAuth } = require("./network-helper.js"); const shared = require("./shared.js"); const f95Selector = require("./constants/css-selector.js"); @@ -13,9 +13,10 @@ const f95Selector = require("./constants/css-selector.js"); * @protected * Search for a game on F95Zone and return a list of URLs, one for each search result. * @param {String} name Game name + * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchGame = async function (name) { +module.exports.searchGame = async function (name, credentials) { shared.logger.info(`Searching games with name ${name}`); // Replace the whitespaces with + @@ -25,16 +26,17 @@ module.exports.searchGame = async function (name) { const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url); + return await fetchResultURLs(url, credentials); }; /** * @protected * Search for a mod on F95Zone and return a list of URLs, one for each search result. * @param {String} name Mod name + * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchMod = async function (name) { +module.exports.searchMod = async function (name, credentials) { shared.logger.info(`Searching mods with name ${name}`); // Replace the whitespaces with + @@ -44,7 +46,7 @@ module.exports.searchMod = async function (name) { const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url); + return await fetchResultURLs(url, credentials); }; //#endregion Public methods @@ -53,13 +55,14 @@ module.exports.searchMod = async function (name) { * @private * Gets the URLs of the threads resulting from the F95Zone search. * @param {String} url Search URL + * @param {Credentials} credentials Platform access credentials * @return {Promise} List of URLs */ -async function fetchResultURLs(url) { +async function fetchResultURLs(url, credentials) { shared.logger.info(`Fetching ${url}...`); // Fetch HTML and prepare Cheerio - const html = await fetchHTML(url); + const html = await fetchHTMLWithAuth(url, credentials); const $ = cheerio.load(html); // Here we get all the DIV that are the body of the various query results diff --git a/package-lock.json b/package-lock.json index d65e78b..2029248 100644 --- a/package-lock.json +++ b/package-lock.json @@ -287,15 +287,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==" }, - "@types/yauzl": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", - "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", - "optional": true, - "requires": { - "@types/node": "*" - } - }, "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -316,11 +307,6 @@ "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", "dev": true }, - "agent-base": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", - "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" - }, "aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -455,12 +441,8 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true }, "binary-extensions": { "version": "2.1.0", @@ -468,16 +450,6 @@ "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", "dev": true }, - "bl": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", - "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", - "requires": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -487,6 +459,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -507,20 +480,6 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, - "buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" - }, "caching-transform": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", @@ -605,11 +564,6 @@ "readdirp": "~3.4.0" } }, - "chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" - }, "clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -651,7 +605,8 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true }, "convert-source-map": { "version": "1.7.0", @@ -746,11 +701,6 @@ "object-keys": "^1.0.12" } }, - "devtools-protocol": { - "version": "0.0.799653", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz", - "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg==" - }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -809,14 +759,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { - "once": "^1.4.0" - } - }, "enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -1120,17 +1062,6 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, - "extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "requires": { - "@types/yauzl": "^2.9.1", - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - } - }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1149,14 +1080,6 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, - "fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", - "requires": { - "pend": "~1.2.0" - } - }, "fetch-blob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.1.tgz", @@ -1195,6 +1118,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1257,11 +1181,6 @@ "integrity": "sha512-Xu2Qh8yqYuDhQGOhD5iJGninErSfI9A3FrriD3tjUgV5VbJFeH8vfgZ9HnC6jWN80QDVNQK5vmxRAmEAp7Mevw==", "dev": true }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -1275,7 +1194,8 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true }, "fsevents": { "version": "2.1.3", @@ -1320,18 +1240,11 @@ "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true }, - "get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "requires": { - "pump": "^3.0.0" - } - }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1401,7 +1314,8 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true }, "html-escaper": { "version": "2.0.2", @@ -1422,20 +1336,6 @@ "readable-stream": "^3.1.1" } }, - "https-proxy-agent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", - "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", - "requires": { - "agent-base": "5", - "debug": "4" - } - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -1476,6 +1376,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1812,6 +1713,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, "requires": { "p-locate": "^4.1.0" } @@ -1913,6 +1815,7 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1932,11 +1835,6 @@ "minimist": "^1.2.5" } }, - "mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" - }, "mocha": { "version": "8.1.3", "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.1.3.tgz", @@ -2198,14 +2096,6 @@ "fetch-blob": "^2.1.1" } }, - "node-html-parser": { - "version": "1.2.21", - "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-1.2.21.tgz", - "integrity": "sha512-6vDhgen6J332syN5HUmeT4FfBG7m6bFRrPN+FXY8Am7FGuVpsIxTASVbeoO5PF2IHbX2s+WEIudb1hgxOjllNQ==", - "requires": { - "he": "1.2.0" - } - }, "node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -2292,6 +2182,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "requires": { "wrappy": "1" } @@ -2314,6 +2205,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -2322,6 +2214,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, "requires": { "p-limit": "^2.2.0" } @@ -2338,7 +2231,8 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true }, "package-hash": { "version": "4.0.0", @@ -2372,12 +2266,14 @@ "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true }, "path-key": { "version": "3.1.1", @@ -2397,11 +2293,6 @@ "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", "dev": true }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" - }, "picomatch": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", @@ -2412,6 +2303,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, "requires": { "find-up": "^4.0.0" } @@ -2434,7 +2326,8 @@ "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true }, "promise.allsettled": { "version": "1.0.2", @@ -2449,44 +2342,12 @@ "iterate-value": "^1.0.0" } }, - "proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, - "pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, - "puppeteer": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz", - "integrity": "sha512-YTM1RaBeYrj6n7IlRXRYLqJHF+GM7tasbvrNFx6w1S16G76NrPq7oYFKLDO+BQsXNtS8kW2GxWCXjIMPvfDyaQ==", - "requires": { - "debug": "^4.1.0", - "devtools-protocol": "0.0.799653", - "extract-zip": "^2.0.0", - "https-proxy-agent": "^4.0.0", - "pkg-dir": "^4.2.0", - "progress": "^2.0.1", - "proxy-from-env": "^1.0.0", - "rimraf": "^3.0.2", - "tar-fs": "^2.0.0", - "unbzip2-stream": "^1.3.3", - "ws": "^7.2.3" - } - }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -2566,6 +2427,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -2817,29 +2679,6 @@ } } }, - "tar-fs": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", - "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", - "requires": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.0.0" - } - }, - "tar-stream": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz", - "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==", - "requires": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - } - }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -2857,11 +2696,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -2907,15 +2741,6 @@ "is-typedarray": "^1.0.0" } }, - "unbzip2-stream": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", - "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", - "requires": { - "buffer": "^5.2.1", - "through": "^2.3.8" - } - }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -3057,7 +2882,8 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "write": { "version": "1.0.3", @@ -3080,11 +2906,6 @@ "typedarray-to-buffer": "^3.1.5" } }, - "ws": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz", - "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==" - }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", @@ -3257,15 +3078,6 @@ } } } - }, - "yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", - "requires": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/test/user-test.js b/test/user-test.js index 22f2046..afbdc94 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -1,15 +1,31 @@ "use strict"; +// Public modules from npm +const dotenv = require("dotenv"); + // Modules from file -const searcher = require("../plain-html/scripts/searcher.js"); -const scraper = require("../plain-html/scripts/scraper.js"); +const searcher = require("../app/scripts/searcher.js"); +const scraper = require("../app/scripts/scraper.js"); +const Credentials = require("../app/scripts/classes/credentials.js"); +const networkHelper = require("../app/scripts/network-helper.js"); + +// Configure the .env reader +dotenv.config(); // Search for Kingdom Of Deception data searchKOD(); async function searchKOD() { + console.log("Token fetch..."); + const creds = new Credentials(process.env.F95_USERNAME, process.env.F95_PASSWORD); + await creds.fetchToken(); + console.log(`Token obtained: ${creds.token}`); + + const html = await networkHelper.fetchHTMLWithAuth("https://f95zone.to/login/login", creds); + console.log(html); + console.log("Searching KOD..."); - const urls = await searcher.searchGame("kingdom of deception"); + const urls = await searcher.searchGame("kingdom of deception", creds); console.log(`Found: ${urls}`); console.log("Scraping data...");