MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Added authentication to platform, game searcher tested

pull/44/head
MillenniumEarl 2020-11-01 14:56:07 +01:00
parent 4c273def44
commit ecfd1784b1
8 changed files with 97 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
app/scripts/classes/credentials.js
View File

@ -7,7 +7,7 @@ class Credentials {
constructor(username, password) {
this.username = username;
this.password = password;
this.token = "";
this.token = null;
}
async fetchToken() {

2
app/scripts/constants/url.js
View File

@ -2,6 +2,6 @@ module.exports = Object.freeze({
F95_BASE_URL: "https://f95zone.to",
F95_SEARCH_URL: "https://f95zone.to/search/?type=post",
F95_LATEST_UPDATES: "https://f95zone.to/latest",
F95_LOGIN_URL: "https://f95zone.to/login",
F95_LOGIN_URL: "https://f95zone.to/login/login",
F95_WATCHED_THREADS: "https://f95zone.to/watched/threads",
});

80
app/scripts/network-helper.js
View File

@ -7,7 +7,8 @@ const ky = require("ky-universal").create({
throwHttpErrors: false,
});
const cheerio = require("cheerio");
const qs = require("querystring");
const axiosCookieJarSupport = require("axios-cookiejar-support").default;
const tough = require("tough-cookie");
// Modules from file
const shared = require("./shared.js");
@ -17,6 +18,8 @@ const f95url = require("./constants/url.js");
const userAgent =
"Mozilla/5.0 (X11; Linux x86_64)" +
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
axiosCookieJarSupport(axios);
const cookieJar = new tough.CookieJar();
/**
* @protected
@ -30,6 +33,8 @@ module.exports.fetchHTML = async function (url) {
headers: {
"User-Agent": userAgent
},
withCredentials: true,
jar: cookieJar
});
return response.data;
} catch (e) {
@ -40,50 +45,44 @@ module.exports.fetchHTML = async function (url) {
/**
* @protected
* Gets the HTML code of a login-protected page.
* @param {String} url URL to fetch
* It authenticates to the platform using the credentials
* and token obtained previously. Save cookies on your
* device after authentication.
* @param {Credentials} credentials Platform access credentials
* @returns {Promise<String>} HTML code or `null` if an error arise
* @returns {Promise<Boolean>} Result of the operation
*/
module.exports.fetchHTMLWithAuth = async function (url, credentials) {
shared.logger.trace(`Fetching ${url} with user ${credentials.username}`);
module.exports.autenticate = async function (credentials) {
shared.logger.info(`Authenticating with user ${credentials.username}`);
if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`);
const data = {
"login": credentials.username,
"url": "",
"password": credentials.password,
"password_confirm": "",
"additional_security": "",
"remember": "1",
"_xfRedirect": "https://f95zone.to/",
"website_code": "",
"_xfToken": credentials.token,
};
// Prepare the parameters to send to the platform to authenticate
const params = new URLSearchParams();
params.append("login", credentials.username);
params.append("url", "");
params.append("password", credentials.password);
params.append("password_confirm", "");
params.append("additional_security", "");
params.append("remember", "1");
params.append("_xfRedirect", "https://f95zone.to/");
params.append("website_code", "");
params.append("_xfToken", credentials.token);
const config = {
headers: {
"User-Agent": userAgent,
"Content-Type": "application/x-www-form-urlencoded"
}
"Content-Type": "application/x-www-form-urlencoded",
"Connection": "keep-alive"
},
withCredentials: true,
jar: cookieJar // Retrieve the stored cookies! What a pain to understand that this is a MUST!
};
try {
console.log(qs.stringify(data));
const response = await axios({
method: "post",
url: url,
data: qs.stringify(data),
headers: {
"user-agent": userAgent,
"content-type": "application/x-www-form-urlencoded;charset=utf-8"
},
withCredentials: true
});
//const response = await axios.post(url, qs.stringify(data), config);
return response.data;
await axios.post(f95url.F95_LOGIN_URL, params, config);
return true;
} catch (e) {
shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`);
return null;
shared.logger.error(`Error ${e.message} occurred while authenticating to ${f95url.F95_LOGIN_URL}`);
return false;
}
};
@ -93,12 +92,17 @@ module.exports.fetchHTMLWithAuth = async function (url, credentials) {
*/
module.exports.getF95Token = async function() {
try {
// Fetch the response of the platform
const response = await axios.get(f95url.F95_LOGIN_URL, {
const config = {
headers: {
"User-Agent": userAgent
"User-Agent": userAgent,
"Connection": "keep-alive"
},
});
withCredentials: true,
jar: cookieJar // Used to store the token in the PC
};
// Fetch the response of the platform
const response = await axios.get(f95url.F95_LOGIN_URL, config);
// The response is a HTML page, we need to find the <input> with name "_xfToken"
const $ = cheerio.load(response.data);

2
app/scripts/scraper.js
View File

@ -75,7 +75,7 @@ function extractInfoFromTitle(body) {
// From the title we can extract: Name, author and version
// TITLE [VERSION] [AUTHOR]
const matches = title.match(/\[(.*?)\]/);
const matches = title.match(/\[(.*?)\]/g);
const endIndex = title.indexOf("["); // The open bracket of the version
const name = title.substring(0, endIndex).trim();
const version = matches[0].trim();

23
app/scripts/searcher.js
View File

@ -4,19 +4,19 @@
const cheerio = require("cheerio");
// Modules from file
const { fetchHTMLWithAuth } = require("./network-helper.js");
const { fetchHTML } = require("./network-helper.js");
const shared = require("./shared.js");
const f95Selector = require("./constants/css-selector.js");
const { F95_BASE_URL } = require("./constants/url.js");
//#region Public methods
/**
* @protected
* Search for a game on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Game name
* @param {Credentials} credentials Platform access credentials
* @returns {Promise<String[]>} URLs of results
*/
module.exports.searchGame = async function (name, credentials) {
module.exports.searchGame = async function (name) {
shared.logger.info(`Searching games with name ${name}`);
// Replace the whitespaces with +
@ -26,17 +26,16 @@ module.exports.searchGame = async function (name, credentials) {
const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`;
// Fetch and parse the result URLs
return await fetchResultURLs(url, credentials);
return await fetchResultURLs(url);
};
/**
* @protected
* Search for a mod on F95Zone and return a list of URLs, one for each search result.
* @param {String} name Mod name
* @param {Credentials} credentials Platform access credentials
* @returns {Promise<String[]>} URLs of results
*/
module.exports.searchMod = async function (name, credentials) {
module.exports.searchMod = async function (name) {
shared.logger.info(`Searching mods with name ${name}`);
// Replace the whitespaces with +
@ -46,7 +45,7 @@ module.exports.searchMod = async function (name, credentials) {
const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`;
// Fetch and parse the result URLs
return await fetchResultURLs(url, credentials);
return await fetchResultURLs(url);
};
//#endregion Public methods
@ -55,14 +54,13 @@ module.exports.searchMod = async function (name, credentials) {
* @private
* Gets the URLs of the threads resulting from the F95Zone search.
* @param {String} url Search URL
* @param {Credentials} credentials Platform access credentials
* @return {Promise<String[]>} List of URLs
*/
async function fetchResultURLs(url, credentials) {
async function fetchResultURLs(url) {
shared.logger.info(`Fetching ${url}...`);
// Fetch HTML and prepare Cheerio
const html = await fetchHTMLWithAuth(url, credentials);
const html = await fetchHTML(url);
const $ = cheerio.load(html);
// Here we get all the DIV that are the body of the various query results
@ -84,11 +82,12 @@ async function fetchResultURLs(url, credentials) {
* @returns {String} URL to thread
*/
function extractLinkFromResult(selector) {
const link = selector
const partialLink = selector
.find(f95Selector.GS_RESULT_THREAD_TITLE)
.attr("href")
.trim();
return link;
// Compose and return the URL
return new URL(partialLink, F95_BASE_URL).toString();
}
//#endregion Private methods

37
package-lock.json generated
View File

@ -416,6 +416,15 @@
"follow-redirects": "^1.10.0"
}
},
"axios-cookiejar-support": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz",
"integrity": "sha512-IZJxnAJ99XxiLqNeMOqrPbfR7fRyIfaoSLdPUf4AMQEGkH8URs0ghJK/xtqBsD+KsSr3pKl4DEQjCn834pHMig==",
"requires": {
"is-redirect": "^1.0.0",
"pify": "^5.0.0"
}
},
"babel-eslint": {
"version": "10.1.0",
"resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz",
@ -1459,6 +1468,11 @@
"integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=",
"dev": true
},
"is-redirect": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz",
"integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ="
},
"is-regex": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz",
@ -2299,6 +2313,11 @@
"integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==",
"dev": true
},
"pify": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz",
"integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA=="
},
"pkg-dir": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
@ -2342,11 +2361,15 @@
"iterate-value": "^1.0.0"
}
},
"psl": {
"version": "1.8.0",
"resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz",
"integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ=="
},
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==",
"dev": true
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
},
"randombytes": {
"version": "2.1.0",
@ -2711,6 +2734,16 @@
"is-number": "^7.0.0"
}
},
"tough-cookie": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz",
"integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==",
"requires": {
"psl": "^1.1.33",
"punycode": "^2.1.1",
"universalify": "^0.1.2"
}
},
"type-check": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz",

4
package.json
View File

@ -32,10 +32,12 @@
},
"dependencies": {
"axios": "^0.21.0",
"axios-cookiejar-support": "^1.0.1",
"cheerio": "^1.0.0-rc.3",
"ky": "^0.24.0",
"ky-universal": "^0.8.2",
"log4js": "^6.3.0"
"log4js": "^6.3.0",
"tough-cookie": "^4.0.0"
},
"devDependencies": {
"babel-eslint": "^10.1.0",

5
test/user-test.js
View File

@ -21,8 +21,9 @@ async function searchKOD() {
await creds.fetchToken();
console.log(`Token obtained: ${creds.token}`);
const html = await networkHelper.fetchHTMLWithAuth("https://f95zone.to/login/login", creds);
console.log(html);
console.log("Authenticating...");
const authenticated = await networkHelper.autenticate(creds);
console.log(`Authentication result: ${authenticated}`);
console.log("Searching KOD...");
const urls = await searcher.searchGame("kingdom of deception", creds);