Merge pull request #18 from MillenniumEarl/deepsource-transform-0c1c8af6
Format code with prettierpull/19/head
Millennium Earl
GitHub
commit
b5444f7057
|
|
@ -482,7 +482,7 @@ async function loginF95(browser, username, password) {
|
|||
await page.waitForNavigation({
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Wait for page to load
|
||||
|
||||
|
||||
// Prepare result
|
||||
let result = new LoginResult();
|
||||
|
||||
|
|
|
|||
|
|
@ -1,29 +1,31 @@
|
|||
module.exports = Object.freeze({
|
||||
AVATAR_INFO: 'span.avatar',
|
||||
AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]',
|
||||
ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span',
|
||||
FILTER_THREADS_BUTTON: 'button[class="button--primary button"]',
|
||||
GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]',
|
||||
GAME_TAGS: 'a.tagItem',
|
||||
GAME_TITLE: 'h1.p-title-value',
|
||||
GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]',
|
||||
LOGIN_BUTTON: 'button.button--icon--login',
|
||||
LOGIN_MESSAGE_ERROR: 'div.blockMessage.blockMessage--error.blockMessage--iconic',
|
||||
ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]',
|
||||
PASSWORD_INPUT: 'input[name="password"]',
|
||||
SEARCH_BUTTON: 'form.block > * button.button--icon--search',
|
||||
SEARCH_FORM_TEXTBOX: 'input[name="keywords"]',
|
||||
STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span',
|
||||
THREAD_POSTS: 'article.message-body:first-child > div.bbWrapper:first-of-type',
|
||||
THREAD_TITLE: 'h3.contentRow-title',
|
||||
TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]',
|
||||
UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]',
|
||||
USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText',
|
||||
USERNAME_INPUT: 'input[name="login"]',
|
||||
WATCHED_THREAD_FILTER_POPUP_BUTTON: 'a.filterBar-menuTrigger',
|
||||
WATCHED_THREAD_NEXT_PAGE: 'a.pageNav-jump--next',
|
||||
WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]',
|
||||
DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]',
|
||||
SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main",
|
||||
SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)"
|
||||
});
|
||||
AVATAR_INFO: "span.avatar",
|
||||
AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]',
|
||||
ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span',
|
||||
FILTER_THREADS_BUTTON: 'button[class="button--primary button"]',
|
||||
GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]',
|
||||
GAME_TAGS: "a.tagItem",
|
||||
GAME_TITLE: "h1.p-title-value",
|
||||
GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]',
|
||||
LOGIN_BUTTON: "button.button--icon--login",
|
||||
LOGIN_MESSAGE_ERROR:
|
||||
"div.blockMessage.blockMessage--error.blockMessage--iconic",
|
||||
ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]',
|
||||
PASSWORD_INPUT: 'input[name="password"]',
|
||||
SEARCH_BUTTON: "form.block > * button.button--icon--search",
|
||||
SEARCH_FORM_TEXTBOX: 'input[name="keywords"]',
|
||||
STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span',
|
||||
THREAD_POSTS:
|
||||
"article.message-body:first-child > div.bbWrapper:first-of-type",
|
||||
THREAD_TITLE: "h3.contentRow-title",
|
||||
TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]',
|
||||
UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]',
|
||||
USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText',
|
||||
USERNAME_INPUT: 'input[name="login"]',
|
||||
WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
|
||||
WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next",
|
||||
WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]',
|
||||
DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]',
|
||||
SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main",
|
||||
SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)",
|
||||
});
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ module.exports.getGameInfo = async function (browser, url) {
|
|||
* @param {GameInfo} info Information about the game
|
||||
* @returns {Promise<String>} Online version of the game
|
||||
*/
|
||||
module.exports.getGameVersionFromTitle = async function(browser, info) {
|
||||
module.exports.getGameVersionFromTitle = async function (browser, info) {
|
||||
let page = await preparePage(browser); // Set new isolated page
|
||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||
await page.goto(info.f95url, {
|
||||
|
|
@ -87,8 +87,7 @@ module.exports.getGameVersionFromTitle = async function(browser, info) {
|
|||
// Get the title
|
||||
let titleHTML = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
(selector) =>
|
||||
document.querySelector(selector).innerHTML,
|
||||
(selector) => document.querySelector(selector).innerHTML,
|
||||
selectors.GAME_TITLE
|
||||
);
|
||||
let title = HTMLParser.parse(titleHTML).childNodes.pop().rawText;
|
||||
|
|
@ -97,9 +96,9 @@ module.exports.getGameVersionFromTitle = async function(browser, info) {
|
|||
let startIndex = title.indexOf("[") + 1;
|
||||
let endIndex = title.indexOf("]", startIndex);
|
||||
let version = title.substring(startIndex, endIndex).trim().toUpperCase();
|
||||
if(version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence
|
||||
if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence
|
||||
return version;
|
||||
}
|
||||
};
|
||||
|
||||
//#region Private methods
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1,15 +1,13 @@
|
|||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
const puppeteer = require('puppeteer');
|
||||
const puppeteer = require("puppeteer");
|
||||
|
||||
// Modules from file
|
||||
const shared = require("./shared.js");
|
||||
const constURLs = require("./constants/urls.js");
|
||||
const selectors = require("./constants/css-selectors.js");
|
||||
const {
|
||||
preparePage,
|
||||
} = require("./puppeteer-helper.js");
|
||||
const { preparePage } = require("./puppeteer-helper.js");
|
||||
|
||||
/**
|
||||
* @protected
|
||||
|
|
@ -18,42 +16,42 @@ const {
|
|||
* @param {String} gamename Name of the game to search for
|
||||
* @returns {Promise<String[]>} List of URL of possible games obtained from the preliminary research on the F95 portal
|
||||
*/
|
||||
module.exports.getSearchGameResults = async function(browser, gamename) {
|
||||
if (shared.debug) console.log("Searching " + gamename + " on F95Zone");
|
||||
module.exports.getSearchGameResults = async function (browser, gamename) {
|
||||
if (shared.debug) console.log("Searching " + gamename + " on F95Zone");
|
||||
|
||||
let page = await preparePage(browser); // Set new isolated page
|
||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||
await page.goto(constURLs.F95_SEARCH_URL, {
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Go to the search form and wait for it
|
||||
let page = await preparePage(browser); // Set new isolated page
|
||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||
await page.goto(constURLs.F95_SEARCH_URL, {
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Go to the search form and wait for it
|
||||
|
||||
// Explicitly wait for the required items to load
|
||||
await page.waitForSelector(selectors.SEARCH_FORM_TEXTBOX);
|
||||
await page.waitForSelector(selectors.TITLE_ONLY_CHECKBOX);
|
||||
await page.waitForSelector(selectors.SEARCH_BUTTON);
|
||||
// Explicitly wait for the required items to load
|
||||
await page.waitForSelector(selectors.SEARCH_FORM_TEXTBOX);
|
||||
await page.waitForSelector(selectors.TITLE_ONLY_CHECKBOX);
|
||||
await page.waitForSelector(selectors.SEARCH_BUTTON);
|
||||
|
||||
await page.type(selectors.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire
|
||||
await page.click(selectors.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles
|
||||
await page.click(selectors.SEARCH_BUTTON); // Execute search
|
||||
await page.waitForNavigation({
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Wait for page to load
|
||||
await page.type(selectors.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire
|
||||
await page.click(selectors.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles
|
||||
await page.click(selectors.SEARCH_BUTTON); // Execute search
|
||||
await page.waitForNavigation({
|
||||
waitUntil: shared.WAIT_STATEMENT,
|
||||
}); // Wait for page to load
|
||||
|
||||
// Select all conversation titles
|
||||
let resultsThread = await page.$$(selectors.SEARCH_THREADS_RESULTS_BODY);
|
||||
// Select all conversation titles
|
||||
let resultsThread = await page.$$(selectors.SEARCH_THREADS_RESULTS_BODY);
|
||||
|
||||
// For each element found extract the info about the conversation
|
||||
if (shared.debug) console.log("Extracting info from conversations");
|
||||
let results = [];
|
||||
for (let element of resultsThread) {
|
||||
let gameUrl = await getOnlyGameThreads(page, element);
|
||||
if (gameUrl !== null) results.push(gameUrl);
|
||||
}
|
||||
if (shared.debug) console.log("Find " + results.length + " conversations");
|
||||
await page.close(); // Close the page
|
||||
// For each element found extract the info about the conversation
|
||||
if (shared.debug) console.log("Extracting info from conversations");
|
||||
let results = [];
|
||||
for (let element of resultsThread) {
|
||||
let gameUrl = await getOnlyGameThreads(page, element);
|
||||
if (gameUrl !== null) results.push(gameUrl);
|
||||
}
|
||||
if (shared.debug) console.log("Find " + results.length + " conversations");
|
||||
await page.close(); // Close the page
|
||||
|
||||
return results;
|
||||
}
|
||||
return results;
|
||||
};
|
||||
|
||||
//#region Private methods
|
||||
/**
|
||||
|
|
@ -64,16 +62,16 @@ module.exports.getSearchGameResults = async function(browser, gamename) {
|
|||
* @return {Promise<String>} URL of the game/mod or null if the URL is not of a game
|
||||
*/
|
||||
async function getOnlyGameThreads(page, divHandle) {
|
||||
// Obtain the elements containing the basic information
|
||||
let titleHandle = await divHandle.$(selectors.THREAD_TITLE);
|
||||
let forumHandle = await divHandle.$(selectors.SEARCH_THREADS_MEMBERSHIP);
|
||||
|
||||
// Get the forum where the thread was posted
|
||||
let forum = await getMembershipForum(page, forumHandle);
|
||||
if(forum !== "GAMES" && forum != "MODS") return null;
|
||||
// Obtain the elements containing the basic information
|
||||
let titleHandle = await divHandle.$(selectors.THREAD_TITLE);
|
||||
let forumHandle = await divHandle.$(selectors.SEARCH_THREADS_MEMBERSHIP);
|
||||
|
||||
// Get the URL of the thread from the title
|
||||
return await getThreadURL(page, titleHandle);
|
||||
// Get the forum where the thread was posted
|
||||
let forum = await getMembershipForum(page, forumHandle);
|
||||
if (forum !== "GAMES" && forum != "MODS") return null;
|
||||
|
||||
// Get the URL of the thread from the title
|
||||
return await getThreadURL(page, titleHandle);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -81,26 +79,26 @@ async function getOnlyGameThreads(page, divHandle) {
|
|||
* Obtain the membership forum of the thread passed throught "handle".
|
||||
* @param {puppeteer.Page} page Page containing the conversation to be analyzed
|
||||
* @param {puppeteer.ElementHandle} handle Handle containing the forum membership
|
||||
* @returns {Promise<String>} Uppercase membership category
|
||||
* @returns {Promise<String>} Uppercase membership category
|
||||
*/
|
||||
async function getMembershipForum(page, handle) {
|
||||
// The link can be something like:
|
||||
// + /forums/request.NUMBER/
|
||||
// + /forums/game-recommendations-identification.NUMBER/
|
||||
// + /forums/games.NUMBER/ <-- We need this
|
||||
// The link can be something like:
|
||||
// + /forums/request.NUMBER/
|
||||
// + /forums/game-recommendations-identification.NUMBER/
|
||||
// + /forums/games.NUMBER/ <-- We need this
|
||||
|
||||
let link = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
(e) => e.getAttribute('href'),
|
||||
handle
|
||||
);
|
||||
let link = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
(e) => e.getAttribute("href"),
|
||||
handle
|
||||
);
|
||||
|
||||
// Parse link
|
||||
link = link.replace("/forums/", "");
|
||||
let endIndex = link.indexOf(".");
|
||||
let forum = link.substring(0, endIndex);
|
||||
// Parse link
|
||||
link = link.replace("/forums/", "");
|
||||
let endIndex = link.indexOf(".");
|
||||
let forum = link.substring(0, endIndex);
|
||||
|
||||
return forum.toUpperCase();
|
||||
return forum.toUpperCase();
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -111,12 +109,12 @@ async function getMembershipForum(page, handle) {
|
|||
* @returns {Promise<String>} URL of the thread
|
||||
*/
|
||||
async function getThreadURL(page, handle) {
|
||||
let relativeURLThread = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
(e) => e.querySelector("a").href,
|
||||
handle
|
||||
);
|
||||
let urlThread = new URL(relativeURLThread, constURLs.F95_BASE_URL).toString();
|
||||
return urlThread;
|
||||
let relativeURLThread = await page.evaluate(
|
||||
/* istanbul ignore next */
|
||||
(e) => e.querySelector("a").href,
|
||||
handle
|
||||
);
|
||||
let urlThread = new URL(relativeURLThread, constURLs.F95_BASE_URL).toString();
|
||||
return urlThread;
|
||||
}
|
||||
//#endregion Private methods
|
||||
|
|
|
|||
|
|
@ -1,25 +1,25 @@
|
|||
'use strict';
|
||||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
const puppeteer = require('puppeteer');
|
||||
const puppeteer = require("puppeteer");
|
||||
|
||||
// Modules from file
|
||||
const shared = require('./shared.js');
|
||||
const shared = require("./shared.js");
|
||||
|
||||
/**
|
||||
* @protected
|
||||
* Create a Chromium instance used to navigate with Puppeteer.
|
||||
* Create a Chromium instance used to navigate with Puppeteer.
|
||||
* By default the browser is headless.
|
||||
* @returns {Promise<puppeteer.Browser>} Created browser
|
||||
*/
|
||||
module.exports.prepareBrowser = async function() {
|
||||
// Create a headless browser
|
||||
let browser = await puppeteer.launch({
|
||||
headless: !shared.debug, // Use GUI when debug = true
|
||||
});
|
||||
module.exports.prepareBrowser = async function () {
|
||||
// Create a headless browser
|
||||
let browser = await puppeteer.launch({
|
||||
headless: !shared.debug, // Use GUI when debug = true
|
||||
});
|
||||
|
||||
return browser;
|
||||
}
|
||||
return browser;
|
||||
};
|
||||
|
||||
/**
|
||||
* @protected
|
||||
|
|
@ -28,24 +28,25 @@ module.exports.prepareBrowser = async function() {
|
|||
* @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created
|
||||
* @returns {Promise<puppeteer.Page>} New page
|
||||
*/
|
||||
module.exports.preparePage = async function(browser) {
|
||||
// Create new page in the browser argument
|
||||
let page = await browser.newPage();
|
||||
module.exports.preparePage = async function (browser) {
|
||||
// Create new page in the browser argument
|
||||
let page = await browser.newPage();
|
||||
|
||||
// Block image download
|
||||
await page.setRequestInterception(true);
|
||||
page.on('request', (request) => {
|
||||
if (request.resourceType() === 'image') request.abort();
|
||||
else if(request.resourceType == 'font') request.abort();
|
||||
// else if (request.resourceType() == 'stylesheet') request.abort();
|
||||
// else if(request.resourceType == 'media') request.abort();
|
||||
else request.continue();
|
||||
});
|
||||
// Block image download
|
||||
await page.setRequestInterception(true);
|
||||
page.on("request", (request) => {
|
||||
if (request.resourceType() === "image") request.abort();
|
||||
else if (request.resourceType == "font") request.abort();
|
||||
// else if (request.resourceType() == 'stylesheet') request.abort();
|
||||
// else if(request.resourceType == 'media') request.abort();
|
||||
else request.continue();
|
||||
});
|
||||
|
||||
// Set custom user-agent
|
||||
let userAgent = 'Mozilla/5.0 (X11; Linux x86_64)' +
|
||||
'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36';
|
||||
await page.setUserAgent(userAgent);
|
||||
// Set custom user-agent
|
||||
let userAgent =
|
||||
"Mozilla/5.0 (X11; Linux x86_64)" +
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36";
|
||||
await page.setUserAgent(userAgent);
|
||||
|
||||
return page;
|
||||
}
|
||||
return page;
|
||||
};
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
const expect = require("chai").expect;
|
||||
const F95API = require("../app/index");
|
||||
const fs = require("fs");
|
||||
const sleep = require('sleep');
|
||||
const dotenv = require('dotenv');
|
||||
const sleep = require("sleep");
|
||||
const dotenv = require("dotenv");
|
||||
dotenv.config();
|
||||
|
||||
const COOKIES_SAVE_PATH = "./f95cache/cookies.json";
|
||||
|
|
@ -26,9 +26,9 @@ describe("Login without cookies", function () {
|
|||
//#region Set-up
|
||||
this.timeout(30000); // All tests in this suite get 30 seconds before timeout
|
||||
|
||||
before("Set isolation", function() {
|
||||
before("Set isolation", function () {
|
||||
F95API.setIsolation(true);
|
||||
})
|
||||
});
|
||||
|
||||
beforeEach("Remove all cookies", function () {
|
||||
// Runs before each test in this block
|
||||
|
|
@ -42,7 +42,7 @@ describe("Login without cookies", function () {
|
|||
it("Test with valid credentials", async function () {
|
||||
// Gain exclusive use of the cookies
|
||||
while (testOrder !== 0) randomSleep();
|
||||
|
||||
|
||||
const result = await F95API.login(USERNAME, PASSWORD);
|
||||
expect(result.success).to.be.true;
|
||||
expect(result.message).equal("Authentication successful");
|
||||
|
|
@ -221,7 +221,8 @@ describe("Check game update", function () {
|
|||
|
||||
// This test depend on the data on F95Zone at
|
||||
// https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/
|
||||
let url = "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/";
|
||||
let url =
|
||||
"https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/";
|
||||
const result = await F95API.getGameDataFromURL(url);
|
||||
result.version = "0.9600";
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue