Merge pull request #34 from MillenniumEarl/deepsource-transform-8574391b
Format code with prettierpull/35/head^2
commit
d330ddef18
11
README.md
11
README.md
|
@ -9,8 +9,9 @@
|
||||||
Unofficial Node JS module for scraping F95Zone platform
|
Unofficial Node JS module for scraping F95Zone platform
|
||||||
|
|
||||||
# Guidelines for errors
|
# Guidelines for errors
|
||||||
+ If you can, return a meaningful value
|
|
||||||
+ Return `null` only if the function should return a complex object (including strings)
|
- If you can, return a meaningful value
|
||||||
+ Return an empty array if the function should return an array
|
- Return `null` only if the function should return a complex object (including strings)
|
||||||
+ Return `false`, `-1` when the function should retrn `boolean` or `number`
|
- Return an empty array if the function should return an array
|
||||||
+ Throw an exception only if it is an error or if a wrong value could mess up the functioning of the library
|
- Return `false`, `-1` when the function should retrn `boolean` or `number`
|
||||||
|
- Throw an exception only if it is an error or if a wrong value could mess up the functioning of the library
|
||||||
|
|
21
app/index.js
21
app/index.js
|
@ -114,7 +114,9 @@ module.exports.login = async function (username, password) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Else, log in throught browser
|
// Else, log in throught browser
|
||||||
shared.logger.info("No saved sessions or expired session, login on the platform");
|
shared.logger.info(
|
||||||
|
"No saved sessions or expired session, login on the platform"
|
||||||
|
);
|
||||||
|
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
||||||
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
const browser = shared.isolation ? await prepareBrowser() : _browser;
|
||||||
|
@ -267,7 +269,8 @@ module.exports.getGameDataFromURL = async function (url) {
|
||||||
// Check URL
|
// Check URL
|
||||||
const exists = await urlHelper.urlExists(url);
|
const exists = await urlHelper.urlExists(url);
|
||||||
if (!exists) throw new URIError(url + " is not a valid URL");
|
if (!exists) throw new URIError(url + " is not a valid URL");
|
||||||
if (!urlHelper.isF95URL(url)) throw new Error(url + " is not a valid F95Zone URL");
|
if (!urlHelper.isF95URL(url))
|
||||||
|
throw new Error(url + " is not a valid F95Zone URL");
|
||||||
|
|
||||||
// Gets the search results of the game being searched for
|
// Gets the search results of the game being searched for
|
||||||
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
if (_browser === null && !shared.isolation) _browser = await prepareBrowser();
|
||||||
|
@ -393,7 +396,9 @@ function isCookieExpired(cookie) {
|
||||||
const expirationDate = new Date(expirationUnixTimestamp * 1000);
|
const expirationDate = new Date(expirationUnixTimestamp * 1000);
|
||||||
|
|
||||||
if (expirationDate < Date.now()) {
|
if (expirationDate < Date.now()) {
|
||||||
shared.logger.warn("Cookie " + cookie.name + " expired, you need to re-authenticate");
|
shared.logger.warn(
|
||||||
|
"Cookie " + cookie.name + " expired, you need to re-authenticate"
|
||||||
|
);
|
||||||
expiredCookies = true;
|
expiredCookies = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -506,10 +511,9 @@ async function loginF95(browser, username, password) {
|
||||||
|
|
||||||
let errorMessageExists = await page.evaluate(
|
let errorMessageExists = await page.evaluate(
|
||||||
/* istanbul ignore next */
|
/* istanbul ignore next */
|
||||||
(selector) =>
|
(selector) => document.querySelector(selector) !== null,
|
||||||
document.querySelector(selector) !== null,
|
|
||||||
selectorK.LOGIN_MESSAGE_ERROR
|
selectorK.LOGIN_MESSAGE_ERROR
|
||||||
)
|
);
|
||||||
|
|
||||||
// Save cookies to avoid re-auth
|
// Save cookies to avoid re-auth
|
||||||
if (success) {
|
if (success) {
|
||||||
|
@ -525,7 +529,10 @@ async function loginF95(browser, username, password) {
|
||||||
|
|
||||||
if (errorMessage === "Incorrect password. Please try again.") {
|
if (errorMessage === "Incorrect password. Please try again.") {
|
||||||
message = "Incorrect password";
|
message = "Incorrect password";
|
||||||
} else if (errorMessage ==='The requested user \'' + username + '\' could not be found.') {
|
} else if (
|
||||||
|
errorMessage ===
|
||||||
|
"The requested user '" + username + "' could not be found."
|
||||||
|
) {
|
||||||
// The escaped quotes are important!
|
// The escaped quotes are important!
|
||||||
message = "Incorrect username";
|
message = "Incorrect username";
|
||||||
} else message = errorMessage;
|
} else message = errorMessage;
|
||||||
|
|
|
@ -27,7 +27,8 @@ module.exports.getGameInfo = async function (browser, url) {
|
||||||
// Verify the correctness of the URL
|
// Verify the correctness of the URL
|
||||||
const exists = await urlHelper.urlExists(url);
|
const exists = await urlHelper.urlExists(url);
|
||||||
if (!exists) throw new URIError(url + " is not a valid URL");
|
if (!exists) throw new URIError(url + " is not a valid URL");
|
||||||
if (!urlHelper.isF95URL(url)) throw new Error(url + " is not a valid F95Zone URL");
|
if (!urlHelper.isF95URL(url))
|
||||||
|
throw new Error(url + " is not a valid F95Zone URL");
|
||||||
|
|
||||||
const page = await preparePage(browser); // Set new isolated page
|
const page = await preparePage(browser); // Set new isolated page
|
||||||
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
await page.setCookie(...shared.cookies); // Set cookies to avoid login
|
||||||
|
@ -309,7 +310,8 @@ async function getLastChangelog(page) {
|
||||||
let parsedText = HTMLParser.parse(changelogHTML).structuredText;
|
let parsedText = HTMLParser.parse(changelogHTML).structuredText;
|
||||||
|
|
||||||
// Clean the text
|
// Clean the text
|
||||||
if (parsedText.startsWith("Spoiler")) parsedText = parsedText.replace("Spoiler", "");
|
if (parsedText.startsWith("Spoiler"))
|
||||||
|
parsedText = parsedText.replace("Spoiler", "");
|
||||||
if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", "");
|
if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", "");
|
||||||
return parsedText.trim();
|
return parsedText.trim();
|
||||||
}
|
}
|
||||||
|
|
|
@ -124,10 +124,7 @@ async function getThreadURL(page, handle) {
|
||||||
if (isF95URL(relativeURLThread)) return relativeURLThread;
|
if (isF95URL(relativeURLThread)) return relativeURLThread;
|
||||||
|
|
||||||
// ... else compose the URL and return
|
// ... else compose the URL and return
|
||||||
const urlThread = new URL(
|
const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString();
|
||||||
relativeURLThread,
|
|
||||||
urlK.F95_BASE_URL
|
|
||||||
).toString();
|
|
||||||
return urlThread;
|
return urlThread;
|
||||||
}
|
}
|
||||||
//#endregion Private methods
|
//#endregion Private methods
|
||||||
|
|
|
@ -257,15 +257,22 @@ describe("Test url-helper", function () {
|
||||||
expect(exists).to.be.false;
|
expect(exists).to.be.false;
|
||||||
|
|
||||||
// Now check for more specific URLs (with redirect)...
|
// Now check for more specific URLs (with redirect)...
|
||||||
exists = urlHelper.urlExists("https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/");
|
exists = urlHelper.urlExists(
|
||||||
|
"https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"
|
||||||
|
);
|
||||||
expect(exists).to.be.true;
|
expect(exists).to.be.true;
|
||||||
|
|
||||||
exists = urlHelper.urlExists("https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", true);
|
exists = urlHelper.urlExists(
|
||||||
|
"https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/",
|
||||||
|
true
|
||||||
|
);
|
||||||
expect(exists).to.be.false;
|
expect(exists).to.be.false;
|
||||||
});
|
});
|
||||||
|
|
||||||
it("Check if URL belong to the platform", async function () {
|
it("Check if URL belong to the platform", async function () {
|
||||||
let belong = urlHelper.isF95URL("https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/");
|
let belong = urlHelper.isF95URL(
|
||||||
|
"https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"
|
||||||
|
);
|
||||||
expect(belong).to.be.true;
|
expect(belong).to.be.true;
|
||||||
|
|
||||||
belong = urlHelper.isF95URL("https://www.google/");
|
belong = urlHelper.isF95URL("https://www.google/");
|
||||||
|
|
Loading…
Reference in New Issue