MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request #81 from MillenniumEarl/simplify-post-parse 

Simplify post parse
2.0.0-ts
Millennium Earl 2021-03-15 21:37:13 +01:00 committed by GitHub
parent 23b0b2d972 ae32a80d2a
commit 94d6f3667b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 275 additions and 236 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.prettierrc
View File

@ -2,5 +2,5 @@
"semi": true,
"trailingComma": "none",
"singleQuote": false,
"printWidth": 90
"printWidth": 100
}

13
.vscode/launch.json vendored
View File

@ -5,7 +5,18 @@
"name": "Test",
"request": "launch",
"command": "npm run test",
"cwd": "${workspaceFolder}"
"cwd": "${workspaceFolder}",
},
{
"type": "node-terminal",
"name": "Example",
"request": "launch",
"command": "npm run run-example",
"cwd": "${workspaceFolder}",
"skipFiles": [
"${workspaceFolder}/node_modules/**/*",
"<node_internals>/**/*"
]
},
]
}

13
package-lock.json generated
View File

@ -22,7 +22,6 @@
"@types/chai": "^4.2.15",
"@types/chai-as-promised": "^7.1.3",
"@types/inquirer": "^7.3.1",
"@types/lodash": "^4.14.168",
"@types/luxon": "^1.25.2",
"@types/mocha": "^8.2.1",
"@types/node": "^14.14.27",
@ -519,12 +518,6 @@
"integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==",
"dev": true
},
"node_modules/@types/lodash": {
"version": "4.14.168",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.168.tgz",
"integrity": "sha512-oVfRvqHV/V6D1yifJbVRU3TMp8OT6o6BG+U9MkwuJ3U8/CsDHvalRpsxBqivn71ztOFZBTfJMvETbqHiaNSj7Q==",
"dev": true
},
"node_modules/@types/luxon": {
"version": "1.25.2",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-1.25.2.tgz",
@ -5482,12 +5475,6 @@
"integrity": "sha512-cxWFQVseBm6O9Gbw1IWb8r6OS4OhSt3hPZLkFApLjM8TEXROBuQGLAH2i2gZpcXdLBIrpXuTDhH7Vbm1iXmNGA==",
"dev": true
},
"@types/lodash": {
"version": "4.14.168",
"resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.14.168.tgz",
"integrity": "sha512-oVfRvqHV/V6D1yifJbVRU3TMp8OT6o6BG+U9MkwuJ3U8/CsDHvalRpsxBqivn71ztOFZBTfJMvETbqHiaNSj7Q==",
"dev": true
},
"@types/luxon": {
"version": "1.25.2",
"resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-1.25.2.tgz",

1
package.json
View File

@ -48,7 +48,6 @@
"@types/chai": "^4.2.15",
"@types/chai-as-promised": "^7.1.3",
"@types/inquirer": "^7.3.1",
"@types/lodash": "^4.14.168",
"@types/luxon": "^1.25.2",
"@types/mocha": "^8.2.1",
"@types/node": "^14.14.27",

10
src/example.ts
View File

@ -60,11 +60,7 @@ async function main() {
// Log in the platform
console.log("Authenticating...");
const result = await login(
process.env.F95_USERNAME,
process.env.F95_PASSWORD,
insert2faCode
);
const result = await login(process.env.F95_USERNAME, process.env.F95_PASSWORD, insert2faCode);
console.log(`Authentication result: ${result.message}\n`);
// Manage failed login
@ -87,9 +83,7 @@ async function main() {
latestQuery.includedTags = ["3d game"];
const latestUpdates = await getLatestUpdates<Game>(latestQuery, 1);
console.log(
`"${latestUpdates.shift().name}" was the last "3d game" tagged game to be updated\n`
);
console.log(`"${latestUpdates.shift().name}" was the last "3d game" tagged game to be updated\n`);
// Get game data
for (const gamename of gameList) {

9
src/scripts/classes/handiwork/handiwork.ts
View File

@ -6,14 +6,7 @@
"use strict";
// Modules from files
import {
TAuthor,
TRating,
IHandiwork,
TEngine,
TCategory,
TStatus
} from "../../interfaces";
import { TAuthor, TRating, IHandiwork, TEngine, TCategory, TStatus } from "../../interfaces";
/**
* It represents a generic work, be it a game, a comic, an animation or an asset.

3
src/scripts/classes/query/handiwork-search-query.ts
View File

@ -96,8 +96,7 @@ export default class HandiworkSearchQuery implements IQuery {
// If the keywords are set or the number
// of included tags is greather than 5,
// we must perform a thread search
if (this.keywords || this.includedTags.length > MAX_TAGS_LATEST_SEARCH)
return "thread";
if (this.keywords || this.includedTags.length > MAX_TAGS_LATEST_SEARCH) return "thread";
return DEFAULT_SEARCH_TYPE;
}

3
src/scripts/classes/query/thread-search-query.ts
View File

@ -130,8 +130,7 @@ export default class ThreadSearchQuery implements IQuery {
if (this.excludedTags) params["c[excludeTags]"] = this.excludedTags.join(",");
// Set minimum reply number
if (this.minimumReplies > 0)
params["c[min_reply_count]"] = this.minimumReplies.toString();
if (this.minimumReplies > 0) params["c[min_reply_count]"] = this.minimumReplies.toString();
// Add prefixes
const parser = new PrefixParser();

4
src/scripts/classes/session.ts
View File

@ -210,8 +210,8 @@ export default class Session {
// Search for expired cookies
const jarValid =
this._cookieJar.getCookiesSync("https://f95zone.to").filter((el) => el.TTL() === 0)
.length === 0;
this._cookieJar.getCookiesSync("https://f95zone.to").filter((el) => el.TTL() === 0).length ===
0;
return dateValid && hashValid && jarValid;
}

13
src/scripts/constants/css-selector.ts
View File

@ -154,7 +154,15 @@ export const POST = {
*
* For use within a `THREAD.POSTS_IN_PAGE` selector.
*/
BOOKMARKED: '* ul.message-attribution-opposite >li > a[title="Bookmark"].is-bookmarked'
BOOKMARKED: '* ul.message-attribution-opposite >li > a[title="Bookmark"].is-bookmarked',
/**
* Button used to hide/show a spoiler element of a post.
*/
SPOILER_BUTTON: "button.bbCodeSpoiler-button",
/**
* Contents of a spoiler element in a post.
*/
SPOILER_CONTENT: "div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content"
};
export const MEMBER = {
@ -205,8 +213,7 @@ export const MEMBER = {
* If the text is `Unfollow` then the user is followed.
* If the text is `Follow` then the user is not followed.
*/
FOLLOWED:
"div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-follow] > span",
FOLLOWED: "div.memberHeader-buttons > div.buttonGroup:first-child > a[data-sk-follow] > span",
/**
* Button used to ignore/unignore the user.
*

5
src/scripts/interfaces.ts
View File

@ -83,10 +83,7 @@ export type TCategory = "games" | "mods" | "comics" | "animations" | "assets";
/**
* Valid names of classes that implement the IQuery interface.
*/
export type TQueryInterface =
| "LatestSearchQuery"
| "ThreadSearchQuery"
| "HandiworkSearchQuery";
export type TQueryInterface = "LatestSearchQuery" | "ThreadSearchQuery" | "HandiworkSearchQuery";
/**
* Collection of values defined for each

21
src/scripts/network-helper.ts
View File

@ -86,9 +86,7 @@ export async function fetchHTML(
error: null
});
return isHTML
? success(response.value.data as string)
: failure(unexpectedResponseError);
return isHTML ? success(response.value.data as string) : failure(unexpectedResponseError);
} else return failure(response.value as GenericAxiosError);
}
@ -105,8 +103,7 @@ export async function authenticate(
force: boolean = false
): Promise<LoginResult> {
shared.logger.info(`Authenticating with user ${credentials.username}`);
if (!credentials.token)
throw new InvalidF95Token(`Invalid token for auth: ${credentials.token}`);
if (!credentials.token) throw new InvalidF95Token(`Invalid token for auth: ${credentials.token}`);
// Secure the URL
const secureURL = enforceHttpsUrl(urls.LOGIN);
@ -216,9 +213,7 @@ export async function fetchGETResponse(
const response = await axios.get(secureURL, commonConfig);
return success(response);
} catch (e) {
shared.logger.error(
`(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`
);
shared.logger.error(`(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`);
const genericError = new GenericAxiosError({
id: 1,
message: `(GET) Error ${e.message} occurred while trying to fetch ${secureURL}`,
@ -305,10 +300,7 @@ export function isStringAValidURL(url: string): boolean {
* If `true`, the function will consider redirects a violation and return `false`.
* Default: `false`
*/
export async function urlExists(
url: string,
checkRedirect: boolean = false
): Promise<boolean> {
export async function urlExists(url: string, checkRedirect: boolean = false): Promise<boolean> {
// Local variables
let valid = false;
@ -376,10 +368,7 @@ function manageLoginPOSTResponse(response: AxiosResponse<any>) {
}
// Get the error message (if any) and remove the new line chars
const errorMessage = $("body")
.find(GENERIC.LOGIN_MESSAGE_ERROR)
.text()
.replace(/\n/g, "");
const errorMessage = $("body").find(GENERIC.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, "");
// Return the result of the authentication
const result = errorMessage.trim() === "";

14
src/scripts/scrape-data/handiwork-parse.ts
View File

@ -122,10 +122,7 @@ function stringToBoolean(s: string): boolean {
*
* Case-insensitive.
*/
function getPostElementByName(
elements: IPostElement[],
name: string
): IPostElement | undefined {
function getPostElementByName(elements: IPostElement[], name: string): IPostElement | undefined {
return elements.find((el) => el.name.toUpperCase() === name.toUpperCase());
}
@ -162,8 +159,7 @@ function fillWithPrefixes(hw: HandiWork, prefixes: string[]) {
// Check what the prefix indicates
if (stringInDict(prefix, shared.prefixes["engines"])) engine = prefix as TEngine;
else if (stringInDict(prefix, shared.prefixes["statuses"]))
status = prefix as TStatus;
else if (stringInDict(prefix, shared.prefixes["statuses"])) status = prefix as TStatus;
else if (stringInDict(prefix, fakeModDict)) mod = true;
// Anyway add the prefix to list
@ -206,8 +202,7 @@ function fillWithPostData(hw: HandiWork, elements: IPostElement[]) {
// Parse the censorship
const censored =
getPostElementByName(elements, "censored") ||
getPostElementByName(elements, "censorship");
getPostElementByName(elements, "censored") || getPostElementByName(elements, "censorship");
if (censored) hw.censored = stringToBoolean(censored.text);
// Get the genres
@ -249,8 +244,7 @@ function fillWithPostData(hw: HandiWork, elements: IPostElement[]) {
//#region Get the changelog
hw.changelog = [];
const changelogElement =
getPostElementByName(elements, "changelog") ||
getPostElementByName(elements, "change-log");
getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log");
if (changelogElement) {
const changelogSpoiler = changelogElement?.content.find((el) => {
return el.type === "Spoiler" && el.content.length > 0;

4
src/scripts/scrape-data/json-ld.ts
View File

@ -59,9 +59,7 @@ function parseJSONLD(element: cheerio.Element): TJsonLD {
const html = cheerio(element).html().trim();
// Obtain the JSON-LD
const data = html
.replace('<script type="application/ld+json">', "")
.replace("</script>", "");
const data = html.replace('<script type="application/ld+json">', "").replace("</script>", "");
// Convert the string to an object
return JSON.parse(data);

382
src/scripts/scrape-data/post-parse.ts
View File

@ -5,6 +5,9 @@
"use strict";
// Import from files
import { POST } from "../constants/css-selector";
//#region Interfaces
export interface IPostElement {
@ -22,13 +25,11 @@ export interface ILink extends IPostElement {
//#endregion Interfaces
//#region Public methods
/**
* Given a post of a thread page it extracts the information contained in the body.
*/
export function parseF95ThreadPost(
$: cheerio.Root,
post: cheerio.Cheerio
): IPostElement[] {
export function parseF95ThreadPost($: cheerio.Root, post: cheerio.Cheerio): IPostElement[] {
// The data is divided between "tag" and "text" elements.
// Simple data is composed of a "tag" element followed
// by a "text" element, while more complex data (contained
@ -40,34 +41,81 @@ export function parseF95ThreadPost(
const elements = post
.contents()
.toArray()
.map((el) => parseCheerioNode($, el))
.filter((node) => node.name || node.text || node.content.length != 0);
.map((el) => parseCheerioNode($, el)) // Parse the nodes
.filter((el) => !isPostElementEmpty(el)) // Ignore the empty nodes
.map((el) => reducePostElement(el)); // Compress the nodes
// ... then parse the elements to create the pairs of title/data
return parsePostElements(elements);
return associateElementsWithName(elements);
}
//#endregion Public methods
//#region Private methods
//#region Node type
/**
* Check if the node passed as a parameter is a formatting one (i.e. `<b>`).
*/
function isFormattingNode(node: cheerio.Element): boolean {
const formattedTags = ["b", "i"];
return node.type === "tag" && formattedTags.includes(node.name);
}
/**
* Check if the node passed as a parameter is of text type.
*/
function isTextNode(node: cheerio.Element): boolean {
return node.type === "text";
}
/**
* Check if the node is a spoiler.
*/
function isSpoilerNode(node: cheerio.Cheerio): boolean {
return node.attr("class") === "bbCodeSpoiler";
}
/**
* Check if the node is a link or a image.
*/
function isLinkNode(node: cheerio.Element): boolean {
// Local variables
let valid = false;
// The node is a valid DOM element
if (node.type === "tag") {
const el = node as cheerio.TagElement;
valid = el.name === "a" || el.name === "img";
}
return valid;
}
/**
* Check if the node is a `noscript` tag.
*/
function isNoScriptNode(node: cheerio.Element): boolean {
return node.type === "tag" && node.name === "noscript";
}
//#endregion Node Type
//#region Parse Cheerio node
/**
* Process a spoiler element by getting its text broken
* down by any other spoiler elements present.
*/
function parseCheerioSpoilerNode(
$: cheerio.Root,
spoiler: cheerio.Cheerio
): IPostElement {
function parseCheerioSpoilerNode($: cheerio.Root, node: cheerio.Cheerio): IPostElement {
// A spoiler block is composed of a div with class "bbCodeSpoiler",
// containing a div "bbCodeSpoiler-content" containing, in cascade,
// a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content".
// This last tag contains the required data.
// Local variables
const BUTTON_CLASS = "button.bbCodeSpoiler-button";
const SPOILER_CONTENT_CLASS =
"div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content";
const content: IPostElement = {
const spoiler: IPostElement = {
type: "Spoiler",
name: "",
text: "",
@ -75,185 +123,219 @@ function parseCheerioSpoilerNode(
};
// Find the title of the spoiler (contained in the button)
const button = spoiler.find(BUTTON_CLASS).toArray().shift();
content.name = $(button).text().trim();
spoiler.name = node.find(POST.SPOILER_BUTTON).first().text().trim();
// Parse the content of the spoiler
spoiler
.find(SPOILER_CONTENT_CLASS)
spoiler.content = node
.find(POST.SPOILER_CONTENT)
.contents()
.map((idx, el) => {
// Convert the element
const element = $(el);
// Parse nested spoiler
if (element.attr("class") === "bbCodeSpoiler") {
const spoiler = parseCheerioSpoilerNode($, element);
content.content.push(spoiler);
} else if (el.type === "text") {
// Append text
content.text += element.text();
}
});
.toArray()
.map((el) => parseCheerioNode($, el));
// Clean text
content.text = content.text.replace(/\s\s+/g, " ").trim();
return content;
spoiler.text = spoiler.text.replace(/\s\s+/g, " ").trim();
return spoiler;
}
/**
* Check if the node passed as a parameter is of text type.
* This also includes formatted nodes (i.e. `<b>`).
* Process a node that contains a link or image.
*/
function isTextNode(node: cheerio.Element): boolean {
const formattedTags = ["b", "i"];
const isText = node.type === "text";
const isFormatted = node.type === "tag" && formattedTags.includes(node.name);
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink {
// Local variable
const link: ILink = {
type: "Link",
name: "",
text: "",
href: "",
content: []
};
return isText || isFormatted;
if (element.is("img")) {
link.type = "Image";
link.text = element.attr("alt");
link.href = element.attr("data-src");
} else if (element.is("a")) {
link.type = "Link";
link.text = element.text().replace(/\s\s+/g, " ").trim();
link.href = element.attr("href");
}
return link;
}
/**
* Process a text only node.
*/
function parseCheerioTextNode(node: cheerio.Cheerio): IPostElement {
const content: IPostElement = {
type: "Text",
name: "",
text: getCheerioNonChildrenText(node),
content: []
};
return content;
}
//#endregion Parse Cheerio node
//#region IPostElement utility
/**
* Check if the node has non empty `name` and `text`.
*/
function isPostElementUnknown(node: IPostElement): boolean {
return node.name.trim() === "" && node.text.trim() === "";
}
/**
* Check if the node has a non empty property
* between `name`, `text` and `content`.
*/
function isPostElementEmpty(node: IPostElement): boolean {
return node.content.length === 0 && isPostElementUnknown(node);
}
/**
* Create a `IPostElement` without name, text or content.
*/
function createEmptyElement(): IPostElement {
return {
type: "Empty",
name: "",
text: "",
content: []
};
}
/**
* Check if the element contains the overview of a thread (post #1).
*/
function elementIsOverview(element: IPostElement): boolean {
// Search the text element that start with "overview"
const result = element.content
.filter((e) => e.type === "Text")
.find((e) => e.text.toUpperCase().startsWith("OVERVIEW"));
return result !== undefined;
}
/**
* If the element contains the overview of a thread, parse it.
*/
function getOverviewFromElement(element: IPostElement): string {
// Local variables
const alphanumericRegex = new RegExp("[a-zA-Z0-9]+");
// Get all the text values of the overview
const textes = element.content
.filter((e) => e.type === "Text")
.filter((e) => {
const cleanValue = e.text.toUpperCase().replace("OVERVIEW", "").trim();
const isAlphanumeric = alphanumericRegex.test(cleanValue);
return cleanValue !== "" && isAlphanumeric;
})
.map((e) => e.text);
// Joins the textes
return textes.join(" ");
}
//#endregion IPostElement utility
/**
* Gets the text of the node only, excluding child nodes.
* Also includes formatted text elements (i.e. `<b>`).
*/
function getCheerioNonChildrenText(node: cheerio.Cheerio): string {
// Find all the text nodes in the node
const text = node
.first()
.contents()
.filter((idx, el) => {
return isTextNode(el);
})
.text();
// Local variable
let text = "";
// If the node has no children, return the node's text
if (node.contents().length === 1) {
// @todo Remove IF after cheerio RC6
text = node.text();
} else {
// Find all the text nodes in the node
text = node
.first()
.contents() // @todo Change to children() after cheerio RC6
.filter((idx, el) => isTextNode(el))
.text();
}
// Clean and return the text
return text.replace(/\s\s+/g, " ").trim();
}
/**
* Process a node and see if it contains a
* link or image. If not, it returns `null`.
*/
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null {
//@ts-ignore
const name = element[0]?.name;
const link: ILink = {
name: "",
type: "Link",
text: "",
href: "",
content: []
};
if (name === "img") {
link.type = "Image";
link.text = element.attr("alt");
link.href = element.attr("data-src");
} else if (name === "a") {
link.type = "Link";
link.text = element.text().replace(/\s\s+/g, " ").trim();
link.href = element.attr("href");
}
return link.href ? link : null;
}
/**
* Collapse an `IPostElement` element with a single subnode
* in the `Content` field in case it has no information.
*/
function reducePostElement(element: IPostElement): IPostElement {
if (element.content.length === 1) {
const content = element.content[0] as IPostElement;
const nullValues =
(!element.name || !content.name) && (!element.text || !content.text);
const sameValues = element.name === content.name || element.text === content.text;
function reducePostElement(element: IPostElement, recursive = true): IPostElement {
// Local variables
const shallowCopy = Object.assign({}, element);
if (nullValues || sameValues) {
element.name = element.name || content.name;
element.text = element.text || content.text;
element.content.push(...content.content);
element.type = content.type;
// If the content is a link, add the HREF to the element
const contentILink = content as ILink;
const elementILink = element as ILink;
if (contentILink.href) elementILink.href = contentILink.href;
}
// Find the posts without name and text
const unknownChildrens = shallowCopy.content.filter((e) => isPostElementUnknown(e));
if (recursive) {
const recursiveUnknownChildrens = unknownChildrens.map((e) => reducePostElement(e));
unknownChildrens.push(...recursiveUnknownChildrens);
}
return element;
// Eliminate non-useful child nodes
if (isPostElementUnknown(shallowCopy) && unknownChildrens.length > 0) {
// Find the valid elements to add to the node
const childContents = unknownChildrens
.filter((e) => !shallowCopy.content.includes(e))
.map((e) => (e.content.length > 0 ? e.content : e));
// Remove the empty elements
shallowCopy.content = shallowCopy.content.filter((e) => !unknownChildrens.includes(e));
// Merge the non-empty children of this node with
// the content of the empty children of this node
const newContent = [].concat(...childContents);
shallowCopy.content.push(...newContent);
}
// If the node has only one child, return it
else if (isPostElementUnknown(shallowCopy) && shallowCopy.content.length === 1) {
return shallowCopy.content[0];
}
return shallowCopy;
}
/**
* Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes.
* @param reduce Compress subsequent subnodes if they contain no information. Default: `true`.
*/
function parseCheerioNode(
$: cheerio.Root,
node: cheerio.Element,
reduce = true
): IPostElement {
function parseCheerioNode($: cheerio.Root, node: cheerio.Element): IPostElement {
// Local variables
const content: IPostElement = {
type: "Empty",
name: "",
text: "",
content: []
};
let post: IPostElement = createEmptyElement();
const cheerioNode = $(node);
if (isTextNode(node)) {
content.text = cheerioNode.text().replace(/\s\s+/g, " ").trim();
content.type = "Text";
} else {
// Get the number of children that the element own
const nChildren = cheerioNode.children().length;
// Parse the node
if (!isNoScriptNode(node)) {
if (isTextNode(node) && !isFormattingNode(node)) post = parseCheerioTextNode(cheerioNode);
else if (isSpoilerNode(cheerioNode)) post = parseCheerioSpoilerNode($, cheerioNode);
else if (isLinkNode(node)) post = parseCheerioLinkNode(cheerioNode);
// Get the text of the element without childrens
content.text = getCheerioNonChildrenText(cheerioNode);
// Parse spoilers
if (cheerioNode.attr("class") === "bbCodeSpoiler") {
const spoiler = parseCheerioSpoilerNode($, cheerioNode);
// Add element if not null
if (spoiler) {
content.content.push(spoiler);
content.type = "Spoiler";
}
}
// Parse links
else if (nChildren === 0 && cheerioNode.length != 0) {
const link = parseCheerioLinkNode(cheerioNode);
// Add element if not null
if (link) {
content.content.push(link);
content.type = "Link";
}
} else {
cheerioNode.children().map((idx, el) => {
// Parse the children of the element passed as parameter
const childElement = parseCheerioNode($, el);
// If the children is valid (not empty) push it
if ((childElement.text || childElement.content.length !== 0) && !isTextNode(el)) {
content.content.push(childElement);
}
});
}
// Parse the node's childrens
const childPosts = cheerioNode
.contents() // @todo Change to children() after cheerio RC6
.toArray()
.filter((el) => el) // Ignore undefined elements
.map((el) => parseCheerioNode($, el))
.filter((el) => !isPostElementEmpty(el));
post.content.push(...childPosts);
}
return reduce ? reducePostElement(content) : content;
return post;
}
/**
* It simplifies the `IPostElement` elements by associating
* the corresponding value to each characterizing element (i.e. author).
*/
function parsePostElements(elements: IPostElement[]): IPostElement[] {
function associateElementsWithName(elements: IPostElement[]): IPostElement[] {
// Local variables
const pairs: IPostElement[] = [];
const specialCharsRegex = /^[-!$%^&*()_+|~=`{}[\]:";'<>?,./]/;
@ -275,11 +357,11 @@ function parsePostElements(elements: IPostElement[]): IPostElement[] {
lastPair.content.push(...elements[i].content);
}
// This is a special case
else if (elements[i].text.startsWith("Overview:\n")) {
else if (elementIsOverview(elements[i])) {
// We add the overview to the pairs as a text element
elements[i].type = "Text";
elements[i].name = "Overview";
elements[i].text = elements[i].text.replace("Overview:\n", "");
elements[i].text = getOverviewFromElement(elements[i]);
pairs.push(elements[i]);
}
// We have an element referred to the previous "title"

4
test/classes/mapping/thread.ts
View File

@ -34,8 +34,6 @@ export function suite(): void {
it("Fetch post with invalid ID", async function fetchWithInvalidID() {
Shared.setIsLogged(true);
const thread = new Thread(-1);
await expect(thread.getPost(0)).to.be.rejectedWith(
"Index must be greater or equal than 1"
);
await expect(thread.getPost(0)).to.be.rejectedWith("Index must be greater or equal than 1");
});
}

10
test/classes/prefix-parser.ts
View File

@ -29,15 +29,7 @@ export function suite(): void {
// Test values
const testIDs = [103, 225, 44, 13, 2, 7, 22];
const testPrefixes = [
"corruption",
"pregnancy",
"slave",
"VN",
"RPGM",
"Ren'Py",
"Abandoned"
];
const testPrefixes = ["corruption", "pregnancy", "slave", "VN", "RPGM", "Ren'Py", "Abandoned"];
// Parse values
const ids = parser.prefixesToIDs(testPrefixes);