MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Fix removeEmptyContent recursion and parse node children only for formmating nodes

pull/83/head
MillenniumEarl 2021-03-21 16:55:15 +01:00
parent 701678b577
commit 14a290468f
1 changed files with 12 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/scripts/scrape-data/post-parse.ts
View File

@ -325,11 +325,13 @@ function removeEmptyContentFromElement(element: IPostElement, recursive = true):
// Create a copy of the element // Create a copy of the element
const copy = Object.assign({}, element); const copy = Object.assign({}, element);
// Find the non-empty nodes
const validNodes = copy.content.filter((e) => !isPostElementEmpty(e));
// Reduce nested contents if recursive // Reduce nested contents if recursive
if (recursive) validNodes.forEach((e) => removeEmptyContentFromElement(e)); const recursiveResult = recursive
? element.content.map((e) => removeEmptyContentFromElement(e))
: copy.content;
// Find the non-empty nodes
const validNodes = recursiveResult.filter((e) => !isPostElementEmpty(e));
// Assign the nodes // Assign the nodes
copy.content = validNodes; copy.content = validNodes;
@ -351,8 +353,10 @@ function parseCheerioNode($: cheerio.Root, node: cheerio.Element): IPostElement
else if (isSpoilerNode(cheerioNode)) post = parseCheerioSpoilerNode($, cheerioNode); else if (isSpoilerNode(cheerioNode)) post = parseCheerioSpoilerNode($, cheerioNode);
else if (isLinkNode(node)) post = parseCheerioLinkNode(cheerioNode); else if (isLinkNode(node)) post = parseCheerioLinkNode(cheerioNode);
// Avoid duplication of link name // Check for childrens only if the node is a <b>/<i> element.
if (!isLinkNode(node)) { // For the link in unnecessary while for the spoilers is
// already done in parseCheerioSpoilerNode
if (isFormattingNode(node)) {
// Parse the node's childrens // Parse the node's childrens
const childPosts = cheerioNode const childPosts = cheerioNode
.contents() // @todo Change to children() after cheerio RC6 .contents() // @todo Change to children() after cheerio RC6
@ -479,7 +483,8 @@ function parseGroupData(
.replace(endsWithSpecialCharsRegex, "") // Remove any special chars at the end .replace(endsWithSpecialCharsRegex, "") // Remove any special chars at the end
.trim() .trim()
) )
.join(" "); // Join with space .join(" ") // Join with space
.trim();
// Append all the content of non-text elements. // Append all the content of non-text elements.
group group