MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reworked script

pull/81/head
MillenniumEarl 2021-03-15 18:43:03 +01:00
parent 1d5836c0d0
commit 7b64528fd0
1 changed files with 234 additions and 146 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

380
src/scripts/scrape-data/post-parse.ts
View File

@ -5,6 +5,9 @@
"use strict"; "use strict";
// Import from files
import { POST } from "../constants/css-selector";
//#region Interfaces //#region Interfaces
export interface IPostElement { export interface IPostElement {
@ -22,6 +25,7 @@ export interface ILink extends IPostElement {
//#endregion Interfaces //#endregion Interfaces
//#region Public methods //#region Public methods
/** /**
* Given a post of a thread page it extracts the information contained in the body. * Given a post of a thread page it extracts the information contained in the body.
*/ */
@ -40,34 +44,81 @@ export function parseF95ThreadPost(
const elements = post const elements = post
.contents() .contents()
.toArray() .toArray()
.map((el) => parseCheerioNode($, el)) .map((el) => parseCheerioNode($, el)) // Parse the nodes
.filter((node) => node.name || node.text || node.content.length != 0); .filter((el) => !isPostElementEmpty(el)) // Ignore the empty nodes
.map((el) => reducePostElement(el)); // Compress the nodes
// ... then parse the elements to create the pairs of title/data // ... then parse the elements to create the pairs of title/data
return parsePostElements(elements); return associateElementsWithName(elements);
} }
//#endregion Public methods //#endregion Public methods
//#region Private methods //#region Private methods
//#region Node type
/**
* Check if the node passed as a parameter is a formatting one (i.e. `<b>`).
*/
function isFormattingNode(node: cheerio.Element): boolean {
const formattedTags = ["b", "i"];
return node.type === "tag" && formattedTags.includes(node.name);
}
/**
* Check if the node passed as a parameter is of text type.
*/
function isTextNode(node: cheerio.Element): boolean {
return node.type === "text";
}
/**
* Check if the node is a spoiler.
*/
function isSpoilerNode(node: cheerio.Cheerio): boolean {
return node.attr("class") === "bbCodeSpoiler";
}
/**
* Check if the node is a link or a image.
*/
function isLinkNode(node: cheerio.Element): boolean {
// Local variables
let valid = false;
// The node is a valid DOM element
if (node.type === "tag") {
const el = node as cheerio.TagElement;
valid = el.name === "a" || el.name === "img";
}
return valid;
}
/**
* Check if the node is a `noscript` tag.
*/
function isNoScriptNode(node: cheerio.Element): boolean {
return node.type === "tag" && node.name === "noscript";
}
//#endregion Node Type
//#region Parse Cheerio node
/** /**
* Process a spoiler element by getting its text broken * Process a spoiler element by getting its text broken
* down by any other spoiler elements present. * down by any other spoiler elements present.
*/ */
function parseCheerioSpoilerNode( function parseCheerioSpoilerNode($: cheerio.Root, node: cheerio.Cheerio): IPostElement {
$: cheerio.Root,
spoiler: cheerio.Cheerio
): IPostElement {
// A spoiler block is composed of a div with class "bbCodeSpoiler", // A spoiler block is composed of a div with class "bbCodeSpoiler",
// containing a div "bbCodeSpoiler-content" containing, in cascade, // containing a div "bbCodeSpoiler-content" containing, in cascade,
// a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content". // a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content".
// This last tag contains the required data. // This last tag contains the required data.
// Local variables // Local variables
const BUTTON_CLASS = "button.bbCodeSpoiler-button"; const spoiler: IPostElement = {
const SPOILER_CONTENT_CLASS =
"div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content";
const content: IPostElement = {
type: "Spoiler", type: "Spoiler",
name: "", name: "",
text: "", text: "",
@ -75,185 +126,222 @@ function parseCheerioSpoilerNode(
}; };
// Find the title of the spoiler (contained in the button) // Find the title of the spoiler (contained in the button)
const button = spoiler.find(BUTTON_CLASS).toArray().shift(); spoiler.name = node.find(POST.SPOILER_BUTTON).first().text().trim();
content.name = $(button).text().trim();
// Parse the content of the spoiler // Parse the content of the spoiler
spoiler spoiler.content = node
.find(SPOILER_CONTENT_CLASS) .find(POST.SPOILER_CONTENT)
.contents() .contents()
.map((idx, el) => { .toArray()
// Convert the element .map((el) => parseCheerioNode($, el));
const element = $(el);
// Parse nested spoiler
if (element.attr("class") === "bbCodeSpoiler") {
const spoiler = parseCheerioSpoilerNode($, element);
content.content.push(spoiler);
} else if (el.type === "text") {
// Append text
content.text += element.text();
}
});
// Clean text // Clean text
content.text = content.text.replace(/\s\s+/g, " ").trim(); spoiler.text = spoiler.text.replace(/\s\s+/g, " ").trim();
return content; return spoiler;
} }
/** /**
* Check if the node passed as a parameter is of text type. * Process a node that contains a link or image.
* This also includes formatted nodes (i.e. `<b>`).
*/ */
function isTextNode(node: cheerio.Element): boolean { function parseCheerioLinkNode(element: cheerio.Cheerio): ILink {
const formattedTags = ["b", "i"]; // Local variable
const isText = node.type === "text"; const link: ILink = {
const isFormatted = node.type === "tag" && formattedTags.includes(node.name); type: "Link",
name: "",
text: "",
href: "",
content: []
};
return isText || isFormatted; if (element.is("img")) {
link.type = "Image";
link.text = element.attr("alt");
link.href = element.attr("data-src");
} else if (element.is("a")) {
link.type = "Link";
link.text = element.text().replace(/\s\s+/g, " ").trim();
link.href = element.attr("href");
}
return link;
} }
/**
* Process a text only node.
*/
function parseCheerioTextNode(node: cheerio.Cheerio): IPostElement {
const content: IPostElement = {
type: "Text",
name: "",
text: getCheerioNonChildrenText(node),
content: []
};
return content;
}
//#endregion Parse Cheerio node
//#region IPostElement utility
/**
* Check if the node has non empty `name` and `text`.
*/
function isPostElementUnknown(node: IPostElement): boolean {
return node.name.trim() === "" && node.text.trim() === "";
}
/**
* Check if the node has a non empty property
* between `name`, `text` and `content`.
*/
function isPostElementEmpty(node: IPostElement): boolean {
return node.content.length === 0 && isPostElementUnknown(node);
}
/**
* Create a `IPostElement` without name, text or content.
*/
function createEmptyElement(): IPostElement {
return {
type: "Empty",
name: "",
text: "",
content: []
};
}
/**
* Check if the element contains the overview of a thread (post #1).
*/
function elementIsOverview(element: IPostElement): boolean {
// Search the text element that start with "overview"
const result = element.content
.filter((e) => e.type === "Text")
.find((e) => e.text.toUpperCase().startsWith("OVERVIEW"));
return result !== undefined;
}
/**
* If the element contains the overview of a thread, parse it.
*/
function getOverviewFromElement(element: IPostElement): string {
// Local variables
const alphanumericRegex = new RegExp("[a-zA-Z0-9]+");
// Get all the text values of the overview
const textes = element.content
.filter((e) => e.type === "Text")
.filter((e) => {
const cleanValue = e.text.toUpperCase().replace("OVERVIEW", "").trim();
const isAlphanumeric = alphanumericRegex.test(cleanValue);
return cleanValue !== "" && isAlphanumeric;
})
.map((e) => e.text);
// Joins the textes
return textes.join(" ");
}
//#endregion IPostElement utility
/** /**
* Gets the text of the node only, excluding child nodes. * Gets the text of the node only, excluding child nodes.
* Also includes formatted text elements (i.e. `<b>`). * Also includes formatted text elements (i.e. `<b>`).
*/ */
function getCheerioNonChildrenText(node: cheerio.Cheerio): string { function getCheerioNonChildrenText(node: cheerio.Cheerio): string {
// Find all the text nodes in the node // Local variable
const text = node let text = "";
.first()
.contents() // If the node has no children, return the node's text
.filter((idx, el) => { if (node.contents().length === 1) {
return isTextNode(el); // @todo Remove IF after cheerio RC6
}) text = node.text();
.text(); } else {
// Find all the text nodes in the node
text = node
.first()
.contents() // @todo Change to children() after cheerio RC6
.filter((idx, el) => isTextNode(el))
.text();
}
// Clean and return the text // Clean and return the text
return text.replace(/\s\s+/g, " ").trim(); return text.replace(/\s\s+/g, " ").trim();
} }
/**
* Process a node and see if it contains a
* link or image. If not, it returns `null`.
*/
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null {
//@ts-ignore
const name = element[0]?.name;
const link: ILink = {
name: "",
type: "Link",
text: "",
href: "",
content: []
};
if (name === "img") {
link.type = "Image";
link.text = element.attr("alt");
link.href = element.attr("data-src");
} else if (name === "a") {
link.type = "Link";
link.text = element.text().replace(/\s\s+/g, " ").trim();
link.href = element.attr("href");
}
return link.href ? link : null;
}
/** /**
* Collapse an `IPostElement` element with a single subnode * Collapse an `IPostElement` element with a single subnode
* in the `Content` field in case it has no information. * in the `Content` field in case it has no information.
*/ */
function reducePostElement(element: IPostElement): IPostElement { function reducePostElement(element: IPostElement, recursive = true): IPostElement {
if (element.content.length === 1) { // Local variables
const content = element.content[0] as IPostElement; const shallowCopy = Object.assign({}, element);
const nullValues =
(!element.name || !content.name) && (!element.text || !content.text);
const sameValues = element.name === content.name || element.text === content.text;
if (nullValues || sameValues) { // Find the posts without name and text
element.name = element.name || content.name; const unknownChildrens = shallowCopy.content.filter((e) => isPostElementUnknown(e));
element.text = element.text || content.text; if (recursive) {
element.content.push(...content.content); const recursiveUnknownChildrens = unknownChildrens.map((e) => reducePostElement(e));
element.type = content.type; unknownChildrens.push(...recursiveUnknownChildrens);
// If the content is a link, add the HREF to the element
const contentILink = content as ILink;
const elementILink = element as ILink;
if (contentILink.href) elementILink.href = contentILink.href;
}
} }
return element; // Eliminate non-useful child nodes
if (isPostElementUnknown(shallowCopy) && unknownChildrens.length > 0) {
// Find the valid elements to add to the node
const childContents = unknownChildrens
.filter((e) => !shallowCopy.content.includes(e))
.map((e) => (e.content.length > 0 ? e.content : e));
// Remove the empty elements
shallowCopy.content = shallowCopy.content.filter(
(e) => !unknownChildrens.includes(e)
);
// Merge the non-empty children of this node with
// the content of the empty children of this node
const newContent = [].concat(...childContents);
shallowCopy.content.push(...newContent);
}
// If the node has only one child, return it
else if (isPostElementUnknown(shallowCopy) && shallowCopy.content.length === 1) {
return shallowCopy.content[0];
}
return shallowCopy;
} }
/** /**
* Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes. * Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes.
* @param reduce Compress subsequent subnodes if they contain no information. Default: `true`.
*/ */
function parseCheerioNode( function parseCheerioNode($: cheerio.Root, node: cheerio.Element): IPostElement {
$: cheerio.Root,
node: cheerio.Element,
reduce = true
): IPostElement {
// Local variables // Local variables
const content: IPostElement = { let post: IPostElement = createEmptyElement();
type: "Empty",
name: "",
text: "",
content: []
};
const cheerioNode = $(node); const cheerioNode = $(node);
if (isTextNode(node)) { // Parse the node
content.text = cheerioNode.text().replace(/\s\s+/g, " ").trim(); if (!isNoScriptNode(node)) {
content.type = "Text"; if (isTextNode(node) && !isFormattingNode(node))
} else { post = parseCheerioTextNode(cheerioNode);
// Get the number of children that the element own else if (isSpoilerNode(cheerioNode)) post = parseCheerioSpoilerNode($, cheerioNode);
const nChildren = cheerioNode.children().length; else if (isLinkNode(node)) post = parseCheerioLinkNode(cheerioNode);
// Get the text of the element without childrens // Parse the node's childrens
content.text = getCheerioNonChildrenText(cheerioNode); const childPosts = cheerioNode
.contents() // @todo Change to children() after cheerio RC6
// Parse spoilers .toArray()
if (cheerioNode.attr("class") === "bbCodeSpoiler") { .filter((el) => el) // Ignore undefined elements
const spoiler = parseCheerioSpoilerNode($, cheerioNode); .map((el) => parseCheerioNode($, el))
.filter((el) => !isPostElementEmpty(el));
// Add element if not null post.content.push(...childPosts);
if (spoiler) {
content.content.push(spoiler);
content.type = "Spoiler";
}
}
// Parse links
else if (nChildren === 0 && cheerioNode.length != 0) {
const link = parseCheerioLinkNode(cheerioNode);
// Add element if not null
if (link) {
content.content.push(link);
content.type = "Link";
}
} else {
cheerioNode.children().map((idx, el) => {
// Parse the children of the element passed as parameter
const childElement = parseCheerioNode($, el);
// If the children is valid (not empty) push it
if ((childElement.text || childElement.content.length !== 0) && !isTextNode(el)) {
content.content.push(childElement);
}
});
}
} }
return reduce ? reducePostElement(content) : content; return post;
} }
/** /**
* It simplifies the `IPostElement` elements by associating * It simplifies the `IPostElement` elements by associating
* the corresponding value to each characterizing element (i.e. author). * the corresponding value to each characterizing element (i.e. author).
*/ */
function parsePostElements(elements: IPostElement[]): IPostElement[] { function associateElementsWithName(elements: IPostElement[]): IPostElement[] {
// Local variables // Local variables
const pairs: IPostElement[] = []; const pairs: IPostElement[] = [];
const specialCharsRegex = /^[-!$%^&*()_+|~=`{}[\]:";'<>?,./]/; const specialCharsRegex = /^[-!$%^&*()_+|~=`{}[\]:";'<>?,./]/;
@ -275,11 +363,11 @@ function parsePostElements(elements: IPostElement[]): IPostElement[] {
lastPair.content.push(...elements[i].content); lastPair.content.push(...elements[i].content);
} }
// This is a special case // This is a special case
else if (elements[i].text.startsWith("Overview:\n")) { else if (elementIsOverview(elements[i])) {
// We add the overview to the pairs as a text element // We add the overview to the pairs as a text element
elements[i].type = "Text"; elements[i].type = "Text";
elements[i].name = "Overview"; elements[i].name = "Overview";
elements[i].text = elements[i].text.replace("Overview:\n", ""); elements[i].text = getOverviewFromElement(elements[i]);
pairs.push(elements[i]); pairs.push(elements[i]);
} }
// We have an element referred to the previous "title" // We have an element referred to the previous "title"