Reworked script
parent
1d5836c0d0
commit
7b64528fd0
|
@ -5,6 +5,9 @@
|
||||||
|
|
||||||
"use strict";
|
"use strict";
|
||||||
|
|
||||||
|
// Import from files
|
||||||
|
import { POST } from "../constants/css-selector";
|
||||||
|
|
||||||
//#region Interfaces
|
//#region Interfaces
|
||||||
|
|
||||||
export interface IPostElement {
|
export interface IPostElement {
|
||||||
|
@ -22,6 +25,7 @@ export interface ILink extends IPostElement {
|
||||||
//#endregion Interfaces
|
//#endregion Interfaces
|
||||||
|
|
||||||
//#region Public methods
|
//#region Public methods
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Given a post of a thread page it extracts the information contained in the body.
|
* Given a post of a thread page it extracts the information contained in the body.
|
||||||
*/
|
*/
|
||||||
|
@ -40,34 +44,81 @@ export function parseF95ThreadPost(
|
||||||
const elements = post
|
const elements = post
|
||||||
.contents()
|
.contents()
|
||||||
.toArray()
|
.toArray()
|
||||||
.map((el) => parseCheerioNode($, el))
|
.map((el) => parseCheerioNode($, el)) // Parse the nodes
|
||||||
.filter((node) => node.name || node.text || node.content.length != 0);
|
.filter((el) => !isPostElementEmpty(el)) // Ignore the empty nodes
|
||||||
|
.map((el) => reducePostElement(el)); // Compress the nodes
|
||||||
|
|
||||||
// ... then parse the elements to create the pairs of title/data
|
// ... then parse the elements to create the pairs of title/data
|
||||||
return parsePostElements(elements);
|
return associateElementsWithName(elements);
|
||||||
}
|
}
|
||||||
|
|
||||||
//#endregion Public methods
|
//#endregion Public methods
|
||||||
|
|
||||||
//#region Private methods
|
//#region Private methods
|
||||||
|
|
||||||
|
//#region Node type
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node passed as a parameter is a formatting one (i.e. `<b>`).
|
||||||
|
*/
|
||||||
|
function isFormattingNode(node: cheerio.Element): boolean {
|
||||||
|
const formattedTags = ["b", "i"];
|
||||||
|
return node.type === "tag" && formattedTags.includes(node.name);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node passed as a parameter is of text type.
|
||||||
|
*/
|
||||||
|
function isTextNode(node: cheerio.Element): boolean {
|
||||||
|
return node.type === "text";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node is a spoiler.
|
||||||
|
*/
|
||||||
|
function isSpoilerNode(node: cheerio.Cheerio): boolean {
|
||||||
|
return node.attr("class") === "bbCodeSpoiler";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node is a link or a image.
|
||||||
|
*/
|
||||||
|
function isLinkNode(node: cheerio.Element): boolean {
|
||||||
|
// Local variables
|
||||||
|
let valid = false;
|
||||||
|
|
||||||
|
// The node is a valid DOM element
|
||||||
|
if (node.type === "tag") {
|
||||||
|
const el = node as cheerio.TagElement;
|
||||||
|
valid = el.name === "a" || el.name === "img";
|
||||||
|
}
|
||||||
|
|
||||||
|
return valid;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node is a `noscript` tag.
|
||||||
|
*/
|
||||||
|
function isNoScriptNode(node: cheerio.Element): boolean {
|
||||||
|
return node.type === "tag" && node.name === "noscript";
|
||||||
|
}
|
||||||
|
|
||||||
|
//#endregion Node Type
|
||||||
|
|
||||||
|
//#region Parse Cheerio node
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Process a spoiler element by getting its text broken
|
* Process a spoiler element by getting its text broken
|
||||||
* down by any other spoiler elements present.
|
* down by any other spoiler elements present.
|
||||||
*/
|
*/
|
||||||
function parseCheerioSpoilerNode(
|
function parseCheerioSpoilerNode($: cheerio.Root, node: cheerio.Cheerio): IPostElement {
|
||||||
$: cheerio.Root,
|
|
||||||
spoiler: cheerio.Cheerio
|
|
||||||
): IPostElement {
|
|
||||||
// A spoiler block is composed of a div with class "bbCodeSpoiler",
|
// A spoiler block is composed of a div with class "bbCodeSpoiler",
|
||||||
// containing a div "bbCodeSpoiler-content" containing, in cascade,
|
// containing a div "bbCodeSpoiler-content" containing, in cascade,
|
||||||
// a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content".
|
// a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content".
|
||||||
// This last tag contains the required data.
|
// This last tag contains the required data.
|
||||||
|
|
||||||
// Local variables
|
// Local variables
|
||||||
const BUTTON_CLASS = "button.bbCodeSpoiler-button";
|
const spoiler: IPostElement = {
|
||||||
const SPOILER_CONTENT_CLASS =
|
|
||||||
"div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content";
|
|
||||||
const content: IPostElement = {
|
|
||||||
type: "Spoiler",
|
type: "Spoiler",
|
||||||
name: "",
|
name: "",
|
||||||
text: "",
|
text: "",
|
||||||
|
@ -75,185 +126,222 @@ function parseCheerioSpoilerNode(
|
||||||
};
|
};
|
||||||
|
|
||||||
// Find the title of the spoiler (contained in the button)
|
// Find the title of the spoiler (contained in the button)
|
||||||
const button = spoiler.find(BUTTON_CLASS).toArray().shift();
|
spoiler.name = node.find(POST.SPOILER_BUTTON).first().text().trim();
|
||||||
content.name = $(button).text().trim();
|
|
||||||
|
|
||||||
// Parse the content of the spoiler
|
// Parse the content of the spoiler
|
||||||
spoiler
|
spoiler.content = node
|
||||||
.find(SPOILER_CONTENT_CLASS)
|
.find(POST.SPOILER_CONTENT)
|
||||||
.contents()
|
.contents()
|
||||||
.map((idx, el) => {
|
.toArray()
|
||||||
// Convert the element
|
.map((el) => parseCheerioNode($, el));
|
||||||
const element = $(el);
|
|
||||||
|
|
||||||
// Parse nested spoiler
|
|
||||||
if (element.attr("class") === "bbCodeSpoiler") {
|
|
||||||
const spoiler = parseCheerioSpoilerNode($, element);
|
|
||||||
content.content.push(spoiler);
|
|
||||||
} else if (el.type === "text") {
|
|
||||||
// Append text
|
|
||||||
content.text += element.text();
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// Clean text
|
// Clean text
|
||||||
content.text = content.text.replace(/\s\s+/g, " ").trim();
|
spoiler.text = spoiler.text.replace(/\s\s+/g, " ").trim();
|
||||||
return content;
|
return spoiler;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Check if the node passed as a parameter is of text type.
|
* Process a node that contains a link or image.
|
||||||
* This also includes formatted nodes (i.e. `<b>`).
|
|
||||||
*/
|
*/
|
||||||
function isTextNode(node: cheerio.Element): boolean {
|
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink {
|
||||||
const formattedTags = ["b", "i"];
|
// Local variable
|
||||||
const isText = node.type === "text";
|
const link: ILink = {
|
||||||
const isFormatted = node.type === "tag" && formattedTags.includes(node.name);
|
type: "Link",
|
||||||
|
name: "",
|
||||||
|
text: "",
|
||||||
|
href: "",
|
||||||
|
content: []
|
||||||
|
};
|
||||||
|
|
||||||
return isText || isFormatted;
|
if (element.is("img")) {
|
||||||
|
link.type = "Image";
|
||||||
|
link.text = element.attr("alt");
|
||||||
|
link.href = element.attr("data-src");
|
||||||
|
} else if (element.is("a")) {
|
||||||
|
link.type = "Link";
|
||||||
|
link.text = element.text().replace(/\s\s+/g, " ").trim();
|
||||||
|
link.href = element.attr("href");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return link;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process a text only node.
|
||||||
|
*/
|
||||||
|
function parseCheerioTextNode(node: cheerio.Cheerio): IPostElement {
|
||||||
|
const content: IPostElement = {
|
||||||
|
type: "Text",
|
||||||
|
name: "",
|
||||||
|
text: getCheerioNonChildrenText(node),
|
||||||
|
content: []
|
||||||
|
};
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
//#endregion Parse Cheerio node
|
||||||
|
|
||||||
|
//#region IPostElement utility
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node has non empty `name` and `text`.
|
||||||
|
*/
|
||||||
|
function isPostElementUnknown(node: IPostElement): boolean {
|
||||||
|
return node.name.trim() === "" && node.text.trim() === "";
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node has a non empty property
|
||||||
|
* between `name`, `text` and `content`.
|
||||||
|
*/
|
||||||
|
function isPostElementEmpty(node: IPostElement): boolean {
|
||||||
|
return node.content.length === 0 && isPostElementUnknown(node);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Create a `IPostElement` without name, text or content.
|
||||||
|
*/
|
||||||
|
function createEmptyElement(): IPostElement {
|
||||||
|
return {
|
||||||
|
type: "Empty",
|
||||||
|
name: "",
|
||||||
|
text: "",
|
||||||
|
content: []
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the element contains the overview of a thread (post #1).
|
||||||
|
*/
|
||||||
|
function elementIsOverview(element: IPostElement): boolean {
|
||||||
|
// Search the text element that start with "overview"
|
||||||
|
const result = element.content
|
||||||
|
.filter((e) => e.type === "Text")
|
||||||
|
.find((e) => e.text.toUpperCase().startsWith("OVERVIEW"));
|
||||||
|
return result !== undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* If the element contains the overview of a thread, parse it.
|
||||||
|
*/
|
||||||
|
function getOverviewFromElement(element: IPostElement): string {
|
||||||
|
// Local variables
|
||||||
|
const alphanumericRegex = new RegExp("[a-zA-Z0-9]+");
|
||||||
|
|
||||||
|
// Get all the text values of the overview
|
||||||
|
const textes = element.content
|
||||||
|
.filter((e) => e.type === "Text")
|
||||||
|
.filter((e) => {
|
||||||
|
const cleanValue = e.text.toUpperCase().replace("OVERVIEW", "").trim();
|
||||||
|
const isAlphanumeric = alphanumericRegex.test(cleanValue);
|
||||||
|
|
||||||
|
return cleanValue !== "" && isAlphanumeric;
|
||||||
|
})
|
||||||
|
.map((e) => e.text);
|
||||||
|
|
||||||
|
// Joins the textes
|
||||||
|
return textes.join(" ");
|
||||||
|
}
|
||||||
|
|
||||||
|
//#endregion IPostElement utility
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Gets the text of the node only, excluding child nodes.
|
* Gets the text of the node only, excluding child nodes.
|
||||||
* Also includes formatted text elements (i.e. `<b>`).
|
* Also includes formatted text elements (i.e. `<b>`).
|
||||||
*/
|
*/
|
||||||
function getCheerioNonChildrenText(node: cheerio.Cheerio): string {
|
function getCheerioNonChildrenText(node: cheerio.Cheerio): string {
|
||||||
|
// Local variable
|
||||||
|
let text = "";
|
||||||
|
|
||||||
|
// If the node has no children, return the node's text
|
||||||
|
if (node.contents().length === 1) {
|
||||||
|
// @todo Remove IF after cheerio RC6
|
||||||
|
text = node.text();
|
||||||
|
} else {
|
||||||
// Find all the text nodes in the node
|
// Find all the text nodes in the node
|
||||||
const text = node
|
text = node
|
||||||
.first()
|
.first()
|
||||||
.contents()
|
.contents() // @todo Change to children() after cheerio RC6
|
||||||
.filter((idx, el) => {
|
.filter((idx, el) => isTextNode(el))
|
||||||
return isTextNode(el);
|
|
||||||
})
|
|
||||||
.text();
|
.text();
|
||||||
|
}
|
||||||
|
|
||||||
// Clean and return the text
|
// Clean and return the text
|
||||||
return text.replace(/\s\s+/g, " ").trim();
|
return text.replace(/\s\s+/g, " ").trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* Process a node and see if it contains a
|
|
||||||
* link or image. If not, it returns `null`.
|
|
||||||
*/
|
|
||||||
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null {
|
|
||||||
//@ts-ignore
|
|
||||||
const name = element[0]?.name;
|
|
||||||
const link: ILink = {
|
|
||||||
name: "",
|
|
||||||
type: "Link",
|
|
||||||
text: "",
|
|
||||||
href: "",
|
|
||||||
content: []
|
|
||||||
};
|
|
||||||
|
|
||||||
if (name === "img") {
|
|
||||||
link.type = "Image";
|
|
||||||
link.text = element.attr("alt");
|
|
||||||
link.href = element.attr("data-src");
|
|
||||||
} else if (name === "a") {
|
|
||||||
link.type = "Link";
|
|
||||||
link.text = element.text().replace(/\s\s+/g, " ").trim();
|
|
||||||
link.href = element.attr("href");
|
|
||||||
}
|
|
||||||
|
|
||||||
return link.href ? link : null;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Collapse an `IPostElement` element with a single subnode
|
* Collapse an `IPostElement` element with a single subnode
|
||||||
* in the `Content` field in case it has no information.
|
* in the `Content` field in case it has no information.
|
||||||
*/
|
*/
|
||||||
function reducePostElement(element: IPostElement): IPostElement {
|
function reducePostElement(element: IPostElement, recursive = true): IPostElement {
|
||||||
if (element.content.length === 1) {
|
// Local variables
|
||||||
const content = element.content[0] as IPostElement;
|
const shallowCopy = Object.assign({}, element);
|
||||||
const nullValues =
|
|
||||||
(!element.name || !content.name) && (!element.text || !content.text);
|
|
||||||
const sameValues = element.name === content.name || element.text === content.text;
|
|
||||||
|
|
||||||
if (nullValues || sameValues) {
|
// Find the posts without name and text
|
||||||
element.name = element.name || content.name;
|
const unknownChildrens = shallowCopy.content.filter((e) => isPostElementUnknown(e));
|
||||||
element.text = element.text || content.text;
|
if (recursive) {
|
||||||
element.content.push(...content.content);
|
const recursiveUnknownChildrens = unknownChildrens.map((e) => reducePostElement(e));
|
||||||
element.type = content.type;
|
unknownChildrens.push(...recursiveUnknownChildrens);
|
||||||
|
|
||||||
// If the content is a link, add the HREF to the element
|
|
||||||
const contentILink = content as ILink;
|
|
||||||
const elementILink = element as ILink;
|
|
||||||
if (contentILink.href) elementILink.href = contentILink.href;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return element;
|
// Eliminate non-useful child nodes
|
||||||
|
if (isPostElementUnknown(shallowCopy) && unknownChildrens.length > 0) {
|
||||||
|
// Find the valid elements to add to the node
|
||||||
|
const childContents = unknownChildrens
|
||||||
|
.filter((e) => !shallowCopy.content.includes(e))
|
||||||
|
.map((e) => (e.content.length > 0 ? e.content : e));
|
||||||
|
|
||||||
|
// Remove the empty elements
|
||||||
|
shallowCopy.content = shallowCopy.content.filter(
|
||||||
|
(e) => !unknownChildrens.includes(e)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Merge the non-empty children of this node with
|
||||||
|
// the content of the empty children of this node
|
||||||
|
const newContent = [].concat(...childContents);
|
||||||
|
shallowCopy.content.push(...newContent);
|
||||||
|
}
|
||||||
|
// If the node has only one child, return it
|
||||||
|
else if (isPostElementUnknown(shallowCopy) && shallowCopy.content.length === 1) {
|
||||||
|
return shallowCopy.content[0];
|
||||||
|
}
|
||||||
|
return shallowCopy;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes.
|
* Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes.
|
||||||
* @param reduce Compress subsequent subnodes if they contain no information. Default: `true`.
|
|
||||||
*/
|
*/
|
||||||
function parseCheerioNode(
|
function parseCheerioNode($: cheerio.Root, node: cheerio.Element): IPostElement {
|
||||||
$: cheerio.Root,
|
|
||||||
node: cheerio.Element,
|
|
||||||
reduce = true
|
|
||||||
): IPostElement {
|
|
||||||
// Local variables
|
// Local variables
|
||||||
const content: IPostElement = {
|
let post: IPostElement = createEmptyElement();
|
||||||
type: "Empty",
|
|
||||||
name: "",
|
|
||||||
text: "",
|
|
||||||
content: []
|
|
||||||
};
|
|
||||||
const cheerioNode = $(node);
|
const cheerioNode = $(node);
|
||||||
|
|
||||||
if (isTextNode(node)) {
|
// Parse the node
|
||||||
content.text = cheerioNode.text().replace(/\s\s+/g, " ").trim();
|
if (!isNoScriptNode(node)) {
|
||||||
content.type = "Text";
|
if (isTextNode(node) && !isFormattingNode(node))
|
||||||
} else {
|
post = parseCheerioTextNode(cheerioNode);
|
||||||
// Get the number of children that the element own
|
else if (isSpoilerNode(cheerioNode)) post = parseCheerioSpoilerNode($, cheerioNode);
|
||||||
const nChildren = cheerioNode.children().length;
|
else if (isLinkNode(node)) post = parseCheerioLinkNode(cheerioNode);
|
||||||
|
|
||||||
// Get the text of the element without childrens
|
// Parse the node's childrens
|
||||||
content.text = getCheerioNonChildrenText(cheerioNode);
|
const childPosts = cheerioNode
|
||||||
|
.contents() // @todo Change to children() after cheerio RC6
|
||||||
// Parse spoilers
|
.toArray()
|
||||||
if (cheerioNode.attr("class") === "bbCodeSpoiler") {
|
.filter((el) => el) // Ignore undefined elements
|
||||||
const spoiler = parseCheerioSpoilerNode($, cheerioNode);
|
.map((el) => parseCheerioNode($, el))
|
||||||
|
.filter((el) => !isPostElementEmpty(el));
|
||||||
// Add element if not null
|
post.content.push(...childPosts);
|
||||||
if (spoiler) {
|
|
||||||
content.content.push(spoiler);
|
|
||||||
content.type = "Spoiler";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// Parse links
|
|
||||||
else if (nChildren === 0 && cheerioNode.length != 0) {
|
|
||||||
const link = parseCheerioLinkNode(cheerioNode);
|
|
||||||
|
|
||||||
// Add element if not null
|
|
||||||
if (link) {
|
|
||||||
content.content.push(link);
|
|
||||||
content.type = "Link";
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
cheerioNode.children().map((idx, el) => {
|
|
||||||
// Parse the children of the element passed as parameter
|
|
||||||
const childElement = parseCheerioNode($, el);
|
|
||||||
|
|
||||||
// If the children is valid (not empty) push it
|
|
||||||
if ((childElement.text || childElement.content.length !== 0) && !isTextNode(el)) {
|
|
||||||
content.content.push(childElement);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return reduce ? reducePostElement(content) : content;
|
return post;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* It simplifies the `IPostElement` elements by associating
|
* It simplifies the `IPostElement` elements by associating
|
||||||
* the corresponding value to each characterizing element (i.e. author).
|
* the corresponding value to each characterizing element (i.e. author).
|
||||||
*/
|
*/
|
||||||
function parsePostElements(elements: IPostElement[]): IPostElement[] {
|
function associateElementsWithName(elements: IPostElement[]): IPostElement[] {
|
||||||
// Local variables
|
// Local variables
|
||||||
const pairs: IPostElement[] = [];
|
const pairs: IPostElement[] = [];
|
||||||
const specialCharsRegex = /^[-!$%^&*()_+|~=`{}[\]:";'<>?,./]/;
|
const specialCharsRegex = /^[-!$%^&*()_+|~=`{}[\]:";'<>?,./]/;
|
||||||
|
@ -275,11 +363,11 @@ function parsePostElements(elements: IPostElement[]): IPostElement[] {
|
||||||
lastPair.content.push(...elements[i].content);
|
lastPair.content.push(...elements[i].content);
|
||||||
}
|
}
|
||||||
// This is a special case
|
// This is a special case
|
||||||
else if (elements[i].text.startsWith("Overview:\n")) {
|
else if (elementIsOverview(elements[i])) {
|
||||||
// We add the overview to the pairs as a text element
|
// We add the overview to the pairs as a text element
|
||||||
elements[i].type = "Text";
|
elements[i].type = "Text";
|
||||||
elements[i].name = "Overview";
|
elements[i].name = "Overview";
|
||||||
elements[i].text = elements[i].text.replace("Overview:\n", "");
|
elements[i].text = getOverviewFromElement(elements[i]);
|
||||||
pairs.push(elements[i]);
|
pairs.push(elements[i]);
|
||||||
}
|
}
|
||||||
// We have an element referred to the previous "title"
|
// We have an element referred to the previous "title"
|
||||||
|
|
Loading…
Reference in New Issue