Add post-parser.ts
MillenniumEarl
parent
1cad5c277f
commit
f7bad33c1f
|
|
@ -0,0 +1,291 @@
|
||||||
|
|
||||||
|
//#region Interfaces
|
||||||
|
|
||||||
|
export interface IPostElement {
|
||||||
|
Type: "Empty" | "Text" | "Link" | "Image" | "Spoiler",
|
||||||
|
Name: string,
|
||||||
|
Text: string,
|
||||||
|
Content: IPostElement[]
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ILink extends IPostElement {
|
||||||
|
Type: "Image" | "Link",
|
||||||
|
Href: string,
|
||||||
|
}
|
||||||
|
|
||||||
|
//#endregion Interfaces
|
||||||
|
|
||||||
|
//#region Public methods
|
||||||
|
/**
|
||||||
|
* Given the main post of the page (#1) it extracts the information contained.
|
||||||
|
*/
|
||||||
|
export function parseCheerioMainPost($: cheerio.Root, post: cheerio.Cheerio): IPostElement[] {
|
||||||
|
// The data is divided between "tag" and "text" elements.
|
||||||
|
// Simple data is composed of a "tag" element followed
|
||||||
|
// by a "text" element, while more complex data (contained
|
||||||
|
// in spoilers) is composed of a "tag" element, followed
|
||||||
|
// by a text containing only ":" and then by an additional
|
||||||
|
// "tag" element having as the first term "Spoiler"
|
||||||
|
|
||||||
|
// First fetch all the elements in the post
|
||||||
|
const elements = post.contents().toArray().map(el => {
|
||||||
|
const node = parseCheerioNode($, el);
|
||||||
|
if (node.Name || node.Text || node.Content.length != 0) {
|
||||||
|
return node;
|
||||||
|
}
|
||||||
|
}).filter(el => el);
|
||||||
|
|
||||||
|
// ... then parse the elements to create the pairs of title/data
|
||||||
|
return parsePostElements(elements);
|
||||||
|
}
|
||||||
|
//#endregion Public methods
|
||||||
|
|
||||||
|
//#region Private methods
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process a spoiler element by getting its text broken
|
||||||
|
* down by any other spoiler elements present.
|
||||||
|
*/
|
||||||
|
function parseCheerioSpoilerNode($: cheerio.Root, spoiler: cheerio.Cheerio): IPostElement {
|
||||||
|
// A spoiler block is composed of a div with class "bbCodeSpoiler",
|
||||||
|
// containing a div "bbCodeSpoiler-content" containing, in cascade,
|
||||||
|
// a div with class "bbCodeBlock--spoiler" and a div with class "bbCodeBlock-content".
|
||||||
|
// This last tag contains the required data.
|
||||||
|
|
||||||
|
// Local variables
|
||||||
|
const BUTTON_CLASS = "button.bbCodeSpoiler-button";
|
||||||
|
const SPOILER_CONTENT_CLASS = "div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content";
|
||||||
|
const content: IPostElement = {
|
||||||
|
Type: "Spoiler",
|
||||||
|
Name: "",
|
||||||
|
Text: "",
|
||||||
|
Content: []
|
||||||
|
};
|
||||||
|
|
||||||
|
// Find the title of the spoiler (contained in the button)
|
||||||
|
const button = spoiler.find(BUTTON_CLASS).toArray().shift();
|
||||||
|
content.Name = $(button).text().trim();
|
||||||
|
|
||||||
|
// Parse the content of the spoiler
|
||||||
|
spoiler.find(SPOILER_CONTENT_CLASS).contents().map((idx, el) => {
|
||||||
|
// Convert the element
|
||||||
|
const element = $(el);
|
||||||
|
|
||||||
|
// Parse nested spoiler
|
||||||
|
if (element.attr("class") === "bbCodeSpoiler") {
|
||||||
|
const spoiler = parseCheerioSpoilerNode($, element);
|
||||||
|
content.Content.push(spoiler);
|
||||||
|
}
|
||||||
|
//@ts-ignore
|
||||||
|
// else if (el.name === "br") {
|
||||||
|
// // Add new line
|
||||||
|
// content.Text += "\n";
|
||||||
|
// }
|
||||||
|
else if (el.type === "text") {
|
||||||
|
// Append text
|
||||||
|
content.Text += element.text();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Clean text
|
||||||
|
content.Text = content.Text.replace(/\s\s+/g, ' ').trim();
|
||||||
|
return content;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Check if the node passed as a parameter is of text type.
|
||||||
|
* This also includes formatted nodes (i.e. `<b>`).
|
||||||
|
*/
|
||||||
|
function isTextNode(node: cheerio.Element): boolean {
|
||||||
|
const formattedTags = ["b", "i"]
|
||||||
|
const isText = node.type === "text";
|
||||||
|
const isFormatted = node.type === "tag" && formattedTags.includes(node.name);
|
||||||
|
|
||||||
|
return isText || isFormatted;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gets the text of the node only, excluding child nodes.
|
||||||
|
* Also includes formatted text elements (i.e. `<b>`).
|
||||||
|
*/
|
||||||
|
function getCheerioNonChildrenText(node: cheerio.Cheerio): string {
|
||||||
|
// Find all the text nodes in the node
|
||||||
|
const text = node.first().contents().filter((idx, el) => {
|
||||||
|
return isTextNode(el);
|
||||||
|
}).text();
|
||||||
|
|
||||||
|
// Clean and return the text
|
||||||
|
return text.replace(/\s\s+/g, ' ').trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Process a node and see if it contains a
|
||||||
|
* link or image. If not, it returns `null`.
|
||||||
|
*/
|
||||||
|
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null {
|
||||||
|
//@ts-ignore
|
||||||
|
const name = element[0]?.name;
|
||||||
|
let returnValue: ILink = null;
|
||||||
|
|
||||||
|
if (name === "img") {
|
||||||
|
returnValue = {
|
||||||
|
Name: "",
|
||||||
|
Type: "Image",
|
||||||
|
Text: element.attr("alt"),
|
||||||
|
Href: element.attr("data-src"),
|
||||||
|
Content: []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (name === "a") {
|
||||||
|
returnValue = {
|
||||||
|
Name: "",
|
||||||
|
Type: "Link",
|
||||||
|
Text: element.text().replace(/\s\s+/g, ' ').trim(),
|
||||||
|
Href: element.attr("href"),
|
||||||
|
Content: []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return returnValue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collapse an `IPostElement` element with a single subnode
|
||||||
|
* in the `Content` field in case it has no information.
|
||||||
|
*/
|
||||||
|
function reducePostElement(element: IPostElement): IPostElement {
|
||||||
|
if (element.Content.length === 1) {
|
||||||
|
const content = element.Content[0] as IPostElement;
|
||||||
|
const nullValues = (!element.Name || !content.Name) && (!element.Text || !content.Text);
|
||||||
|
const sameValues = (element.Name === content.Name) || (element.Text === content.Text)
|
||||||
|
|
||||||
|
if (nullValues || sameValues) {
|
||||||
|
element.Name = element.Name || content.Name;
|
||||||
|
element.Text = element.Text || content.Text;
|
||||||
|
element.Content = content.Content;
|
||||||
|
element.Type = content.Type;
|
||||||
|
|
||||||
|
// If the content is a link, add the HREF to the element
|
||||||
|
const contentILink = content as ILink;
|
||||||
|
const elementILink = element as ILink;
|
||||||
|
if (contentILink.Href) elementILink.Href = contentILink.Href;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return element;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Transform a `cheerio.Cheerio` node into an `IPostElement` element with its subnodes.
|
||||||
|
* @param reduce Compress subsequent subnodes if they contain no information. Default: `true`.
|
||||||
|
*/
|
||||||
|
function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true): IPostElement {
|
||||||
|
// Local variables
|
||||||
|
let content: IPostElement = {
|
||||||
|
Type: "Empty",
|
||||||
|
Name: "",
|
||||||
|
Text: "",
|
||||||
|
Content: []
|
||||||
|
};
|
||||||
|
const cheerioNode = $(node);
|
||||||
|
|
||||||
|
if (isTextNode(node)) {
|
||||||
|
content.Text = cheerioNode.text().replace(/\s\s+/g, ' ').trim();
|
||||||
|
content.Type = "Text";
|
||||||
|
} else {
|
||||||
|
// Get the number of children that the element own
|
||||||
|
const nChildren = cheerioNode.children().length;
|
||||||
|
|
||||||
|
// Get the text of the element without childrens
|
||||||
|
content.Text = getCheerioNonChildrenText(cheerioNode);
|
||||||
|
|
||||||
|
// Parse spoilers
|
||||||
|
if (cheerioNode.attr("class") === "bbCodeSpoiler") {
|
||||||
|
const spoiler = parseCheerioSpoilerNode($, cheerioNode);
|
||||||
|
|
||||||
|
// Add element if not null
|
||||||
|
if (spoiler) {
|
||||||
|
content.Content.push(spoiler);
|
||||||
|
content.Type = "Spoiler";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Parse links
|
||||||
|
else if (nChildren === 0 && cheerioNode.length != 0) {
|
||||||
|
const link = parseCheerioLinkNode(cheerioNode);
|
||||||
|
|
||||||
|
// Add element if not null
|
||||||
|
if (link) {
|
||||||
|
content.Content.push(link);
|
||||||
|
content.Type = "Link";
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
cheerioNode.children().map((idx, el) => {
|
||||||
|
// Parse the children of the element passed as parameter
|
||||||
|
const childElement = parseCheerioNode($, el);
|
||||||
|
|
||||||
|
// If the children is valid (not empty) push it
|
||||||
|
if ((childElement.Text || childElement.Content.length !== 0) && !isTextNode(el)) {
|
||||||
|
content.Content.push(childElement);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return reduce ? reducePostElement(content) : content;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* It simplifies the `IPostElement` elements by associating
|
||||||
|
* the corresponding value to each characterizing element (i.e. author).
|
||||||
|
*/
|
||||||
|
function parsePostElements(elements: IPostElement[]): IPostElement[] {
|
||||||
|
// Local variables
|
||||||
|
const pairs: IPostElement[] = [];
|
||||||
|
const specialCharsRegex = /^[-!$%^&*()_+|~=`{}\[\]:";'<>?,.\/]/;
|
||||||
|
const specialRegex = new RegExp(specialCharsRegex);
|
||||||
|
|
||||||
|
for (let i = 0; i < elements.length; i++) {
|
||||||
|
// If the text starts with a special char, clean it
|
||||||
|
const startWithSpecial = specialRegex.test(elements[i].Text);
|
||||||
|
|
||||||
|
// /^[-!$%^&*()_+|~=`{}\[\]:";'<>?,.\/]/
|
||||||
|
// Get the uppercase text
|
||||||
|
const upperText = elements[i].Text.toUpperCase();
|
||||||
|
|
||||||
|
// Get the latest IPostElement in "pairs"
|
||||||
|
const lastIndex = pairs.length - 1;
|
||||||
|
const lastPair = pairs[lastIndex];
|
||||||
|
|
||||||
|
// If this statement is valid, we have a "data"
|
||||||
|
if (elements[i].Type === "Text" && startWithSpecial && pairs.length > 0) {
|
||||||
|
// We merge this element with the last element appended to 'pairs'
|
||||||
|
const cleanText = elements[i].Text.replace(specialCharsRegex, "").trim();
|
||||||
|
lastPair.Text = lastPair.Text || cleanText;
|
||||||
|
lastPair.Content.push(...elements[i].Content);
|
||||||
|
}
|
||||||
|
// This is a special case
|
||||||
|
else if (elements[i].Text.startsWith("Overview:\n")) {
|
||||||
|
// We add the overview to the pairs as a text element
|
||||||
|
elements[i].Type = "Text";
|
||||||
|
elements[i].Name = "Overview";
|
||||||
|
elements[i].Text = elements[i].Text.replace("Overview:\n", "");
|
||||||
|
pairs.push(elements[i]);
|
||||||
|
}
|
||||||
|
// We have an element referred to the previous "title"
|
||||||
|
else if (elements[i].Type != "Text" && pairs.length > 0) {
|
||||||
|
// We append this element to the content of the last title
|
||||||
|
lastPair.Content.push(elements[i]);
|
||||||
|
}
|
||||||
|
// ... else we have a "title" (we need to swap the text to the name because it is a title)
|
||||||
|
else {
|
||||||
|
const swap: IPostElement = Object.assign({}, elements[i]);
|
||||||
|
swap.Name = elements[i].Text;
|
||||||
|
swap.Text = "";
|
||||||
|
pairs.push(swap);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return pairs;
|
||||||
|
}
|
||||||
|
|
||||||
|
//#endregion Private methods
|
||||||
Loading…
Reference in New Issue