MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Clean element before validation to remove useless elements

pull/83/head
MillenniumEarl 2021-03-22 12:38:23 +01:00
parent 5112143a0e
commit b4b83f36e1
1 changed files with 28 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
src/scripts/scrape-data/post-parse.ts
View File

@ -293,6 +293,29 @@ function createGenericElement(): IPostElement {
}; };
} }
/**
* Clean the element `name` and `text` removing initial and final special characters.
*/
function cleanElement(element: IPostElement): IPostElement {
// Local variables
const shallow = Object.assign({}, element);
const specialCharSet = /[-!$%^&*()_+|~=`{}[\]:";'<>?,./]/;
const startsWithSpecialCharsRegex = new RegExp("^" + specialCharSet.source);
const endsWithSpecialCharsRegex = new RegExp(specialCharSet.source + "$");
shallow.name = shallow.name
.replace(startsWithSpecialCharsRegex, "")
.replace(endsWithSpecialCharsRegex, "")
.trim();
shallow.text = shallow.text
.replace(startsWithSpecialCharsRegex, "")
.replace(endsWithSpecialCharsRegex, "")
.trim();
return shallow;
}
//#endregion IPostElement utility //#endregion IPostElement utility
/** /**
@ -327,7 +350,9 @@ function removeEmptyContentFromElement(element: IPostElement, recursive = true):
: copy.content; : copy.content;
// Find the non-empty nodes // Find the non-empty nodes
const validNodes = recursiveResult.filter((e) => !isPostElementEmpty(e)); const validNodes = recursiveResult
.filter((e) => !isPostElementEmpty(e)) // Remove the empty nodes
.filter((e) => !isPostElementEmpty(cleanElement(e))); // Remove the useless nodes
// Assign the nodes // Assign the nodes
copy.content = validNodes; copy.content = validNodes;
@ -402,11 +427,11 @@ function pairUpElements(elements: IPostElement[]): IPostElement[] {
}); });
// Than we find all the IDs of the elements that are "titles". // Than we find all the IDs of the elements that are "titles".
let indexes = shallow const indexes = shallow
.filter((e, i) => isValidTitleElement(e, i, shallow)) .filter((e, i) => isValidTitleElement(e, i, shallow))
.map((e) => shallow.indexOf(e)); .map((e) => shallow.indexOf(e));
if (indexes.length === 0) indexes = shallow.map((e, i) => i); //if (indexes.length === 0) indexes = shallow.map((e, i) => i);
// Now we find all the elements between indexes and // Now we find all the elements between indexes and
// associate them with the previous "title" element // associate them with the previous "title" element