From 2acf7699700462a958c19358b8fe11e36c0f58a2 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 22 Mar 2021 14:18:39 +0100 Subject: [PATCH 1/3] Add specific type for changelog --- src/scripts/classes/handiwork/animation.ts | 4 ++-- src/scripts/classes/handiwork/asset.ts | 4 ++-- src/scripts/classes/handiwork/comic.ts | 4 ++-- src/scripts/classes/handiwork/game.ts | 4 ++-- src/scripts/classes/handiwork/handiwork.ts | 12 ++++++++++-- src/scripts/interfaces.ts | 16 +++++++++++++++- 6 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/scripts/classes/handiwork/animation.ts b/src/scripts/classes/handiwork/animation.ts index d097cc8..65c8447 100644 --- a/src/scripts/classes/handiwork/animation.ts +++ b/src/scripts/classes/handiwork/animation.ts @@ -6,7 +6,7 @@ "use strict"; // Modules from files -import { TAuthor, IAnimation, TRating, TCategory } from "../../interfaces"; +import { TAuthor, IAnimation, TRating, TCategory, TChangelog } from "../../interfaces"; export default class Animation implements IAnimation { //#region Properties @@ -19,7 +19,7 @@ export default class Animation implements IAnimation { resolution: string[]; authors: TAuthor[]; category: TCategory; - changelog: string[]; + changelog: TChangelog[]; cover: string; id: number; lastThreadUpdate: Date; diff --git a/src/scripts/classes/handiwork/asset.ts b/src/scripts/classes/handiwork/asset.ts index b18a9e5..5a434be 100644 --- a/src/scripts/classes/handiwork/asset.ts +++ b/src/scripts/classes/handiwork/asset.ts @@ -6,7 +6,7 @@ "use strict"; // Modules from files -import { TAuthor, IAsset, TRating, TCategory } from "../../interfaces"; +import { TAuthor, IAsset, TRating, TCategory, TChangelog } from "../../interfaces"; export default class Asset implements IAsset { //#region Properties @@ -18,7 +18,7 @@ export default class Asset implements IAsset { sku: string; authors: TAuthor[]; category: TCategory; - changelog: string[]; + changelog: TChangelog[]; cover: string; id: number; lastThreadUpdate: Date; diff --git a/src/scripts/classes/handiwork/comic.ts b/src/scripts/classes/handiwork/comic.ts index 6bf713a..ff5b795 100644 --- a/src/scripts/classes/handiwork/comic.ts +++ b/src/scripts/classes/handiwork/comic.ts @@ -6,7 +6,7 @@ "use strict"; // Modules from files -import { TAuthor, IComic, TRating, TCategory } from "../../interfaces"; +import { TAuthor, IComic, TRating, TCategory, TChangelog } from "../../interfaces"; export default class Comic implements IComic { //#region Properties @@ -15,7 +15,7 @@ export default class Comic implements IComic { resolution: string[]; authors: TAuthor[]; category: TCategory; - changelog: string[]; + changelog: TChangelog[]; cover: string; id: number; lastThreadUpdate: Date; diff --git a/src/scripts/classes/handiwork/game.ts b/src/scripts/classes/handiwork/game.ts index d422668..80a71d5 100644 --- a/src/scripts/classes/handiwork/game.ts +++ b/src/scripts/classes/handiwork/game.ts @@ -6,7 +6,7 @@ "use strict"; // Modules from files -import { TAuthor, TEngine, IGame, TRating, TStatus, TCategory } from "../../interfaces"; +import { TAuthor, TEngine, IGame, TRating, TStatus, TCategory, TChangelog } from "../../interfaces"; export default class Game implements IGame { //#region Properties @@ -22,7 +22,7 @@ export default class Game implements IGame { version: string; authors: TAuthor[]; category: TCategory; - changelog: string[]; + changelog: TChangelog[]; cover: string; id: number; lastThreadUpdate: Date; diff --git a/src/scripts/classes/handiwork/handiwork.ts b/src/scripts/classes/handiwork/handiwork.ts index cb68d27..c558c1a 100644 --- a/src/scripts/classes/handiwork/handiwork.ts +++ b/src/scripts/classes/handiwork/handiwork.ts @@ -6,7 +6,15 @@ "use strict"; // Modules from files -import { TAuthor, TRating, IHandiwork, TEngine, TCategory, TStatus } from "../../interfaces"; +import { + TAuthor, + TRating, + IHandiwork, + TEngine, + TCategory, + TStatus, + TChangelog +} from "../../interfaces"; /** * It represents a generic work, be it a game, a comic, an animation or an asset. @@ -25,7 +33,7 @@ export default class HandiWork implements IHandiwork { version: string; authors: TAuthor[]; category: TCategory; - changelog: string[]; + changelog: TChangelog[]; cover: string; id: number; lastThreadUpdate: Date; diff --git a/src/scripts/interfaces.ts b/src/scripts/interfaces.ts index 33916e6..64b1bae 100644 --- a/src/scripts/interfaces.ts +++ b/src/scripts/interfaces.ts @@ -51,6 +51,20 @@ export type TRating = { count: number; }; +/** + * Information about a single version of the product. + */ +export type TChangelog = { + /** + * Product version. + */ + version: string; + /** + * Version information. + */ + information: string[]; +}; + /** * List of possible graphics engines used for game development. */ @@ -101,7 +115,7 @@ export interface IBasic { /** * List of changes of the work for each version. */ - changelog: string[]; + changelog: TChangelog[]; /** * link to the cover image of the work. */ From bc683b2387d437783e7e3d9bbb247588a970a2a9 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 22 Mar 2021 14:18:57 +0100 Subject: [PATCH 2/3] Add special values in isValidTitleElement --- src/scripts/scrape-data/post-parse.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/scripts/scrape-data/post-parse.ts b/src/scripts/scrape-data/post-parse.ts index 798f0f7..3837005 100644 --- a/src/scripts/scrape-data/post-parse.ts +++ b/src/scripts/scrape-data/post-parse.ts @@ -431,8 +431,6 @@ function pairUpElements(elements: IPostElement[]): IPostElement[] { .filter((e, i) => isValidTitleElement(e, i, shallow)) .map((e) => shallow.indexOf(e)); - //if (indexes.length === 0) indexes = shallow.map((e, i) => i); - // Now we find all the elements between indexes and // associate them with the previous "title" element return indexes.map((i, j) => parseGroupData(i, j, indexes, shallow)); @@ -452,7 +450,7 @@ function isValidTitleElement(element: IPostElement, index: number, array: IPostE element.type === "Text" && (isPostfixDoublePoints || nextElementIsValue); // Special values tha must be set has "title" - const specialValues = ["DOWNLOAD"]; + const specialValues = ["DOWNLOAD", "CHANGELOG", "CHANGE-LOG", "GENRE"]; const specialTypes = ["Image"]; // Used to ignore already merged elements with name (ignore spoilers) From f4e9575930c9e4f8b47c8c45a819f28ce2f933dd Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 22 Mar 2021 14:50:12 +0100 Subject: [PATCH 3/3] Add method for changelog parse --- src/scripts/scrape-data/handiwork-parse.ts | 73 +++++++++++++++------- 1 file changed, 52 insertions(+), 21 deletions(-) diff --git a/src/scripts/scrape-data/handiwork-parse.ts b/src/scripts/scrape-data/handiwork-parse.ts index c99c55d..1699a19 100644 --- a/src/scripts/scrape-data/handiwork-parse.ts +++ b/src/scripts/scrape-data/handiwork-parse.ts @@ -11,7 +11,7 @@ import { DateTime } from "luxon"; // Modules from files import HandiWork from "../classes/handiwork/handiwork"; import Thread from "../classes/mapping/thread"; -import { IBasic, TAuthor, TEngine, TExternalPlatform, TStatus } from "../interfaces"; +import { IBasic, TAuthor, TChangelog, TEngine, TExternalPlatform, TStatus } from "../interfaces"; import shared, { TPrefixDict } from "../shared"; import { ILink, IPostElement } from "./post-parse"; @@ -223,26 +223,8 @@ function fillWithPostData(hw: HandiWork, elements: IPostElement[]) { // Get the author hw.authors = parseAuthor(elements); - //#region Get the changelog - hw.changelog = []; - const changelogElement = - getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log"); - - if (false && changelogElement?.content) { - const changelogSpoiler = changelogElement.content.find( - (el) => el.type === "Spoiler" && el.content.length > 0 - ); - - // Add to the changelog the single spoilers - const spoilers = changelogSpoiler.content - .filter((e) => e.text.trim() !== "") - .map((e) => e.text); - hw.changelog.push(...spoilers); - - // Add at the end also the text of the "changelog" element - hw.changelog.push(changelogSpoiler.text); - } - //#endregion Get the changelog + // Get the changelog + hw.changelog = parseChangelog(elements); } /** @@ -283,4 +265,53 @@ function parseAuthor(elements: IPostElement[]): TAuthor[] { return [author]; } +/** + * Parse the changelog from the post's data. + */ +function parseChangelog(elements: IPostElement[]): TChangelog[] { + // Local variables + const changelog = []; + const changelogElement = + getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log"); + + if (changelogElement) { + // regex used to match version tags + const versionRegex = /^v[0-9]+\.[0-9]+.*/; + + // Get the indexes of the version tags + const indexesVersion = changelogElement.content + .filter((e) => e.type === "Text" && versionRegex.test(e.text)) + .map((e) => changelogElement.content.indexOf(e)); + + const results = indexesVersion.map((i, j) => { + // In-loop variable + const versionChangelog: TChangelog = { + version: "", + information: [] + }; + + // Get the difference in indexes between this and the next version tag + const diff = indexesVersion[j + 1] ?? changelogElement.content.length; + + // fetch the group of data of this version tag + const group = changelogElement.content.slice(i, diff); + versionChangelog.version = group.shift().text.replace("v", "").trim(); + + // parse the data + group.forEach((e) => { + if (e.type === "Generic" || e.type === "Spoiler") { + const textes = e.content.map((c) => c.text); + versionChangelog.information.push(...textes); + } else versionChangelog.information.push(e.text); + }); + + return versionChangelog; + }); + + changelog.push(...results); + } + + return changelog; +} + //#endregion Private methods