MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Renamed types and fields

pull/73/head
MillenniumEarl 2021-02-21 14:40:05 +01:00
parent dac5e9f16b
commit 91f809f249
9 changed files with 330 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/scripts/classes/handiwork/animation.ts
View File

@ -1,31 +1,31 @@
"use strict"; "use strict";
// Modules from files // Modules from files
import { AuthorType, IAnimation, RatingType, CategoryType } from "../../interfaces"; import { TAuthor, IAnimation, TRating, TCategory } from "../../interfaces";
export default class Animation implements IAnimation { export default class Animation implements IAnimation {
//#region Properties //#region Properties
Censored: boolean; censored: boolean;
Genre: string[]; genre: string[];
Installation: string; installation: string;
Language: string[]; language: string[];
Lenght: string; lenght: string;
Pages: string; pages: string;
Resolution: string[]; resolution: string[];
Authors: AuthorType[]; authors: TAuthor[];
Category: CategoryType; category: TCategory;
Changelog: string[]; changelog: string[];
Cover: string; cover: string;
ID: number; id: number;
LastThreadUpdate: Date; lastThreadUpdate: Date;
Name: string; name: string;
Overview: string; overview: string;
Prefixes: string[]; prefixes: string[];
Rating: RatingType; rating: TRating;
Tags: string[]; tags: string[];
ThreadPublishingDate: Date; threadPublishingDate: Date;
Url: string; url: string;
//#endregion Properties //#endregion Properties
} }

40
src/scripts/classes/handiwork/asset.ts
View File

@ -1,30 +1,30 @@
"use strict"; "use strict";
// Modules from files // Modules from files
import { AuthorType, IAsset, RatingType, CategoryType } from "../../interfaces"; import { TAuthor, IAsset, TRating, TCategory } from "../../interfaces";
export default class Asset implements IAsset { export default class Asset implements IAsset {
//#region Properties //#region Properties
AssetLink: string; assetLink: string;
AssociatedAssets: string[]; associatedAssets: string[];
CompatibleSoftware: string; compatibleSoftware: string;
IncludedAssets: string[]; includedAssets: string[];
OfficialLinks: string[]; officialLinks: string[];
SKU: string; sku: string;
Authors: AuthorType[]; authors: TAuthor[];
Category: CategoryType; category: TCategory;
Changelog: string[]; changelog: string[];
Cover: string; cover: string;
ID: number; id: number;
LastThreadUpdate: Date; lastThreadUpdate: Date;
Name: string; name: string;
Overview: string; overview: string;
Prefixes: string[]; prefixes: string[];
Rating: RatingType; rating: TRating;
Tags: string[]; tags: string[];
ThreadPublishingDate: Date; threadPublishingDate: Date;
Url: string; url: string;
//#endregion Properties //#endregion Properties
} }

34
src/scripts/classes/handiwork/comic.ts
View File

@ -1,26 +1,26 @@
"use strict"; "use strict";
// Modules from files // Modules from files
import { AuthorType, IComic, RatingType, CategoryType } from "../../interfaces"; import { TAuthor, IComic, TRating, TCategory } from "../../interfaces";
export default class Comic implements IComic { export default class Comic implements IComic {
//#region Properties //#region Properties
Genre: string[]; genre: string[];
Pages: string; pages: string;
Resolution: string[]; resolution: string[];
Authors: AuthorType[]; authors: TAuthor[];
Category: CategoryType; category: TCategory;
Changelog: string[]; changelog: string[];
Cover: string; cover: string;
ID: number; id: number;
LastThreadUpdate: Date; lastThreadUpdate: Date;
Name: string; name: string;
Overview: string; overview: string;
Prefixes: string[]; prefixes: string[];
Rating: RatingType; rating: TRating;
Tags: string[]; tags: string[];
ThreadPublishingDate: Date; threadPublishingDate: Date;
Url: string; url: string;
//#endregion Properties //#endregion Properties
} }

48
src/scripts/classes/handiwork/game.ts
View File

@ -1,34 +1,34 @@
"use strict"; "use strict";
// Modules from files // Modules from files
import { AuthorType, EngineType, IGame, RatingType, StatusType, CategoryType } from "../../interfaces"; import { TAuthor, TEngine, IGame, TRating, TStatus, TCategory } from "../../interfaces";
export default class Game implements IGame { export default class Game implements IGame {
//#region Properties //#region Properties
Censored: boolean; censored: boolean;
Genre: string[]; engine: TEngine;
Installation: string; genre: string[];
Language: string[]; installation: string;
LastRelease: Date; language: string[];
OS: string[]; lastRelease: Date;
Version: string; mod: boolean;
Authors: AuthorType[]; os: string[];
Category: CategoryType; status: TStatus;
Changelog: string[]; version: string;
Cover: string; authors: TAuthor[];
ID: number; category: TCategory;
LastThreadUpdate: Date; changelog: string[];
Name: string; cover: string;
Overview: string; id: number;
Prefixes: string[]; lastThreadUpdate: Date;
Rating: RatingType; name: string;
Tags: string[]; overview: string;
ThreadPublishingDate: Date; prefixes: string[];
Url: string; rating: TRating;
Engine: EngineType; tags: string[];
Mod: boolean; threadPublishingDate: Date;
Status: StatusType; url: string;
//#endregion Properties //#endregion Properties
} }

67
src/scripts/classes/handiwork/handiwork.ts
View File

@ -1,7 +1,7 @@
"use strict"; "use strict";
// Modules from files // Modules from files
import { AuthorType, RatingType, IHandiwork , EngineType, CategoryType, StatusType} from "../../interfaces"; import { TAuthor, TRating, IHandiwork, TEngine, TCategory, TStatus } from "../../interfaces";
/** /**
* It represents a generic work, be it a game, a comic, an animation or an asset. * It represents a generic work, be it a game, a comic, an animation or an asset.
@ -9,39 +9,38 @@ import { AuthorType, RatingType, IHandiwork , EngineType, CategoryType, StatusTy
export default class HandiWork implements IHandiwork { export default class HandiWork implements IHandiwork {
//#region Properties //#region Properties
AssetLink: string; censored: boolean;
AssociatedAssets: string[]; engine: TEngine;
Censored: boolean; genre: string[];
Changelog: string[]; installation: string;
CompatibleSoftware: string; language: string[];
Genre: string[]; lastRelease: Date;
IncludedAssets: string[]; mod: boolean;
Installation: string; os: string[];
Language: string[]; status: TStatus;
LastRelease: Date; version: string;
Lenght: string; authors: TAuthor[];
OfficialLinks: string[]; category: TCategory;
OS: string[]; changelog: string[];
Pages: string; cover: string;
Password: string; id: number;
Resolution: string[]; lastThreadUpdate: Date;
SKU: string; name: string;
Version: string; overview: string;
Authors: AuthorType[]; prefixes: string[];
Category: CategoryType; rating: TRating;
Cover: string; tags: string[];
ID: number; threadPublishingDate: Date;
LastThreadUpdate: Date; url: string;
Name: string; pages: string;
Overview: string; resolution: string[];
Prefixes: string[]; lenght: string;
Rating: RatingType; assetLink: string;
Tags: string[]; associatedAssets: string[];
ThreadPublishingDate: Date; compatibleSoftware: string;
Url: string; includedAssets: string[];
Engine: EngineType; officialLinks: string[];
Mod: boolean; sku: string;
Status: StatusType;
//#endregion Properties //#endregion Properties
} }

112
src/scripts/interfaces.d.ts vendored
View File

@ -1,63 +1,63 @@
/** /**
* Data relating to an external platform (i.e. Patreon). * Data relating to an external platform (i.e. Patreon).
*/ */
export type ExternalPlatformType = { export type TExternalPlatform = {
/** /**
* Name of the platform. * name of the platform.
*/ */
Name: string, name: string,
/** /**
* Link to the platform. * link to the platform.
*/ */
Link: string link: string
} }
/** /**
* Information about the author of a work. * Information about the author of a work.
*/ */
export type AuthorType = { export type TAuthor = {
/** /**
* Plain name or username of the author. * Plain name or username of the author.
*/ */
Name: string, name: string,
/** /**
* *
*/ */
Platforms: ExternalPlatformType[], platforms: TExternalPlatform[],
} }
/** /**
* Information on the evaluation of a work. * Information on the evaluation of a work.
*/ */
export type RatingType = { export type TRating = {
/** /**
* Average value of evaluations. * average value of evaluations.
*/ */
Average: number, average: number,
/** /**
* Best rating received. * Best rating received.
*/ */
Best: number, best: number,
/** /**
* Number of ratings made by users. * Number of ratings made by users.
*/ */
Count: number, count: number,
} }
/** /**
* List of possible graphics engines used for game development. * List of possible graphics engines used for game development.
*/ */
export type EngineType = "QSP" | "RPGM" | "Unity" | "HTML" | "RAGS" | "Java" | "Ren'Py" | "Flash" | "ADRIFT" | "Others" | "Tads" | "Wolf RPG" | "Unreal Engine" | "WebGL"; export type TEngine = "QSP" | "RPGM" | "Unity" | "HTML" | "RAGS" | "Java" | "Ren'Py" | "Flash" | "ADRIFT" | "Others" | "Tads" | "Wolf RPG" | "Unreal Engine" | "WebGL";
/** /**
* List of possible progress states associated with a game. * List of possible progress states associated with a game.
*/ */
export type StatusType = "Completed" | "Ongoing" | "Abandoned" | "Onhold"; export type TStatus = "Completed" | "Ongoing" | "Abandoned" | "Onhold";
/** /**
* List of possible categories of a particular work. * List of possible categories of a particular work.
*/ */
export type CategoryType = "games" | "comics" | "animations" | "assets"; export type TCategory = "games" | "comics" | "animations" | "assets";
/** /**
* Collection of values defined for each * Collection of values defined for each
@ -67,55 +67,55 @@ export interface IBasic {
/** /**
* Authors of the work. * Authors of the work.
*/ */
Authors: AuthorType[], authors: TAuthor[],
/** /**
* Category of the work.. * Category of the work..
*/ */
Category: CategoryType, category: TCategory,
/** /**
* List of changes of the work for each version. * List of changes of the work for each version.
*/ */
Changelog: string[], changelog: string[],
/** /**
* Link to the cover image of the work. * link to the cover image of the work.
*/ */
Cover: string, cover: string,
/** /**
* Unique ID of the work on the platform. * Unique ID of the work on the platform.
*/ */
ID: number, id: number,
/** /**
* Last update of the opera thread. * Last update of the opera thread.
*/ */
LastThreadUpdate: Date, lastThreadUpdate: Date,
/** /**
* Plain name of the work (without tags and/or prefixes) * Plain name of the work (without tags and/or prefixes)
*/ */
Name: string, name: string,
/** /**
* Work description * Work description
*/ */
Overview: string, overview: string,
/** /**
* List of prefixes associated with the work. * List of prefixes associated with the work.
*/ */
Prefixes: string[], prefixes: string[],
/** /**
* Evaluation of the work by the users of the platform. * Evaluation of the work by the users of the platform.
*/ */
Rating: RatingType, rating: TRating,
/** /**
* List of tags associated with the work. * List of tags associated with the work.
*/ */
Tags: string[], tags: string[],
/** /**
* Date of publication of the thread associated with the work. * Date of publication of the thread associated with the work.
*/ */
ThreadPublishingDate: Date, threadPublishingDate: Date,
/** /**
* URL to the work's official conversation on the F95Zone portal. * URL to the work's official conversation on the F95Zone portal.
*/ */
Url: string, url: string,
} }
/** /**
@ -126,43 +126,43 @@ export interface IGame extends IBasic {
* Specify whether the work has censorship * Specify whether the work has censorship
* measures regarding NSFW scenes * measures regarding NSFW scenes
*/ */
Censored: boolean, censored: boolean,
/** /**
* Graphics engine used for game development. * Graphics engine used for game development.
*/ */
Engine: EngineType, engine: TEngine,
/** /**
* List of genres associated with the work. * List of genres associated with the work.
*/ */
Genre: string[], genre: string[],
/** /**
* Author's Guide to Installation. * Author's Guide to Installation.
*/ */
Installation: string, installation: string,
/** /**
* List of available languages. * List of available languages.
*/ */
Language: string[], language: string[],
/** /**
* Last time the work underwent updates. * Last time the work underwent updates.
*/ */
LastRelease: Date, lastRelease: Date,
/** /**
* Indicates that this item represents a mod. * Indicates that this item represents a mod.
*/ */
Mod: boolean, mod: boolean,
/** /**
* List of OS for which the work is compatible. * List of OS for which the work is compatible.
*/ */
OS: string[], os: string[],
/** /**
* Indicates the progress of a game. * Indicates the progress of a game.
*/ */
Status: StatusType, status: TStatus,
/** /**
* Version of the work. * Version of the work.
*/ */
Version: string, version: string,
} }
/** /**
@ -172,15 +172,15 @@ export interface IComic extends IBasic {
/** /**
* List of genres associated with the work. * List of genres associated with the work.
*/ */
Genre: string[], genre: string[],
/** /**
* Number of pages or elements that make up the work. * Number of pages or elements that make up the work.
*/ */
Pages: string, pages: string,
/** /**
* List of resolutions available for the work. * List of resolutions available for the work.
*/ */
Resolution: string[], resolution: string[],
} }
/** /**
@ -191,31 +191,31 @@ export interface IAnimation extends IBasic {
* Specify whether the work has censorship * Specify whether the work has censorship
* measures regarding NSFW scenes * measures regarding NSFW scenes
*/ */
Censored: boolean, censored: boolean,
/** /**
* List of genres associated with the work. * List of genres associated with the work.
*/ */
Genre: string[], genre: string[],
/** /**
* Author's Guide to Installation. * Author's Guide to Installation.
*/ */
Installation: string, installation: string,
/** /**
* List of available languages. * List of available languages.
*/ */
Language: string[], language: string[],
/** /**
* Length of the animation. * Length of the animation.
*/ */
Lenght: string, lenght: string,
/** /**
* Number of pages or elements that make up the work. * Number of pages or elements that make up the work.
*/ */
Pages: string, pages: string,
/** /**
* List of resolutions available for the work. * List of resolutions available for the work.
*/ */
Resolution: string[], resolution: string[],
} }
/** /**
@ -225,28 +225,28 @@ export interface IAsset extends IBasic {
/** /**
* External URL of the asset. * External URL of the asset.
*/ */
AssetLink: string, assetLink: string,
/** /**
* List of URLs of assets associated with the work * List of URLs of assets associated with the work
* (for example same collection). * (for example same collection).
*/ */
AssociatedAssets: string[], associatedAssets: string[],
/** /**
* Software compatible with the work. * Software compatible with the work.
*/ */
CompatibleSoftware: string, compatibleSoftware: string,
/** /**
* List of assets url included in the work or used to develop it. * List of assets url included in the work or used to develop it.
*/ */
IncludedAssets: string[], includedAssets: string[],
/** /**
* List of official links of the work, external to the platform. * List of official links of the work, external to the platform.
*/ */
OfficialLinks: string[], officialLinks: string[],
/** /**
* Unique SKU value of the work. * Unique SKU value of the work.
*/ */
SKU: string, sku: string,
} }
/** /**

36
src/scripts/platform-data.ts
View File

@ -16,28 +16,28 @@ import { fetchHTML } from "./network-helper.js";
/** /**
* Represents the single element contained in the data categories. * Represents the single element contained in the data categories.
*/ */
interface SingleOptionObj { interface ISingleOption {
ID: number, id: number,
Name: string, name: string,
Class: string class: string
} }
/** /**
* Represents the set of values associated with a specific category of data. * Represents the set of values associated with a specific category of data.
*/ */
interface CategoryResObj { interface ICategoryResource {
ID: number, id: number,
Name: string, name: string,
Prefixes: SingleOptionObj[] prefixes: ISingleOption[]
} }
/** /**
* Represents the set of tags present on the platform- * Represents the set of tags present on the platform-
*/ */
interface LatestResObj { interface ILatestResource {
Prefixes: CategoryResObj[], prefixes: ICategoryResource[],
Tags: DictType, tags: DictType,
Options: string options: string
} }
//#endregion Interface definitions //#endregion Interface definitions
@ -107,7 +107,7 @@ function saveCache(path: string): void {
* Given the HTML code of the response from the F95Zone, * Given the HTML code of the response from the F95Zone,
* parse it and return the result. * parse it and return the result.
*/ */
function parseLatestPlatformHTML(html: string): LatestResObj{ function parseLatestPlatformHTML(html: string): ILatestResource{
const $ = cheerio.load(html); const $ = cheerio.load(html);
// Clean the JSON string // Clean the JSON string
@ -122,24 +122,24 @@ function parseLatestPlatformHTML(html: string): LatestResObj{
* @private * @private
* Assign to the local variables the values from the F95Zone. * Assign to the local variables the values from the F95Zone.
*/ */
function assignLatestPlatformData(data: LatestResObj): void { function assignLatestPlatformData(data: ILatestResource): void {
// Local variables // Local variables
const scrapedData = {}; const scrapedData = {};
// Parse and assign the values that are NOT tags // Parse and assign the values that are NOT tags
for (const p of data.Prefixes) { for (const p of data.prefixes) {
// Prepare the dict // Prepare the dict
const dict: DictType = {}; const dict: DictType = {};
for (const e of p.Prefixes) dict[e.ID] = e.Name.replace("'", "'"); for (const e of p.prefixes) dict[e.id] = e.name.replace("'", "'");
// Save the property // Save the property
scrapedData[p.Name] = dict; scrapedData[p.name] = dict;
} }
// Save the values // Save the values
shared.setPrefixPair("engines", Object.assign({}, scrapedData["Engine"])); shared.setPrefixPair("engines", Object.assign({}, scrapedData["Engine"]));
shared.setPrefixPair("statuses", Object.assign({}, scrapedData["Status"])); shared.setPrefixPair("statuses", Object.assign({}, scrapedData["Status"]));
shared.setPrefixPair("others", Object.assign({}, scrapedData["Other"])); shared.setPrefixPair("others", Object.assign({}, scrapedData["Other"]));
shared.setPrefixPair("tags", data.Tags); shared.setPrefixPair("tags", data.tags);
} }
//#endregion //#endregion

134
src/scripts/post-parser.ts
View File

@ -2,15 +2,15 @@
//#region Interfaces //#region Interfaces
export interface IPostElement { export interface IPostElement {
Type: "Empty" | "Text" | "Link" | "Image" | "Spoiler", type: "Empty" | "Text" | "Link" | "Image" | "Spoiler",
Name: string, name: string,
Text: string, text: string,
Content: IPostElement[] content: IPostElement[]
} }
export interface ILink extends IPostElement { export interface ILink extends IPostElement {
Type: "Image" | "Link", type: "Image" | "Link",
Href: string, href: string,
} }
//#endregion Interfaces //#endregion Interfaces
@ -30,7 +30,7 @@ export function parseCheerioMainPost($: cheerio.Root, post: cheerio.Cheerio): IP
// First fetch all the elements in the post // First fetch all the elements in the post
const elements = post.contents().toArray().map(el => { const elements = post.contents().toArray().map(el => {
const node = parseCheerioNode($, el); const node = parseCheerioNode($, el);
if (node.Name || node.Text || node.Content.length != 0) { if (node.name || node.text || node.content.length != 0) {
return node; return node;
} }
}).filter(el => el); }).filter(el => el);
@ -56,15 +56,15 @@ function parseCheerioSpoilerNode($: cheerio.Root, spoiler: cheerio.Cheerio): IPo
const BUTTON_CLASS = "button.bbCodeSpoiler-button"; const BUTTON_CLASS = "button.bbCodeSpoiler-button";
const SPOILER_CONTENT_CLASS = "div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content"; const SPOILER_CONTENT_CLASS = "div.bbCodeSpoiler-content > div.bbCodeBlock--spoiler > div.bbCodeBlock-content";
const content: IPostElement = { const content: IPostElement = {
Type: "Spoiler", type: "Spoiler",
Name: "", name: "",
Text: "", text: "",
Content: [] content: []
}; };
// Find the title of the spoiler (contained in the button) // Find the title of the spoiler (contained in the button)
const button = spoiler.find(BUTTON_CLASS).toArray().shift(); const button = spoiler.find(BUTTON_CLASS).toArray().shift();
content.Name = $(button).text().trim(); content.name = $(button).text().trim();
// Parse the content of the spoiler // Parse the content of the spoiler
spoiler.find(SPOILER_CONTENT_CLASS).contents().map((idx, el) => { spoiler.find(SPOILER_CONTENT_CLASS).contents().map((idx, el) => {
@ -74,21 +74,21 @@ function parseCheerioSpoilerNode($: cheerio.Root, spoiler: cheerio.Cheerio): IPo
// Parse nested spoiler // Parse nested spoiler
if (element.attr("class") === "bbCodeSpoiler") { if (element.attr("class") === "bbCodeSpoiler") {
const spoiler = parseCheerioSpoilerNode($, element); const spoiler = parseCheerioSpoilerNode($, element);
content.Content.push(spoiler); content.content.push(spoiler);
} }
//@ts-ignore //@ts-ignore
// else if (el.name === "br") { // else if (el.name === "br") {
// // Add new line // // Add new line
// content.Text += "\n"; // content.text += "\n";
// } // }
else if (el.type === "text") { else if (el.type === "text") {
// Append text // Append text
content.Text += element.text(); content.text += element.text();
} }
}); });
// Clean text // Clean text
content.Text = content.Text.replace(/\s\s+/g, ' ').trim(); content.text = content.text.replace(/\s\s+/g, ' ').trim();
return content; return content;
} }
@ -125,28 +125,26 @@ function getCheerioNonChildrenText(node: cheerio.Cheerio): string {
function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null { function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null {
//@ts-ignore //@ts-ignore
const name = element[0]?.name; const name = element[0]?.name;
let returnValue: ILink = null; const link: ILink = {
name: "",
type: "Link",
text: "",
href: "",
content: []
};
if (name === "img") { if (name === "img") {
returnValue = { link.type = "Image";
Name: "", link.text = element.attr("alt");
Type: "Image", link.href = element.attr("data-src");
Text: element.attr("alt"),
Href: element.attr("data-src"),
Content: []
}
} }
else if (name === "a") { else if (name === "a") {
returnValue = { link.type = "Link";
Name: "", link.text = element.text().replace(/\s\s+/g, ' ').trim();
Type: "Link", link.href = element.attr("href");
Text: element.text().replace(/\s\s+/g, ' ').trim(),
Href: element.attr("href"),
Content: []
}
} }
return returnValue; return link.href ? link : null;
} }
/** /**
@ -154,21 +152,21 @@ function parseCheerioLinkNode(element: cheerio.Cheerio): ILink | null {
* in the `Content` field in case it has no information. * in the `Content` field in case it has no information.
*/ */
function reducePostElement(element: IPostElement): IPostElement { function reducePostElement(element: IPostElement): IPostElement {
if (element.Content.length === 1) { if (element.content.length === 1) {
const content = element.Content[0] as IPostElement; const content = element.content[0] as IPostElement;
const nullValues = (!element.Name || !content.Name) && (!element.Text || !content.Text); const nullValues = (!element.name || !content.name) && (!element.text || !content.text);
const sameValues = (element.Name === content.Name) || (element.Text === content.Text) const sameValues = (element.name === content.name) || (element.text === content.text)
if (nullValues || sameValues) { if (nullValues || sameValues) {
element.Name = element.Name || content.Name; element.name = element.name || content.name;
element.Text = element.Text || content.Text; element.text = element.text || content.text;
element.Content = content.Content; element.content = content.content;
element.Type = content.Type; element.type = content.type;
// If the content is a link, add the HREF to the element // If the content is a link, add the HREF to the element
const contentILink = content as ILink; const contentILink = content as ILink;
const elementILink = element as ILink; const elementILink = element as ILink;
if (contentILink.Href) elementILink.Href = contentILink.Href; if (contentILink.href) elementILink.href = contentILink.href;
} }
} }
@ -182,22 +180,22 @@ function reducePostElement(element: IPostElement): IPostElement {
function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true): IPostElement { function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true): IPostElement {
// Local variables // Local variables
let content: IPostElement = { let content: IPostElement = {
Type: "Empty", type: "Empty",
Name: "", name: "",
Text: "", text: "",
Content: [] content: []
}; };
const cheerioNode = $(node); const cheerioNode = $(node);
if (isTextNode(node)) { if (isTextNode(node)) {
content.Text = cheerioNode.text().replace(/\s\s+/g, ' ').trim(); content.text = cheerioNode.text().replace(/\s\s+/g, ' ').trim();
content.Type = "Text"; content.type = "Text";
} else { } else {
// Get the number of children that the element own // Get the number of children that the element own
const nChildren = cheerioNode.children().length; const nChildren = cheerioNode.children().length;
// Get the text of the element without childrens // Get the text of the element without childrens
content.Text = getCheerioNonChildrenText(cheerioNode); content.text = getCheerioNonChildrenText(cheerioNode);
// Parse spoilers // Parse spoilers
if (cheerioNode.attr("class") === "bbCodeSpoiler") { if (cheerioNode.attr("class") === "bbCodeSpoiler") {
@ -205,8 +203,8 @@ function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true)
// Add element if not null // Add element if not null
if (spoiler) { if (spoiler) {
content.Content.push(spoiler); content.content.push(spoiler);
content.Type = "Spoiler"; content.type = "Spoiler";
} }
} }
// Parse links // Parse links
@ -215,8 +213,8 @@ function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true)
// Add element if not null // Add element if not null
if (link) { if (link) {
content.Content.push(link); content.content.push(link);
content.Type = "Link"; content.type = "Link";
} }
} else { } else {
cheerioNode.children().map((idx, el) => { cheerioNode.children().map((idx, el) => {
@ -224,8 +222,8 @@ function parseCheerioNode($: cheerio.Root, node: cheerio.Element, reduce = true)
const childElement = parseCheerioNode($, el); const childElement = parseCheerioNode($, el);
// If the children is valid (not empty) push it // If the children is valid (not empty) push it
if ((childElement.Text || childElement.Content.length !== 0) && !isTextNode(el)) { if ((childElement.text || childElement.content.length !== 0) && !isTextNode(el)) {
content.Content.push(childElement); content.content.push(childElement);
} }
}); });
} }
@ -246,41 +244,41 @@ function parsePostElements(elements: IPostElement[]): IPostElement[] {
for (let i = 0; i < elements.length; i++) { for (let i = 0; i < elements.length; i++) {
// If the text starts with a special char, clean it // If the text starts with a special char, clean it
const startWithSpecial = specialRegex.test(elements[i].Text); const startWithSpecial = specialRegex.test(elements[i].text);
// /^[-!$%^&*()_+|~=`{}\[\]:";'<>?,.\/]/ // /^[-!$%^&*()_+|~=`{}\[\]:";'<>?,.\/]/
// Get the uppercase text // Get the uppercase text
const upperText = elements[i].Text.toUpperCase(); const upperText = elements[i].text.toUpperCase();
// Get the latest IPostElement in "pairs" // Get the latest IPostElement in "pairs"
const lastIndex = pairs.length - 1; const lastIndex = pairs.length - 1;
const lastPair = pairs[lastIndex]; const lastPair = pairs[lastIndex];
// If this statement is valid, we have a "data" // If this statement is valid, we have a "data"
if (elements[i].Type === "Text" && startWithSpecial && pairs.length > 0) { if (elements[i].type === "Text" && startWithSpecial && pairs.length > 0) {
// We merge this element with the last element appended to 'pairs' // We merge this element with the last element appended to 'pairs'
const cleanText = elements[i].Text.replace(specialCharsRegex, "").trim(); const cleanText = elements[i].text.replace(specialCharsRegex, "").trim();
lastPair.Text = lastPair.Text || cleanText; lastPair.text = lastPair.text || cleanText;
lastPair.Content.push(...elements[i].Content); lastPair.content.push(...elements[i].content);
} }
// This is a special case // This is a special case
else if (elements[i].Text.startsWith("Overview:\n")) { else if (elements[i].text.startsWith("Overview:\n")) {
// We add the overview to the pairs as a text element // We add the overview to the pairs as a text element
elements[i].Type = "Text"; elements[i].type = "Text";
elements[i].Name = "Overview"; elements[i].name = "Overview";
elements[i].Text = elements[i].Text.replace("Overview:\n", ""); elements[i].text = elements[i].text.replace("Overview:\n", "");
pairs.push(elements[i]); pairs.push(elements[i]);
} }
// We have an element referred to the previous "title" // We have an element referred to the previous "title"
else if (elements[i].Type != "Text" && pairs.length > 0) { else if (elements[i].type != "Text" && pairs.length > 0) {
// We append this element to the content of the last title // We append this element to the content of the last title
lastPair.Content.push(elements[i]); lastPair.content.push(elements[i]);
} }
// ... else we have a "title" (we need to swap the text to the name because it is a title) // ... else we have a "title" (we need to swap the text to the name because it is a title)
else { else {
const swap: IPostElement = Object.assign({}, elements[i]); const swap: IPostElement = Object.assign({}, elements[i]);
swap.Name = elements[i].Text; swap.name = elements[i].text;
swap.Text = ""; swap.text = "";
pairs.push(swap); pairs.push(swap);
} }
} }

140
src/scripts/scraper.ts
View File

@ -7,13 +7,11 @@ import luxon from "luxon";
// Modules from file // Modules from file
import shared from "./shared.js"; import shared from "./shared.js";
import { fetchHTML } from "./network-helper.js"; import { fetchHTML } from "./network-helper.js";
import { getJSONLD, JSONLD } from "./json-ld.js"; import { getJSONLD, TJsonLD } from "./json-ld.js";
import { selectors as f95Selector } from "./constants/css-selector.js"; import { selectors as f95Selector } from "./constants/css-selector.js";
import HandiWork from "./classes/handiwork/handiwork.js"; import HandiWork from "./classes/handiwork/handiwork.js";
import { RatingType, IBasic, AuthorType, ExternalPlatformType, EngineType, StatusType, CategoryType } from "./interfaces.js"; import { TRating, IBasic, TAuthor, TExternalPlatform, TEngine, TStatus, TCategory } from "./interfaces.js";
import { login } from "../index.js";
import { ILink, IPostElement, parseCheerioMainPost } from "./post-parser.js"; import { ILink, IPostElement, parseCheerioMainPost } from "./post-parser.js";
import Game from "./classes/handiwork/game.js";
//#region Public methods //#region Public methods
/** /**
@ -34,16 +32,16 @@ export async function getPostInformation<T extends IBasic>(url: string): Promise
// Extract data // Extract data
const postData = parseCheerioMainPost($, mainPost); const postData = parseCheerioMainPost($, mainPost);
const JSONLD = getJSONLD($, body); const TJsonLD = getJSONLD(body);
// Fill in the HandiWork element with the information obtained // Fill in the HandiWork element with the information obtained
const hw: HandiWork = {} as HandiWork; const hw: HandiWork = {} as HandiWork;
fillWithJSONLD(hw, JSONLD); fillWithJSONLD(hw, TJsonLD);
fillWithPostData(hw, postData); fillWithPostData(hw, postData);
fillWithPrefixes(hw, body); fillWithPrefixes(hw, body);
hw.Tags = extractTags(body); hw.tags = extractTags(body);
shared.logger.info(`Founded data for ${hw.Name}`); shared.logger.info(`Founded data for ${hw.name}`);
return <T><unknown>hw; return <T><unknown>hw;
}; };
//#endregion Public methods //#endregion Public methods
@ -52,6 +50,10 @@ export async function getPostInformation<T extends IBasic>(url: string): Promise
//#region Generic Utility //#region Generic Utility
/**
* Convert a string to a boolean.
* Check also for `yes`/`no` and `1`/`0`.
*/
function stringToBoolean(s: string): boolean { function stringToBoolean(s: string): boolean {
// Local variables // Local variables
const positiveTerms = ["true", "yes", "1"]; const positiveTerms = ["true", "yes", "1"];
@ -67,15 +69,15 @@ function stringToBoolean(s: string): boolean {
/** /**
* It processes the evaluations of a particular work starting from the data contained in the JSON+LD tag. * It processes the evaluations of a particular work starting from the data contained in the JSON+LD tag.
*/ */
function parseRating(data: JSONLD): RatingType { function parseRating(data: TJsonLD): TRating {
shared.logger.trace("Parsing rating..."); shared.logger.trace("Parsing rating...");
// Local variables // Local variables
const ratingTree = data["aggregateRating"] as JSONLD; const ratingTree = data["aggregateRating"] as TJsonLD;
const rating: RatingType = { const rating: TRating = {
Average: parseFloat(ratingTree["ratingValue"] as string), average: parseFloat(ratingTree["ratingValue"] as string),
Best: parseInt(ratingTree["bestRating"] as string), best: parseInt(ratingTree["bestRating"] as string),
Count: parseInt(ratingTree["ratingCount"] as string), count: parseInt(ratingTree["ratingCount"] as string),
}; };
return rating; return rating;
@ -96,6 +98,11 @@ function extractIDFromURL(url: string): number {
return parseInt(match[0], 10); return parseInt(match[0], 10);
} }
/**
* Clean the title of a HandiWork, removing prefixes
* and generic elements between square brackets, and
* returns the clean title of the work.
*/
function cleanHeadline(headline: string): string { function cleanHeadline(headline: string): string {
shared.logger.trace("Cleaning headline..."); shared.logger.trace("Cleaning headline...");
@ -105,9 +112,7 @@ function cleanHeadline(headline: string): string {
// Get the title name // Get the title name
let name = headline; let name = headline;
matches.forEach(function replaceElementsInTitle(e) { matches.forEach(e => name = name.replace(e, ""));
name = name.replace(e, "");
});
return name.trim(); return name.trim();
} }
@ -117,7 +122,7 @@ function cleanHeadline(headline: string): string {
*/ */
function getPostElementByName(elements: IPostElement[], name: string): IPostElement | undefined { function getPostElementByName(elements: IPostElement[], name: string): IPostElement | undefined {
return elements.find(el => { return elements.find(el => {
return el.Name.toUpperCase() === name.toUpperCase(); return el.name.toUpperCase() === name.toUpperCase();
}); });
} }
@ -164,6 +169,7 @@ function isMod(prefix: string): boolean {
} }
//#endregion Prefix Utility //#endregion Prefix Utility
/** /**
* Compiles a HandiWork object with the data extracted * Compiles a HandiWork object with the data extracted
* from the JSON+LD tags related to the object itself. * from the JSON+LD tags related to the object itself.
@ -171,25 +177,25 @@ function isMod(prefix: string): boolean {
* `URL`, `ID`, `Category`, `Rating`, * `URL`, `ID`, `Category`, `Rating`,
* `Name`, `ThreadPublishingDate`, `LastThreadUpdate`. * `Name`, `ThreadPublishingDate`, `LastThreadUpdate`.
*/ */
function fillWithJSONLD(hw: HandiWork, data: JSONLD) { function fillWithJSONLD(hw: HandiWork, data: TJsonLD) {
shared.logger.trace("Extracting data from JSON+LD..."); shared.logger.trace("Extracting data from JSON+LD...");
// Set the basic values // Set the basic values
hw.Url = data["@id"] as string; hw.url = data["@id"] as string;
hw.ID = extractIDFromURL(hw.Url); hw.id = extractIDFromURL(hw.url);
hw.Category = data["articleSection"] as CategoryType; hw.category = data["articleSection"] as TCategory;
hw.Rating = parseRating(data); hw.rating = parseRating(data);
hw.Name = cleanHeadline(data["headline"] as string); hw.name = cleanHeadline(data["headline"] as string);
// Check and set the dates // Check and set the dates
const published = data["datePublished"] as string; const published = data["datePublished"] as string;
if (luxon.DateTime.fromISO(published).isValid) { if (luxon.DateTime.fromISO(published).isValid) {
hw.ThreadPublishingDate = new Date(published); hw.threadPublishingDate = new Date(published);
} }
const modified = data["dateModified"] as string; const modified = data["dateModified"] as string;
if (luxon.DateTime.fromISO(modified).isValid) { if (luxon.DateTime.fromISO(modified).isValid) {
hw.LastThreadUpdate = new Date(modified); hw.lastThreadUpdate = new Date(modified);
} }
} }
@ -199,66 +205,70 @@ function fillWithJSONLD(hw: HandiWork, data: JSONLD) {
* The values that will be added are: * The values that will be added are:
* `Overview`, `OS`, `Language`, `Version`, `Installation`, * `Overview`, `OS`, `Language`, `Version`, `Installation`,
* `Pages`, `Resolution`, `Lenght`, `Genre`, `Censored`, * `Pages`, `Resolution`, `Lenght`, `Genre`, `Censored`,
* `LastRelease`, `Authors`, `Changelog`. * `LastRelease`, `Authors`, `Changelog`, `Cover`.
*/ */
function fillWithPostData(hw: HandiWork, elements: IPostElement[]) { function fillWithPostData(hw: HandiWork, elements: IPostElement[]) {
// First fill the "simple" elements // First fill the "simple" elements
hw.Overview = getPostElementByName(elements, "overview")?.Text; hw.overview = getPostElementByName(elements, "overview")?.text;
hw.OS = getPostElementByName(elements, "os")?.Text?.split(",").map(s => s.trim()); hw.os = getPostElementByName(elements, "os")?.text?.split(",").map(s => s.trim());
hw.Language = getPostElementByName(elements, "language")?.Text?.split(",").map(s => s.trim()); hw.language = getPostElementByName(elements, "language")?.text?.split(",").map(s => s.trim());
hw.Version = getPostElementByName(elements, "version")?.Text; hw.version = getPostElementByName(elements, "version")?.text;
hw.Installation = getPostElementByName(elements, "installation")?.Content.shift()?.Text; hw.installation = getPostElementByName(elements, "installation")?.content.shift()?.text;
hw.Pages = getPostElementByName(elements, "pages")?.Text; hw.pages = getPostElementByName(elements, "pages")?.text;
hw.Resolution = getPostElementByName(elements, "resolution")?.Text?.split(",").map(s => s.trim()); hw.resolution = getPostElementByName(elements, "resolution")?.text?.split(",").map(s => s.trim());
hw.Lenght = getPostElementByName(elements, "lenght")?.Text; hw.lenght = getPostElementByName(elements, "lenght")?.text;
// Parse the censorship // Parse the censorship
const censored = getPostElementByName(elements, "censored") || getPostElementByName(elements, "censorship"); const censored = getPostElementByName(elements, "censored") || getPostElementByName(elements, "censorship");
if (censored) hw.Censored = stringToBoolean(censored.Text); if (censored) hw.censored = stringToBoolean(censored.text);
// Get the genres // Get the genres
const genre = getPostElementByName(elements, "genre")?.Content.shift()?.Text; const genre = getPostElementByName(elements, "genre")?.content.shift()?.text;
hw.Genre = genre?.split(",").map(s => s.trim()); hw.genre = genre?.split(",").map(s => s.trim());
// Get the cover
const cover = getPostElementByName(elements, "overview")?.content.find(el => el.type === "Image") as ILink;
hw.cover = cover?.href;
// Fill the dates // Fill the dates
const releaseDate = getPostElementByName(elements, "release date")?.Text; const releaseDate = getPostElementByName(elements, "release date")?.text;
if (luxon.DateTime.fromISO(releaseDate).isValid) hw.LastRelease = new Date(releaseDate); if (luxon.DateTime.fromISO(releaseDate).isValid) hw.lastRelease = new Date(releaseDate);
//#region Convert the author //#region Convert the author
const authorElement = getPostElementByName(elements, "developer") || const authorElement = getPostElementByName(elements, "developer") ||
getPostElementByName(elements, "developer/publisher") || getPostElementByName(elements, "developer/publisher") ||
getPostElementByName(elements, "artist"); getPostElementByName(elements, "artist");
const author: AuthorType = { const author: TAuthor = {
Name: authorElement.Text, name: authorElement.text,
Platforms: [] platforms: []
}; };
// Add the found platforms // Add the found platforms
authorElement?.Content.forEach((el: ILink, idx) => { authorElement?.content.forEach((el: ILink, idx) => {
const platform: ExternalPlatformType = { const platform: TExternalPlatform = {
Name: el.Text, name: el.text,
Link: el.Href, link: el.href,
}; };
author.Platforms.push(platform); author.platforms.push(platform);
}); });
hw.Authors = [author]; hw.authors = [author];
//#endregion Convert the author //#endregion Convert the author
//#region Get the changelog //#region Get the changelog
hw.Changelog = []; hw.changelog = [];
const changelogElement = getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log"); const changelogElement = getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log");
const changelogSpoiler = changelogElement?.Content.find(el => { const changelogSpoiler = changelogElement?.content.find(el => {
return el.Type === "Spoiler" && el.Content.length > 0; return el.type === "Spoiler" && el.content.length > 0;
}); });
// Add to the changelog the single spoilers // Add to the changelog the single spoilers
changelogSpoiler.Content.forEach(el => { changelogSpoiler.content.forEach(el => {
if (el.Text.trim()) hw.Changelog.push(el.Text); if (el.text.trim()) hw.changelog.push(el.text);
}); });
// Add at the ened also the text of the "changelog" element // Add at the ened also the text of the "changelog" element
hw.Changelog.push(changelogSpoiler.Text); hw.changelog.push(changelogSpoiler.text);
//#endregion Get the changelog //#endregion Get the changelog
} }
@ -288,11 +298,11 @@ function fillWithPrefixes(hw: HandiWork, body: cheerio.Cheerio) {
// Local variables // Local variables
let mod = false; let mod = false;
let engine: EngineType = null; let engine: TEngine = null;
let status: StatusType = null; let status: TStatus = null;
// Initialize the array // Initialize the array
hw.Prefixes = []; hw.prefixes = [];
// Obtain the title prefixes // Obtain the title prefixes
const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES); const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES);
@ -305,20 +315,20 @@ function fillWithPrefixes(hw: HandiWork, body: cheerio.Cheerio) {
prefix = prefix.replace("[", "").replace("]", ""); prefix = prefix.replace("[", "").replace("]", "");
// Check what the prefix indicates // Check what the prefix indicates
if (isEngine(prefix)) engine = prefix as EngineType; if (isEngine(prefix)) engine = prefix as TEngine;
else if (isStatus(prefix)) status = prefix as StatusType; else if (isStatus(prefix)) status = prefix as TStatus;
else if (isMod(prefix)) mod = true; else if (isMod(prefix)) mod = true;
// Anyway add the prefix to list // Anyway add the prefix to list
hw.Prefixes.push(prefix); hw.prefixes.push(prefix);
}); });
// If the status is not set, then the game is in development (Ongoing) // If the status is not set, then the game is in development (Ongoing)
status = (!status && hw.Category === "games") ? status : "Ongoing"; status = (!status && hw.category === "games") ? status : "Ongoing";
hw.Engine = engine; hw.engine = engine;
hw.Status = status; hw.status = status;
hw.Mod = mod; hw.mod = mod;
} }
//#endregion //#endregion