Remove file
MillenniumEarl
parent
f4af0c43f1
commit
f60606e5e8
|
|
@ -1,332 +0,0 @@
|
|||
"use strict";
|
||||
|
||||
// Public modules from npm
|
||||
import cheerio from "cheerio";
|
||||
import luxon from "luxon";
|
||||
|
||||
// Modules from file
|
||||
import shared from "../shared.js";
|
||||
import { fetchHTML } from "../network-helper.js";
|
||||
import { getJSONLD, TJsonLD } from "./json-ld.js";
|
||||
import { selectors as f95Selector, THREAD } from "../constants/css-selector.js";
|
||||
import HandiWork from "../classes/handiwork/handiwork.js";
|
||||
import { TRating, IBasic, TAuthor, TExternalPlatform, TEngine, TStatus, TCategory } from "../interfaces.js";
|
||||
import { ILink, IPostElement, parseF95ThreadPost } from "./post-parse.js";
|
||||
|
||||
//#region Public methods
|
||||
/**
|
||||
* Gets information from the post of a particular handiwork.
|
||||
* If you don't want to specify the object type, use `HandiWork`.
|
||||
* @todo It does not currently support assets.
|
||||
*/
|
||||
export async function getHandiworkInformation<T extends IBasic>(url: string): Promise<T> {
|
||||
shared.logger.info(`Obtaining post info from ${url}`);
|
||||
|
||||
// Fetch HTML and prepare Cheerio
|
||||
const html = await fetchHTML(url);
|
||||
|
||||
if (html.isSuccess()) {
|
||||
const $ = cheerio.load(html.value);
|
||||
const body = $("body");
|
||||
const mainPost = $(f95Selector.GS_POSTS).first();
|
||||
|
||||
// Extract data
|
||||
const postData = parseF95ThreadPost($, mainPost);
|
||||
const JSONLD = getJSONLD(body);
|
||||
|
||||
// Fill in the HandiWork element with the information obtained
|
||||
const hw: HandiWork = {} as HandiWork;
|
||||
fillWithJSONLD(hw, JSONLD);
|
||||
fillWithPostData(hw, postData);
|
||||
fillWithPrefixes(hw, body);
|
||||
hw.tags = extractTags(body);
|
||||
|
||||
shared.logger.info(`Founded data for ${hw.name}`);
|
||||
return <T><unknown>hw;
|
||||
} else throw html.value;
|
||||
};
|
||||
//#endregion Public methods
|
||||
|
||||
//#region Private methods
|
||||
|
||||
//#region Generic Utility
|
||||
|
||||
/**
|
||||
* Convert a string to a boolean.
|
||||
* Check also for `yes`/`no` and `1`/`0`.
|
||||
*/
|
||||
function stringToBoolean(s: string): boolean {
|
||||
// Local variables
|
||||
const positiveTerms = ["true", "yes", "1"];
|
||||
const negativeTerms = ["false", "no", "0"];
|
||||
const cleanString = s.toLowerCase().trim();
|
||||
let result = Boolean(s);
|
||||
|
||||
if (positiveTerms.includes(cleanString)) result = true;
|
||||
else if (negativeTerms.includes(cleanString)) result = false;
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* It processes the evaluations of a particular work starting from the data contained in the JSON+LD tag.
|
||||
*/
|
||||
function parseRating(data: TJsonLD): TRating {
|
||||
shared.logger.trace("Parsing rating...");
|
||||
|
||||
// Local variables
|
||||
const ratingTree = data["aggregateRating"] as TJsonLD;
|
||||
const rating: TRating = {
|
||||
average: parseFloat(ratingTree["ratingValue"] as string),
|
||||
best: parseInt(ratingTree["bestRating"] as string),
|
||||
count: parseInt(ratingTree["ratingCount"] as string),
|
||||
};
|
||||
|
||||
return rating;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts the work's unique ID from its URL.
|
||||
*/
|
||||
function extractIDFromURL(url: string): number {
|
||||
shared.logger.trace("Extracting ID from URL...");
|
||||
|
||||
// URL are in the format https://f95zone.to/threads/GAMENAME-VERSION-DEVELOPER.ID/
|
||||
// or https://f95zone.to/threads/ID/
|
||||
const match = url.match(/([0-9]+)(?=\/|\b)(?!-|\.)/);
|
||||
if (!match) return -1;
|
||||
|
||||
// Parse and return number
|
||||
return parseInt(match[0], 10);
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean the title of a HandiWork, removing prefixes
|
||||
* and generic elements between square brackets, and
|
||||
* returns the clean title of the work.
|
||||
*/
|
||||
function cleanHeadline(headline: string): string {
|
||||
shared.logger.trace("Cleaning headline...");
|
||||
|
||||
// From the title we can extract: Name, author and version
|
||||
// [PREFIXES] TITLE [VERSION] [AUTHOR]
|
||||
const matches = headline.match(/\[(.*?)\]/g);
|
||||
|
||||
// Get the title name
|
||||
let name = headline;
|
||||
matches.forEach(e => name = name.replace(e, ""));
|
||||
return name.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the element with the given name or `undefined`.
|
||||
* Case-insensitive.
|
||||
*/
|
||||
function getPostElementByName(elements: IPostElement[], name: string): IPostElement | undefined {
|
||||
return elements.find(el => {
|
||||
return el.name.toUpperCase() === name.toUpperCase();
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Makes an array of strings uppercase.
|
||||
*/
|
||||
function toUpperCaseArray(a: string[]): string[] {
|
||||
/**
|
||||
* Makes a string uppercase.
|
||||
*/
|
||||
function toUpper(s: string): string {
|
||||
return s.toUpperCase();
|
||||
}
|
||||
return a.map(toUpper);
|
||||
}
|
||||
|
||||
//#endregion Generic Utility
|
||||
|
||||
//#region Prefix Utility
|
||||
|
||||
/**
|
||||
* Check if the prefix is a game's engine.
|
||||
*/
|
||||
function isEngine(prefix: string): boolean {
|
||||
const engines = toUpperCaseArray(Object.values(shared.prefixes["engines"]));
|
||||
return engines.includes(prefix.toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the prefix is a game's status.
|
||||
*/
|
||||
function isStatus(prefix: string): boolean {
|
||||
const statuses = toUpperCaseArray(Object.values(shared.prefixes["statuses"]));
|
||||
return statuses.includes(prefix.toUpperCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the prefix indicates a mod.
|
||||
*/
|
||||
function isMod(prefix: string): boolean {
|
||||
const modPrefixes = ["MOD", "CHEAT MOD"];
|
||||
return modPrefixes.includes(prefix.toUpperCase());
|
||||
}
|
||||
//#endregion Prefix Utility
|
||||
|
||||
/**
|
||||
* Compiles a HandiWork object with the data extracted
|
||||
* from the JSON+LD tags related to the object itself.
|
||||
* The values that will be added are:
|
||||
* `URL`, `ID`, `Category`, `Rating`,
|
||||
* `Name`, `ThreadPublishingDate`, `LastThreadUpdate`.
|
||||
*/
|
||||
function fillWithJSONLD(hw: HandiWork, data: TJsonLD) {
|
||||
shared.logger.trace("Extracting data from JSON+LD...");
|
||||
|
||||
// Set the basic values
|
||||
hw.url = data["@id"] as string;
|
||||
hw.id = extractIDFromURL(hw.url);
|
||||
hw.category = data["articleSection"] as TCategory;
|
||||
hw.rating = parseRating(data);
|
||||
hw.name = cleanHeadline(data["headline"] as string);
|
||||
|
||||
// Check and set the dates
|
||||
const published = data["datePublished"] as string;
|
||||
if (luxon.DateTime.fromISO(published).isValid) {
|
||||
hw.threadPublishingDate = new Date(published);
|
||||
}
|
||||
|
||||
const modified = data["dateModified"] as string;
|
||||
if (luxon.DateTime.fromISO(modified).isValid) {
|
||||
hw.lastThreadUpdate = new Date(modified);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compiles a HandiWork object with the data extracted
|
||||
* from the main post of the HandiWork page.
|
||||
* The values that will be added are:
|
||||
* `Overview`, `OS`, `Language`, `Version`, `Installation`,
|
||||
* `Pages`, `Resolution`, `Lenght`, `Genre`, `Censored`,
|
||||
* `LastRelease`, `Authors`, `Changelog`, `Cover`.
|
||||
*/
|
||||
function fillWithPostData(hw: HandiWork, elements: IPostElement[]) {
|
||||
// First fill the "simple" elements
|
||||
hw.overview = getPostElementByName(elements, "overview")?.text;
|
||||
hw.os = getPostElementByName(elements, "os")?.text?.split(",").map(s => s.trim());
|
||||
hw.language = getPostElementByName(elements, "language")?.text?.split(",").map(s => s.trim());
|
||||
hw.version = getPostElementByName(elements, "version")?.text;
|
||||
hw.installation = getPostElementByName(elements, "installation")?.content.shift()?.text;
|
||||
hw.pages = getPostElementByName(elements, "pages")?.text;
|
||||
hw.resolution = getPostElementByName(elements, "resolution")?.text?.split(",").map(s => s.trim());
|
||||
hw.lenght = getPostElementByName(elements, "lenght")?.text;
|
||||
|
||||
// Parse the censorship
|
||||
const censored = getPostElementByName(elements, "censored") || getPostElementByName(elements, "censorship");
|
||||
if (censored) hw.censored = stringToBoolean(censored.text);
|
||||
|
||||
// Get the genres
|
||||
const genre = getPostElementByName(elements, "genre")?.content.shift()?.text;
|
||||
hw.genre = genre?.split(",").map(s => s.trim());
|
||||
|
||||
// Get the cover
|
||||
const cover = getPostElementByName(elements, "overview")?.content.find(el => el.type === "Image") as ILink;
|
||||
hw.cover = cover?.href;
|
||||
|
||||
// Fill the dates
|
||||
const releaseDate = getPostElementByName(elements, "release date")?.text;
|
||||
if (luxon.DateTime.fromISO(releaseDate).isValid) hw.lastRelease = new Date(releaseDate);
|
||||
|
||||
//#region Convert the author
|
||||
const authorElement = getPostElementByName(elements, "developer") ||
|
||||
getPostElementByName(elements, "developer/publisher") ||
|
||||
getPostElementByName(elements, "artist");
|
||||
const author: TAuthor = {
|
||||
name: authorElement.text,
|
||||
platforms: []
|
||||
};
|
||||
|
||||
// Add the found platforms
|
||||
authorElement?.content.forEach((el: ILink, idx) => {
|
||||
const platform: TExternalPlatform = {
|
||||
name: el.text,
|
||||
link: el.href,
|
||||
};
|
||||
|
||||
author.platforms.push(platform);
|
||||
});
|
||||
hw.authors = [author];
|
||||
//#endregion Convert the author
|
||||
|
||||
//#region Get the changelog
|
||||
hw.changelog = [];
|
||||
const changelogElement = getPostElementByName(elements, "changelog") || getPostElementByName(elements, "change-log");
|
||||
const changelogSpoiler = changelogElement?.content.find(el => {
|
||||
return el.type === "Spoiler" && el.content.length > 0;
|
||||
});
|
||||
|
||||
// Add to the changelog the single spoilers
|
||||
changelogSpoiler.content.forEach(el => {
|
||||
if (el.text.trim()) hw.changelog.push(el.text);
|
||||
});
|
||||
|
||||
// Add at the ened also the text of the "changelog" element
|
||||
hw.changelog.push(changelogSpoiler.text);
|
||||
//#endregion Get the changelog
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the tags used to classify the game.
|
||||
* @param {cheerio.Cheerio} body Page `body` selector
|
||||
*/
|
||||
function extractTags(body: cheerio.Cheerio): string[] {
|
||||
shared.logger.trace("Extracting tags...");
|
||||
|
||||
// Get the game tags
|
||||
const tagResults = body.find(THREAD.TAGS);
|
||||
return tagResults.map(function parseGameTags(idx, el) {
|
||||
return cheerio(el).text().trim();
|
||||
}).get();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the post prefixes.
|
||||
* In particular, it elaborates the following prefixes for games:
|
||||
* `Engine`, `Status`, `Mod`.
|
||||
* @param {cheerio.Cheerio} body Page `body` selector
|
||||
*/
|
||||
function fillWithPrefixes(hw: HandiWork, body: cheerio.Cheerio) {
|
||||
shared.logger.trace("Parsing prefixes...");
|
||||
|
||||
// Local variables
|
||||
let mod = false;
|
||||
let engine: TEngine = null;
|
||||
let status: TStatus = null;
|
||||
|
||||
// Initialize the array
|
||||
hw.prefixes = [];
|
||||
|
||||
// Obtain the title prefixes
|
||||
const prefixeElements = body.find(THREAD.PREFIXES);
|
||||
|
||||
prefixeElements.each(function parseGamePrefix(idx, el) {
|
||||
// Obtain the prefix text
|
||||
let prefix = cheerio(el).text().trim();
|
||||
|
||||
// Remove the square brackets
|
||||
prefix = prefix.replace("[", "").replace("]", "");
|
||||
|
||||
// Check what the prefix indicates
|
||||
if (isEngine(prefix)) engine = prefix as TEngine;
|
||||
else if (isStatus(prefix)) status = prefix as TStatus;
|
||||
else if (isMod(prefix)) mod = true;
|
||||
|
||||
// Anyway add the prefix to list
|
||||
hw.prefixes.push(prefix);
|
||||
});
|
||||
|
||||
// If the status is not set, then the game is in development (Ongoing)
|
||||
status = (!status && hw.category === "games") ? status : "Ongoing";
|
||||
|
||||
hw.engine = engine;
|
||||
hw.status = status;
|
||||
hw.mod = mod;
|
||||
}
|
||||
|
||||
//#endregion
|
||||
Loading…
Reference in New Issue