F95API/app/scripts/game-scraper.js

340 lines
12 KiB
JavaScript
Raw Normal View History

'use strict';
// Public modules from npm
const HTMLParser = require('node-html-parser');
const puppeteer = require('puppeteer');
const urlExist = require('url-exist');
// Modules from file
const shared = require('./shared.js');
const selectors = require('./costants/css-selectors.js');
const { preparePage } = require('./puppeteer-helper.js');
const GameDownload = require('./classes/game-download.js');
const GameInfo = require('./classes/game-info.js');
const { isStringAValidURL, isF95URL } = require('./urls-helper.js');
/**
* @protected
* Get information from the game's main page.
* @param {puppeteer.Browser} browser Browser object used for navigation
* @param {URL} url URL of the game/mod to extract data from
* @return {Promise<GameInfo>} Complete information about the game you are looking for
*/
module.exports.getGameInfo = async function(browser, url) {
if (shared.debug) console.log('Obtaining game info');
// Verify the correctness of the URL
if (!isF95URL(url)) throw url + ' is not a valid F95Zone URL';
let exists = await urlExist(url.toString());
if (!exists) return new GameInfo();
let page = await preparePage(browser); // Set new isolated page
await page.setCookie(...shared.cookies); // Set cookies to avoid login
await page.goto(url.toString(), {
waitUntil: shared.WAIT_STATEMENT
}); // Go to the game page and wait until it loads
// It asynchronously searches for the elements and
// then waits at the end to compile the object to be returned
let info = new GameInfo();
let title = getGameTitle(page);
let author = getGameAuthor(page);
let tags = getGameTags(page);
let previewSource = getGamePreviewSource(page);
let downloadData = getGameDownloadLink(page);
info = await parsePrefixes(page, info); // Fill status/engines/isMod
let structuredText = await getMainPostStructuredText(page);
let overview = getOverview(structuredText, info.isMod);
let parsedInfos = parseConversationPage(structuredText);
// Fill in the GameInfo element with the information obtained
info.name = await title;
info.author = await author;
info.overview = overview;
info.tags = await tags;
info.f95url = url;
info.version = info.isMod ? parsedInfos['MOD VERSION'] : parsedInfos['VERSION'];
info.lastUpdate = info.isMod ? parsedInfos['UPDATED'] : parsedInfos['THREAD UPDATED'];
info.previewSource = await previewSource;
info.downloadInfo = await downloadData;
await page.close(); // Close the page
if (shared.debug) console.log('Founded data for ' + info.name);
return info;
}
//#region Private methods
/**
* @private
* Get the game description from its web page.
* Different processing depending on whether the game is a mod or not.
* @param {String} text Structured text extracted from the game's web page
* @param {Boolean} isMod Specify if it is a game or a mod
* @returns {Promise<String>} Game description
*/
function getOverview(text, isMod) {
// Get overview (different parsing for game and mod)
let overviewEndIndex;
if (isMod) overviewEndIndex = text.indexOf('Updated');
else overviewEndIndex = text.indexOf('Thread Updated');
return text.substring(0, overviewEndIndex).replace('Overview:\n', '').trim();
}
/**
* @private
* Extrapolate the page structure by removing the element tags
* and leaving only the text and its spacing.
* @param {puppeteer.Page} page Page containing the text
* @returns {Promise<String>} Structured text
*/
async function getMainPostStructuredText(page) {
// Gets the first post, where are listed all the game's informations
let post = (await page.$$(selectors.THREAD_POSTS))[0];
// The info are plain text so we need to parse the HTML code
let bodyHTML = await page.evaluate( /* istanbul ignore next */ (mainPost) => mainPost.innerHTML, post);
return HTMLParser.parse(bodyHTML).structuredText;
}
/**
* @private
* Extrapolates and cleans the author from the page passed by parameter.
* @param {puppeteer.Page} page Page containing the author to be extrapolated
* @returns {Promise<String>} Game author
*/
async function getGameAuthor(page) {
// Get the game/mod name (without square brackets)
let titleHTML = await page.evaluate( /* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerHTML,
selectors.GAME_TITLE);
let structuredTitle = HTMLParser.parse(titleHTML);
// The last element **shoud be** the title without prefixes (engines, status, other...)
let gameTitle = structuredTitle.childNodes.pop().rawText;
// The last square brackets contain the author
let startTitleIndex = gameTitle.lastIndexOf('[') + 1;
return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim();
}
/**
* @private
* Process the post text to get all the useful
* information in the format *DESCRIPTOR : VALUE*.
* @param {String} text Structured text of the post
* @returns {Object} Dictionary of information
*/
function parseConversationPage(text) {
let dataPairs = {};
// The information searched in the game post are one per line
let splittedText = text.split('\n');
for (let line of splittedText) {
if (!line.includes(':')) continue;
// Create pair key/value
let splitted = line.split(':');
let key = splitted[0].trim().toUpperCase(); // Uppercase to avoid mismatch
let value = splitted[1].trim();
// Add pair to the dict if valid
if (value != '') dataPairs[key] = value;
}
return dataPairs;
}
/**
* @private
* Gets the URL of the image used as a preview for the game in the conversation.
* @param {puppeteer.Page} page Page containing the URL to be extrapolated
* @returns {Promise<URL>} URL of the image or null if failed to get it
*/
async function getGamePreviewSource(page) {
let src = await page.evaluate( /* istanbul ignore next */ (selector) => {
// Get the firs image available
let img = document.querySelector(selector);
if (img) return img.getAttribute('src');
else return null;
}, selectors.GAME_IMAGES);
// Check if the URL is valid
return isStringAValidURL(src) ? new URL(src) : null;
}
/**
* @private
* Extrapolates and cleans the title from the page passed by parameter.
* @param {puppeteer.Page} page Page containing the title to be extrapolated
* @returns {Promise<String>} Game title
*/
async function getGameTitle(page) {
// Get the game/mod name (without square brackets)
let titleHTML = await page.evaluate( /* istanbul ignore next */ (selector) =>
document.querySelector(selector).innerHTML,
selectors.GAME_TITLE);
let structuredTitle = HTMLParser.parse(titleHTML);
// The last element **shoud be** the title without prefixes (engines, status, other...)
let gameTitle = structuredTitle.childNodes.pop().rawText;
let endTitleIndex = gameTitle.indexOf('[');
return gameTitle.substring(0, endTitleIndex).trim();
}
/**
* @private
* Get the alphabetically sorted list of tags associated with the game.
* @param {puppeteer.Page} page Page containing the tags to be extrapolated
* @returns {Promise<String[]>} List of uppercase tags
*/
async function getGameTags(page) {
let tags = [];
// Get the game tags
for (let handle of await page.$$(selectors.GAME_TAGS)) {
let tag = await page.evaluate( /* istanbul ignore next */ (element) => element.innerText, handle);
tags.push(tag.toUpperCase());
}
return tags.sort();
}
/**
* @private
* Process the game title prefixes to extract information such as game status,
* graphics engine used, and whether it is a mod or original game.
* @param {puppeteer.Page} page Page containing the prefixes to be extrapolated
* @param {GameInfo} info Object to assign the identified information to
* @returns {Promise<GameInfo>} GameInfo object passed in to which the identified information has been added
*/
async function parsePrefixes(page, info) {
const MOD_PREFIX = 'MOD';
// The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete'
info.status = 'Ongoing';
for (let handle of await page.$$(selectors.GAME_TITLE_PREFIXES)) {
let value = await page.evaluate( /* istanbul ignore next */ (element) => element.innerText, handle);
// Clean the prefix
let prefix = value.toUpperCase().replace('[', '').replace(']', '').trim();
// Getting infos...
if (shared.statuses.includes(prefix)) info.status = prefix;
else if (shared.engines.includes(prefix)) info.engine = prefix;
// This is not a game but a mod
else if (prefix === MOD_PREFIX) info.isMod = true;
}
return info;
}
/**
* @private
* Get game download links for different platforms.
* @param {puppeteer.Page} page Page containing the links to be extrapolated
* @returns {Promise<GameDownload[]>} List of objects used for game download
*/
async function getGameDownloadLink(page) {
// Most used hosting platforms
let hostingPlatforms = ['MEGA', 'NOPY', 'FILESUPLOAD', 'MIXDROP', 'UPLOADHAVEN', 'PIXELDRAIN', 'FILESFM'];
// Supported OS platforms
let platformOS = ['WIN', 'LINUX', 'MAC', 'ALL']
// Gets the <span> which contains the download links
let temp = await page.$$(selectors.DOWNLOAD_LINKS_CONTAINER);
if(temp.length === 0) return [];
// Look for the container that contains the links
// It is necessary because the same css selector
// also identifies other elements on the page
let container = null;
for(let candidate of temp) {
if (container !== null) break;
let upperText = (await page.evaluate( /* istanbul ignore next */ (e) => e.innerText, candidate)).toUpperCase();
// Search if the container contains the name of a hosting platform
for (let p of hostingPlatforms) {
if(upperText.includes(p)) {
container = candidate;
break;
}
}
}
if(container === null) return [];
// Extract the HTML text from the container
let searchText = (await page.evaluate( /* istanbul ignore next */ (e) => e.innerHTML, container)).toLowerCase();
// Parse the download links
let downloadData = [];
for(let platform of platformOS) {
let data = extractGameHostingData(platform, searchText);
downloadData.push(...data);
}
return downloadData;
}
/**
* @private
* From the HTML text it extracts the game download links for the specified operating system.
* @param {String} platform Name of the operating system to look for a compatible link to.
* It can only be *WIN/LINUX/MAC/ALL*
* @param {String} text HTML string to extract links from
* @returns {GameDownload[]} List of game download links for the selected platform
*/
function extractGameHostingData(platform, text) {
const PLATFORM_BOLD_OPEN = '<b>';
const CONTAINER_SPAN_CLOSE = '</span>';
const LINK_OPEN = '<a';
const LINK_CLOSE = '</a>';
const HREF_START = 'href="';
const HREF_END = '\"';
const TAG_CLOSE = '>';
// Identify the individual platforms
let startIndex = text.indexOf(platform.toLowerCase());
if(startIndex === -1) return [];
else startIndex += platform.length
// Find the <b>platform</b>
let endIndex = text.indexOf(
PLATFORM_BOLD_OPEN,
startIndex) + PLATFORM_BOLD_OPEN.length;
// Find the end of the container
if (endIndex === -1) text.indexOf(
CONTAINER_SPAN_CLOSE,
startIndex) + CONTAINER_SPAN_CLOSE.length;
text = text.substring(startIndex, endIndex);
let downloadData = [];
let linkTags = text.split(LINK_OPEN);
for(let tag of linkTags) {
// Ignore non-link string
if (!tag.includes(HREF_START)) continue;
// Find the hosting platform name
startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length;
endIndex = tag.indexOf(LINK_CLOSE, startIndex);
let hosting = tag.substring(startIndex, endIndex);
// Find the 'href' attribute
startIndex = tag.indexOf(HREF_START) + HREF_START.length;
endIndex = tag.indexOf(HREF_END, startIndex);
let link = tag.substring(startIndex, endIndex);
if (isStringAValidURL(link)) {
let gd = new GameDownload();
gd.hosting = hosting.toUpperCase();
gd.link = new URL(link);
gd.supportedOS = platform.toUpperCase();
downloadData.push(gd);
}
}
return downloadData;
}
//#endregion Private methods