Extract platform data fetch

pull/59/head
MillenniumEarl 2020-12-15 14:16:18 +01:00
parent 29ee0a9d73
commit 7b8eb5b357
2 changed files with 129 additions and 116 deletions

View File

@ -1,8 +1,5 @@
"use strict"; "use strict";
// Core modules
const {readFileSync, writeFileSync, existsSync} = require("fs");
// Public modules from npm // Public modules from npm
const axios = require("axios").default; const axios = require("axios").default;
const cheerio = require("cheerio"); const cheerio = require("cheerio");
@ -14,6 +11,7 @@ const shared = require("./shared.js");
const f95url = require("./constants/url.js"); const f95url = require("./constants/url.js");
const f95selector = require("./constants/css-selector.js"); const f95selector = require("./constants/css-selector.js");
const LoginResult = require("./classes/login-result.js"); const LoginResult = require("./classes/login-result.js");
const fetchPlatformData = require("./platform-data.js").fetchPlatformData;
// Global variables // Global variables
const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " +
@ -98,7 +96,7 @@ module.exports.authenticate = async function (credentials, force) {
// Return the result of the authentication // Return the result of the authentication
if (errorMessage === "") { if (errorMessage === "") {
// Fetch data // Fetch data
await exports.fetchPlatformData(); await fetchPlatformData();
return new LoginResult(true, "Authentication successful"); return new LoginResult(true, "Authentication successful");
} }
else return new LoginResult(false, errorMessage); else return new LoginResult(false, errorMessage);
@ -126,28 +124,6 @@ module.exports.getF95Token = async function() {
return token; return token;
}; };
/**
* @protected
* Gets the basic data used for game data processing
* (such as graphics engines and progress statuses)
*/
module.exports.fetchPlatformData = async function() {
// Check if the data are cached
if(!_readCache(shared.cachePath)) {
// Load the HTML
const html = await exports.fetchHTML(f95url.F95_LATEST_UPDATES);
// Parse data
const data = _parseLatestPlatformHTML(html);
// Assign data
_assignLatestPlatformData(data);
// Cache data
_saveCache(shared.cachePath);
}
};
//#region Utility methods //#region Utility methods
/** /**
* @protected * @protected
@ -259,94 +235,4 @@ async function _axiosUrlExists(url) {
} }
return valid; return valid;
} }
/**
* @private
* Read the platform cache (if available)
* @param {String} path Path to cache
*/
function _readCache(path) {
// Local variables
let returnValue = false;
if (existsSync(path)) {
const data = readFileSync(path);
const json = JSON.parse(data);
shared.engines = json.engines;
shared.statuses = json.statuses;
shared.tags = json.tags;
shared.others = json.others;
returnValue = true;
}
return returnValue;
}
/**
* @private
* Save the current platform variables to disk.
* @param {String} path Path to cache
*/
function _saveCache(path) {
const saveDict = {
engines: shared.engines,
statuses: shared.statuses,
tags: shared.tags,
others: shared.others,
};
const json = JSON.stringify(saveDict);
writeFileSync(path, json);
}
/**
* @private
* Given the HTML code of the response from the F95Zone,
* parse it and return the result.
* @param {String} html
* @returns {Object.<string, object>} Parsed data
*/
function _parseLatestPlatformHTML(html) {
const $ = cheerio.load(html);
// Clean the JSON string
const unparsedText = $(f95selector.LU_TAGS_SCRIPT).html().trim();
const startIndex = unparsedText.indexOf("{");
const endIndex = unparsedText.lastIndexOf("}");
const parsedText = unparsedText.substring(startIndex, endIndex + 1);
return JSON.parse(parsedText);
}
/**
* @private
* Assign to the local variables the values from the F95Zone.
* @param {Object.<string, object>} data
*/
function _assignLatestPlatformData(data) {
// Local variables
const propertiesMap = {
"Engine": shared.engines,
"Status": shared.statuses,
"Other": shared.others,
};
// Extract and parse the data
const prefixes = data.prefixes.games.map(e => {
return {
element: e.name,
data: e.prefixes
};
});
// Parse and assign the values that are NOT tags
for (const p of prefixes) {
// Prepare the dict
const dict = {};
for (const e of p.data) dict[parseInt(e.id)] = e.name.replace("&#039;", "'");
// Save the property
propertiesMap[p] = dict;
}
// Parse the tags
shared.tags = data.tags;
}
//#endregion //#endregion

View File

@ -0,0 +1,127 @@
"use strict";
// Core modules
const {readFileSync, writeFileSync, existsSync} = require("fs");
// Public modules from npm
const cheerio = require("cheerio");
// Modules from file
const shared = require("./shared.js");
const f95url = require("./constants/url.js");
const f95selector = require("./constants/css-selector.js");
const {fetchHTML} = require("./network-helper.js");
/**
* @protected
* Gets the basic data used for game data processing
* (such as graphics engines and progress statuses)
*/
module.exports.fetchPlatformData = async function () {
// Check if the data are cached
if (!_readCache(shared.cachePath)) {
// Load the HTML
const html = await fetchHTML(f95url.F95_LATEST_UPDATES);
// Parse data
const data = _parseLatestPlatformHTML(html);
// Assign data
_assignLatestPlatformData(data);
// Cache data
_saveCache(shared.cachePath);
}
};
//#region Private methods
/**
* @private
* Read the platform cache (if available)
* @param {String} path Path to cache
*/
function _readCache(path) {
// Local variables
let returnValue = false;
if (existsSync(path)) {
const data = readFileSync(path);
const json = JSON.parse(data);
shared.engines = json.engines;
shared.statuses = json.statuses;
shared.tags = json.tags;
shared.others = json.others;
returnValue = true;
}
return returnValue;
}
/**
* @private
* Save the current platform variables to disk.
* @param {String} path Path to cache
*/
function _saveCache(path) {
const saveDict = {
engines: shared.engines,
statuses: shared.statuses,
tags: shared.tags,
others: shared.others,
};
const json = JSON.stringify(saveDict);
writeFileSync(path, json);
}
/**
* @private
* Given the HTML code of the response from the F95Zone,
* parse it and return the result.
* @param {String} html
* @returns {Object.<string, object>} Parsed data
*/
function _parseLatestPlatformHTML(html) {
const $ = cheerio.load(html);
// Clean the JSON string
const unparsedText = $(f95selector.LU_TAGS_SCRIPT).html().trim();
const startIndex = unparsedText.indexOf("{");
const endIndex = unparsedText.lastIndexOf("}");
const parsedText = unparsedText.substring(startIndex, endIndex + 1);
return JSON.parse(parsedText);
}
/**
* @private
* Assign to the local variables the values from the F95Zone.
* @param {Object.<string, object>} data
*/
function _assignLatestPlatformData(data) {
// Local variables
const propertiesMap = {
"Engine": shared.engines,
"Status": shared.statuses,
"Other": shared.others,
};
// Extract and parse the data
const prefixes = data.prefixes.games.map(e => {
return {
element: e.name,
data: e.prefixes
};
});
// Parse and assign the values that are NOT tags
for (const p of prefixes) {
// Prepare the dict
const dict = {};
for (const e of p.data) dict[parseInt(e.id)] = e.name.replace("&#039;", "'");
// Save the property
propertiesMap[p] = dict;
}
// Parse the tags
shared.tags = data.tags;
}
//#endregion