MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Reduced cyclomatick complexity

pull/59/head
MillenniumEarl 2020-12-12 12:39:16 +01:00
parent 84aff42c6c
commit 0db26c6e26
1 changed files with 27 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
app/scripts/scraper.js
View File

@ -22,6 +22,7 @@ module.exports.getGameInfo = async function (url) {
// Fetch HTML and prepare Cheerio // Fetch HTML and prepare Cheerio
const html = await fetchHTML(url); const html = await fetchHTML(url);
if(!html) return null;
const $ = cheerio.load(html); const $ = cheerio.load(html);
const body = $("body"); const body = $("body");
const mainPost = $(f95Selector.GS_POSTS).first(); const mainPost = $(f95Selector.GS_POSTS).first();
@ -99,7 +100,7 @@ function parseGamePrefixes(body) {
}); });
// If the status is not set, then the game in in development (Ongoing) // If the status is not set, then the game in in development (Ongoing)
if (!status) status = "Ongoing"; status = status ?? "Ongoing";
return { return {
engine, engine,
@ -144,7 +145,6 @@ function extractInfoFromTitle(body) {
// Remove the trailing "v" // Remove the trailing "v"
if (version[0] === "v") version = version.replace("v", ""); if (version[0] === "v") version = version.replace("v", "");
} }
else shared.logger.trace(`Malformed title: ${title}`);
// Last element (the regex [[\]]+ remove the square brackets) // Last element (the regex [[\]]+ remove the square brackets)
const author = matches[matches.length - 1].replace(/[[\]]+/g, "").trim(); const author = matches[matches.length - 1].replace(/[[\]]+/g, "").trim();
@ -295,16 +295,12 @@ function parseMainPostText(text) {
/** /**
* @private * @private
* Extracts and processes the JSON-LD values found at the bottom of the page. * Parse a JSON-LD element.
* @param {cheerio.Cheerio} body Page `body` selector * @param {cheerio.Element} element
* @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
*/ */
function extractStructuredData(body) { function parseScriptTag(element) {
shared.logger.trace("Extracting JSON-LD data...");
const structuredDataElements = body.find(f95Selector.GT_JSONLD);
const json = structuredDataElements.map(function parseScriptTag(idx, el) {
// Get the element HTML // Get the element HTML
const html = cheerio.load(el).html().trim(); const html = cheerio.load(element).html().trim();
// Obtain the JSON-LD // Obtain the JSON-LD
const data = html const data = html
@ -316,7 +312,22 @@ function extractStructuredData(body) {
// Return only the data of the game // Return only the data of the game
if (json["@type"] === "Book") return json; if (json["@type"] === "Book") return json;
}).get(); }
/**
* @private
* Extracts and processes the JSON-LD values found at the bottom of the page.
* @param {cheerio.Cheerio} body Page `body` selector
* @returns {Object.<string, string>} JSON-LD or `null` if no valid JSON is found
*/
function extractStructuredData(body) {
shared.logger.trace("Extracting JSON-LD data...");
// Fetch the JSON-LD data
const structuredDataElements = body.find(f95Selector.GT_JSONLD);
// Parse the data
const json = structuredDataElements.map((idx, el) => parseScriptTag(el)).get();
return json.lenght !== 0 ? json[0] : null; return json.lenght !== 0 ? json[0] : null;
} }
@ -389,12 +400,8 @@ function extractIDFromURL(url) {
* @private * @private
* Makes an array of strings uppercase. * Makes an array of strings uppercase.
* @param {String[]} a * @param {String[]} a
* @returns {String[]}
*/ */
function toUpperCaseArray(a) { function toUpperCaseArray(a) {
// If the array is empty, return
if(a.length === 0) return [];
/** /**
* Makes a string uppercase. * Makes a string uppercase.
* @param {String} s * @param {String} s