MickSlash
/
F95API
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Renamed selectors

pull/73/head
MillenniumEarl 2021-03-02 11:49:24 +01:00
parent e3eefe72f7
commit e5cdb9f65d
2 changed files with 7 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/scripts/scrape-data/json-ld.ts
View File

@ -5,7 +5,7 @@ import cheerio from "cheerio";
// Modules from file // Modules from file
import shared from "../shared.js"; import shared from "../shared.js";
import { selectors as f95Selector } from "../constants/css-selector.js"; import { THREAD } from "../constants/css-selector.js";
/** /**
* Represents information contained in a JSON+LD tag. * Represents information contained in a JSON+LD tag.
@ -21,7 +21,7 @@ export function getJSONLD(body: cheerio.Cheerio): TJsonLD {
shared.logger.trace("Extracting JSON-LD data..."); shared.logger.trace("Extracting JSON-LD data...");
// Fetch the JSON-LD data // Fetch the JSON-LD data
const structuredDataElements = body.find(f95Selector.GT_JSONLD); const structuredDataElements = body.find(THREAD.JSONLD);
// Parse the data // Parse the data
const values = structuredDataElements.map((idx, el) => parseJSONLD(el)).get(); const values = structuredDataElements.map((idx, el) => parseJSONLD(el)).get();

11
src/scripts/scrape-data/scrape-thread.ts
View File

@ -8,7 +8,7 @@ import luxon from "luxon";
import shared from "../shared.js"; import shared from "../shared.js";
import { fetchHTML } from "../network-helper.js"; import { fetchHTML } from "../network-helper.js";
import { getJSONLD, TJsonLD } from "./json-ld.js"; import { getJSONLD, TJsonLD } from "./json-ld.js";
import { selectors as f95Selector } from "../constants/css-selector.js"; import { selectors as f95Selector, THREAD } from "../constants/css-selector.js";
import HandiWork from "../classes/handiwork/handiwork.js"; import HandiWork from "../classes/handiwork/handiwork.js";
import { TRating, IBasic, TAuthor, TExternalPlatform, TEngine, TStatus, TCategory } from "../interfaces.js"; import { TRating, IBasic, TAuthor, TExternalPlatform, TEngine, TStatus, TCategory } from "../interfaces.js";
import { ILink, IPostElement, parseF95ThreadPost } from "./post-parse.js"; import { ILink, IPostElement, parseF95ThreadPost } from "./post-parse.js";
@ -32,11 +32,11 @@ export async function getHandiworkInformation<T extends IBasic>(url: string): Pr
// Extract data // Extract data
const postData = parseF95ThreadPost($, mainPost); const postData = parseF95ThreadPost($, mainPost);
const TJsonLD = getJSONLD(body); const JSONLD = getJSONLD(body);
// Fill in the HandiWork element with the information obtained // Fill in the HandiWork element with the information obtained
const hw: HandiWork = {} as HandiWork; const hw: HandiWork = {} as HandiWork;
fillWithJSONLD(hw, TJsonLD); fillWithJSONLD(hw, JSONLD);
fillWithPostData(hw, postData); fillWithPostData(hw, postData);
fillWithPrefixes(hw, body); fillWithPrefixes(hw, body);
hw.tags = extractTags(body); hw.tags = extractTags(body);
@ -142,7 +142,6 @@ function toUpperCaseArray(a: string[]): string[] {
//#endregion Generic Utility //#endregion Generic Utility
//#region Prefix Utility //#region Prefix Utility
/** /**
@ -280,7 +279,7 @@ function extractTags(body: cheerio.Cheerio): string[] {
shared.logger.trace("Extracting tags..."); shared.logger.trace("Extracting tags...");
// Get the game tags // Get the game tags
const tagResults = body.find(f95Selector.GT_TAGS); const tagResults = body.find(THREAD.TAGS);
return tagResults.map(function parseGameTags(idx, el) { return tagResults.map(function parseGameTags(idx, el) {
return cheerio(el).text().trim(); return cheerio(el).text().trim();
}).get(); }).get();
@ -304,7 +303,7 @@ function fillWithPrefixes(hw: HandiWork, body: cheerio.Cheerio) {
hw.prefixes = []; hw.prefixes = [];
// Obtain the title prefixes // Obtain the title prefixes
const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES); const prefixeElements = body.find(THREAD.PREFIXES);
prefixeElements.each(function parseGamePrefix(idx, el) { prefixeElements.each(function parseGamePrefix(idx, el) {
// Obtain the prefix text // Obtain the prefix text