Clena css-selectors
							parent
							
								
									cb3370c212
								
							
						
					
					
						commit
						0a57d2b2ec
					
				| 
						 | 
				
			
			@ -3,31 +3,6 @@
 | 
			
		|||
// This software is released under the MIT License.
 | 
			
		||||
// https://opensource.org/licenses/MIT
 | 
			
		||||
 | 
			
		||||
export const selectors = {
 | 
			
		||||
  WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger",
 | 
			
		||||
  WT_NEXT_PAGE: "a.pageNav-jump--next",
 | 
			
		||||
  WT_URLS: 'a[href^="/threads/"][data-tp-primary]',
 | 
			
		||||
  WT_UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]',
 | 
			
		||||
  GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type",
 | 
			
		||||
  GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a",
 | 
			
		||||
  GS_RESULT_BODY: "div.contentRow-main",
 | 
			
		||||
  GS_MEMBERSHIP: "li > a:not(.username)",
 | 
			
		||||
  GET_REQUEST_TOKEN: 'input[name="_xfToken"]',
 | 
			
		||||
  UD_USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText',
 | 
			
		||||
  UD_AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]',
 | 
			
		||||
  LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic",
 | 
			
		||||
  LU_TAGS_SCRIPT: "script:contains('latestUpdates')",
 | 
			
		||||
  BK_RESULTS: "ol.listPlain > *  div.contentRow-main",
 | 
			
		||||
  BK_POST_URL: "div.contentRow-title > a",
 | 
			
		||||
  BK_DESCRIPTION: "div.contentRow-snippet",
 | 
			
		||||
  BK_POST_OWNER: "div.contentRow-minor > * a.username",
 | 
			
		||||
  BK_TAGS: "div.contentRow-minor > * a.tagItem",
 | 
			
		||||
  /**
 | 
			
		||||
   * Attribute `datetime` contains an ISO date.
 | 
			
		||||
   */
 | 
			
		||||
  BK_TIME: "div.contentRow-minor > * time"
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
export const GENERIC = {
 | 
			
		||||
  /**
 | 
			
		||||
   * The ID of the user currently logged into
 | 
			
		||||
| 
						 | 
				
			
			@ -37,7 +12,19 @@ export const GENERIC = {
 | 
			
		|||
  /**
 | 
			
		||||
   * Banner containing any error messages as text.
 | 
			
		||||
   */
 | 
			
		||||
  ERROR_BANNER: "div.p-body-pageContent > div.blockMessage"
 | 
			
		||||
  ERROR_BANNER: "div.p-body-pageContent > div.blockMessage",
 | 
			
		||||
  /**
 | 
			
		||||
   * Locate the token used for the session.
 | 
			
		||||
   */
 | 
			
		||||
  GET_REQUEST_TOKEN: 'input[name="_xfToken"]',
 | 
			
		||||
  /**
 | 
			
		||||
   * Block containing the text of any errors that occurred during the login.
 | 
			
		||||
   */
 | 
			
		||||
  LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic",
 | 
			
		||||
  /**
 | 
			
		||||
   * Locate the script containing the tags and prefixes of the platform content in JSON format.
 | 
			
		||||
   */
 | 
			
		||||
  LATEST_UPDATES_TAGS_SCRIPT: "script:contains('latestUpdates')"
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
export const WATCHED_THREAD = {
 | 
			
		||||
| 
						 | 
				
			
			@ -110,6 +97,17 @@ export const THREAD = {
 | 
			
		|||
  POSTS_IN_PAGE: "article.message"
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
export const THREAD_SEARCH = {
 | 
			
		||||
  /**
 | 
			
		||||
   * Thread title resulting from research.
 | 
			
		||||
   */
 | 
			
		||||
  THREAD_TITLE: "h3.contentRow-title > a",
 | 
			
		||||
  /**
 | 
			
		||||
   *Thread body resulting from research.
 | 
			
		||||
   */
 | 
			
		||||
  BODY: "div.contentRow-main"
 | 
			
		||||
};
 | 
			
		||||
 | 
			
		||||
export const POST = {
 | 
			
		||||
  /**
 | 
			
		||||
   * Unique post number for the current thread.
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -14,10 +14,11 @@ import cheerio from "cheerio";
 | 
			
		|||
// Modules from file
 | 
			
		||||
import shared, { TPrefixDict } from "../shared.js";
 | 
			
		||||
import { urls as f95url } from "../constants/url.js";
 | 
			
		||||
import { selectors as f95selector } from "../constants/css-selector.js";
 | 
			
		||||
import { GENERIC } from "../constants/css-selector.js";
 | 
			
		||||
import { fetchHTML } from "../network-helper.js";
 | 
			
		||||
 | 
			
		||||
//#region Interface definitions
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * Represents the single element contained in the data categories.
 | 
			
		||||
 */
 | 
			
		||||
| 
						 | 
				
			
			@ -44,9 +45,11 @@ interface ILatestResource {
 | 
			
		|||
  tags: TPrefixDict;
 | 
			
		||||
  options: string;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
//#endregion Interface definitions
 | 
			
		||||
 | 
			
		||||
//#region Public methods
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * Gets the basic data used for game data processing
 | 
			
		||||
 * (such as graphics engines and progress statuses)
 | 
			
		||||
| 
						 | 
				
			
			@ -69,12 +72,13 @@ export default async function fetchPlatformData(): Promise<void> {
 | 
			
		|||
    } else throw html.value;
 | 
			
		||||
  }
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
//#endregion Public methods
 | 
			
		||||
 | 
			
		||||
//#region Private methods
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @private
 | 
			
		||||
 * Read the platform cache (if available)
 | 
			
		||||
 * Read the platform cache (if available).
 | 
			
		||||
 */
 | 
			
		||||
function readCache(path: string) {
 | 
			
		||||
  // Local variables
 | 
			
		||||
| 
						 | 
				
			
			@ -95,7 +99,6 @@ function readCache(path: string) {
 | 
			
		|||
}
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @private
 | 
			
		||||
 * Save the current platform variables to disk.
 | 
			
		||||
 */
 | 
			
		||||
function saveCache(path: string): void {
 | 
			
		||||
| 
						 | 
				
			
			@ -110,7 +113,6 @@ function saveCache(path: string): void {
 | 
			
		|||
}
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @private
 | 
			
		||||
 * Given the HTML code of the response from the F95Zone,
 | 
			
		||||
 * parse it and return the result.
 | 
			
		||||
 */
 | 
			
		||||
| 
						 | 
				
			
			@ -118,7 +120,7 @@ function parseLatestPlatformHTML(html: string): ILatestResource {
 | 
			
		|||
  const $ = cheerio.load(html);
 | 
			
		||||
 | 
			
		||||
  // Clean the JSON string
 | 
			
		||||
  const unparsedText = $(f95selector.LU_TAGS_SCRIPT).html().trim();
 | 
			
		||||
  const unparsedText = $(GENERIC.LATEST_UPDATES_TAGS_SCRIPT).html().trim();
 | 
			
		||||
  const startIndex = unparsedText.indexOf("{");
 | 
			
		||||
  const endIndex = unparsedText.lastIndexOf("}");
 | 
			
		||||
  const parsedText = unparsedText.substring(startIndex, endIndex + 1);
 | 
			
		||||
| 
						 | 
				
			
			@ -126,7 +128,6 @@ function parseLatestPlatformHTML(html: string): ILatestResource {
 | 
			
		|||
}
 | 
			
		||||
 | 
			
		||||
/**
 | 
			
		||||
 * @private
 | 
			
		||||
 * Assign to the local variables the values from the F95Zone.
 | 
			
		||||
 */
 | 
			
		||||
function assignLatestPlatformData(data: ILatestResource): void {
 | 
			
		||||
| 
						 | 
				
			
			@ -154,4 +155,5 @@ function assignLatestPlatformData(data: ILatestResource): void {
 | 
			
		|||
  shared.setPrefixPair("others", Object.assign({}, scrapedData["Other"]));
 | 
			
		||||
  shared.setPrefixPair("tags", data.tags);
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
//#endregion
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -10,7 +10,7 @@ import cheerio from "cheerio";
 | 
			
		|||
 | 
			
		||||
// Modules from file
 | 
			
		||||
import shared from "../shared.js";
 | 
			
		||||
import { selectors as f95Selector } from "../constants/css-selector.js";
 | 
			
		||||
import { THREAD_SEARCH } from "../constants/css-selector.js";
 | 
			
		||||
import { urls as f95urls } from "../constants/url.js";
 | 
			
		||||
import ThreadSearchQuery from "../classes/query/thread-search-query.js";
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -28,7 +28,7 @@ import ThreadSearchQuery from "../classes/query/thread-search-query.js";
 | 
			
		|||
 */
 | 
			
		||||
export default async function fetchThreadHandiworkURLs(
 | 
			
		||||
  query: ThreadSearchQuery,
 | 
			
		||||
  limit = 30
 | 
			
		||||
  limit: number = 30
 | 
			
		||||
): Promise<string[]> {
 | 
			
		||||
  // Execute the query
 | 
			
		||||
  const response = await query.execute();
 | 
			
		||||
| 
						 | 
				
			
			@ -47,12 +47,12 @@ export default async function fetchThreadHandiworkURLs(
 | 
			
		|||
 * @param {number} limit
 | 
			
		||||
 * Maximum number of items to get. Default: 30
 | 
			
		||||
 */
 | 
			
		||||
async function fetchResultURLs(html: string, limit = 30): Promise<string[]> {
 | 
			
		||||
async function fetchResultURLs(html: string, limit: number = 30): Promise<string[]> {
 | 
			
		||||
  // Prepare cheerio
 | 
			
		||||
  const $ = cheerio.load(html);
 | 
			
		||||
 | 
			
		||||
  // Here we get all the DIV that are the body of the various query results
 | 
			
		||||
  const results = $("body").find(f95Selector.GS_RESULT_BODY);
 | 
			
		||||
  const results = $("body").find(THREAD_SEARCH.BODY);
 | 
			
		||||
 | 
			
		||||
  // Than we extract the URLs
 | 
			
		||||
  const urls = results
 | 
			
		||||
| 
						 | 
				
			
			@ -74,10 +74,7 @@ async function fetchResultURLs(html: string, limit = 30): Promise<string[]> {
 | 
			
		|||
function extractLinkFromResult(selector: cheerio.Cheerio): string {
 | 
			
		||||
  shared.logger.trace("Extracting thread link from result...");
 | 
			
		||||
 | 
			
		||||
  const partialLink = selector
 | 
			
		||||
    .find(f95Selector.GS_RESULT_THREAD_TITLE)
 | 
			
		||||
    .attr("href")
 | 
			
		||||
    .trim();
 | 
			
		||||
  const partialLink = selector.find(THREAD_SEARCH.THREAD_TITLE).attr("href").trim();
 | 
			
		||||
 | 
			
		||||
  // Compose and return the URL
 | 
			
		||||
  return new URL(partialLink, f95urls.F95_BASE_URL).toString();
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -13,7 +13,7 @@ import axiosCookieJarSupport from "axios-cookiejar-support";
 | 
			
		|||
// Modules from file
 | 
			
		||||
import shared from "./shared.js";
 | 
			
		||||
import { urls } from "./constants/url.js";
 | 
			
		||||
import { selectors as f95selector } from "./constants/css-selector.js";
 | 
			
		||||
import { GENERIC } from "./constants/css-selector.js";
 | 
			
		||||
import LoginResult from "./classes/login-result.js";
 | 
			
		||||
import { failure, Result, success } from "./classes/result.js";
 | 
			
		||||
import {
 | 
			
		||||
| 
						 | 
				
			
			@ -187,7 +187,7 @@ export async function getF95Token(): Promise<string> {
 | 
			
		|||
  if (response.isSuccess()) {
 | 
			
		||||
    // The response is a HTML page, we need to find the <input> with name "_xfToken"
 | 
			
		||||
    const $ = cheerio.load(response.value.data as string);
 | 
			
		||||
    return $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value");
 | 
			
		||||
    return $("body").find(GENERIC.GET_REQUEST_TOKEN).attr("value");
 | 
			
		||||
  } else throw response.value;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -368,7 +368,7 @@ function manageLoginPOSTResponse(response: AxiosResponse<any>) {
 | 
			
		|||
 | 
			
		||||
  // Get the error message (if any) and remove the new line chars
 | 
			
		||||
  const errorMessage = $("body")
 | 
			
		||||
    .find(f95selector.LOGIN_MESSAGE_ERROR)
 | 
			
		||||
    .find(GENERIC.LOGIN_MESSAGE_ERROR)
 | 
			
		||||
    .text()
 | 
			
		||||
    .replace(/\n/g, "");
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
		Reference in New Issue