From 08b23b0c6f2223061fa12eb9799dc74610e0fce3 Mon Sep 17 00:00:00 2001 From: "samuele.berlusconi" Date: Thu, 29 Oct 2020 22:10:20 +0100 Subject: [PATCH 01/17] Validate eslintrc file --- .eslintrc.json | 6 +- package-lock.json | 605 ++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 + 3 files changed, 611 insertions(+), 2 deletions(-) diff --git a/.eslintrc.json b/.eslintrc.json index 0083c2b..0438ccf 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -6,7 +6,7 @@ "node": true }, "extends": "eslint:recommended", - "parser": "./node_modules/babel-eslint", + "parser": "babel-eslint", "parserOptions": { "ecmaVersion": 12 }, @@ -29,7 +29,9 @@ ], "no-unused-vars": [ "error", - "after-used" + { + "args": "after-used" + } ] } } diff --git a/package-lock.json b/package-lock.json index 205a06f..3018479 100644 --- a/package-lock.json +++ b/package-lock.json @@ -222,6 +222,41 @@ "to-fast-properties": "^2.0.0" } }, + "@eslint/eslintrc": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.2.1.tgz", + "integrity": "sha512-XRUeBZ5zBWLYgSANMpThFddrZZkEbGHgUdt5UJjZfnlN9BGCiUBrf+nvbRupSjMvqzwnQN0qwCmOxITt1cfywA==", + "dev": true, + "requires": { + "ajv": "^6.12.4", + "debug": "^4.1.1", + "espree": "^7.3.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.2.1", + "js-yaml": "^3.13.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "strip-json-comments": "^3.1.1" + }, + "dependencies": { + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true + } + } + }, "@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -270,6 +305,18 @@ "event-target-shim": "^5.0.0" } }, + "acorn": { + "version": "7.4.1", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", + "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", + "dev": true + }, + "acorn-jsx": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", + "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", + "dev": true + }, "agent-base": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", @@ -285,6 +332,18 @@ "indent-string": "^4.0.0" } }, + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "ansi-colors": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.1.tgz", @@ -358,6 +417,34 @@ "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", "dev": true }, + "astral-regex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-1.0.0.tgz", + "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", + "dev": true + }, + "babel-eslint": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", + "integrity": "sha512-ifWaTHQ0ce+448CYop8AdrQiBsGrnC+bMgfyKFdi6EsPLTAWG+QfyDeM6OH+FmWnKvEq5NnBMLvlBUPKQZoDSg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@babel/parser": "^7.7.0", + "@babel/traverse": "^7.7.0", + "@babel/types": "^7.7.0", + "eslint-visitor-keys": "^1.0.0", + "resolve": "^1.12.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } + } + }, "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", @@ -434,6 +521,12 @@ "write-file-atomic": "^3.0.0" } }, + "callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "dev": true + }, "camelcase": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-5.3.1.tgz", @@ -588,6 +681,12 @@ "type-detect": "^4.0.0" } }, + "deep-is": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.3.tgz", + "integrity": "sha1-s2nW+128E+7PUk+RsHD+7cNXzzQ=", + "dev": true + }, "default-require-extensions": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/default-require-extensions/-/default-require-extensions-3.0.0.tgz", @@ -617,6 +716,15 @@ "integrity": "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A==", "dev": true }, + "doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dev": true, + "requires": { + "esutils": "^2.0.2" + } + }, "dotenv": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", @@ -637,6 +745,15 @@ "once": "^1.4.0" } }, + "enquirer": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", + "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", + "dev": true, + "requires": { + "ansi-colors": "^4.1.1" + } + }, "es-abstract": { "version": "1.17.6", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", @@ -700,12 +817,227 @@ "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", "dev": true }, + "eslint": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.12.1.tgz", + "integrity": "sha512-HlMTEdr/LicJfN08LB3nM1rRYliDXOmfoO4vj39xN6BLpFzF00hbwBoqHk8UcJ2M/3nlARZWy/mslvGEuZFvsg==", + "dev": true, + "requires": { + "@babel/code-frame": "^7.0.0", + "@eslint/eslintrc": "^0.2.1", + "ajv": "^6.10.0", + "chalk": "^4.0.0", + "cross-spawn": "^7.0.2", + "debug": "^4.0.1", + "doctrine": "^3.0.0", + "enquirer": "^2.3.5", + "eslint-scope": "^5.1.1", + "eslint-utils": "^2.1.0", + "eslint-visitor-keys": "^2.0.0", + "espree": "^7.3.0", + "esquery": "^1.2.0", + "esutils": "^2.0.2", + "file-entry-cache": "^5.0.1", + "functional-red-black-tree": "^1.0.1", + "glob-parent": "^5.0.0", + "globals": "^12.1.0", + "ignore": "^4.0.6", + "import-fresh": "^3.0.0", + "imurmurhash": "^0.1.4", + "is-glob": "^4.0.0", + "js-yaml": "^3.13.1", + "json-stable-stringify-without-jsonify": "^1.0.1", + "levn": "^0.4.1", + "lodash": "^4.17.19", + "minimatch": "^3.0.4", + "natural-compare": "^1.4.0", + "optionator": "^0.9.1", + "progress": "^2.0.0", + "regexpp": "^3.1.0", + "semver": "^7.2.1", + "strip-ansi": "^6.0.0", + "strip-json-comments": "^3.1.0", + "table": "^5.2.3", + "text-table": "^0.2.0", + "v8-compile-cache": "^2.0.3" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "chalk": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.0.tgz", + "integrity": "sha512-qwx12AxXe2Q5xQ43Ac//I6v5aXTipYrSESdOgzrN+9XjgEpyjpKuvSGaN4qE93f7TQTlerQQ8S+EQ0EyDoVL1A==", + "dev": true, + "requires": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "globals": { + "version": "12.4.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz", + "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==", + "dev": true, + "requires": { + "type-fest": "^0.8.1" + } + }, + "has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true + }, + "semver": { + "version": "7.3.2", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz", + "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==", + "dev": true + }, + "strip-json-comments": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", + "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", + "dev": true + }, + "supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "requires": { + "has-flag": "^4.0.0" + } + } + } + }, + "eslint-scope": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", + "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "dev": true, + "requires": { + "esrecurse": "^4.3.0", + "estraverse": "^4.1.1" + } + }, + "eslint-utils": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", + "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "dev": true, + "requires": { + "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } + } + }, + "eslint-visitor-keys": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz", + "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==", + "dev": true + }, + "espree": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz", + "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==", + "dev": true, + "requires": { + "acorn": "^7.4.0", + "acorn-jsx": "^5.2.0", + "eslint-visitor-keys": "^1.3.0" + }, + "dependencies": { + "eslint-visitor-keys": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", + "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "dev": true + } + } + }, "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", "dev": true }, + "esquery": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/esquery/-/esquery-1.3.1.tgz", + "integrity": "sha512-olpvt9QG0vniUBZspVRN6lwB7hOZoTRtT+jzR+tS4ffYx2mzbw+z0XCOk44aaLYKApNX5nMm+E+P6o25ip/DHQ==", + "dev": true, + "requires": { + "estraverse": "^5.1.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "esrecurse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", + "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==", + "dev": true, + "requires": { + "estraverse": "^5.2.0" + }, + "dependencies": { + "estraverse": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz", + "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==", + "dev": true + } + } + }, + "estraverse": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", + "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "dev": true + }, + "esutils": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", + "dev": true + }, "event-target-shim": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", @@ -722,6 +1054,24 @@ "yauzl": "^2.10.0" } }, + "fast-deep-equal": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dev": true + }, + "fast-json-stable-stringify": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true + }, + "fast-levenshtein": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", + "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", + "dev": true + }, "fd-slicer": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", @@ -735,6 +1085,15 @@ "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.1.tgz", "integrity": "sha512-Uf+gxPCe1hTOFXwkxYyckn8iUSk6CFXGy5VENZKifovUTZC9eUODWSBhOBS7zICGrAetKzdwLMr85KhIcePMAQ==" }, + "file-entry-cache": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/file-entry-cache/-/file-entry-cache-5.0.1.tgz", + "integrity": "sha512-bCg29ictuBaKUwwArK4ouCaqDgLZcysCFLmM/Yn/FDoqndh/9vNuQfXRDvTuXKLxfD/JtZQGKFT8MGcJBK644g==", + "dev": true, + "requires": { + "flat-cache": "^2.0.1" + } + }, "fill-range": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz", @@ -773,6 +1132,28 @@ "is-buffer": "~2.0.3" } }, + "flat-cache": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/flat-cache/-/flat-cache-2.0.1.tgz", + "integrity": "sha512-LoQe6yDuUMDzQAEH8sgmh4Md6oZnc/7PjtwjNFSzveXqSHt6ka9fPBuso7IGf9Rz4uqnSnWiFH2B/zj24a5ReA==", + "dev": true, + "requires": { + "flatted": "^2.0.0", + "rimraf": "2.6.3", + "write": "1.0.3" + }, + "dependencies": { + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "dev": true, + "requires": { + "glob": "^7.1.3" + } + } + } + }, "flatted": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", @@ -827,6 +1208,12 @@ "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", "dev": true }, + "functional-red-black-tree": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz", + "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", + "dev": true + }, "gensync": { "version": "1.0.0-beta.1", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz", @@ -954,6 +1341,30 @@ "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "ignore": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", + "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "dev": true + }, + "import-fresh": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.2.1.tgz", + "integrity": "sha512-6e1q1cnWP2RXD9/keSkxHScg508CdXqXWgWBaETNhyuBFz+kUZlKboh+ISK+bU++DmbHimVBrOz/zzPe0sZ3sQ==", + "dev": true, + "requires": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "dependencies": { + "resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "dev": true + } + } + }, "imurmurhash": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/imurmurhash/-/imurmurhash-0.1.4.tgz", @@ -1249,6 +1660,18 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + }, + "json-stable-stringify-without-jsonify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/json-stable-stringify-without-jsonify/-/json-stable-stringify-without-jsonify-1.0.1.tgz", + "integrity": "sha1-nbe1lJatPzz+8wp1FC0tkwrXJlE=", + "dev": true + }, "json5": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.1.3.tgz", @@ -1280,6 +1703,16 @@ "node-fetch": "3.0.0-beta.9" } }, + "levn": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", + "integrity": "sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==", + "dev": true, + "requires": { + "prelude-ls": "^1.2.1", + "type-check": "~0.4.0" + } + }, "locate-path": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", @@ -1396,6 +1829,15 @@ "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==", "dev": true }, + "mkdirp": { + "version": "0.5.5", + "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", + "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -1647,6 +2089,12 @@ "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", "dev": true }, + "natural-compare": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", + "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", + "dev": true + }, "node-fetch": { "version": "3.0.0-beta.9", "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.0.0-beta.9.tgz", @@ -1746,6 +2194,20 @@ "wrappy": "1" } }, + "optionator": { + "version": "0.9.1", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.1.tgz", + "integrity": "sha512-74RlY5FCnhq4jRxVUPKDaRwrVNXMqsGsiW6AJw4XK8hmtm10wC0ypZBLw5IIp85NZMr91+qd1RvvENwg7jjRFw==", + "dev": true, + "requires": { + "deep-is": "^0.1.3", + "fast-levenshtein": "^2.0.6", + "levn": "^0.4.1", + "prelude-ls": "^1.2.1", + "type-check": "^0.4.0", + "word-wrap": "^1.2.3" + } + }, "p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -1788,6 +2250,15 @@ "release-zalgo": "^1.0.0" } }, + "parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "dev": true, + "requires": { + "callsites": "^3.0.0" + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", @@ -1835,6 +2306,12 @@ "find-up": "^4.0.0" } }, + "prelude-ls": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.2.1.tgz", + "integrity": "sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==", + "dev": true + }, "process-on-spawn": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/process-on-spawn/-/process-on-spawn-1.0.0.tgz", @@ -1876,6 +2353,12 @@ "once": "^1.3.1" } }, + "punycode": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", + "dev": true + }, "puppeteer": { "version": "5.3.1", "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz", @@ -1922,6 +2405,12 @@ "picomatch": "^2.2.1" } }, + "regexpp": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/regexpp/-/regexpp-3.1.0.tgz", + "integrity": "sha512-ZOIzd8yVsQQA7j8GCSlPGXwg5PfmA1mrq0JP4nGhh54LaKN3xdai/vHUDu74pKwV8OxseMS65u2NImosQcSD0Q==", + "dev": true + }, "release-zalgo": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/release-zalgo/-/release-zalgo-1.0.0.tgz", @@ -2028,6 +2517,25 @@ "nan": "^2.14.1" } }, + "slice-ansi": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", + "integrity": "sha512-Qu+VC3EwYLldKa1fCxuuvULvSJOKEgk9pi8dZeCVK7TqBfUNTH4sFkk4joj8afVSfAYgJoSOetjx9QWOJ5mYoQ==", + "dev": true, + "requires": { + "ansi-styles": "^3.2.0", + "astral-regex": "^1.0.0", + "is-fullwidth-code-point": "^2.0.0" + }, + "dependencies": { + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + } + } + }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", @@ -2147,6 +2655,58 @@ "has-flag": "^3.0.0" } }, + "table": { + "version": "5.4.6", + "resolved": "https://registry.npmjs.org/table/-/table-5.4.6.tgz", + "integrity": "sha512-wmEc8m4fjnob4gt5riFRtTu/6+4rSe12TpAELNSqHMfF3IqnA+CH37USM6/YR3qRZv7e56kAEAtd6nKZaxe0Ug==", + "dev": true, + "requires": { + "ajv": "^6.10.2", + "lodash": "^4.17.14", + "slice-ansi": "^2.1.0", + "string-width": "^3.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-4.1.0.tgz", + "integrity": "sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg==", + "dev": true + }, + "emoji-regex": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-7.0.3.tgz", + "integrity": "sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA==", + "dev": true + }, + "is-fullwidth-code-point": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", + "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=", + "dev": true + }, + "string-width": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-3.1.0.tgz", + "integrity": "sha512-vafcv6KjVZKSgz06oM/H6GDBrAtz8vdhQakGjFIvNrHA6y3HCF1CInLy+QLq8dTJPQ1b+KDUqDFctkdRW44e1w==", + "dev": true, + "requires": { + "emoji-regex": "^7.0.1", + "is-fullwidth-code-point": "^2.0.0", + "strip-ansi": "^5.1.0" + } + }, + "strip-ansi": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz", + "integrity": "sha512-DuRs1gKbBqsMKIZlrffwlug8MHkcnpjs5VPmL1PAh+mA30U0DTotfDZ0d2UUsXpPmPmMMJ6W773MaA3J+lbiWA==", + "dev": true, + "requires": { + "ansi-regex": "^4.1.0" + } + } + } + }, "tar-fs": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", @@ -2181,6 +2741,12 @@ "minimatch": "^3.0.4" } }, + "text-table": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", + "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", + "dev": true + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", @@ -2201,6 +2767,15 @@ "is-number": "^7.0.0" } }, + "type-check": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", + "integrity": "sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==", + "dev": true, + "requires": { + "prelude-ls": "^1.2.1" + } + }, "type-detect": { "version": "4.0.8", "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", @@ -2236,6 +2811,15 @@ "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==" }, + "uri-js": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz", + "integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==", + "dev": true, + "requires": { + "punycode": "^2.1.0" + } + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -2247,6 +2831,12 @@ "integrity": "sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==", "dev": true }, + "v8-compile-cache": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.2.0.tgz", + "integrity": "sha512-gTpR5XQNKFwOd4clxfnhaqvfqMpqEwr4tOtCyz4MtYZX2JYhfr1JvBFKdS+7K/9rfpZR3VLX+YWBbKoxCgS43Q==", + "dev": true + }, "which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -2304,6 +2894,12 @@ } } }, + "word-wrap": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", + "dev": true + }, "workerpool": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.0.0.tgz", @@ -2353,6 +2949,15 @@ "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, + "write": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/write/-/write-1.0.3.tgz", + "integrity": "sha512-/lg70HAjtkUgWPVZhZcm+T4hkL8Zbtp1nFNOn3lRrxnlv50SRBv7cR7RqR+GMsd3hUXy9hWBo4CHTbFTcOYwig==", + "dev": true, + "requires": { + "mkdirp": "^0.5.1" + } + }, "write-file-atomic": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/write-file-atomic/-/write-file-atomic-3.0.3.tgz", diff --git a/package.json b/package.json index c14afea..2836d66 100644 --- a/package.json +++ b/package.json @@ -38,8 +38,10 @@ "puppeteer": "^5.3.1" }, "devDependencies": { + "babel-eslint": "^10.1.0", "chai": "^4.2.0", "dotenv": "^8.2.0", + "eslint": "^7.12.1", "lodash": "^4.17.20", "mocha": "^8.1.3", "nyc": "^15.1.0", From f7dd2d8e4ec5005cb59fee1d04b354508f8d8a12 Mon Sep 17 00:00:00 2001 From: "samuele.berlusconi" Date: Thu, 29 Oct 2020 22:14:40 +0100 Subject: [PATCH 02/17] Style fix --- app/index.js | 652 +++++++++++++------------- app/scripts/classes/game-download.js | 115 ++--- app/scripts/classes/game-info.js | 112 ++--- app/scripts/classes/login-result.js | 24 +- app/scripts/classes/user-data.js | 14 +- app/scripts/constants/css-selector.js | 58 +-- app/scripts/constants/url.js | 10 +- app/scripts/game-scraper.js | 512 ++++++++++---------- app/scripts/game-searcher.js | 128 ++--- app/scripts/puppeteer-helper.js | 56 +-- app/scripts/shared.js | 38 +- app/scripts/url-helper.js | 46 +- test/index-test.js | 364 +++++++------- test/user-test.js | 14 +- 14 files changed, 1072 insertions(+), 1071 deletions(-) diff --git a/app/index.js b/app/index.js index d03987b..1c99110 100644 --- a/app/index.js +++ b/app/index.js @@ -10,8 +10,8 @@ const selectorK = require("./scripts/constants/css-selector.js"); const urlHelper = require("./scripts/url-helper.js"); const scraper = require("./scripts/game-scraper.js"); const { - prepareBrowser, - preparePage, + prepareBrowser, + preparePage, } = require("./scripts/puppeteer-helper.js"); const searcher = require("./scripts/game-searcher.js"); @@ -32,10 +32,10 @@ module.exports.UserData = UserData; * @param {Boolean} value */ module.exports.debug = function (value) { - shared.debug = value; + shared.debug = value; - // Configure logger - shared.logger.level = value ? "debug" : "warn"; + // Configure logger + shared.logger.level = value ? "debug" : "warn"; }; /** * @public @@ -43,7 +43,7 @@ module.exports.debug = function (value) { * @returns {String} */ module.exports.isLogged = function () { - return shared.isLogged; + return shared.isLogged; }; /** * @public @@ -52,7 +52,7 @@ module.exports.isLogged = function () { * @returns {String} */ module.exports.setIsolation = function (value) { - shared.isolation = value; + shared.isolation = value; }; /** * @public @@ -60,7 +60,7 @@ module.exports.setIsolation = function (value) { * @returns {String} */ module.exports.getCacheDir = function () { - return shared.cacheDir; + return shared.cacheDir; }; /** * @public @@ -68,10 +68,10 @@ module.exports.getCacheDir = function () { * @returns {String} */ module.exports.setCacheDir = function (value) { - shared.cacheDir = value; + shared.cacheDir = value; - // Create directory if it doesn't exist - if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); + // Create directory if it doesn't exist + if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); }; /** * @public @@ -79,7 +79,7 @@ module.exports.setCacheDir = function (value) { * @returns {String} */ module.exports.setChromiumPath = function (value) { - shared.chromiumLocalPath = value; + shared.chromiumLocalPath = value; }; //#endregion Export properties @@ -98,41 +98,41 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue"; * @returns {Promise} Result of the operation */ module.exports.login = async function (username, password) { - if (shared.isLogged) { - shared.logger.info("Already logged in"); - const result = new LoginResult(true, "Already logged in"); - return result; - } + if (shared.isLogged) { + shared.logger.info("Already logged in"); + const result = new LoginResult(true, "Already logged in"); + return result; + } - // If cookies are loaded, use them to authenticate - shared.cookies = loadCookies(); - if (shared.cookies !== null) { - shared.logger.info("Valid session, no need to re-authenticate"); - shared.isLogged = true; - const result = new LoginResult(true, "Logged with cookies"); - return result; - } - - // Else, log in throught browser - shared.logger.info( - "No saved sessions or expired session, login on the platform" - ); - - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const result = await loginF95(browser, username, password); - shared.isLogged = result.success; - - if (result.success) { - // Reload cookies + // If cookies are loaded, use them to authenticate shared.cookies = loadCookies(); - shared.logger.info("User logged in through the platform"); - } else { - shared.logger.warn("Error during authentication: " + result.message); - } - if (shared.isolation) await browser.close(); - return result; + if (shared.cookies !== null) { + shared.logger.info("Valid session, no need to re-authenticate"); + shared.isLogged = true; + const result = new LoginResult(true, "Logged with cookies"); + return result; + } + + // Else, log in throught browser + shared.logger.info( + "No saved sessions or expired session, login on the platform" + ); + + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + + const result = await loginF95(browser, username, password); + shared.isLogged = result.success; + + if (result.success) { + // Reload cookies + shared.cookies = loadCookies(); + shared.logger.info("User logged in through the platform"); + } else { + shared.logger.warn("Error during authentication: " + result.message); + } + if (shared.isolation) await browser.close(); + return result; }; /** * @public @@ -142,47 +142,47 @@ module.exports.login = async function (username, password) { * @returns {Promise} Result of the operation */ module.exports.loadF95BaseData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return false; + } - shared.logger.info("Loading base data..."); + shared.logger.info("Loading base data..."); - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; + // Prepare a new web page + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login - // Go to latest update page and wait for it to load - await page.goto(urlK.F95_LATEST_UPDATES, { - waitUntil: shared.WAIT_STATEMENT, - }); + // Go to latest update page and wait for it to load + await page.goto(urlK.F95_LATEST_UPDATES, { + waitUntil: shared.WAIT_STATEMENT, + }); - // Obtain engines (disk/online) - await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); - shared.engines = await loadValuesFromLatestPage( - page, - shared.enginesCachePath, - selectorK.ENGINE_ID_SELECTOR, - "engines" - ); + // Obtain engines (disk/online) + await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); + shared.engines = await loadValuesFromLatestPage( + page, + shared.enginesCachePath, + selectorK.ENGINE_ID_SELECTOR, + "engines" + ); - // Obtain statuses (disk/online) - await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); - shared.statuses = await loadValuesFromLatestPage( - page, - shared.statusesCachePath, - selectorK.STATUS_ID_SELECTOR, - "statuses" - ); + // Obtain statuses (disk/online) + await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); + shared.statuses = await loadValuesFromLatestPage( + page, + shared.statusesCachePath, + selectorK.STATUS_ID_SELECTOR, + "statuses" + ); - await page.close(); - if (shared.isolation) await browser.close(); - shared.logger.info("Base data loaded"); - return true; + await page.close(); + if (shared.isolation) await browser.close(); + shared.logger.info("Base data loaded"); + return true; }; /** * @public @@ -192,25 +192,25 @@ module.exports.loadF95BaseData = async function () { * @returns {Promise} true if an update is available, false otherwise */ module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return false; + } - // F95 change URL at every game update, - // so if the URL is different an update is available - const exists = await urlHelper.urlExists(info.f95url, true); - if (!exists) return true; + // F95 change URL at every game update, + // so if the URL is different an update is available + const exists = await urlHelper.urlExists(info.f95url, true); + if (!exists) return true; - // Parse version from title - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; + // Parse version from title + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; - const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); + const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); - if (shared.isolation) await browser.close(); + if (shared.isolation) await browser.close(); - return onlineVersion.toUpperCase() !== info.version.toUpperCase(); + return onlineVersion.toUpperCase() !== info.version.toUpperCase(); }; /** * @public @@ -222,36 +222,36 @@ module.exports.chekIfGameHasUpdate = async function (info) { * an identified game (in the case of homonymy of titles) */ module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const urlList = await searcher.getSearchGameResults(browser, name); + // Gets the search results of the game being searched for + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + const urlList = await searcher.getSearchGameResults(browser, name); - // Process previous partial results - const promiseList = []; - for (const url of urlList) { + // Process previous partial results + const promiseList = []; + for (const url of urlList) { // Start looking for information - promiseList.push(scraper.getGameInfo(browser, url)); - } + promiseList.push(scraper.getGameInfo(browser, url)); + } - // Filter for mods - const result = []; - for (const info of await Promise.all(promiseList)) { + // Filter for mods + const result = []; + for (const info of await Promise.all(promiseList)) { // Ignore empty results - if (!info) continue; - // Skip mods if not required - if (info.isMod && !includeMods) continue; - // Else save data - result.push(info); - } + if (!info) continue; + // Skip mods if not required + if (info.isMod && !includeMods) continue; + // Else save data + result.push(info); + } - if (shared.isolation) await browser.close(); - return result; + if (shared.isolation) await browser.close(); + return result; }; /** * @public @@ -261,26 +261,26 @@ module.exports.getGameData = async function (name, includeMods) { * @returns {Promise} Information about the game. If no game was found, null is returned */ module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Check URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); - if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); + // Check URL + const exists = await urlHelper.urlExists(url); + if (!exists) throw new URIError(url + " is not a valid URL"); + if (!urlHelper.isF95URL(url)) + throw new Error(url + " is not a valid F95Zone URL"); - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; + // Gets the search results of the game being searched for + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; - // Get game data - const result = await scraper.getGameInfo(browser, url); + // Get game data + const result = await scraper.getGameInfo(browser, url); - if (shared.isolation) await browser.close(); - return result; + if (shared.isolation) await browser.close(); + return result; }; /** * @public @@ -289,47 +289,47 @@ module.exports.getGameDataFromURL = async function (url) { * @returns {Promise} Data of the user currently logged in */ module.exports.getUserData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_BASE_URL); // Go to base page + // Prepare a new web page + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(urlK.F95_BASE_URL); // Go to base page - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_ELEMENT), - page.waitForSelector(selectorK.AVATAR_PIC), - ]); + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.USERNAME_ELEMENT), + page.waitForSelector(selectorK.AVATAR_PIC), + ]); - const threads = getUserWatchedGameThreads(browser); + const threads = getUserWatchedGameThreads(browser); - const username = await page.evaluate( + const username = await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.USERNAME_ELEMENT - ); + document.querySelector(selector).innerText, + selectorK.USERNAME_ELEMENT + ); - const avatarSrc = await page.evaluate( + const avatarSrc = await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector).getAttribute("src"), - selectorK.AVATAR_PIC - ); + document.querySelector(selector).getAttribute("src"), + selectorK.AVATAR_PIC + ); - const ud = new UserData(); - ud.username = username; - ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = await threads; + const ud = new UserData(); + ud.username = username; + ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; + ud.watchedThreads = await threads; - await page.close(); - if (shared.isolation) await browser.close(); + await page.close(); + if (shared.isolation) await browser.close(); - return ud; + return ud; }; /** * @public @@ -337,19 +337,19 @@ module.exports.getUserData = async function () { * You **must** be logged in to the portal before calling this method. */ module.exports.logout = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return; - } + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return; + } - // Logout - shared.isLogged = false; + // Logout + shared.isLogged = false; - // Gracefully close shared browser - if (!shared.isolation && _browser !== null) { - await _browser.close(); - _browser = null; - } + // Gracefully close shared browser + if (!shared.isolation && _browser !== null) { + await _browser.close(); + _browser = null; + } }; //#endregion @@ -363,20 +363,20 @@ module.exports.logout = async function () { * @return {object[]} List of dictionaries or null if cookies don't exist */ function loadCookies() { - // Check the existence of the cookie file - if (fs.existsSync(shared.cookiesCachePath)) { + // Check the existence of the cookie file + if (fs.existsSync(shared.cookiesCachePath)) { // Read cookies - const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); - const cookies = JSON.parse(cookiesJSON); + const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); + const cookies = JSON.parse(cookiesJSON); - // Check if the cookies have expired - for (const cookie of cookies) { - if (isCookieExpired(cookie)) return null; - } + // Check if the cookies have expired + for (const cookie of cookies) { + if (isCookieExpired(cookie)) return null; + } - // Cookies loaded and verified - return cookies; - } else return null; + // Cookies loaded and verified + return cookies; + } else return null; } /** * @private @@ -385,25 +385,25 @@ function loadCookies() { * @returns {Boolean} true if the cookie has expired, false otherwise */ function isCookieExpired(cookie) { - // Local variables - let expiredCookies = false; + // Local variables + let expiredCookies = false; - // Ignore cookies that never expire - const expirationUnixTimestamp = cookie.expire; + // Ignore cookies that never expire + const expirationUnixTimestamp = cookie.expire; - if (expirationUnixTimestamp !== "-1") { + if (expirationUnixTimestamp !== "-1") { // Convert UNIX epoch timestamp to normal Date - const expirationDate = new Date(expirationUnixTimestamp * 1000); + const expirationDate = new Date(expirationUnixTimestamp * 1000); - if (expirationDate < Date.now()) { - shared.logger.warn( - "Cookie " + cookie.name + " expired, you need to re-authenticate" - ); - expiredCookies = true; + if (expirationDate < Date.now()) { + shared.logger.warn( + "Cookie " + cookie.name + " expired, you need to re-authenticate" + ); + expiredCookies = true; + } } - } - return expiredCookies; + return expiredCookies; } //#endregion Cookies functions @@ -420,27 +420,27 @@ function isCookieExpired(cookie) { * @returns {Promise} List of required values in uppercase */ async function loadValuesFromLatestPage( - page, - path, - selector, - elementRequested -) { - // If the values already exist they are loaded from disk without having to connect to F95 - shared.logger.info("Load " + elementRequested + " from disk..."); - if (fs.existsSync(path)) { - const valueJSON = fs.readFileSync(path); - return JSON.parse(valueJSON); - } - - // Otherwise, connect and download the data from the portal - shared.logger.info("No " + elementRequested + " cached, downloading..."); - const values = await getValuesFromLatestPage( page, + path, selector, - "Getting " + elementRequested + " from page" - ); - fs.writeFileSync(path, JSON.stringify(values)); - return values; + elementRequested +) { + // If the values already exist they are loaded from disk without having to connect to F95 + shared.logger.info("Load " + elementRequested + " from disk..."); + if (fs.existsSync(path)) { + const valueJSON = fs.readFileSync(path); + return JSON.parse(valueJSON); + } + + // Otherwise, connect and download the data from the portal + shared.logger.info("No " + elementRequested + " cached, downloading..."); + const values = await getValuesFromLatestPage( + page, + selector, + "Getting " + elementRequested + " from page" + ); + fs.writeFileSync(path, JSON.stringify(values)); + return values; } /** * @private @@ -453,20 +453,20 @@ async function loadValuesFromLatestPage( * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector */ async function getValuesFromLatestPage(page, selector, logMessage) { - shared.logger.info(logMessage); + shared.logger.info(logMessage); - const result = []; - const elements = await page.$$(selector); + const result = []; + const elements = await page.$$(selector); - for (const element of elements) { - const text = await element.evaluate( - /* istanbul ignore next */ (e) => e.innerText - ); + for (const element of elements) { + const text = await element.evaluate( + /* istanbul ignore next */ (e) => e.innerText + ); - // Save as upper text for better match if used in query - result.push(text.toUpperCase()); - } - return result; + // Save as upper text for better match if used in query + result.push(text.toUpperCase()); + } + return result; } //#endregion @@ -480,66 +480,66 @@ async function getValuesFromLatestPage(page, selector, logMessage) { * @returns {Promise} Result of the operation */ async function loginF95(browser, username, password) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_LOGIN_URL); // Go to login page + const page = await preparePage(browser); // Set new isolated page + await page.goto(urlK.F95_LOGIN_URL); // Go to login page - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_INPUT), - page.waitForSelector(selectorK.PASSWORD_INPUT), - page.waitForSelector(selectorK.LOGIN_BUTTON), - ]); + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.USERNAME_INPUT), + page.waitForSelector(selectorK.PASSWORD_INPUT), + page.waitForSelector(selectorK.LOGIN_BUTTON), + ]); - await page.type(selectorK.USERNAME_INPUT, username); // Insert username - await page.type(selectorK.PASSWORD_INPUT, password); // Insert password - await Promise.all([ - page.click(selectorK.LOGIN_BUTTON), // Click on the login button - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); + await page.type(selectorK.USERNAME_INPUT, username); // Insert username + await page.type(selectorK.PASSWORD_INPUT, password); // Insert password + await Promise.all([ + page.click(selectorK.LOGIN_BUTTON), // Click on the login button + page.waitForNavigation({ + waitUntil: shared.WAIT_STATEMENT, + }), // Wait for page to load + ]); - // Prepare result - let message = ""; + // Prepare result + let message = ""; - // Check if the user is logged in - const success = await page.evaluate( + // Check if the user is logged in + const success = await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector) !== null, - selectorK.AVATAR_INFO - ); - - const errorMessageExists = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector) !== null, - selectorK.LOGIN_MESSAGE_ERROR - ); - - // Save cookies to avoid re-auth - if (success) { - const c = await page.cookies(); - fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); - message = "Authentication successful"; - } else if (errorMessageExists) { - const errorMessage = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.LOGIN_MESSAGE_ERROR + document.querySelector(selector) !== null, + selectorK.AVATAR_INFO ); - if (errorMessage === "Incorrect password. Please try again.") { - message = "Incorrect password"; - } else if ( - errorMessage === - "The requested user '" + username + "' could not be found." - ) { - // The escaped quotes are important! - message = "Incorrect username"; - } else message = errorMessage; - } else message = "Unknown error"; + const errorMessageExists = await page.evaluate( + /* istanbul ignore next */ + (selector) => document.querySelector(selector) !== null, + selectorK.LOGIN_MESSAGE_ERROR + ); - await page.close(); // Close the page - return new LoginResult(success, message); + // Save cookies to avoid re-auth + if (success) { + const c = await page.cookies(); + fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); + message = "Authentication successful"; + } else if (errorMessageExists) { + const errorMessage = await page.evaluate( + /* istanbul ignore next */ (selector) => + document.querySelector(selector).innerText, + selectorK.LOGIN_MESSAGE_ERROR + ); + + if (errorMessage === "Incorrect password. Please try again.") { + message = "Incorrect password"; + } else if ( + errorMessage === + "The requested user '" + username + "' could not be found." + ) { + // The escaped quotes are important! + message = "Incorrect username"; + } else message = errorMessage; + } else message = "Unknown error"; + + await page.close(); // Close the page + return new LoginResult(success, message); } /** * @private @@ -548,61 +548,61 @@ async function loginF95(browser, username, password) { * @returns {Promise} URL list */ async function getUserWatchedGameThreads(browser) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page + const page = await preparePage(browser); // Set new isolated page + await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page - // Explicitly wait for the required items to load - await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); + // Explicitly wait for the required items to load + await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); - // Show the popup - await Promise.all([ - page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), - ]); + // Show the popup + await Promise.all([ + page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), + page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), + page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), + page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), + ]); - // Set the filters - await page.evaluate( + // Set the filters + await page.evaluate( /* istanbul ignore next */ (selector) => - document.querySelector(selector).removeAttribute("checked"), - selectorK.UNREAD_THREAD_CHECKBOX - ); // Also read the threads already read + document.querySelector(selector).removeAttribute("checked"), + selectorK.UNREAD_THREAD_CHECKBOX + ); // Also read the threads already read - // Filter the threads - await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); - await page.click(selectorK.FILTER_THREADS_BUTTON); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + // Filter the threads + await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); + await page.click(selectorK.FILTER_THREADS_BUTTON); + await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - // Get the threads urls - const urls = []; - let nextPageExists = false; - do { + // Get the threads urls + const urls = []; + let nextPageExists = false; + do { // Get all the URLs - for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { - const src = await page.evaluate( - /* istanbul ignore next */ (element) => element.href, - handle - ); - // If 'unread' is left, it will redirect to the last unread post - const url = src.replace("/unread", ""); - urls.push(url); - } + for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { + const src = await page.evaluate( + /* istanbul ignore next */ (element) => element.href, + handle + ); + // If 'unread' is left, it will redirect to the last unread post + const url = src.replace("/unread", ""); + urls.push(url); + } - nextPageExists = await page.evaluate( - /* istanbul ignore next */ (selector) => document.querySelector(selector), - selectorK.WATCHED_THREAD_NEXT_PAGE - ); + nextPageExists = await page.evaluate( + /* istanbul ignore next */ (selector) => document.querySelector(selector), + selectorK.WATCHED_THREAD_NEXT_PAGE + ); - // Click to next page - if (nextPageExists) { - await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - } - } while (nextPageExists); + // Click to next page + if (nextPageExists) { + await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); + await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + } + } while (nextPageExists); - await page.close(); - return urls; + await page.close(); + return urls; } //#endregion User diff --git a/app/scripts/classes/game-download.js b/app/scripts/classes/game-download.js index 4cc5868..e1240fb 100644 --- a/app/scripts/classes/game-download.js +++ b/app/scripts/classes/game-download.js @@ -4,6 +4,7 @@ // Core modules const fs = require("fs"); +const path = require("path"); // Public modules from npm // const { File } = require('megajs'); @@ -13,98 +14,98 @@ const { prepareBrowser, preparePage } = require("../puppeteer-helper.js"); const shared = require("../shared.js"); class GameDownload { - constructor() { + constructor() { /** * @public * Platform that hosts game files * @type String */ - this.hosting = ""; - /** + this.hosting = ""; + /** * @public * Link to game files * @type String */ - this.link = null; - /** + this.link = null; + /** * @public * Operating systems supported by the game version indicated in this class. * Can be *WINDOWS/LINUX/MACOS* * @type String[] */ - this.supportedOS = []; - } + this.supportedOS = []; + } - /** + /** * @public * Download the game data in the indicated path. * Supported hosting platforms: MEGA, NOPY * @param {String} path Save path * @return {Promise} Result of the operation */ - async download(path) { - if (this.link.includes("mega.nz")) - return await downloadMEGA(this.link, path); - else if (this.link.includes("nopy.to")) - return await downloadNOPY(this.link, path); - } + async download(path) { + if (this.link.includes("mega.nz")) + return await downloadMEGA(this.link, path); + else if (this.link.includes("nopy.to")) + return await downloadNOPY(this.link, path); + } } module.exports = GameDownload; async function downloadMEGA(url, savepath) { - // The URL is masked - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url); - await page.waitForSelector("a.host_link"); + // The URL is masked + const browser = await prepareBrowser(); + const page = await preparePage(browser); + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(url); + await page.waitForSelector("a.host_link"); - // Obtain the link for the unmasked page and click it - const link = await page.$("a.host_link"); - await link.click(); - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads + // Obtain the link for the unmasked page and click it + const link = await page.$("a.host_link"); + await link.click(); + await page.goto(url, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the game page and wait until it loads - // Obtain the URL after the redirect - const downloadURL = page.url(); + // Obtain the URL after the redirect + const downloadURL = page.url(); - // Close browser and page - await page.close(); - await browser.close(); + // Close browser and page + await page.close(); + await browser.close(); - const stream = fs.createWriteStream(savepath); - const file = File.fromURL(downloadURL); - file.download().pipe(stream); - return fs.existsSync(savepath); + const stream = fs.createWriteStream(savepath); + const file = File.fromURL(downloadURL); + file.download().pipe(stream); + return fs.existsSync(savepath); } async function downloadNOPY(url, savepath) { - // Prepare browser - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.goto(url); - await page.waitForSelector("#download"); + // Prepare browser + const browser = await prepareBrowser(); + const page = await preparePage(browser); + await page.goto(url); + await page.waitForSelector("#download"); - // Set the save path - await page._client.send("Page.setDownloadBehavior", { - behavior: "allow", - downloadPath: path.basename(path.dirname(savepath)), // It's a directory - }); + // Set the save path + await page._client.send("Page.setDownloadBehavior", { + behavior: "allow", + downloadPath: path.basename(path.dirname(savepath)), // It's a directory + }); - // Obtain the download button and click it - const downloadButton = await page.$("#download"); - await downloadButton.click(); + // Obtain the download button and click it + const downloadButton = await page.$("#download"); + await downloadButton.click(); - // Await for all the connections to close - await page.waitForNavigation({ - waitUntil: "networkidle0", - timeout: 0, // Disable timeout - }); + // Await for all the connections to close + await page.waitForNavigation({ + waitUntil: "networkidle0", + timeout: 0, // Disable timeout + }); - // Close browser and page - await page.close(); - await browser.close(); + // Close browser and page + await page.close(); + await browser.close(); - return fs.existsSync(savepath); + return fs.existsSync(savepath); } diff --git a/app/scripts/classes/game-info.js b/app/scripts/classes/game-info.js index 2b26de7..c9fcac5 100644 --- a/app/scripts/classes/game-info.js +++ b/app/scripts/classes/game-info.js @@ -1,119 +1,119 @@ "use strict"; class GameInfo { - constructor() { + constructor() { //#region Properties /** * Game name * @type String */ - this.name = null; - /** + this.name = null; + /** * Game author * @type String */ - this.author = null; - /** + this.author = null; + /** * URL to the game's official conversation on the F95Zone portal * @type String */ - this.f95url = null; - /** + this.f95url = null; + /** * Game description * @type String */ - this.overview = null; - /** + this.overview = null; + /** * List of tags associated with the game * @type String[] */ - this.tags = []; - /** + this.tags = []; + /** * Graphics engine used for game development * @type String */ - this.engine = null; - /** + this.engine = null; + /** * Progress of the game * @type String */ - this.status = null; - /** + this.status = null; + /** * Game description image URL * @type String */ - this.previewSource = null; - /** + this.previewSource = null; + /** * Game version * @type String */ - this.version = null; - /** + this.version = null; + /** * Last time the game underwent updates * @type String */ - this.lastUpdate = null; - /** + this.lastUpdate = null; + /** * Last time the local copy of the game was run * @type String */ - this.lastPlayed = null; - /** + this.lastPlayed = null; + /** * Specifies if the game is original or a mod * @type Boolean */ - this.isMod = false; - /** + this.isMod = false; + /** * Changelog for the last version. * @type String */ - this.changelog = null; - /** + this.changelog = null; + /** * Directory containing the local copy of the game * @type String */ - this.gameDir = null; - /** + this.gameDir = null; + /** * Information on game file download links, * including information on hosting platforms * and operating system supported by the specific link * @type GameDownload[] */ - this.downloadInfo = []; + this.downloadInfo = []; //#endregion Properties - } + } - /** + /** * Converts the object to a dictionary used for JSON serialization */ - /* istanbul ignore next */ - toJSON() { - return { - name: this.name, - author: this.author, - f95url: this.f95url, - overview: this.overview, - engine: this.engine, - status: this.status, - previewSource: this.previewSource, - version: this.version, - lastUpdate: this.lastUpdate, - lastPlayed: this.lastPlayed, - isMod: this.isMod, - changelog: this.changelog, - gameDir: this.gameDir, - downloadInfo: this.downloadInfo, - }; - } + /* istanbul ignore next */ + toJSON() { + return { + name: this.name, + author: this.author, + f95url: this.f95url, + overview: this.overview, + engine: this.engine, + status: this.status, + previewSource: this.previewSource, + version: this.version, + lastUpdate: this.lastUpdate, + lastPlayed: this.lastPlayed, + isMod: this.isMod, + changelog: this.changelog, + gameDir: this.gameDir, + downloadInfo: this.downloadInfo, + }; + } - /** + /** * Return a new GameInfo from a JSON string * @param {String} json JSON string used to create the new object * @returns {GameInfo} */ - /* istanbul ignore next */ - static fromJSON(json) { - return Object.assign(new GameInfo(), json); - } + /* istanbul ignore next */ + static fromJSON(json) { + return Object.assign(new GameInfo(), json); + } } module.exports = GameInfo; diff --git a/app/scripts/classes/login-result.js b/app/scripts/classes/login-result.js index 5eb9646..51fd9bc 100644 --- a/app/scripts/classes/login-result.js +++ b/app/scripts/classes/login-result.js @@ -4,17 +4,17 @@ * Object obtained in response to an attempt to login to the portal. */ class LoginResult { - constructor(success, message) { - /** - * Result of the login operation - * @type Boolean - */ - this.success = success; - /** - * Login response message - * @type String - */ - this.message = message; - } + constructor(success, message) { + /** + * Result of the login operation + * @type Boolean + */ + this.success = success; + /** + * Login response message + * @type String + */ + this.message = message; + } } module.exports = LoginResult; diff --git a/app/scripts/classes/user-data.js b/app/scripts/classes/user-data.js index d6fead0..7edb904 100644 --- a/app/scripts/classes/user-data.js +++ b/app/scripts/classes/user-data.js @@ -4,23 +4,23 @@ * Class containing the data of the user currently connected to the F95Zone platform. */ class UserData { - constructor() { + constructor() { /** * User username. * @type String */ - this.username = ""; - /** + this.username = ""; + /** * Path to the user's profile picture. * @type String */ - this.avatarSrc = null; - /** + this.avatarSrc = null; + /** * List of followed thread URLs. * @type URL[] */ - this.watchedThreads = []; - } + this.watchedThreads = []; + } } module.exports = UserData; diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index 6ef347e..bf0f60b 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -1,33 +1,33 @@ module.exports = Object.freeze({ - AVATAR_INFO: "span.avatar", - AVATAR_PIC: 'a[href="/account/"] > span.avatar > img[class^="avatar"]', - ENGINE_ID_SELECTOR: 'div[id^="btn-prefix_1_"]>span', - FILTER_THREADS_BUTTON: 'button[class="button--primary button"]', - GAME_IMAGES: 'img[src^="https://attachments.f95zone.to"]', - GAME_TAGS: "a.tagItem", - GAME_TITLE: "h1.p-title-value", - GAME_TITLE_PREFIXES: 'h1.p-title-value > a.labelLink > span[dir="auto"]', - LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: + AVATAR_INFO: "span.avatar", + AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", + ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", + FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", + GAME_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", + GAME_TAGS: "a.tagItem", + GAME_TITLE: "h1.p-title-value", + GAME_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", + LOGIN_BUTTON: "button.button--icon--login", + LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", - ONLY_GAMES_THREAD_OPTION: 'select[name="nodes[]"] > option[value="2"]', - PASSWORD_INPUT: 'input[name="password"]', - SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: 'input[name="keywords"][type="search"]', - SEARCH_ONLY_GAMES_OPTION: 'select[name="c[nodes][]"] > option[value="1"]', - STATUS_ID_SELECTOR: 'div[id^="btn-prefix_4_"]>span', - THREAD_POSTS: + ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", + PASSWORD_INPUT: "input[name=\"password\"]", + SEARCH_BUTTON: "form.block > * button.button--icon--search", + SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", + SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", + STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", + THREAD_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", - THREAD_TITLE: "h3.contentRow-title", - TITLE_ONLY_CHECKBOX: 'form.block > * input[name="c[title_only]"]', - UNREAD_THREAD_CHECKBOX: 'input[type="checkbox"][name="unread"]', - USERNAME_ELEMENT: 'a[href="/account/"] > span.p-navgroup-linkText', - USERNAME_INPUT: 'input[name="login"]', - WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", - WATCHED_THREAD_URLS: 'a[href^="/threads/"][data-tp-primary]', - DOWNLOAD_LINKS_CONTAINER: 'span[style="font-size: 18px"]', - SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", - SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", - THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", + THREAD_TITLE: "h3.contentRow-title", + TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", + UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", + USERNAME_INPUT: "input[name=\"login\"]", + WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", + WATCHED_THREAD_URLS: "a[href^=\"/threads/\"][data-tp-primary]", + DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", + SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", + SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", + THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", }); diff --git a/app/scripts/constants/url.js b/app/scripts/constants/url.js index 87a8847..fac63ba 100644 --- a/app/scripts/constants/url.js +++ b/app/scripts/constants/url.js @@ -1,7 +1,7 @@ module.exports = Object.freeze({ - F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search/?type=post", - F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_LOGIN_URL: "https://f95zone.to/login", - F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", + F95_BASE_URL: "https://f95zone.to", + F95_SEARCH_URL: "https://f95zone.to/search/?type=post", + F95_LATEST_UPDATES: "https://f95zone.to/latest", + F95_LOGIN_URL: "https://f95zone.to/login", + F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", }); diff --git a/app/scripts/game-scraper.js b/app/scripts/game-scraper.js index dd52d52..7b20e47 100644 --- a/app/scripts/game-scraper.js +++ b/app/scripts/game-scraper.js @@ -21,57 +21,57 @@ const urlHelper = require("./url-helper.js"); * looking for */ module.exports.getGameInfo = async function (browser, url) { - shared.logger.info("Obtaining game info"); + shared.logger.info("Obtaining game info"); - // Verify the correctness of the URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(`${url} is not a valid URL`); - if (!urlHelper.isF95URL(url)) - throw new Error(`${url} is not a valid F95Zone URL`); + // Verify the correctness of the URL + const exists = await urlHelper.urlExists(url); + if (!exists) throw new URIError(`${url} is not a valid URL`); + if (!urlHelper.isF95URL(url)) + throw new Error(`${url} is not a valid F95Zone URL`); - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(url, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the game page and wait until it loads - // It asynchronously searches for the elements and - // then waits at the end to compile the object to be returned - let info = new GameInfo(); - const title = getGameTitle(page); - const author = getGameAuthor(page); - const tags = getGameTags(page); - const redirectUrl = urlHelper.getUrlRedirect(url); - info = await parsePrefixes(page, info); // Fill status/engines/isMod - const structuredText = await getMainPostStructuredText(page); - const overview = getOverview(structuredText, info.isMod); - const parsedInfos = parseConversationPage(structuredText); - const previewSource = getGamePreviewSource(page); - const changelog = getLastChangelog(page); + // It asynchronously searches for the elements and + // then waits at the end to compile the object to be returned + let info = new GameInfo(); + const title = getGameTitle(page); + const author = getGameAuthor(page); + const tags = getGameTags(page); + const redirectUrl = urlHelper.getUrlRedirect(url); + info = await parsePrefixes(page, info); // Fill status/engines/isMod + const structuredText = await getMainPostStructuredText(page); + const overview = getOverview(structuredText, info.isMod); + const parsedInfos = parseConversationPage(structuredText); + const previewSource = getGamePreviewSource(page); + const changelog = getLastChangelog(page); - // Fill in the GameInfo element with the information obtained - info.name = await title; - info.author = await author; - info.tags = await tags; - info.f95url = await redirectUrl; - info.overview = overview; - info.lastUpdate = info.isMod - ? parsedInfos.UPDATED - : parsedInfos.THREAD_UPDATED; - info.previewSource = await previewSource; - info.changelog = await changelog; - info.version = await exports.getGameVersionFromTitle(browser, info); + // Fill in the GameInfo element with the information obtained + info.name = await title; + info.author = await author; + info.tags = await tags; + info.f95url = await redirectUrl; + info.overview = overview; + info.lastUpdate = info.isMod + ? parsedInfos.UPDATED + : parsedInfos.THREAD_UPDATED; + info.previewSource = await previewSource; + info.changelog = await changelog; + info.version = await exports.getGameVersionFromTitle(browser, info); - //let downloadData = getGameDownloadLink(page); - //info.downloadInfo = await downloadData; - /* Downloading games without going directly to + //let downloadData = getGameDownloadLink(page); + //info.downloadInfo = await downloadData; + /* Downloading games without going directly to * the platform appears to be prohibited by * the guidelines. It is therefore useless to * keep the links for downloading the games. */ - await page.close(); // Close the page - shared.logger.info("Founded data for " + info.name); - return info; + await page.close(); // Close the page + shared.logger.info("Founded data for " + info.name); + return info; }; /** @@ -81,27 +81,27 @@ module.exports.getGameInfo = async function (browser, url) { * @returns {Promise} Online version of the game */ module.exports.getGameVersionFromTitle = async function (browser, info) { - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(info.f95url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(info.f95url, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the game page and wait until it loads - // Get the title - const titleHTML = await page.evaluate( + // Get the title + const titleHTML = await page.evaluate( /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; + (selector) => document.querySelector(selector).innerHTML, + selectorK.GAME_TITLE + ); + const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; - // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] - const startIndex = title.indexOf("[") + 1; - const endIndex = title.indexOf("]", startIndex); - let version = title.substring(startIndex, endIndex).trim().toUpperCase(); - if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence - await page.close(); - return cleanFSString(version); + // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] + const startIndex = title.indexOf("[") + 1; + const endIndex = title.indexOf("]", startIndex); + let version = title.substring(startIndex, endIndex).trim().toUpperCase(); + if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence + await page.close(); + return cleanFSString(version); }; //#region Private methods @@ -111,8 +111,8 @@ module.exports.getGameVersionFromTitle = async function (browser, info) { * @returns {String} */ function cleanFSString(s) { - const rx = /[/\\?%*:|"<>]/g; - return s.replace(rx, ""); + const rx = /[/\\?%*:|"<>]/g; + return s.replace(rx, ""); } /** @@ -124,11 +124,11 @@ function cleanFSString(s) { * @returns {Promise} Game description */ function getOverview(text, isMod) { - // Get overview (different parsing for game and mod) - let overviewEndIndex; - if (isMod) overviewEndIndex = text.indexOf("Updated"); - else overviewEndIndex = text.indexOf("Thread Updated"); - return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); + // Get overview (different parsing for game and mod) + let overviewEndIndex; + if (isMod) overviewEndIndex = text.indexOf("Updated"); + else overviewEndIndex = text.indexOf("Thread Updated"); + return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); } /** @@ -139,16 +139,16 @@ function getOverview(text, isMod) { * @returns {Promise} Structured text */ async function getMainPostStructuredText(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; + // Gets the first post, where are listed all the game's informations + const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - // The info are plain text so we need to parse the HTML code - const bodyHTML = await page.evaluate( + // The info are plain text so we need to parse the HTML code + const bodyHTML = await page.evaluate( /* istanbul ignore next */ - (mainPost) => mainPost.innerHTML, - post - ); - return HTMLParser.parse(bodyHTML).structuredText; + (mainPost) => mainPost.innerHTML, + post + ); + return HTMLParser.parse(bodyHTML).structuredText; } /** @@ -158,20 +158,20 @@ async function getMainPostStructuredText(page) { * @returns {Promise} Game author */ async function getGameAuthor(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( + // Get the game/mod name (without square brackets) + const titleHTML = await page.evaluate( /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); + (selector) => document.querySelector(selector).innerHTML, + selectorK.GAME_TITLE + ); + const structuredTitle = HTMLParser.parse(titleHTML); - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; + // The last element **shoud be** the title without prefixes (engines, status, other...) + const gameTitle = structuredTitle.childNodes.pop().rawText; - // The last square brackets contain the author - const startTitleIndex = gameTitle.lastIndexOf("[") + 1; - return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); + // The last square brackets contain the author + const startTitleIndex = gameTitle.lastIndexOf("[") + 1; + return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); } /** @@ -182,23 +182,23 @@ async function getGameAuthor(page) { * @returns {Object} Dictionary of information */ function parseConversationPage(text) { - const dataPairs = {}; + const dataPairs = {}; - // The information searched in the game post are one per line - const splittedText = text.split("\n"); - for (const line of splittedText) { - if (!line.includes(":")) continue; + // The information searched in the game post are one per line + const splittedText = text.split("\n"); + for (const line of splittedText) { + if (!line.includes(":")) continue; - // Create pair key/value - const splitted = line.split(":"); - const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch - const value = splitted[1].trim(); + // Create pair key/value + const splitted = line.split(":"); + const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch + const value = splitted[1].trim(); - // Add pair to the dict if valid - if (value !== "") dataPairs[key] = value; - } + // Add pair to the dict if valid + if (value !== "") dataPairs[key] = value; + } - return dataPairs; + return dataPairs; } /** @@ -208,27 +208,27 @@ function parseConversationPage(text) { * @returns {Promise} URL (String) of the image or null if failed to get it */ async function getGamePreviewSource(page) { - // Wait for the selector or return an empty value - try { - await page.waitForSelector(selectorK.GAME_IMAGES); - } catch { - return null; - } + // Wait for the selector or return an empty value + try { + await page.waitForSelector(selectorK.GAME_IMAGES); + } catch { + return null; + } - const src = await page.evaluate( + const src = await page.evaluate( /* istanbul ignore next */ - (selector) => { - // Get the firs image available - const img = document.querySelector(selector); + (selector) => { + // Get the firs image available + const img = document.querySelector(selector); - if (img) return img.getAttribute("src"); - else return null; - }, - selectorK.GAME_IMAGES - ); + if (img) return img.getAttribute("src"); + else return null; + }, + selectorK.GAME_IMAGES + ); - // Check if the URL is valid - return urlHelper.isStringAValidURL(src) ? src : null; + // Check if the URL is valid + return urlHelper.isStringAValidURL(src) ? src : null; } /** @@ -238,18 +238,18 @@ async function getGamePreviewSource(page) { * @returns {Promise} Game title */ async function getGameTitle(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( + // Get the game/mod name (without square brackets) + const titleHTML = await page.evaluate( /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); + (selector) => document.querySelector(selector).innerHTML, + selectorK.GAME_TITLE + ); + const structuredTitle = HTMLParser.parse(titleHTML); - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - const endTitleIndex = gameTitle.indexOf("["); - return gameTitle.substring(0, endTitleIndex).trim(); + // The last element **shoud be** the title without prefixes (engines, status, other...) + const gameTitle = structuredTitle.childNodes.pop().rawText; + const endTitleIndex = gameTitle.indexOf("["); + return gameTitle.substring(0, endTitleIndex).trim(); } /** @@ -259,18 +259,18 @@ async function getGameTitle(page) { * @returns {Promise} List of uppercase tags */ async function getGameTags(page) { - const tags = []; + const tags = []; - // Get the game tags - for (const handle of await page.$$(selectorK.GAME_TAGS)) { - const tag = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - tags.push(tag.toUpperCase()); - } - return tags.sort(); + // Get the game tags + for (const handle of await page.$$(selectorK.GAME_TAGS)) { + const tag = await page.evaluate( + /* istanbul ignore next */ + (element) => element.innerText, + handle + ); + tags.push(tag.toUpperCase()); + } + return tags.sort(); } /** @@ -282,25 +282,25 @@ async function getGameTags(page) { * @returns {Promise} GameInfo object passed in to which the identified information has been added */ async function parsePrefixes(page, info) { - // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' - info.status = "ONGOING"; - for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { - const value = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); + // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' + info.status = "ONGOING"; + for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { + const value = await page.evaluate( + /* istanbul ignore next */ + (element) => element.innerText, + handle + ); - // Clean the prefix - const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); + // Clean the prefix + const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); - // Getting infos... - if (shared.statuses.includes(prefix)) info.status = prefix; - else if (shared.engines.includes(prefix)) info.engine = prefix; - // This is not a game but a mod - else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; - } - return info; + // Getting infos... + if (shared.statuses.includes(prefix)) info.status = prefix; + else if (shared.engines.includes(prefix)) info.engine = prefix; + // This is not a game but a mod + else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; + } + return info; } /** @@ -310,24 +310,24 @@ async function parsePrefixes(page, info) { * @returns {Promise} Changelog for the last version or a empty string if no changelog is found */ async function getLastChangelog(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; + // Gets the first post, where are listed all the game's informations + const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG); - if (!spoiler) return ""; + const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG); + if (!spoiler) return ""; - const changelogHTML = await page.evaluate( + const changelogHTML = await page.evaluate( /* istanbul ignore next */ - (e) => e.innerText, - spoiler - ); - let parsedText = HTMLParser.parse(changelogHTML).structuredText; + (e) => e.innerText, + spoiler + ); + let parsedText = HTMLParser.parse(changelogHTML).structuredText; - // Clean the text - if (parsedText.startsWith("Spoiler")) - parsedText = parsedText.replace("Spoiler", ""); - if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", ""); - return parsedText.trim(); + // Clean the text + if (parsedText.startsWith("Spoiler")) + parsedText = parsedText.replace("Spoiler", ""); + if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", ""); + return parsedText.trim(); } /** @@ -340,64 +340,64 @@ async function getLastChangelog(page) { /* istanbul ignore next */ // skipcq: JS-0128 async function getGameDownloadLink(page) { - // Most used hosting platforms - const hostingPlatforms = [ - "MEGA", - "NOPY", - "FILESUPLOAD", - "MIXDROP", - "UPLOADHAVEN", - "PIXELDRAIN", - "FILESFM", - ]; + // Most used hosting platforms + const hostingPlatforms = [ + "MEGA", + "NOPY", + "FILESUPLOAD", + "MIXDROP", + "UPLOADHAVEN", + "PIXELDRAIN", + "FILESFM", + ]; - // Supported OS platforms - const platformOS = ["WIN", "LINUX", "MAC", "ALL"]; + // Supported OS platforms + const platformOS = ["WIN", "LINUX", "MAC", "ALL"]; - // Gets the which contains the download links - const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER); - if (temp.length === 0) return []; + // Gets the which contains the download links + const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER); + if (temp.length === 0) return []; - // Look for the container that contains the links - // It is necessary because the same css selector - // also identifies other elements on the page - let container = null; - for (const candidate of temp) { - if (container !== null) break; - const upperText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - candidate - ) - ).toUpperCase(); + // Look for the container that contains the links + // It is necessary because the same css selector + // also identifies other elements on the page + let container = null; + for (const candidate of temp) { + if (container !== null) break; + const upperText = ( + await page.evaluate( + /* istanbul ignore next */ + (e) => e.innerText, + candidate + ) + ).toUpperCase(); - // Search if the container contains the name of a hosting platform - for (const p of hostingPlatforms) { - if (upperText.includes(p)) { - container = candidate; - break; - } + // Search if the container contains the name of a hosting platform + for (const p of hostingPlatforms) { + if (upperText.includes(p)) { + container = candidate; + break; + } + } } - } - if (container === null) return []; + if (container === null) return []; - // Extract the HTML text from the container - const searchText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerHTML, - container - ) - ).toLowerCase(); + // Extract the HTML text from the container + const searchText = ( + await page.evaluate( + /* istanbul ignore next */ + (e) => e.innerHTML, + container + ) + ).toLowerCase(); - // Parse the download links - const downloadData = []; - for (const platform of platformOS) { - const data = extractGameHostingData(platform, searchText); - downloadData.push(...data); - } - return downloadData; + // Parse the download links + const downloadData = []; + for (const platform of platformOS) { + const data = extractGameHostingData(platform, searchText); + downloadData.push(...data); + } + return downloadData; } /** @@ -411,57 +411,57 @@ async function getGameDownloadLink(page) { */ /* istanbul ignore next */ function extractGameHostingData(platform, text) { - const PLATFORM_BOLD_OPEN = ""; - const CONTAINER_SPAN_CLOSE = ""; - const LINK_OPEN = "platform - let endIndex = + // Find the platform + let endIndex = text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length; - // Find the end of the container - if (endIndex === -1) - endIndex = + // Find the end of the container + if (endIndex === -1) + endIndex = text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) + CONTAINER_SPAN_CLOSE.length; - text = text.substring(startIndex, endIndex); + text = text.substring(startIndex, endIndex); - const downloadData = []; - const linkTags = text.split(LINK_OPEN); - for (const tag of linkTags) { + const downloadData = []; + const linkTags = text.split(LINK_OPEN); + for (const tag of linkTags) { // Ignore non-link string - if (!tag.includes(HREF_START)) continue; + if (!tag.includes(HREF_START)) continue; - // Find the hosting platform name - startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length; - endIndex = tag.indexOf(LINK_CLOSE, startIndex); - const hosting = tag.substring(startIndex, endIndex); + // Find the hosting platform name + startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length; + endIndex = tag.indexOf(LINK_CLOSE, startIndex); + const hosting = tag.substring(startIndex, endIndex); - // Find the 'href' attribute - startIndex = tag.indexOf(HREF_START) + HREF_START.length; - endIndex = tag.indexOf(HREF_END, startIndex); - const link = tag.substring(startIndex, endIndex); + // Find the 'href' attribute + startIndex = tag.indexOf(HREF_START) + HREF_START.length; + endIndex = tag.indexOf(HREF_END, startIndex); + const link = tag.substring(startIndex, endIndex); - if (urlHelper.isStringAValidURL(link)) { - const gd = new GameDownload(); - gd.hosting = hosting.toUpperCase(); - gd.link = link; - gd.supportedOS = platform.toUpperCase(); + if (urlHelper.isStringAValidURL(link)) { + const gd = new GameDownload(); + gd.hosting = hosting.toUpperCase(); + gd.link = link; + gd.supportedOS = platform.toUpperCase(); - downloadData.push(gd); + downloadData.push(gd); + } } - } - return downloadData; + return downloadData; } //#endregion Private methods diff --git a/app/scripts/game-searcher.js b/app/scripts/game-searcher.js index 592b490..497edfe 100644 --- a/app/scripts/game-searcher.js +++ b/app/scripts/game-searcher.js @@ -18,46 +18,46 @@ const { isF95URL } = require("./url-helper.js"); * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal */ module.exports.getSearchGameResults = async function (browser, gamename) { - shared.logger.info(`Searching ${gamename} on F95Zone`); + shared.logger.info(`Searching ${gamename} on F95Zone`); - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_SEARCH_URL, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the search form and wait for it + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(urlK.F95_SEARCH_URL, { + waitUntil: shared.WAIT_STATEMENT, + }); // Go to the search form and wait for it - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), - page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), - page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), - page.waitForSelector(selectorK.SEARCH_BUTTON), - ]); + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), + page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), + page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), + page.waitForSelector(selectorK.SEARCH_BUTTON), + ]); - await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire - await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles - await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod - await Promise.all([ - page.click(selectorK.SEARCH_BUTTON), // Execute search - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); + await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire + await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles + await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod + await Promise.all([ + page.click(selectorK.SEARCH_BUTTON), // Execute search + page.waitForNavigation({ + waitUntil: shared.WAIT_STATEMENT, + }), // Wait for page to load + ]); - // Select all conversation titles - const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY); + // Select all conversation titles + const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY); - // For each element found extract the info about the conversation - shared.logger.info("Extracting info from conversations"); - const results = []; - for (const element of resultsThread) { - const gameUrl = await getOnlyGameThreads(page, element); - if (gameUrl !== null) results.push(gameUrl); - } - shared.logger.info(`Find ${results.length} conversations`); - await page.close(); // Close the page + // For each element found extract the info about the conversation + shared.logger.info("Extracting info from conversations"); + const results = []; + for (const element of resultsThread) { + const gameUrl = await getOnlyGameThreads(page, element); + if (gameUrl !== null) results.push(gameUrl); + } + shared.logger.info(`Find ${results.length} conversations`); + await page.close(); // Close the page - return results; + return results; }; //#region Private methods @@ -69,16 +69,16 @@ module.exports.getSearchGameResults = async function (browser, gamename) { * @return {Promise} URL of the game/mod or null if the URL is not of a game */ async function getOnlyGameThreads(page, divHandle) { - // Obtain the elements containing the basic information - const titleHandle = await divHandle.$(selectorK.THREAD_TITLE); - const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP); + // Obtain the elements containing the basic information + const titleHandle = await divHandle.$(selectorK.THREAD_TITLE); + const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP); - // Get the forum where the thread was posted - const forum = await getMembershipForum(page, forumHandle); - if (forum !== "GAMES" && forum !== "MODS") return null; + // Get the forum where the thread was posted + const forum = await getMembershipForum(page, forumHandle); + if (forum !== "GAMES" && forum !== "MODS") return null; - // Get the URL of the thread from the title - return await getThreadURL(page, titleHandle); + // Get the URL of the thread from the title + return await getThreadURL(page, titleHandle); } /** @@ -89,23 +89,23 @@ async function getOnlyGameThreads(page, divHandle) { * @returns {Promise} Uppercase membership category */ async function getMembershipForum(page, handle) { - // The link can be something like: - // + /forums/request.NUMBER/ - // + /forums/game-recommendations-identification.NUMBER/ - // + /forums/games.NUMBER/ <-- We need this + // The link can be something like: + // + /forums/request.NUMBER/ + // + /forums/game-recommendations-identification.NUMBER/ + // + /forums/games.NUMBER/ <-- We need this - let link = await page.evaluate( + let link = await page.evaluate( /* istanbul ignore next */ - (e) => e.getAttribute("href"), - handle - ); + (e) => e.getAttribute("href"), + handle + ); - // Parse link - link = link.replace("/forums/", ""); - const endIndex = link.indexOf("."); - const forum = link.substring(0, endIndex); + // Parse link + link = link.replace("/forums/", ""); + const endIndex = link.indexOf("."); + const forum = link.substring(0, endIndex); - return forum.toUpperCase(); + return forum.toUpperCase(); } /** @@ -116,17 +116,17 @@ async function getMembershipForum(page, handle) { * @returns {Promise} URL of the thread */ async function getThreadURL(page, handle) { - const relativeURLThread = await page.evaluate( + const relativeURLThread = await page.evaluate( /* istanbul ignore next */ - (e) => e.querySelector("a").href, - handle - ); + (e) => e.querySelector("a").href, + handle + ); - // Some game already have a full URL... - if (isF95URL(relativeURLThread)) return relativeURLThread; + // Some game already have a full URL... + if (isF95URL(relativeURLThread)) return relativeURLThread; - // ... else compose the URL and return - const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString(); - return urlThread; + // ... else compose the URL and return + const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString(); + return urlThread; } //#endregion Private methods diff --git a/app/scripts/puppeteer-helper.js b/app/scripts/puppeteer-helper.js index 18a40a4..942da84 100644 --- a/app/scripts/puppeteer-helper.js +++ b/app/scripts/puppeteer-helper.js @@ -13,20 +13,20 @@ const shared = require("./shared.js"); * @returns {Promise} Created browser */ module.exports.prepareBrowser = async function () { - // Create a headless browser - let browser = null; - if (shared.chromiumLocalPath) { - browser = await puppeteer.launch({ - executablePath: shared.chromiumLocalPath, - headless: !shared.debug, // Use GUI when debug = true - }); - } else { - browser = await puppeteer.launch({ - headless: !shared.debug, // Use GUI when debug = true - }); - } + // Create a headless browser + let browser = null; + if (shared.chromiumLocalPath) { + browser = await puppeteer.launch({ + executablePath: shared.chromiumLocalPath, + headless: !shared.debug, // Use GUI when debug = true + }); + } else { + browser = await puppeteer.launch({ + headless: !shared.debug, // Use GUI when debug = true + }); + } - return browser; + return browser; }; /** @@ -37,24 +37,24 @@ module.exports.prepareBrowser = async function () { * @returns {Promise} New page */ module.exports.preparePage = async function (browser) { - // Create new page in the browser argument - const page = await browser.newPage(); + // Create new page in the browser argument + const page = await browser.newPage(); - // Block image download - await page.setRequestInterception(true); - page.on("request", (request) => { - if (request.resourceType() === "image") request.abort(); - else if (request.resourceType === "font") request.abort(); - // else if (request.resourceType() == 'stylesheet') request.abort(); - // else if(request.resourceType == 'media') request.abort(); - else request.continue(); - }); + // Block image download + await page.setRequestInterception(true); + page.on("request", (request) => { + if (request.resourceType() === "image") request.abort(); + else if (request.resourceType === "font") request.abort(); + // else if (request.resourceType() == 'stylesheet') request.abort(); + // else if(request.resourceType == 'media') request.abort(); + else request.continue(); + }); - // Set custom user-agent - const userAgent = + // Set custom user-agent + const userAgent = "Mozilla/5.0 (X11; Linux x86_64)" + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; - await page.setUserAgent(userAgent); + await page.setUserAgent(userAgent); - return page; + return page; }; diff --git a/app/scripts/shared.js b/app/scripts/shared.js index ead66fc..85a5ba4 100644 --- a/app/scripts/shared.js +++ b/app/scripts/shared.js @@ -9,7 +9,7 @@ const log4js = require("log4js"); * Class containing variables shared between modules. */ class Shared { - //#region Properties + //#region Properties /** * Shows log messages and other useful functions for module debugging. * @type Boolean @@ -64,63 +64,63 @@ class Shared { * @returns {Boolean} */ static get debug() { - return this.#_debug; + return this.#_debug; } /** * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {Boolean} */ static get isLogged() { - return this.#_isLogged; + return this.#_isLogged; } /** * List of cookies obtained from the F95Zone platform. * @returns {Object[]} */ static get cookies() { - return this.#_cookies; + return this.#_cookies; } /** * List of possible game engines used for development. * @returns {String[]} */ static get engines() { - return this.#_engines; + return this.#_engines; } /** * List of possible development states that a game can assume. * @returns {String[]} */ static get statuses() { - return this.#_statuses; + return this.#_statuses; } /** * Directory to save the API cache. * @returns {String} */ static get cacheDir() { - return this.#_cacheDir; + return this.#_cacheDir; } /** * Path to the F95 platform cache. * @returns {String} */ static get cookiesCachePath() { - return join(this.#_cacheDir, "cookies.json"); + return join(this.#_cacheDir, "cookies.json"); } /** * Path to the game engine cache. * @returns {String} */ static get enginesCachePath() { - return join(this.#_cacheDir, "engines.json"); + return join(this.#_cacheDir, "engines.json"); } /** * Path to the cache of possible game states. * @returns {String} */ static get statusesCachePath() { - return join(this.#_cacheDir, "statuses.json"); + return join(this.#_cacheDir, "statuses.json"); } /** * If true, it opens a new browser for each request @@ -128,44 +128,44 @@ class Shared { * @returns {Boolean} */ static get isolation() { - return this.#_isolation; + return this.#_isolation; } /** * Logger object used to write to both file and console. * @returns {log4js.Logger} */ static get logger() { - return this.#_logger; + return this.#_logger; } //#endregion Getters //#region Setters static set cookies(val) { - this.#_cookies = val; + this.#_cookies = val; } static set engines(val) { - this.#_engines = val; + this.#_engines = val; } static set statuses(val) { - this.#_statuses = val; + this.#_statuses = val; } static set cacheDir(val) { - this.#_cacheDir = val; + this.#_cacheDir = val; } static set debug(val) { - this.#_debug = val; + this.#_debug = val; } static set isLogged(val) { - this.#_isLogged = val; + this.#_isLogged = val; } static set isolation(val) { - this.#_isolation = val; + this.#_isolation = val; } //#endregion Setters } diff --git a/app/scripts/url-helper.js b/app/scripts/url-helper.js index 221a56c..aae60a0 100644 --- a/app/scripts/url-helper.js +++ b/app/scripts/url-helper.js @@ -2,7 +2,7 @@ // Public modules from npm const ky = require("ky-universal").create({ - throwHttpErrors: false, + throwHttpErrors: false, }); // Modules from file @@ -15,8 +15,8 @@ const { F95_BASE_URL } = require("./constants/url.js"); * @returns {Boolean} true if the url belongs to the domain, false otherwise */ module.exports.isF95URL = function (url) { - if (url.toString().startsWith(F95_BASE_URL)) return true; - else return false; + if (url.toString().startsWith(F95_BASE_URL)) return true; + else return false; }; /** @@ -26,12 +26,12 @@ module.exports.isF95URL = function (url) { * @returns {Boolean} true if the string is a valid URL, false otherwise */ module.exports.isStringAValidURL = function (url) { - try { - new URL(url); // skipcq: JS-0078 - return true; - } catch (err) { - return false; - } + try { + new URL(url); // skipcq: JS-0078 + return true; + } catch (err) { + return false; + } }; /** @@ -42,22 +42,22 @@ module.exports.isStringAValidURL = function (url) { * @returns {Promise} true if the URL exists, false otherwise */ module.exports.urlExists = async function (url, checkRedirect) { - if (!exports.isStringAValidURL(url)) { - return false; - } + if (!exports.isStringAValidURL(url)) { + return false; + } - const response = await ky.head(url); - let valid = response !== undefined && !/4\d\d/.test(response.status); + const response = await ky.head(url); + let valid = response !== undefined && !/4\d\d/.test(response.status); - if (!valid) return false; + if (!valid) return false; - if (checkRedirect) { - const redirectUrl = await exports.getUrlRedirect(url); - if (redirectUrl === url) valid = true; - else valid = false; - } + if (checkRedirect) { + const redirectUrl = await exports.getUrlRedirect(url); + if (redirectUrl === url) valid = true; + else valid = false; + } - return valid; + return valid; }; /** @@ -67,6 +67,6 @@ module.exports.urlExists = async function (url, checkRedirect) { * @returns {Promise} Redirect URL or the passed URL */ module.exports.getUrlRedirect = async function (url) { - const response = await ky.head(url); - return response.url; + const response = await ky.head(url); + return response.url; }; diff --git a/test/index-test.js b/test/index-test.js index 4d89e73..7b69d4c 100644 --- a/test/index-test.js +++ b/test/index-test.js @@ -27,264 +27,264 @@ const FAKE_PASSWORD = "fake_password"; //F95API.debug(false); function randomSleep() { - const random = Math.floor(Math.random() * 500) + 50; - sleep.msleep(500 + random); + const random = Math.floor(Math.random() * 500) + 50; + sleep.msleep(500 + random); } describe("Login without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout - before("Set isolation", function () { - F95API.setIsolation(true); - }); + before("Set isolation", function () { + F95API.setIsolation(true); + }); - beforeEach("Remove all cookies", function () { + beforeEach("Remove all cookies", function () { // Runs before each test in this block - if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up + if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up - let testOrder = 0; + let testOrder = 0; - it("Test with valid credentials", async function () { + it("Test with valid credentials", async function () { // Gain exclusive use of the cookies - while (testOrder !== 0) randomSleep(); + while (testOrder !== 0) randomSleep(); - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Authentication successful"); + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(result.message).equal("Authentication successful"); - testOrder = 1; - }); - it("Test with invalid username", async function () { + testOrder = 1; + }); + it("Test with invalid username", async function () { // Gain exclusive use of the cookies - while (testOrder !== 1) randomSleep(); + while (testOrder !== 1) randomSleep(); - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect username"); - testOrder = 2; - }); - it("Test with invalid password", async function () { + testOrder = 2; + }); + it("Test with invalid password", async function () { // Gain exclusive use of the cookies - while (testOrder !== 2) randomSleep(); + while (testOrder !== 2) randomSleep(); - const result = await F95API.login(USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect password"); + const result = await F95API.login(USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect password"); - testOrder = 3; - }); - it("Test with invalid credentials", async function () { + testOrder = 3; + }); + it("Test with invalid credentials", async function () { // Gain exclusive use of the cookies - while (testOrder !== 3) randomSleep(); + while (testOrder !== 3) randomSleep(); - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); // It should first check the username + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect username"); // It should first check the username - testOrder = 4; - }); + testOrder = 4; + }); }); describe("Login with cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout - before("Log in to create cookies then logout", async function () { + before("Log in to create cookies then logout", async function () { // Runs once before the first test in this block - if (!fs.existsSync(COOKIES_SAVE_PATH)) - await F95API.login(USERNAME, PASSWORD); // Download cookies - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up + if (!fs.existsSync(COOKIES_SAVE_PATH)) + await F95API.login(USERNAME, PASSWORD); // Download cookies + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up - it("Test with valid credentials", async function () { - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Logged with cookies"); - }); + it("Test with valid credentials", async function () { + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(result.message).equal("Logged with cookies"); + }); }); describe("Load base data without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout - before("Delete cache if exists", function () { + before("Delete cache if exists", function () { // Runs once before the first test in this block - if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); - if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); - }); - //#endregion Set-up + if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); + if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); + }); + //#endregion Set-up - it("With login", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("With login", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - const result = await F95API.loadF95BaseData(); + const result = await F95API.loadF95BaseData(); - const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); - const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); + const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); + const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); - expect(result).to.be.true; - expect(enginesCacheExists).to.be.true; - expect(statusesCacheExists).to.be.true; - }); + expect(result).to.be.true; + expect(enginesCacheExists).to.be.true; + expect(statusesCacheExists).to.be.true; + }); - it("Without login", async function () { - if (F95API.isLogged()) F95API.logout(); - const result = await F95API.loadF95BaseData(); - expect(result).to.be.false; - }); + it("Without login", async function () { + if (F95API.isLogged()) F95API.logout(); + const result = await F95API.loadF95BaseData(); + expect(result).to.be.false; + }); }); describe("Search game data", function () { - //#region Set-up - this.timeout(60000); // All tests in this suite get 60 seconds before timeout + //#region Set-up + this.timeout(60000); // All tests in this suite get 60 seconds before timeout - beforeEach("Prepare API", function () { + beforeEach("Prepare API", function () { // Runs once before the first test in this block - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up - let testGame = null; + let testGame = null; - it("Search game when logged", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("Search game when logged", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; + const loadResult = await F95API.loadF95BaseData(); + expect(loadResult).to.be.true; - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const gamesList = await F95API.getGameData("Kingdom of Deception", false); - expect(gamesList.length, "Should find only the game").to.equal(1); - const result = gamesList[0]; - const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ + const gamesList = await F95API.getGameData("Kingdom of Deception", false); + expect(gamesList.length, "Should find only the game").to.equal(1); + const result = gamesList[0]; + const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; - // Test only the main information - expect(result.name).to.equal("Kingdom of Deception"); - expect(result.author).to.equal("Hreinn Games"); - expect(result.isMod, "Should be false").to.be.false; - expect(result.engine).to.equal("REN'PY"); - expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? - testGame = Object.assign({}, result); - }); - it("Search game when not logged", async function () { - const result = await F95API.getGameData("Kingdom of Deception", false); - expect(result, "Without being logged should return null").to.be.null; - }); - it("Test game serialization", function () { - const json = JSON.stringify(testGame); - const parsedGameInfo = JSON.parse(json); - const result = _.isEqual(parsedGameInfo, testGame); - expect(result).to.be.true; - }); + // Test only the main information + expect(result.name).to.equal("Kingdom of Deception"); + expect(result.author).to.equal("Hreinn Games"); + expect(result.isMod, "Should be false").to.be.false; + expect(result.engine).to.equal("REN'PY"); + expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? + testGame = Object.assign({}, result); + }); + it("Search game when not logged", async function () { + const result = await F95API.getGameData("Kingdom of Deception", false); + expect(result, "Without being logged should return null").to.be.null; + }); + it("Test game serialization", function () { + const json = JSON.stringify(testGame); + const parsedGameInfo = JSON.parse(json); + const result = _.isEqual(parsedGameInfo, testGame); + expect(result).to.be.true; + }); }); describe("Load user data", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up - it("Retrieve when logged", async function () { + it("Retrieve when logged", async function () { // Login - await F95API.login(USERNAME, PASSWORD); + await F95API.login(USERNAME, PASSWORD); - // Then retrieve user data - const data = await F95API.getUserData(); + // Then retrieve user data + const data = await F95API.getUserData(); - expect(data).to.exist; - expect(data.username).to.equal(USERNAME); - }); - it("Retrieve when not logged", async function () { + expect(data).to.exist; + expect(data.username).to.equal(USERNAME); + }); + it("Retrieve when not logged", async function () { // Logout - if (F95API.isLogged()) F95API.logout(); + if (F95API.isLogged()) F95API.logout(); - // Try to retrieve user data - const data = await F95API.getUserData(); + // Try to retrieve user data + const data = await F95API.getUserData(); - expect(data).to.be.null; - }); + expect(data).to.be.null; + }); }); describe("Check game update", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up - it("Get online game and verify that no update exists", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("Get online game and verify that no update exists", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; + const loadResult = await F95API.loadF95BaseData(); + expect(loadResult).to.be.true; - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ + const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.false; - }); + const update = await F95API.chekIfGameHasUpdate(result); + expect(update).to.be.false; + }); - it("Verify that update exists from old URL", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; + it("Verify that update exists from old URL", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ - const url = + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ + const url = "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; - const result = await F95API.getGameDataFromURL(url); - result.version = "0.9600"; + const result = await F95API.getGameDataFromURL(url); + result.version = "0.9600"; - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.true; - }); + const update = await F95API.chekIfGameHasUpdate(result); + expect(update).to.be.true; + }); }); describe("Test url-helper", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up - it("Check if URL exists", async function () { + it("Check if URL exists", async function () { // Check generic URLs... - let exists = await urlHelper.urlExists("https://www.google.com/"); - expect(exists, "Complete valid URL").to.be.true; + let exists = await urlHelper.urlExists("https://www.google.com/"); + expect(exists, "Complete valid URL").to.be.true; - exists = await urlHelper.urlExists("www.google.com"); - expect(exists, "URl without protocol prefix").to.be.false; + exists = await urlHelper.urlExists("www.google.com"); + expect(exists, "URl without protocol prefix").to.be.false; - exists = await urlHelper.urlExists("https://www.google/"); - expect(exists, "URL without third level domain").to.be.false; + exists = await urlHelper.urlExists("https://www.google/"); + expect(exists, "URL without third level domain").to.be.false; - // Now check for more specific URLs (with redirect)... - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(exists, "URL with redirect without check").to.be.true; + // Now check for more specific URLs (with redirect)... + exists = await urlHelper.urlExists( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" + ); + expect(exists, "URL with redirect without check").to.be.true; - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", - true - ); - expect(exists, "URL with redirect with check").to.be.false; - }); + exists = await urlHelper.urlExists( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", + true + ); + expect(exists, "URL with redirect with check").to.be.false; + }); - it("Check if URL belong to the platform", async function () { - let belong = urlHelper.isF95URL( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(belong).to.be.true; + it("Check if URL belong to the platform", async function () { + let belong = urlHelper.isF95URL( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" + ); + expect(belong).to.be.true; - belong = urlHelper.isF95URL("https://www.google/"); - expect(belong).to.be.false; - }); + belong = urlHelper.isF95URL("https://www.google/"); + expect(belong).to.be.false; + }); }); diff --git a/test/user-test.js b/test/user-test.js index c6f6b29..9819d66 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -4,12 +4,12 @@ F95API.debug(true); main(); async function main() { - const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); + const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); - if (loginResult.success) { - await F95API.loadF95BaseData(); - const gameData = await F95API.getGameData("a struggle with sin", false); - console.log(gameData); - } - F95API.logout(); + if (loginResult.success) { + await F95API.loadF95BaseData(); + const gameData = await F95API.getGameData("a struggle with sin", false); + console.log(gameData); + } + F95API.logout(); } From 20fea5c3152d00f18e2a1872b3f3c410fb032332 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Fri, 30 Oct 2020 20:41:56 +0100 Subject: [PATCH 03/17] Implemented search for games and mods --- app/scriptsV2/classes/game-info.js | 111 ++++++++++++++++++++++ app/scriptsV2/classes/login-result.js | 20 ++++ app/scriptsV2/classes/user-data.js | 26 ++++++ app/scriptsV2/constants/css-selector.js | 31 +++++++ app/scriptsV2/constants/url.js | 7 ++ app/scriptsV2/network-helper.js | 35 +++++++ app/scriptsV2/search.js | 88 ++++++++++++++++++ package-lock.json | 118 +++++++++++++++++++++++- package.json | 2 + 9 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 app/scriptsV2/classes/game-info.js create mode 100644 app/scriptsV2/classes/login-result.js create mode 100644 app/scriptsV2/classes/user-data.js create mode 100644 app/scriptsV2/constants/css-selector.js create mode 100644 app/scriptsV2/constants/url.js create mode 100644 app/scriptsV2/network-helper.js create mode 100644 app/scriptsV2/search.js diff --git a/app/scriptsV2/classes/game-info.js b/app/scriptsV2/classes/game-info.js new file mode 100644 index 0000000..44e7137 --- /dev/null +++ b/app/scriptsV2/classes/game-info.js @@ -0,0 +1,111 @@ +"use strict"; + +class GameInfo { + constructor() { + //#region Properties + /** + * Game name + * @type String + */ + this.name = null; + /** + * Game author + * @type String + */ + this.author = null; + /** + * URL to the game's official conversation on the F95Zone portal + * @type String + */ + this.url = null; + /** + * Game description + * @type String + */ + this.overview = null; + /** + * List of tags associated with the game + * @type String[] + */ + this.tags = []; + /** + * Graphics engine used for game development + * @type String + */ + this.engine = null; + /** + * Progress of the game + * @type String + */ + this.status = null; + /** + * Game description image URL + * @type String + */ + this.previewSrc = null; + /** + * Game version + * @type String + */ + this.version = null; + /** + * Last time the game underwent updates + * @type String + */ + this.lastUpdate = null; + /** + * Last time the local copy of the game was run + * @type String + */ + this.lastPlayed = null; + /** + * Specifies if the game is original or a mod + * @type Boolean + */ + this.isMod = false; + /** + * Changelog for the last version. + * @type String + */ + this.changelog = null; + /** + * Directory containing the local copy of the game + * @type String + */ + this.gameDir = null; + //#endregion Properties + } + + /** + * Converts the object to a dictionary used for JSON serialization. + */ + /* istanbul ignore next */ + toJSON() { + return { + name: this.name, + author: this.author, + url: this.url, + overview: this.overview, + engine: this.engine, + status: this.status, + previewSrc: this.previewSrc, + version: this.version, + lastUpdate: this.lastUpdate, + lastPlayed: this.lastPlayed, + isMod: this.isMod, + changelog: this.changelog, + gameDir: this.gameDir, + }; + } + + /** + * Return a new GameInfo from a JSON string. + * @param {String} json JSON string used to create the new object + * @returns {GameInfo} + */ + /* istanbul ignore next */ + static fromJSON(json) { + return Object.assign(new GameInfo(), json); + } +} +module.exports = GameInfo; diff --git a/app/scriptsV2/classes/login-result.js b/app/scriptsV2/classes/login-result.js new file mode 100644 index 0000000..51fd9bc --- /dev/null +++ b/app/scriptsV2/classes/login-result.js @@ -0,0 +1,20 @@ +"use strict"; + +/** + * Object obtained in response to an attempt to login to the portal. + */ +class LoginResult { + constructor(success, message) { + /** + * Result of the login operation + * @type Boolean + */ + this.success = success; + /** + * Login response message + * @type String + */ + this.message = message; + } +} +module.exports = LoginResult; diff --git a/app/scriptsV2/classes/user-data.js b/app/scriptsV2/classes/user-data.js new file mode 100644 index 0000000..7edb904 --- /dev/null +++ b/app/scriptsV2/classes/user-data.js @@ -0,0 +1,26 @@ +"use strict"; + +/** + * Class containing the data of the user currently connected to the F95Zone platform. + */ +class UserData { + constructor() { + /** + * User username. + * @type String + */ + this.username = ""; + /** + * Path to the user's profile picture. + * @type String + */ + this.avatarSrc = null; + /** + * List of followed thread URLs. + * @type URL[] + */ + this.watchedThreads = []; + } +} + +module.exports = UserData; diff --git a/app/scriptsV2/constants/css-selector.js b/app/scriptsV2/constants/css-selector.js new file mode 100644 index 0000000..032e6c9 --- /dev/null +++ b/app/scriptsV2/constants/css-selector.js @@ -0,0 +1,31 @@ +module.exports = Object.freeze({ + AVATAR_INFO: "span.avatar", + AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", + ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", + FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", + GT_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", + GT_TAGS: "a.tagItem", + GT_TITLE: "h1.p-title-value", + GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", + LOGIN_BUTTON: "button.button--icon--login", + LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", + ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", + PASSWORD_INPUT: "input[name=\"password\"]", + SEARCH_BUTTON: "form.block > * button.button--icon--search", + SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", + SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", + STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", + THREAD_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", + GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", + TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", + WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", + USERNAME_INPUT: "input[name=\"login\"]", + WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WT_NEXT_PAGE: "a.pageNav-jump--next", + WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]", + DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", + GS_RESULT_BODY: "div.contentRow-main", + GS_MEMBERSHIP: "li > a:not(.username)", + THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", +}); diff --git a/app/scriptsV2/constants/url.js b/app/scriptsV2/constants/url.js new file mode 100644 index 0000000..fac63ba --- /dev/null +++ b/app/scriptsV2/constants/url.js @@ -0,0 +1,7 @@ +module.exports = Object.freeze({ + F95_BASE_URL: "https://f95zone.to", + F95_SEARCH_URL: "https://f95zone.to/search/?type=post", + F95_LATEST_UPDATES: "https://f95zone.to/latest", + F95_LOGIN_URL: "https://f95zone.to/login", + F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", +}); diff --git a/app/scriptsV2/network-helper.js b/app/scriptsV2/network-helper.js new file mode 100644 index 0000000..6bf95b9 --- /dev/null +++ b/app/scriptsV2/network-helper.js @@ -0,0 +1,35 @@ +"use strict"; + +// Public modules from npm +const axios = require("axios").default; +const _ = require("lodash"); + +// Modules from file +const shared = require("./scripts/shared.js"); + +/** + * @protected + * Gets the HTML code of a page. + * @param {String} url URL to fetch + * @returns {Promise} HTML code or `null` if an error arise + */ +module.exports = async function fetchHTML(url) { + try { + const response = await axios.get(url); + return response.data; + } catch { + shared.logger.error(`An error occurred while trying to fetch the URL: ${url}`); + return null; + } +}; + +/** + * @protected + * Enforces the scheme of the URL is https and returns the new URL. + * @param {String} url + * @returns {String} + */ +module.exports = function enforceHttpsUrl(url) { + const value = _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; + return value; +}; \ No newline at end of file diff --git a/app/scriptsV2/search.js b/app/scriptsV2/search.js new file mode 100644 index 0000000..d348169 --- /dev/null +++ b/app/scriptsV2/search.js @@ -0,0 +1,88 @@ +"use strict"; + +// Public modules from npm +const cheerio = require("cheerio"); + +// Modules from file +const { fetchHTML } = require("./network-helper.js"); +const shared = require("./scripts/shared.js"); +const f95Selector = require("./constants/css-selector.js"); + +/** + * @protected + * Search for a game on F95Zone and return a list of URLs, one for each search result. + * @param {String} name Game name + * @returns {Promise} URLs of results + */ +module.exports = async function searchGame(name) { + shared.logger.info(`Searching games with name ${name}`); + + // Replace the whitespaces with + + const searchName = name.replaceAll(" ", "+").toUpperCase(); + + // Prepare the URL (only title, search in the "Games" section, order by relevance) + const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; + + // Fetch and parse the result URLs + return await fetchResultURLs(url); +}; + +/** + * @protected + * Search for a mod on F95Zone and return a list of URLs, one for each search result. + * @param {String} name Mod name + * @returns {Promise} URLs of results + */ +module.exports = async function searchMod(name) { + shared.logger.info(`Searching mods with name ${name}`); + // Replace the whitespaces with + + const searchName = name.replaceAll(" ", "+").toUpperCase(); + + // Prepare the URL (only title, search in the "Mods" section, order by relevance) + const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; + + // Fetch and parse the result URLs + return await fetchResultURLs(url); +}; + +//#region Private methods +/** + * @private + * Gets the URLs of the threads resulting from the F95Zone search. + * @param {String} url Search URL + * @return {Promise} List of URLs + */ +async function fetchResultURLs(url) { + shared.logger.info(`Fetching ${url}...`); + + // Fetch HTML and prepare Cheerio + const html = await fetchHTML(url); + const $ = cheerio.load(html); + + // Here we get all the DIV that are the body of the various query results + const results = $("body").find(f95Selector.GS_RESULT_BODY); + + // Than we extract the URLs + const urls = results.map((idx, el) => { + const elementSelector = $(el); + return extractLinkFromResult(elementSelector); + }).get(); + + return urls; +} + +/** + * @private + * Look for the URL to the thread referenced by the item. + * @param {cheerio.Cheerio} selector Element to search + * @returns {String} URL to thread + */ +function extractLinkFromResult(selector) { + const link = selector + .find(f95Selector.GS_RESULT_THREAD_TITLE) + .attr("href") + .trim(); + + return link; +} +//#endregion Private methods diff --git a/package-lock.json b/package-lock.json index 3018479..d65e78b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -285,8 +285,7 @@ "@types/node": { "version": "14.11.2", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", - "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==", - "optional": true + "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==" }, "@types/yauzl": { "version": "2.9.1", @@ -423,6 +422,14 @@ "integrity": "sha512-+Ryf6g3BKoRc7jfp7ad8tM4TtMiaWvbF/1/sQcZPkkS7ag3D5nMBCe2UfOTONtAkaG0tO0ij3C5Lwmf1EiyjHg==", "dev": true }, + "axios": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.0.tgz", + "integrity": "sha512-fmkJBknJKoZwem3/IKSSLpkdNXZeBu5Q7GA/aRsr2btgrptmSCxi2oFjZHqGdK9DoTil9PIHlPIZw2EcRJXRvw==", + "requires": { + "follow-redirects": "^1.10.0" + } + }, "babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -471,6 +478,11 @@ "readable-stream": "^3.4.0" } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=" + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -564,6 +576,19 @@ "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", "dev": true }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "chokidar": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.4.2.tgz", @@ -648,6 +673,22 @@ "which": "^2.0.1" } }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==" + }, "data-uri-to-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-3.0.1.tgz", @@ -725,6 +766,37 @@ "esutils": "^2.0.2" } }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==" + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "dotenv": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-8.2.0.tgz", @@ -754,6 +826,11 @@ "ansi-colors": "^4.1.1" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==" + }, "es-abstract": { "version": "1.17.6", "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.6.tgz", @@ -1159,6 +1236,11 @@ "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" }, + "follow-redirects": { + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.13.0.tgz", + "integrity": "sha512-aq6gF1BEKje4a9i9+5jimNFIpq4Q1WiwBToeRK5NvZBd/TRsmW8BsJfOEGkr76TbOyPVD3OVDN910EcUNtRYEA==" + }, "foreground-child": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-2.0.0.tgz", @@ -1327,6 +1409,19 @@ "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", "dev": true }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + } + }, "https-proxy-agent": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", @@ -1724,8 +1819,7 @@ "lodash": { "version": "4.17.20", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", - "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==", - "dev": true + "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" }, "lodash.flattendeep": { "version": "4.4.0", @@ -2127,6 +2221,14 @@ "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", "dev": true }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "requires": { + "boolbase": "~1.0.0" + } + }, "nyc": { "version": "15.1.0", "resolved": "https://registry.npmjs.org/nyc/-/nyc-15.1.0.tgz", @@ -2259,6 +2361,14 @@ "callsites": "^3.0.0" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "requires": { + "@types/node": "*" + } + }, "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", diff --git a/package.json b/package.json index 2836d66..c178177 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "node": ">=10.0" }, "dependencies": { + "axios": "^0.21.0", + "cheerio": "^1.0.0-rc.3", "ky": "^0.24.0", "ky-universal": "^0.8.2", "log4js": "^6.3.0", From 652fe0d3d6b2311f9dbb4867020c08cd97467e23 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Sat, 31 Oct 2020 16:00:26 +0100 Subject: [PATCH 04/17] Addes scraper, need authentication --- .eslintrc.json | 3 +- app/index.js | 20 +- app/scripts/classes/game-info.js | 118 ++-- app/scripts/classes/user-data.js | 20 +- app/scripts/constants/css-selector.js | 30 +- app/scripts/network-helper.js | 112 ++++ app/scripts/scraper.js | 248 +++++++ .../search.js => scripts/searcher.js} | 13 +- app/scriptsV2/classes/game-info.js | 111 ---- app/scriptsV2/network-helper.js | 35 - legacy/index.js | 609 ++++++++++++++++++ .../scripts/classes/game-download.js | 0 legacy/scripts/classes/game-info.js | 119 ++++ .../scripts}/classes/login-result.js | 0 .../scripts}/classes/user-data.js | 0 .../scripts}/constants/css-selector.js | 28 +- .../scripts}/constants/url.js | 0 {app => legacy}/scripts/game-scraper.js | 0 {app => legacy}/scripts/game-searcher.js | 0 {app => legacy}/scripts/puppeteer-helper.js | 0 legacy/scripts/shared.js | 173 +++++ {app => legacy}/scripts/url-helper.js | 0 package.json | 4 +- test/user-test.js | 26 +- 24 files changed, 1392 insertions(+), 277 deletions(-) create mode 100644 app/scripts/network-helper.js create mode 100644 app/scripts/scraper.js rename app/{scriptsV2/search.js => scripts/searcher.js} (88%) delete mode 100644 app/scriptsV2/classes/game-info.js delete mode 100644 app/scriptsV2/network-helper.js create mode 100644 legacy/index.js rename {app => legacy}/scripts/classes/game-download.js (100%) create mode 100644 legacy/scripts/classes/game-info.js rename {app/scriptsV2 => legacy/scripts}/classes/login-result.js (100%) rename {app/scriptsV2 => legacy/scripts}/classes/user-data.js (100%) rename {app/scriptsV2 => legacy/scripts}/constants/css-selector.js (56%) rename {app/scriptsV2 => legacy/scripts}/constants/url.js (100%) rename {app => legacy}/scripts/game-scraper.js (100%) rename {app => legacy}/scripts/game-searcher.js (100%) rename {app => legacy}/scripts/puppeteer-helper.js (100%) create mode 100644 legacy/scripts/shared.js rename {app => legacy}/scripts/url-helper.js (100%) diff --git a/.eslintrc.json b/.eslintrc.json index 0438ccf..10f4dca 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -3,7 +3,8 @@ "browser": true, "commonjs": true, "es2021": true, - "node": true + "node": true, + "mocha": true }, "extends": "eslint:recommended", "parser": "babel-eslint", diff --git a/app/index.js b/app/index.js index 1c99110..cb26ad8 100644 --- a/app/index.js +++ b/app/index.js @@ -4,21 +4,21 @@ const fs = require("fs"); // Modules from file -const shared = require("./scripts/shared.js"); -const urlK = require("./scripts/constants/url.js"); -const selectorK = require("./scripts/constants/css-selector.js"); -const urlHelper = require("./scripts/url-helper.js"); -const scraper = require("./scripts/game-scraper.js"); +const shared = require("../app/scripts/shared.js"); +const urlK = require("../app/scripts/constants/url.js"); +const selectorK = require("../app/scripts/constants/css-selector.js"); +const urlHelper = require("../app/scripts/url-helper.js"); +const scraper = require("../app/scripts/game-scraper.js"); const { prepareBrowser, preparePage, -} = require("./scripts/puppeteer-helper.js"); -const searcher = require("./scripts/game-searcher.js"); +} = require("../app/scripts/puppeteer-helper.js"); +const searcher = require("../app/scripts/game-searcher.js"); // Classes from file -const GameInfo = require("./scripts/classes/game-info.js"); -const LoginResult = require("./scripts/classes/login-result.js"); -const UserData = require("./scripts/classes/user-data.js"); +const GameInfo = require("../app/scripts/classes/game-info.js"); +const LoginResult = require("../app/scripts/classes/login-result.js"); +const UserData = require("../app/scripts/classes/user-data.js"); //#region Export classes module.exports.GameInfo = GameInfo; diff --git a/app/scripts/classes/game-info.js b/app/scripts/classes/game-info.js index c9fcac5..44e7137 100644 --- a/app/scripts/classes/game-info.js +++ b/app/scripts/classes/game-info.js @@ -2,115 +2,107 @@ class GameInfo { constructor() { - //#region Properties - /** - * Game name - * @type String - */ + //#region Properties + /** + * Game name + * @type String + */ this.name = null; /** - * Game author - * @type String - */ + * Game author + * @type String + */ this.author = null; /** - * URL to the game's official conversation on the F95Zone portal - * @type String - */ - this.f95url = null; + * URL to the game's official conversation on the F95Zone portal + * @type String + */ + this.url = null; /** - * Game description - * @type String - */ + * Game description + * @type String + */ this.overview = null; /** - * List of tags associated with the game - * @type String[] - */ + * List of tags associated with the game + * @type String[] + */ this.tags = []; /** - * Graphics engine used for game development - * @type String - */ + * Graphics engine used for game development + * @type String + */ this.engine = null; /** - * Progress of the game - * @type String - */ + * Progress of the game + * @type String + */ this.status = null; /** - * Game description image URL - * @type String - */ - this.previewSource = null; + * Game description image URL + * @type String + */ + this.previewSrc = null; /** - * Game version - * @type String - */ + * Game version + * @type String + */ this.version = null; /** - * Last time the game underwent updates - * @type String - */ + * Last time the game underwent updates + * @type String + */ this.lastUpdate = null; /** - * Last time the local copy of the game was run - * @type String - */ + * Last time the local copy of the game was run + * @type String + */ this.lastPlayed = null; /** - * Specifies if the game is original or a mod - * @type Boolean - */ + * Specifies if the game is original or a mod + * @type Boolean + */ this.isMod = false; /** - * Changelog for the last version. - * @type String - */ + * Changelog for the last version. + * @type String + */ this.changelog = null; /** - * Directory containing the local copy of the game - * @type String - */ + * Directory containing the local copy of the game + * @type String + */ this.gameDir = null; - /** - * Information on game file download links, - * including information on hosting platforms - * and operating system supported by the specific link - * @type GameDownload[] - */ - this.downloadInfo = []; - //#endregion Properties + //#endregion Properties } /** - * Converts the object to a dictionary used for JSON serialization - */ + * Converts the object to a dictionary used for JSON serialization. + */ /* istanbul ignore next */ toJSON() { return { name: this.name, author: this.author, - f95url: this.f95url, + url: this.url, overview: this.overview, engine: this.engine, status: this.status, - previewSource: this.previewSource, + previewSrc: this.previewSrc, version: this.version, lastUpdate: this.lastUpdate, lastPlayed: this.lastPlayed, isMod: this.isMod, changelog: this.changelog, gameDir: this.gameDir, - downloadInfo: this.downloadInfo, }; } /** - * Return a new GameInfo from a JSON string - * @param {String} json JSON string used to create the new object - * @returns {GameInfo} - */ + * Return a new GameInfo from a JSON string. + * @param {String} json JSON string used to create the new object + * @returns {GameInfo} + */ /* istanbul ignore next */ static fromJSON(json) { return Object.assign(new GameInfo(), json); diff --git a/app/scripts/classes/user-data.js b/app/scripts/classes/user-data.js index 7edb904..bdc2457 100644 --- a/app/scripts/classes/user-data.js +++ b/app/scripts/classes/user-data.js @@ -5,20 +5,20 @@ */ class UserData { constructor() { - /** - * User username. - * @type String - */ + /** + * User name. + * @type String + */ this.username = ""; /** - * Path to the user's profile picture. - * @type String - */ + * Path to the user's profile picture. + * @type String + */ this.avatarSrc = null; /** - * List of followed thread URLs. - * @type URL[] - */ + * List of followed thread URLs. + * @type String[] + */ this.watchedThreads = []; } } diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index bf0f60b..3d27cea 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -3,31 +3,29 @@ module.exports = Object.freeze({ AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", - GAME_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", - GAME_TAGS: "a.tagItem", - GAME_TITLE: "h1.p-title-value", - GAME_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", + GT_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", + GT_TAGS: "a.tagItem", + GT_TITLE: "h1.p-title-value", + GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: - "div.blockMessage.blockMessage--error.blockMessage--iconic", + LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", PASSWORD_INPUT: "input[name=\"password\"]", SEARCH_BUTTON: "form.block > * button.button--icon--search", SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", - THREAD_POSTS: - "article.message-body:first-child > div.bbWrapper:first-of-type", - THREAD_TITLE: "h3.contentRow-title", + GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", + GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", - UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", USERNAME_INPUT: "input[name=\"login\"]", - WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", - WATCHED_THREAD_URLS: "a[href^=\"/threads/\"][data-tp-primary]", + WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WT_NEXT_PAGE: "a.pageNav-jump--next", + WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]", DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", - SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", - SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", - THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", + GS_RESULT_BODY: "div.contentRow-main", + GS_MEMBERSHIP: "li > a:not(.username)", + GT_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", }); diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js new file mode 100644 index 0000000..682e4bd --- /dev/null +++ b/app/scripts/network-helper.js @@ -0,0 +1,112 @@ +"use strict"; + +// Public modules from npm +const axios = require("axios").default; +const _ = require("lodash"); +const ky = require("ky-universal").create({ + throwHttpErrors: false, +}); + +// Modules from file +const shared = require("./shared.js"); +const { + F95_BASE_URL +} = require("./constants/url.js"); + +/** + * @protected + * Gets the HTML code of a page. + * @param {String} url URL to fetch + * @returns {Promise} HTML code or `null` if an error arise + */ +module.exports.fetchHTML = async function (url) { + const userAgent = + "Mozilla/5.0 (X11; Linux x86_64)" + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; + + try { + const response = await axios.get(url, { + headers: { + "User-Agent": userAgent + } + }); + return response.data; + } catch (e) { + shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`); + return null; + } +}; + +/** + * @protected + * Enforces the scheme of the URL is https and returns the new URL. + * @param {String} url + * @returns {String} + */ +module.exports.enforceHttpsUrl = function (url) { + const value = _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; + return value; +}; + +/** + * @protected + * Check if the url belongs to the domain of the F95 platform. + * @param {String} url URL to check + * @returns {Boolean} true if the url belongs to the domain, false otherwise + */ +module.exports.isF95URL = function (url) { + if (url.toString().startsWith(F95_BASE_URL)) return true; + else return false; +}; + +/** + * @protected + * Checks if the string passed by parameter has a properly formatted and valid path to a URL. + * @param {String} url String to check for correctness + * @returns {Boolean} true if the string is a valid URL, false otherwise + */ +module.exports.isStringAValidURL = function (url) { + try { + new URL(url); // skipcq: JS-0078 + return true; + } catch (err) { + return false; + } +}; + +/** + * @protected + * Check if a particular URL is valid and reachable on the web. + * @param {String} url URL to check + * @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false + * @returns {Promise} true if the URL exists, false otherwise + */ +module.exports.urlExists = async function (url, checkRedirect) { + if (!exports.isStringAValidURL(url)) { + return false; + } + + const response = await ky.head(url); + let valid = response !== undefined && !/4\d\d/.test(response.status); + + if (!valid) return false; + + if (checkRedirect) { + const redirectUrl = await exports.getUrlRedirect(url); + if (redirectUrl === url) valid = true; + else valid = false; + } + + return valid; +}; + +/** + * @protected + * Check if the URL has a redirect to another page. + * @param {String} url URL to check for redirect + * @returns {Promise} Redirect URL or the passed URL + */ +module.exports.getUrlRedirect = async function (url) { + const response = await ky.head(url); + return response.url; +}; \ No newline at end of file diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js new file mode 100644 index 0000000..a59eb7c --- /dev/null +++ b/app/scripts/scraper.js @@ -0,0 +1,248 @@ +"use strict"; + +// Public modules from npm +const cheerio = require("cheerio"); + +// Modules from file +const { fetchHTML, getUrlRedirect } = require("./network-helper.js"); +const shared = require("./shared.js"); +const GameInfo = require("./classes/game-info.js"); +const f95Selector = require("./constants/css-selector.js"); + +/** + * @protected + * Get information from the game's main page. + * @param {String} url URL of the game/mod to extract data from + * @return {Promise} Complete information about the game you are + * looking for + */ +module.exports.getGameInfo = async function (url) { + shared.logger.info("Obtaining game info"); + + // Fetch HTML and prepare Cheerio + const html = await fetchHTML(url); + const $ = cheerio.load(html); + const body = $("body"); + const mainPost = $(f95Selector.GS_POSTS).first(); + + // Extract data + const titleData = extractInfoFromTitle(body); + console.log(titleData); + const tags = extractTags(body); + console.log(tags); + const mainPostData = extractInfoFromMainPost(mainPost); + console.log(mainPostData); + const structuredData = extractStructuredData(body); + + // Obtain the updated URL + const redirectUrl = await getUrlRedirect(url); + + // TODO: Check to change + const parsedInfos = parseMainPostText(mainPost.text()); + const overview = getOverview(mainPost.text(), info.isMod); + + // Fill in the GameInfo element with the information obtained + const info = new GameInfo(); + info.name = titleData.name; + info.author = titleData.author; + info.isMod = titleData.mod; + info.engine = titleData.engine; + info.status = titleData.status; + info.tags = tags; + info.url = redirectUrl; + info.overview = overview; + info.lastUpdate = titleData.mod ? parsedInfos.UPDATED : parsedInfos.THREAD_UPDATED; + info.previewSource = mainPostData.previewSource; + info.changelog = mainPostData.changelog; + info.version = titleData.version; + + shared.logger.info(`Founded data for ${info.name}`); + return info; +}; + +//#region Private methods +/** + * @private + * Extracts all the possible informations from the title, including the prefixes. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {Object} Dictionary of values + */ +function extractInfoFromTitle(body) { + const title = body + .find(f95Selector.GT_TITLE) + .text() + .trim(); + + // From the title we can extract: Name, author and version + // TITLE [VERSION] [AUTHOR] + const matches = title.match(/\[(.*?)\]/); + const endIndex = title.indexOf("["); // The open bracket of the version + const name = title.substring(0, endIndex).trim(); + const version = matches[0].trim(); + const author = matches[1].trim(); + + // Parse the title prefixes + const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES); + let mod = false, engine = null, status = null; + prefixeElements.each(function parseGamePrefixes(el) { + const prefix = el.text().trim(); + if(isEngine(prefix)) engine = prefix; + else if(isStatus(prefix)) status = prefix; + else if (isMod(prefix)) mod = true; + }); + + return { + name, + version, + author, + engine, + status, + mod + }; +} + +/** + * @private + * Gets the tags used to classify the game. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String[]} List of tags + */ +function extractTags(body) { + // Get the game tags + const tagResults = body.find(f95Selector.GT_TAGS); + return tagResults.map((idx, el) => { + return el.text().trim(); + }).get(); +} + +/** + * @private + * Extracts the name of the game, its author and its current version from the title of the page. + * @param {cheerio.Cheerio} mainPost Selector of the main post + * @returns {Object} Dictionary of values + */ +function extractInfoFromMainPost(mainPost) { + // Get the preview image + const previewElement = mainPost.find(f95Selector.GT_IMAGES); + const previewSource = previewElement ? previewElement.first().attr("src") : null; + + // Get the latest changelog + const changelogElement = mainPost.find(f95Selector.GT_LAST_CHANGELOG); + const changelog = changelogElement ? changelogElement.text().trim() : null; + + return { + previewSource, + changelog + }; +} + +/** + * @private + * Process the main post text to get all the useful + * information in the format *DESCRIPTOR : VALUE*. + * @param {String} text Structured text of the post + * @returns {Object} Dictionary of information + */ +function parseMainPostText(text) { + const dataPairs = {}; + + // The information searched in the game post are one per line + const splittedText = text.split("\n"); + for (const line of splittedText) { + if (!line.includes(":")) continue; + + // Create pair key/value + const splitted = line.split(":"); + const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch + const value = splitted[1].trim(); + + // Add pair to the dict if valid + if (value !== "") dataPairs[key] = value; + } + + return dataPairs; +} + +/** + * @private + * Extracts and processes the JSON-LD values found at the bottom of the page. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns ??? + */ +function extractStructuredData(body) { + const structuredDataElements = body.find("..."); + for (const el in structuredDataElements) { + for (const child in structuredDataElements[el].children) { + const data = structuredDataElements[el].children[child].data; + console.log(data); + // TODO: The @type should be "Book" + // TODO: Test here + } + } +} + +/** + * @private + * Get the game description from its web page. + * Different processing depending on whether the game is a mod or not. + * @param {String} text Structured text extracted from the game's web page + * @param {Boolean} mod Specify if it is a game or a mod + * @returns {Promise} Game description + */ +function getOverview(text, mod) { + // Get overview (different parsing for game and mod) + const overviewEndIndex = mod ? text.indexOf("Updated") : text.indexOf("Thread Updated"); + return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); +} + +/** + * @private + * Check if the prefix is a game's engine. + * @param {String} prefix Prefix to check + * @return {Boolean} + */ +function isEngine(prefix) { + const engines = toUpperCaseArray(shared.engines); + return engines.includes(prefix.toUpperCase()); +} + +/** + * @private + * Check if the prefix is a game's status. + * @param {String} prefix Prefix to check + * @return {Boolean} + */ +function isStatus(prefix) { + const statuses = toUpperCaseArray(shared.statuses); + return statuses.includes(prefix.toUpperCase()); +} + +/** + * @private + * Check if the prefix indicates a mod. + * @param {String} prefix Prefix to check + * @return {Boolean} + */ +function isMod(prefix) { + const modPrefixes = ["MOD", "CHEAT MOD"]; + return modPrefixes.includes(prefix.toUpperCase()); +} + +/** + * @private + * Makes an array of strings uppercase. + * @param {String[]} a + * @returns {String[]} + */ +function toUpperCaseArray(a) { + /** + * Makes a string uppercase. + * @param {String} s + * @returns {String} + */ + function toUpper(s) { + return s.toUpperCase(); + } + return a.map(toUpper); +} +//#endregion Private methods \ No newline at end of file diff --git a/app/scriptsV2/search.js b/app/scripts/searcher.js similarity index 88% rename from app/scriptsV2/search.js rename to app/scripts/searcher.js index d348169..eac6f55 100644 --- a/app/scriptsV2/search.js +++ b/app/scripts/searcher.js @@ -5,20 +5,21 @@ const cheerio = require("cheerio"); // Modules from file const { fetchHTML } = require("./network-helper.js"); -const shared = require("./scripts/shared.js"); +const shared = require("./shared.js"); const f95Selector = require("./constants/css-selector.js"); +//#region Public methods /** * @protected * Search for a game on F95Zone and return a list of URLs, one for each search result. * @param {String} name Game name * @returns {Promise} URLs of results */ -module.exports = async function searchGame(name) { +module.exports.searchGame = async function (name) { shared.logger.info(`Searching games with name ${name}`); // Replace the whitespaces with + - const searchName = name.replaceAll(" ", "+").toUpperCase(); + const searchName = encodeURIComponent(name.toUpperCase()); // Prepare the URL (only title, search in the "Games" section, order by relevance) const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; @@ -33,10 +34,11 @@ module.exports = async function searchGame(name) { * @param {String} name Mod name * @returns {Promise} URLs of results */ -module.exports = async function searchMod(name) { +module.exports.searchMod = async function (name) { shared.logger.info(`Searching mods with name ${name}`); + // Replace the whitespaces with + - const searchName = name.replaceAll(" ", "+").toUpperCase(); + const searchName = encodeURIComponent(name.toUpperCase()); // Prepare the URL (only title, search in the "Mods" section, order by relevance) const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; @@ -44,6 +46,7 @@ module.exports = async function searchMod(name) { // Fetch and parse the result URLs return await fetchResultURLs(url); }; +//#endregion Public methods //#region Private methods /** diff --git a/app/scriptsV2/classes/game-info.js b/app/scriptsV2/classes/game-info.js deleted file mode 100644 index 44e7137..0000000 --- a/app/scriptsV2/classes/game-info.js +++ /dev/null @@ -1,111 +0,0 @@ -"use strict"; - -class GameInfo { - constructor() { - //#region Properties - /** - * Game name - * @type String - */ - this.name = null; - /** - * Game author - * @type String - */ - this.author = null; - /** - * URL to the game's official conversation on the F95Zone portal - * @type String - */ - this.url = null; - /** - * Game description - * @type String - */ - this.overview = null; - /** - * List of tags associated with the game - * @type String[] - */ - this.tags = []; - /** - * Graphics engine used for game development - * @type String - */ - this.engine = null; - /** - * Progress of the game - * @type String - */ - this.status = null; - /** - * Game description image URL - * @type String - */ - this.previewSrc = null; - /** - * Game version - * @type String - */ - this.version = null; - /** - * Last time the game underwent updates - * @type String - */ - this.lastUpdate = null; - /** - * Last time the local copy of the game was run - * @type String - */ - this.lastPlayed = null; - /** - * Specifies if the game is original or a mod - * @type Boolean - */ - this.isMod = false; - /** - * Changelog for the last version. - * @type String - */ - this.changelog = null; - /** - * Directory containing the local copy of the game - * @type String - */ - this.gameDir = null; - //#endregion Properties - } - - /** - * Converts the object to a dictionary used for JSON serialization. - */ - /* istanbul ignore next */ - toJSON() { - return { - name: this.name, - author: this.author, - url: this.url, - overview: this.overview, - engine: this.engine, - status: this.status, - previewSrc: this.previewSrc, - version: this.version, - lastUpdate: this.lastUpdate, - lastPlayed: this.lastPlayed, - isMod: this.isMod, - changelog: this.changelog, - gameDir: this.gameDir, - }; - } - - /** - * Return a new GameInfo from a JSON string. - * @param {String} json JSON string used to create the new object - * @returns {GameInfo} - */ - /* istanbul ignore next */ - static fromJSON(json) { - return Object.assign(new GameInfo(), json); - } -} -module.exports = GameInfo; diff --git a/app/scriptsV2/network-helper.js b/app/scriptsV2/network-helper.js deleted file mode 100644 index 6bf95b9..0000000 --- a/app/scriptsV2/network-helper.js +++ /dev/null @@ -1,35 +0,0 @@ -"use strict"; - -// Public modules from npm -const axios = require("axios").default; -const _ = require("lodash"); - -// Modules from file -const shared = require("./scripts/shared.js"); - -/** - * @protected - * Gets the HTML code of a page. - * @param {String} url URL to fetch - * @returns {Promise} HTML code or `null` if an error arise - */ -module.exports = async function fetchHTML(url) { - try { - const response = await axios.get(url); - return response.data; - } catch { - shared.logger.error(`An error occurred while trying to fetch the URL: ${url}`); - return null; - } -}; - -/** - * @protected - * Enforces the scheme of the URL is https and returns the new URL. - * @param {String} url - * @returns {String} - */ -module.exports = function enforceHttpsUrl(url) { - const value = _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; - return value; -}; \ No newline at end of file diff --git a/legacy/index.js b/legacy/index.js new file mode 100644 index 0000000..cb26ad8 --- /dev/null +++ b/legacy/index.js @@ -0,0 +1,609 @@ +"use strict"; + +// Core modules +const fs = require("fs"); + +// Modules from file +const shared = require("../app/scripts/shared.js"); +const urlK = require("../app/scripts/constants/url.js"); +const selectorK = require("../app/scripts/constants/css-selector.js"); +const urlHelper = require("../app/scripts/url-helper.js"); +const scraper = require("../app/scripts/game-scraper.js"); +const { + prepareBrowser, + preparePage, +} = require("../app/scripts/puppeteer-helper.js"); +const searcher = require("../app/scripts/game-searcher.js"); + +// Classes from file +const GameInfo = require("../app/scripts/classes/game-info.js"); +const LoginResult = require("../app/scripts/classes/login-result.js"); +const UserData = require("../app/scripts/classes/user-data.js"); + +//#region Export classes +module.exports.GameInfo = GameInfo; +module.exports.LoginResult = LoginResult; +module.exports.UserData = UserData; +//#endregion Export classes + +//#region Export properties +/** + * Shows log messages and other useful functions for module debugging. + * @param {Boolean} value + */ +module.exports.debug = function (value) { + shared.debug = value; + + // Configure logger + shared.logger.level = value ? "debug" : "warn"; +}; +/** + * @public + * Indicates whether a user is logged in to the F95Zone platform or not. + * @returns {String} + */ +module.exports.isLogged = function () { + return shared.isLogged; +}; +/** + * @public + * If true, it opens a new browser for each request + * to the F95Zone platform, otherwise it reuses the same. + * @returns {String} + */ +module.exports.setIsolation = function (value) { + shared.isolation = value; +}; +/** + * @public + * Path to the cache directory + * @returns {String} + */ +module.exports.getCacheDir = function () { + return shared.cacheDir; +}; +/** + * @public + * Set path to the cache directory + * @returns {String} + */ +module.exports.setCacheDir = function (value) { + shared.cacheDir = value; + + // Create directory if it doesn't exist + if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); +}; +/** + * @public + * Set local chromium path. + * @returns {String} + */ +module.exports.setChromiumPath = function (value) { + shared.chromiumLocalPath = value; +}; +//#endregion Export properties + +//#region Global variables +var _browser = null; +const USER_NOT_LOGGED = "User not authenticated, unable to continue"; +//#endregion + +//#region Export methods +/** + * @public + * Log in to the F95Zone platform. + * This **must** be the first operation performed before accessing any other script functions. + * @param {String} username Username used for login + * @param {String} password Password used for login + * @returns {Promise} Result of the operation + */ +module.exports.login = async function (username, password) { + if (shared.isLogged) { + shared.logger.info("Already logged in"); + const result = new LoginResult(true, "Already logged in"); + return result; + } + + // If cookies are loaded, use them to authenticate + shared.cookies = loadCookies(); + if (shared.cookies !== null) { + shared.logger.info("Valid session, no need to re-authenticate"); + shared.isLogged = true; + const result = new LoginResult(true, "Logged with cookies"); + return result; + } + + // Else, log in throught browser + shared.logger.info( + "No saved sessions or expired session, login on the platform" + ); + + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + + const result = await loginF95(browser, username, password); + shared.isLogged = result.success; + + if (result.success) { + // Reload cookies + shared.cookies = loadCookies(); + shared.logger.info("User logged in through the platform"); + } else { + shared.logger.warn("Error during authentication: " + result.message); + } + if (shared.isolation) await browser.close(); + return result; +}; +/** + * @public + * This method loads the main data from the F95 portal + * used to provide game information. You **must** be logged + * in to the portal before calling this method. + * @returns {Promise} Result of the operation + */ +module.exports.loadF95BaseData = async function () { + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return false; + } + + shared.logger.info("Loading base data..."); + + // Prepare a new web page + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + + // Go to latest update page and wait for it to load + await page.goto(urlK.F95_LATEST_UPDATES, { + waitUntil: shared.WAIT_STATEMENT, + }); + + // Obtain engines (disk/online) + await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); + shared.engines = await loadValuesFromLatestPage( + page, + shared.enginesCachePath, + selectorK.ENGINE_ID_SELECTOR, + "engines" + ); + + // Obtain statuses (disk/online) + await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); + shared.statuses = await loadValuesFromLatestPage( + page, + shared.statusesCachePath, + selectorK.STATUS_ID_SELECTOR, + "statuses" + ); + + await page.close(); + if (shared.isolation) await browser.close(); + shared.logger.info("Base data loaded"); + return true; +}; +/** + * @public + * Chek if exists a new version of the game. + * You **must** be logged in to the portal before calling this method. + * @param {GameInfo} info Information about the game to get the version for + * @returns {Promise} true if an update is available, false otherwise + */ +module.exports.chekIfGameHasUpdate = async function (info) { + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return false; + } + + // F95 change URL at every game update, + // so if the URL is different an update is available + const exists = await urlHelper.urlExists(info.f95url, true); + if (!exists) return true; + + // Parse version from title + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + + const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); + + if (shared.isolation) await browser.close(); + + return onlineVersion.toUpperCase() !== info.version.toUpperCase(); +}; +/** + * @public + * Starting from the name, it gets all the information about the game you are looking for. + * You **must** be logged in to the portal before calling this method. + * @param {String} name Name of the game searched + * @param {Boolean} includeMods Indicates whether to also take mods into account when searching + * @returns {Promise} List of information obtained where each item corresponds to + * an identified game (in the case of homonymy of titles) + */ +module.exports.getGameData = async function (name, includeMods) { + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } + + // Gets the search results of the game being searched for + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + const urlList = await searcher.getSearchGameResults(browser, name); + + // Process previous partial results + const promiseList = []; + for (const url of urlList) { + // Start looking for information + promiseList.push(scraper.getGameInfo(browser, url)); + } + + // Filter for mods + const result = []; + for (const info of await Promise.all(promiseList)) { + // Ignore empty results + if (!info) continue; + // Skip mods if not required + if (info.isMod && !includeMods) continue; + // Else save data + result.push(info); + } + + if (shared.isolation) await browser.close(); + return result; +}; +/** + * @public + * Starting from the url, it gets all the information about the game you are looking for. + * You **must** be logged in to the portal before calling this method. + * @param {String} url URL of the game to obtain information of + * @returns {Promise} Information about the game. If no game was found, null is returned + */ +module.exports.getGameDataFromURL = async function (url) { + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } + + // Check URL + const exists = await urlHelper.urlExists(url); + if (!exists) throw new URIError(url + " is not a valid URL"); + if (!urlHelper.isF95URL(url)) + throw new Error(url + " is not a valid F95Zone URL"); + + // Gets the search results of the game being searched for + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + + // Get game data + const result = await scraper.getGameInfo(browser, url); + + if (shared.isolation) await browser.close(); + return result; +}; +/** + * @public + * Gets the data of the currently logged in user. + * You **must** be logged in to the portal before calling this method. + * @returns {Promise} Data of the user currently logged in + */ +module.exports.getUserData = async function () { + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return null; + } + + // Prepare a new web page + if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); + const browser = shared.isolation ? await prepareBrowser() : _browser; + const page = await preparePage(browser); // Set new isolated page + await page.setCookie(...shared.cookies); // Set cookies to avoid login + await page.goto(urlK.F95_BASE_URL); // Go to base page + + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.USERNAME_ELEMENT), + page.waitForSelector(selectorK.AVATAR_PIC), + ]); + + const threads = getUserWatchedGameThreads(browser); + + const username = await page.evaluate( + /* istanbul ignore next */ (selector) => + document.querySelector(selector).innerText, + selectorK.USERNAME_ELEMENT + ); + + const avatarSrc = await page.evaluate( + /* istanbul ignore next */ (selector) => + document.querySelector(selector).getAttribute("src"), + selectorK.AVATAR_PIC + ); + + const ud = new UserData(); + ud.username = username; + ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; + ud.watchedThreads = await threads; + + await page.close(); + if (shared.isolation) await browser.close(); + + return ud; +}; +/** + * @public + * Logout from the current user and gracefully close shared browser. + * You **must** be logged in to the portal before calling this method. + */ +module.exports.logout = async function () { + if (!shared.isLogged || !shared.cookies) { + shared.logger.warn(USER_NOT_LOGGED); + return; + } + + // Logout + shared.isLogged = false; + + // Gracefully close shared browser + if (!shared.isolation && _browser !== null) { + await _browser.close(); + _browser = null; + } +}; +//#endregion + +//#region Private methods + +//#region Cookies functions +/** + * @private + * Loads and verifies the expiration of previously stored cookies from disk + * if they exist, otherwise it returns null. + * @return {object[]} List of dictionaries or null if cookies don't exist + */ +function loadCookies() { + // Check the existence of the cookie file + if (fs.existsSync(shared.cookiesCachePath)) { + // Read cookies + const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); + const cookies = JSON.parse(cookiesJSON); + + // Check if the cookies have expired + for (const cookie of cookies) { + if (isCookieExpired(cookie)) return null; + } + + // Cookies loaded and verified + return cookies; + } else return null; +} +/** + * @private + * Check the validity of a cookie. + * @param {object} cookie Cookies to verify the validity. It's a dictionary + * @returns {Boolean} true if the cookie has expired, false otherwise + */ +function isCookieExpired(cookie) { + // Local variables + let expiredCookies = false; + + // Ignore cookies that never expire + const expirationUnixTimestamp = cookie.expire; + + if (expirationUnixTimestamp !== "-1") { + // Convert UNIX epoch timestamp to normal Date + const expirationDate = new Date(expirationUnixTimestamp * 1000); + + if (expirationDate < Date.now()) { + shared.logger.warn( + "Cookie " + cookie.name + " expired, you need to re-authenticate" + ); + expiredCookies = true; + } + } + + return expiredCookies; +} +//#endregion Cookies functions + +//#region Latest Updates page parserer +/** + * @private + * If present, it reads the file containing the searched values (engines or states) + * from the disk, otherwise it connects to the F95 portal (at the page + * https://f95zone.to/latest) and downloads them. + * @param {puppeteer.Page} page Page used to locate the required elements + * @param {String} path Path to disk of the JSON file containing the data to read / write + * @param {String} selector CSS selector of the required elements + * @param {String} elementRequested Required element (engines or states) used to detail log messages + * @returns {Promise} List of required values in uppercase + */ +async function loadValuesFromLatestPage( + page, + path, + selector, + elementRequested +) { + // If the values already exist they are loaded from disk without having to connect to F95 + shared.logger.info("Load " + elementRequested + " from disk..."); + if (fs.existsSync(path)) { + const valueJSON = fs.readFileSync(path); + return JSON.parse(valueJSON); + } + + // Otherwise, connect and download the data from the portal + shared.logger.info("No " + elementRequested + " cached, downloading..."); + const values = await getValuesFromLatestPage( + page, + selector, + "Getting " + elementRequested + " from page" + ); + fs.writeFileSync(path, JSON.stringify(values)); + return values; +} +/** + * @private + * Gets all the textual values of the elements present + * in the F95 portal page and identified by the selector + * passed by parameter + * @param {puppeteer.Page} page Page used to locate items specified by the selector + * @param {String} selector CSS selector + * @param {String} logMessage Log message indicating which items the selector is requesting + * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector + */ +async function getValuesFromLatestPage(page, selector, logMessage) { + shared.logger.info(logMessage); + + const result = []; + const elements = await page.$$(selector); + + for (const element of elements) { + const text = await element.evaluate( + /* istanbul ignore next */ (e) => e.innerText + ); + + // Save as upper text for better match if used in query + result.push(text.toUpperCase()); + } + return result; +} +//#endregion + +//#region User +/** + * @private + * Log in to the F95Zone portal and, if successful, save the cookies. + * @param {puppeteer.Browser} browser Browser object used for navigation + * @param {String} username Username to use during login + * @param {String} password Password to use during login + * @returns {Promise} Result of the operation + */ +async function loginF95(browser, username, password) { + const page = await preparePage(browser); // Set new isolated page + await page.goto(urlK.F95_LOGIN_URL); // Go to login page + + // Explicitly wait for the required items to load + await Promise.all([ + page.waitForSelector(selectorK.USERNAME_INPUT), + page.waitForSelector(selectorK.PASSWORD_INPUT), + page.waitForSelector(selectorK.LOGIN_BUTTON), + ]); + + await page.type(selectorK.USERNAME_INPUT, username); // Insert username + await page.type(selectorK.PASSWORD_INPUT, password); // Insert password + await Promise.all([ + page.click(selectorK.LOGIN_BUTTON), // Click on the login button + page.waitForNavigation({ + waitUntil: shared.WAIT_STATEMENT, + }), // Wait for page to load + ]); + + // Prepare result + let message = ""; + + // Check if the user is logged in + const success = await page.evaluate( + /* istanbul ignore next */ (selector) => + document.querySelector(selector) !== null, + selectorK.AVATAR_INFO + ); + + const errorMessageExists = await page.evaluate( + /* istanbul ignore next */ + (selector) => document.querySelector(selector) !== null, + selectorK.LOGIN_MESSAGE_ERROR + ); + + // Save cookies to avoid re-auth + if (success) { + const c = await page.cookies(); + fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); + message = "Authentication successful"; + } else if (errorMessageExists) { + const errorMessage = await page.evaluate( + /* istanbul ignore next */ (selector) => + document.querySelector(selector).innerText, + selectorK.LOGIN_MESSAGE_ERROR + ); + + if (errorMessage === "Incorrect password. Please try again.") { + message = "Incorrect password"; + } else if ( + errorMessage === + "The requested user '" + username + "' could not be found." + ) { + // The escaped quotes are important! + message = "Incorrect username"; + } else message = errorMessage; + } else message = "Unknown error"; + + await page.close(); // Close the page + return new LoginResult(success, message); +} +/** + * @private + * Gets the list of URLs of threads the user follows. + * @param {puppeteer.Browser} browser Browser object used for navigation + * @returns {Promise} URL list + */ +async function getUserWatchedGameThreads(browser) { + const page = await preparePage(browser); // Set new isolated page + await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page + + // Explicitly wait for the required items to load + await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); + + // Show the popup + await Promise.all([ + page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), + page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), + page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), + page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), + ]); + + // Set the filters + await page.evaluate( + /* istanbul ignore next */ (selector) => + document.querySelector(selector).removeAttribute("checked"), + selectorK.UNREAD_THREAD_CHECKBOX + ); // Also read the threads already read + + // Filter the threads + await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); + await page.click(selectorK.FILTER_THREADS_BUTTON); + await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + + // Get the threads urls + const urls = []; + let nextPageExists = false; + do { + // Get all the URLs + for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { + const src = await page.evaluate( + /* istanbul ignore next */ (element) => element.href, + handle + ); + // If 'unread' is left, it will redirect to the last unread post + const url = src.replace("/unread", ""); + urls.push(url); + } + + nextPageExists = await page.evaluate( + /* istanbul ignore next */ (selector) => document.querySelector(selector), + selectorK.WATCHED_THREAD_NEXT_PAGE + ); + + // Click to next page + if (nextPageExists) { + await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); + await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + } + } while (nextPageExists); + + await page.close(); + return urls; +} +//#endregion User + +//#endregion Private methods diff --git a/app/scripts/classes/game-download.js b/legacy/scripts/classes/game-download.js similarity index 100% rename from app/scripts/classes/game-download.js rename to legacy/scripts/classes/game-download.js diff --git a/legacy/scripts/classes/game-info.js b/legacy/scripts/classes/game-info.js new file mode 100644 index 0000000..c9fcac5 --- /dev/null +++ b/legacy/scripts/classes/game-info.js @@ -0,0 +1,119 @@ +"use strict"; + +class GameInfo { + constructor() { + //#region Properties + /** + * Game name + * @type String + */ + this.name = null; + /** + * Game author + * @type String + */ + this.author = null; + /** + * URL to the game's official conversation on the F95Zone portal + * @type String + */ + this.f95url = null; + /** + * Game description + * @type String + */ + this.overview = null; + /** + * List of tags associated with the game + * @type String[] + */ + this.tags = []; + /** + * Graphics engine used for game development + * @type String + */ + this.engine = null; + /** + * Progress of the game + * @type String + */ + this.status = null; + /** + * Game description image URL + * @type String + */ + this.previewSource = null; + /** + * Game version + * @type String + */ + this.version = null; + /** + * Last time the game underwent updates + * @type String + */ + this.lastUpdate = null; + /** + * Last time the local copy of the game was run + * @type String + */ + this.lastPlayed = null; + /** + * Specifies if the game is original or a mod + * @type Boolean + */ + this.isMod = false; + /** + * Changelog for the last version. + * @type String + */ + this.changelog = null; + /** + * Directory containing the local copy of the game + * @type String + */ + this.gameDir = null; + /** + * Information on game file download links, + * including information on hosting platforms + * and operating system supported by the specific link + * @type GameDownload[] + */ + this.downloadInfo = []; + //#endregion Properties + } + + /** + * Converts the object to a dictionary used for JSON serialization + */ + /* istanbul ignore next */ + toJSON() { + return { + name: this.name, + author: this.author, + f95url: this.f95url, + overview: this.overview, + engine: this.engine, + status: this.status, + previewSource: this.previewSource, + version: this.version, + lastUpdate: this.lastUpdate, + lastPlayed: this.lastPlayed, + isMod: this.isMod, + changelog: this.changelog, + gameDir: this.gameDir, + downloadInfo: this.downloadInfo, + }; + } + + /** + * Return a new GameInfo from a JSON string + * @param {String} json JSON string used to create the new object + * @returns {GameInfo} + */ + /* istanbul ignore next */ + static fromJSON(json) { + return Object.assign(new GameInfo(), json); + } +} +module.exports = GameInfo; diff --git a/app/scriptsV2/classes/login-result.js b/legacy/scripts/classes/login-result.js similarity index 100% rename from app/scriptsV2/classes/login-result.js rename to legacy/scripts/classes/login-result.js diff --git a/app/scriptsV2/classes/user-data.js b/legacy/scripts/classes/user-data.js similarity index 100% rename from app/scriptsV2/classes/user-data.js rename to legacy/scripts/classes/user-data.js diff --git a/app/scriptsV2/constants/css-selector.js b/legacy/scripts/constants/css-selector.js similarity index 56% rename from app/scriptsV2/constants/css-selector.js rename to legacy/scripts/constants/css-selector.js index 032e6c9..bf0f60b 100644 --- a/app/scriptsV2/constants/css-selector.js +++ b/legacy/scripts/constants/css-selector.js @@ -3,29 +3,31 @@ module.exports = Object.freeze({ AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", - GT_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", - GT_TAGS: "a.tagItem", - GT_TITLE: "h1.p-title-value", - GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", + GAME_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", + GAME_TAGS: "a.tagItem", + GAME_TITLE: "h1.p-title-value", + GAME_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", + LOGIN_MESSAGE_ERROR: + "div.blockMessage.blockMessage--error.blockMessage--iconic", ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", PASSWORD_INPUT: "input[name=\"password\"]", SEARCH_BUTTON: "form.block > * button.button--icon--search", SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", - THREAD_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", - GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", + THREAD_POSTS: + "article.message-body:first-child > div.bbWrapper:first-of-type", + THREAD_TITLE: "h3.contentRow-title", TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", - WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", USERNAME_INPUT: "input[name=\"login\"]", - WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WT_NEXT_PAGE: "a.pageNav-jump--next", - WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]", + WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", + WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", + WATCHED_THREAD_URLS: "a[href^=\"/threads/\"][data-tp-primary]", DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", - GS_RESULT_BODY: "div.contentRow-main", - GS_MEMBERSHIP: "li > a:not(.username)", + SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", + SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", }); diff --git a/app/scriptsV2/constants/url.js b/legacy/scripts/constants/url.js similarity index 100% rename from app/scriptsV2/constants/url.js rename to legacy/scripts/constants/url.js diff --git a/app/scripts/game-scraper.js b/legacy/scripts/game-scraper.js similarity index 100% rename from app/scripts/game-scraper.js rename to legacy/scripts/game-scraper.js diff --git a/app/scripts/game-searcher.js b/legacy/scripts/game-searcher.js similarity index 100% rename from app/scripts/game-searcher.js rename to legacy/scripts/game-searcher.js diff --git a/app/scripts/puppeteer-helper.js b/legacy/scripts/puppeteer-helper.js similarity index 100% rename from app/scripts/puppeteer-helper.js rename to legacy/scripts/puppeteer-helper.js diff --git a/legacy/scripts/shared.js b/legacy/scripts/shared.js new file mode 100644 index 0000000..85a5ba4 --- /dev/null +++ b/legacy/scripts/shared.js @@ -0,0 +1,173 @@ +"use strict"; + +// Core modules +const { join } = require("path"); + +const log4js = require("log4js"); + +/** + * Class containing variables shared between modules. + */ +class Shared { + //#region Properties + /** + * Shows log messages and other useful functions for module debugging. + * @type Boolean + */ + static #_debug = false; + /** + * Indicates whether a user is logged in to the F95Zone platform or not. + * @type Boolean + */ + static #_isLogged = false; + /** + * List of cookies obtained from the F95Zone platform. + * @type Object[] + */ + static #_cookies = null; + /** + * List of possible game engines used for development. + * @type String[] + */ + static #_engines = null; + /** + * List of possible development statuses that a game can assume. + * @type String[] + */ + static #_statuses = null; + /** + * Wait instruction for the browser created by puppeteer. + * @type String + */ + static WAIT_STATEMENT = "domcontentloaded"; + /** + * Path to the directory to save the cache generated by the API. + * @type String + */ + static #_cacheDir = "./f95cache"; + /** + * If true, it opens a new browser for each request to + * the F95Zone platform, otherwise it reuses the same. + * @type Boolean + */ + static #_isolation = false; + /** + * Logger object used to write to both file and console. + * @type log4js.Logger + */ + static #_logger = log4js.getLogger(); + //#endregion Properties + + //#region Getters + /** + * Shows log messages and other useful functions for module debugging. + * @returns {Boolean} + */ + static get debug() { + return this.#_debug; + } + /** + * Indicates whether a user is logged in to the F95Zone platform or not. + * @returns {Boolean} + */ + static get isLogged() { + return this.#_isLogged; + } + /** + * List of cookies obtained from the F95Zone platform. + * @returns {Object[]} + */ + static get cookies() { + return this.#_cookies; + } + /** + * List of possible game engines used for development. + * @returns {String[]} + */ + static get engines() { + return this.#_engines; + } + /** + * List of possible development states that a game can assume. + * @returns {String[]} + */ + static get statuses() { + return this.#_statuses; + } + /** + * Directory to save the API cache. + * @returns {String} + */ + static get cacheDir() { + return this.#_cacheDir; + } + /** + * Path to the F95 platform cache. + * @returns {String} + */ + static get cookiesCachePath() { + return join(this.#_cacheDir, "cookies.json"); + } + /** + * Path to the game engine cache. + * @returns {String} + */ + static get enginesCachePath() { + return join(this.#_cacheDir, "engines.json"); + } + /** + * Path to the cache of possible game states. + * @returns {String} + */ + static get statusesCachePath() { + return join(this.#_cacheDir, "statuses.json"); + } + /** + * If true, it opens a new browser for each request + * to the F95Zone platform, otherwise it reuses the same. + * @returns {Boolean} + */ + static get isolation() { + return this.#_isolation; + } + /** + * Logger object used to write to both file and console. + * @returns {log4js.Logger} + */ + static get logger() { + return this.#_logger; + } + //#endregion Getters + + //#region Setters + static set cookies(val) { + this.#_cookies = val; + } + + static set engines(val) { + this.#_engines = val; + } + + static set statuses(val) { + this.#_statuses = val; + } + + static set cacheDir(val) { + this.#_cacheDir = val; + } + + static set debug(val) { + this.#_debug = val; + } + + static set isLogged(val) { + this.#_isLogged = val; + } + + static set isolation(val) { + this.#_isolation = val; + } + //#endregion Setters +} + +module.exports = Shared; diff --git a/app/scripts/url-helper.js b/legacy/scripts/url-helper.js similarity index 100% rename from app/scripts/url-helper.js rename to legacy/scripts/url-helper.js diff --git a/package.json b/package.json index c178177..c57c96c 100644 --- a/package.json +++ b/package.json @@ -35,9 +35,7 @@ "cheerio": "^1.0.0-rc.3", "ky": "^0.24.0", "ky-universal": "^0.8.2", - "log4js": "^6.3.0", - "node-html-parser": "^1.2.21", - "puppeteer": "^5.3.1" + "log4js": "^6.3.0" }, "devDependencies": { "babel-eslint": "^10.1.0", diff --git a/test/user-test.js b/test/user-test.js index 9819d66..22f2046 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -1,15 +1,21 @@ -const F95API = require("../app/index.js"); +"use strict"; -F95API.debug(true); -main(); +// Modules from file +const searcher = require("../plain-html/scripts/searcher.js"); +const scraper = require("../plain-html/scripts/scraper.js"); -async function main() { - const loginResult = await F95API.login("MillenniumEarl", "f9vTcRNuvxj4YpK"); +// Search for Kingdom Of Deception data +searchKOD(); - if (loginResult.success) { - await F95API.loadF95BaseData(); - const gameData = await F95API.getGameData("a struggle with sin", false); - console.log(gameData); +async function searchKOD() { + console.log("Searching KOD..."); + const urls = await searcher.searchGame("kingdom of deception"); + console.log(`Found: ${urls}`); + + console.log("Scraping data..."); + for (const url of urls) { + const gamedata = await scraper.getGameInfo(url); + console.log(gamedata); } - F95API.logout(); + console.log("Scraping completed!"); } From 4c273def446495dc98cef66de2a32537b4b256b4 Mon Sep 17 00:00:00 2001 From: "samuele.berlusconi" Date: Sun, 1 Nov 2020 09:54:59 +0100 Subject: [PATCH 05/17] Try to auth in the platform via POST request --- app/scripts/classes/credentials.js | 18 +++ app/scripts/network-helper.js | 91 +++++++++-- app/scripts/searcher.js | 17 +- package-lock.json | 246 ++++------------------------- test/user-test.js | 22 ++- 5 files changed, 158 insertions(+), 236 deletions(-) create mode 100644 app/scripts/classes/credentials.js diff --git a/app/scripts/classes/credentials.js b/app/scripts/classes/credentials.js new file mode 100644 index 0000000..6eb7ab2 --- /dev/null +++ b/app/scripts/classes/credentials.js @@ -0,0 +1,18 @@ +"use strict"; + +// Modules from file +const { getF95Token } = require("../network-helper.js"); + +class Credentials { + constructor(username, password) { + this.username = username; + this.password = password; + this.token = ""; + } + + async fetchToken() { + this.token = await getF95Token(); + } +} + +module.exports = Credentials; \ No newline at end of file diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 682e4bd..d2a160b 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -6,12 +6,17 @@ const _ = require("lodash"); const ky = require("ky-universal").create({ throwHttpErrors: false, }); +const cheerio = require("cheerio"); +const qs = require("querystring"); // Modules from file const shared = require("./shared.js"); -const { - F95_BASE_URL -} = require("./constants/url.js"); +const f95url = require("./constants/url.js"); + +// Global variables +const userAgent = + "Mozilla/5.0 (X11; Linux x86_64)" + + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; /** * @protected @@ -20,15 +25,11 @@ const { * @returns {Promise} HTML code or `null` if an error arise */ module.exports.fetchHTML = async function (url) { - const userAgent = - "Mozilla/5.0 (X11; Linux x86_64)" + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; - try { const response = await axios.get(url, { headers: { "User-Agent": userAgent - } + }, }); return response.data; } catch (e) { @@ -37,6 +38,78 @@ module.exports.fetchHTML = async function (url) { } }; +/** + * @protected + * Gets the HTML code of a login-protected page. + * @param {String} url URL to fetch + * @param {Credentials} credentials Platform access credentials + * @returns {Promise} HTML code or `null` if an error arise + */ +module.exports.fetchHTMLWithAuth = async function (url, credentials) { + shared.logger.trace(`Fetching ${url} with user ${credentials.username}`); + + const data = { + "login": credentials.username, + "url": "", + "password": credentials.password, + "password_confirm": "", + "additional_security": "", + "remember": "1", + "_xfRedirect": "https://f95zone.to/", + "website_code": "", + "_xfToken": credentials.token, + }; + + const config = { + headers: { + "User-Agent": userAgent, + "Content-Type": "application/x-www-form-urlencoded" + } + }; + + try { + console.log(qs.stringify(data)); + const response = await axios({ + method: "post", + url: url, + data: qs.stringify(data), + headers: { + "user-agent": userAgent, + "content-type": "application/x-www-form-urlencoded;charset=utf-8" + }, + withCredentials: true + }); + //const response = await axios.post(url, qs.stringify(data), config); + return response.data; + } catch (e) { + shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`); + return null; + } +}; + +/** + * Obtain the token used to authenticate the user to the platform. + * @returns {Promise} Token or `null` if an error arise + */ +module.exports.getF95Token = async function() { + try { + // Fetch the response of the platform + const response = await axios.get(f95url.F95_LOGIN_URL, { + headers: { + "User-Agent": userAgent + }, + }); + + // The response is a HTML page, we need to find the with name "_xfToken" + const $ = cheerio.load(response.data); + const token = $("body").find("input[name='_xfToken']").attr("value"); + return token; + } catch (e) { + shared.logger.error(`Error ${e.message} occurred while trying to fetch F95 token`); + return null; + } +}; + /** * @protected * Enforces the scheme of the URL is https and returns the new URL. @@ -55,7 +128,7 @@ module.exports.enforceHttpsUrl = function (url) { * @returns {Boolean} true if the url belongs to the domain, false otherwise */ module.exports.isF95URL = function (url) { - if (url.toString().startsWith(F95_BASE_URL)) return true; + if (url.toString().startsWith(f95url.F95_BASE_URL)) return true; else return false; }; diff --git a/app/scripts/searcher.js b/app/scripts/searcher.js index eac6f55..a7ba734 100644 --- a/app/scripts/searcher.js +++ b/app/scripts/searcher.js @@ -4,7 +4,7 @@ const cheerio = require("cheerio"); // Modules from file -const { fetchHTML } = require("./network-helper.js"); +const { fetchHTMLWithAuth } = require("./network-helper.js"); const shared = require("./shared.js"); const f95Selector = require("./constants/css-selector.js"); @@ -13,9 +13,10 @@ const f95Selector = require("./constants/css-selector.js"); * @protected * Search for a game on F95Zone and return a list of URLs, one for each search result. * @param {String} name Game name + * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchGame = async function (name) { +module.exports.searchGame = async function (name, credentials) { shared.logger.info(`Searching games with name ${name}`); // Replace the whitespaces with + @@ -25,16 +26,17 @@ module.exports.searchGame = async function (name) { const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url); + return await fetchResultURLs(url, credentials); }; /** * @protected * Search for a mod on F95Zone and return a list of URLs, one for each search result. * @param {String} name Mod name + * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchMod = async function (name) { +module.exports.searchMod = async function (name, credentials) { shared.logger.info(`Searching mods with name ${name}`); // Replace the whitespaces with + @@ -44,7 +46,7 @@ module.exports.searchMod = async function (name) { const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url); + return await fetchResultURLs(url, credentials); }; //#endregion Public methods @@ -53,13 +55,14 @@ module.exports.searchMod = async function (name) { * @private * Gets the URLs of the threads resulting from the F95Zone search. * @param {String} url Search URL + * @param {Credentials} credentials Platform access credentials * @return {Promise} List of URLs */ -async function fetchResultURLs(url) { +async function fetchResultURLs(url, credentials) { shared.logger.info(`Fetching ${url}...`); // Fetch HTML and prepare Cheerio - const html = await fetchHTML(url); + const html = await fetchHTMLWithAuth(url, credentials); const $ = cheerio.load(html); // Here we get all the DIV that are the body of the various query results diff --git a/package-lock.json b/package-lock.json index d65e78b..2029248 100644 --- a/package-lock.json +++ b/package-lock.json @@ -287,15 +287,6 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz", "integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA==" }, - "@types/yauzl": { - "version": "2.9.1", - "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", - "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", - "optional": true, - "requires": { - "@types/node": "*" - } - }, "abort-controller": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", @@ -316,11 +307,6 @@ "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==", "dev": true }, - "agent-base": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-5.1.1.tgz", - "integrity": "sha512-TMeqbNl2fMW0nMjTEPOwe3J/PRFP4vqeoNuQMG0HlMrtm5QxKqdvAkZ1pRBQ/ulIyDD5Yq0nJ7YbdD8ey0TO3g==" - }, "aggregate-error": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", @@ -455,12 +441,8 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" - }, - "base64-js": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.3.1.tgz", - "integrity": "sha512-mLQ4i2QO1ytvGWFWmcngKO//JXAQueZvwEKtjgQFM4jIK0kU+ytMfplL8j+n5mspOfjHwoAg+9yhb7BwAHm36g==" + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", + "dev": true }, "binary-extensions": { "version": "2.1.0", @@ -468,16 +450,6 @@ "integrity": "sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ==", "dev": true }, - "bl": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz", - "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==", - "requires": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -487,6 +459,7 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -507,20 +480,6 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, - "buffer": { - "version": "5.6.0", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz", - "integrity": "sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==", - "requires": { - "base64-js": "^1.0.2", - "ieee754": "^1.1.4" - } - }, - "buffer-crc32": { - "version": "0.2.13", - "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", - "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" - }, "caching-transform": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-4.0.0.tgz", @@ -605,11 +564,6 @@ "readdirp": "~3.4.0" } }, - "chownr": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" - }, "clean-stack": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", @@ -651,7 +605,8 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", + "dev": true }, "convert-source-map": { "version": "1.7.0", @@ -746,11 +701,6 @@ "object-keys": "^1.0.12" } }, - "devtools-protocol": { - "version": "0.0.799653", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.799653.tgz", - "integrity": "sha512-t1CcaZbvm8pOlikqrsIM9GOa7Ipp07+4h/q9u0JXBWjPCjHdBl9KkddX87Vv9vBHoBGtwV79sYQNGnQM6iS5gg==" - }, "diff": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/diff/-/diff-4.0.2.tgz", @@ -809,14 +759,6 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "dev": true }, - "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "requires": { - "once": "^1.4.0" - } - }, "enquirer": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", @@ -1120,17 +1062,6 @@ "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==" }, - "extract-zip": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", - "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", - "requires": { - "@types/yauzl": "^2.9.1", - "debug": "^4.1.1", - "get-stream": "^5.1.0", - "yauzl": "^2.10.0" - } - }, "fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -1149,14 +1080,6 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, - "fd-slicer": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", - "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", - "requires": { - "pend": "~1.2.0" - } - }, "fetch-blob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-2.1.1.tgz", @@ -1195,6 +1118,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "dev": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -1257,11 +1181,6 @@ "integrity": "sha512-Xu2Qh8yqYuDhQGOhD5iJGninErSfI9A3FrriD3tjUgV5VbJFeH8vfgZ9HnC6jWN80QDVNQK5vmxRAmEAp7Mevw==", "dev": true }, - "fs-constants": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" - }, "fs-extra": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", @@ -1275,7 +1194,8 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", + "dev": true }, "fsevents": { "version": "2.1.3", @@ -1320,18 +1240,11 @@ "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==", "dev": true }, - "get-stream": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", - "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", - "requires": { - "pump": "^3.0.0" - } - }, "glob": { "version": "7.1.6", "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "dev": true, "requires": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", @@ -1401,7 +1314,8 @@ "he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", - "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==" + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "dev": true }, "html-escaper": { "version": "2.0.2", @@ -1422,20 +1336,6 @@ "readable-stream": "^3.1.1" } }, - "https-proxy-agent": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz", - "integrity": "sha512-zoDhWrkR3of1l9QAL8/scJZyLu8j/gBkcwcaQOZh7Gyh/+uJQzGVETdgT30akuwkpL8HTRfssqI3BZuV18teDg==", - "requires": { - "agent-base": "5", - "debug": "4" - } - }, - "ieee754": { - "version": "1.1.13", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", - "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" - }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", @@ -1476,6 +1376,7 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", + "dev": true, "requires": { "once": "^1.3.0", "wrappy": "1" @@ -1812,6 +1713,7 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "dev": true, "requires": { "p-locate": "^4.1.0" } @@ -1913,6 +1815,7 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "dev": true, "requires": { "brace-expansion": "^1.1.7" } @@ -1932,11 +1835,6 @@ "minimist": "^1.2.5" } }, - "mkdirp-classic": { - "version": "0.5.3", - "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" - }, "mocha": { "version": "8.1.3", "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.1.3.tgz", @@ -2198,14 +2096,6 @@ "fetch-blob": "^2.1.1" } }, - "node-html-parser": { - "version": "1.2.21", - "resolved": "https://registry.npmjs.org/node-html-parser/-/node-html-parser-1.2.21.tgz", - "integrity": "sha512-6vDhgen6J332syN5HUmeT4FfBG7m6bFRrPN+FXY8Am7FGuVpsIxTASVbeoO5PF2IHbX2s+WEIudb1hgxOjllNQ==", - "requires": { - "he": "1.2.0" - } - }, "node-preload": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/node-preload/-/node-preload-0.2.1.tgz", @@ -2292,6 +2182,7 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "dev": true, "requires": { "wrappy": "1" } @@ -2314,6 +2205,7 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "dev": true, "requires": { "p-try": "^2.0.0" } @@ -2322,6 +2214,7 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "dev": true, "requires": { "p-limit": "^2.2.0" } @@ -2338,7 +2231,8 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", + "dev": true }, "package-hash": { "version": "4.0.0", @@ -2372,12 +2266,14 @@ "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "dev": true }, "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", + "dev": true }, "path-key": { "version": "3.1.1", @@ -2397,11 +2293,6 @@ "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", "dev": true }, - "pend": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", - "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" - }, "picomatch": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.2.2.tgz", @@ -2412,6 +2303,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "dev": true, "requires": { "find-up": "^4.0.0" } @@ -2434,7 +2326,8 @@ "progress": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==" + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "dev": true }, "promise.allsettled": { "version": "1.0.2", @@ -2449,44 +2342,12 @@ "iterate-value": "^1.0.0" } }, - "proxy-from-env": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", - "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" - }, - "pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", - "requires": { - "end-of-stream": "^1.1.0", - "once": "^1.3.1" - } - }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", "dev": true }, - "puppeteer": { - "version": "5.3.1", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-5.3.1.tgz", - "integrity": "sha512-YTM1RaBeYrj6n7IlRXRYLqJHF+GM7tasbvrNFx6w1S16G76NrPq7oYFKLDO+BQsXNtS8kW2GxWCXjIMPvfDyaQ==", - "requires": { - "debug": "^4.1.0", - "devtools-protocol": "0.0.799653", - "extract-zip": "^2.0.0", - "https-proxy-agent": "^4.0.0", - "pkg-dir": "^4.2.0", - "progress": "^2.0.1", - "proxy-from-env": "^1.0.0", - "rimraf": "^3.0.2", - "tar-fs": "^2.0.0", - "unbzip2-stream": "^1.3.3", - "ws": "^7.2.3" - } - }, "randombytes": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/randombytes/-/randombytes-2.1.0.tgz", @@ -2566,6 +2427,7 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "dev": true, "requires": { "glob": "^7.1.3" } @@ -2817,29 +2679,6 @@ } } }, - "tar-fs": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.0.tgz", - "integrity": "sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg==", - "requires": { - "chownr": "^1.1.1", - "mkdirp-classic": "^0.5.2", - "pump": "^3.0.0", - "tar-stream": "^2.0.0" - } - }, - "tar-stream": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.1.4.tgz", - "integrity": "sha512-o3pS2zlG4gxr67GmFYBLlq+dM8gyRGUOvsrHclSkvtVtQbjV0s/+ZE8OpICbaj8clrX3tjeHngYGP7rweaBnuw==", - "requires": { - "bl": "^4.0.3", - "end-of-stream": "^1.4.1", - "fs-constants": "^1.0.0", - "inherits": "^2.0.3", - "readable-stream": "^3.1.1" - } - }, "test-exclude": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz", @@ -2857,11 +2696,6 @@ "integrity": "sha1-f17oI66AUgfACvLfSoTsP8+lcLQ=", "dev": true }, - "through": { - "version": "2.3.8", - "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" - }, "to-fast-properties": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", @@ -2907,15 +2741,6 @@ "is-typedarray": "^1.0.0" } }, - "unbzip2-stream": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.4.3.tgz", - "integrity": "sha512-mlExGW4w71ebDJviH16lQLtZS32VKqsSfk80GCfUlwT/4/hNRFsoscrF/c++9xinkMzECL1uL9DDwXqFWkruPg==", - "requires": { - "buffer": "^5.2.1", - "through": "^2.3.8" - } - }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -3057,7 +2882,8 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", + "dev": true }, "write": { "version": "1.0.3", @@ -3080,11 +2906,6 @@ "typedarray-to-buffer": "^3.1.5" } }, - "ws": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/ws/-/ws-7.3.1.tgz", - "integrity": "sha512-D3RuNkynyHmEJIpD2qrgVkc9DQ23OrN/moAwZX4L8DfvszsJxpjQuUq3LMx6HoYji9fbIOBY18XWBsAux1ZZUA==" - }, "y18n": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz", @@ -3257,15 +3078,6 @@ } } } - }, - "yauzl": { - "version": "2.10.0", - "resolved": "https://registry.npmjs.org/yauzl/-/yauzl-2.10.0.tgz", - "integrity": "sha1-x+sXyT4RLLEIb6bY5R+wZnt5pfk=", - "requires": { - "buffer-crc32": "~0.2.3", - "fd-slicer": "~1.1.0" - } } } } diff --git a/test/user-test.js b/test/user-test.js index 22f2046..afbdc94 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -1,15 +1,31 @@ "use strict"; +// Public modules from npm +const dotenv = require("dotenv"); + // Modules from file -const searcher = require("../plain-html/scripts/searcher.js"); -const scraper = require("../plain-html/scripts/scraper.js"); +const searcher = require("../app/scripts/searcher.js"); +const scraper = require("../app/scripts/scraper.js"); +const Credentials = require("../app/scripts/classes/credentials.js"); +const networkHelper = require("../app/scripts/network-helper.js"); + +// Configure the .env reader +dotenv.config(); // Search for Kingdom Of Deception data searchKOD(); async function searchKOD() { + console.log("Token fetch..."); + const creds = new Credentials(process.env.F95_USERNAME, process.env.F95_PASSWORD); + await creds.fetchToken(); + console.log(`Token obtained: ${creds.token}`); + + const html = await networkHelper.fetchHTMLWithAuth("https://f95zone.to/login/login", creds); + console.log(html); + console.log("Searching KOD..."); - const urls = await searcher.searchGame("kingdom of deception"); + const urls = await searcher.searchGame("kingdom of deception", creds); console.log(`Found: ${urls}`); console.log("Scraping data..."); From ecfd1784b12eb47c4b01d98306a01a3b4aee3832 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Sun, 1 Nov 2020 14:56:07 +0100 Subject: [PATCH 06/17] Added authentication to platform, game searcher tested --- app/scripts/classes/credentials.js | 2 +- app/scripts/constants/url.js | 2 +- app/scripts/network-helper.js | 80 ++++++++++++++++-------------- app/scripts/scraper.js | 2 +- app/scripts/searcher.js | 23 ++++----- package-lock.json | 37 +++++++++++++- package.json | 4 +- test/user-test.js | 5 +- 8 files changed, 97 insertions(+), 58 deletions(-) diff --git a/app/scripts/classes/credentials.js b/app/scripts/classes/credentials.js index 6eb7ab2..3041f03 100644 --- a/app/scripts/classes/credentials.js +++ b/app/scripts/classes/credentials.js @@ -7,7 +7,7 @@ class Credentials { constructor(username, password) { this.username = username; this.password = password; - this.token = ""; + this.token = null; } async fetchToken() { diff --git a/app/scripts/constants/url.js b/app/scripts/constants/url.js index fac63ba..6931f82 100644 --- a/app/scripts/constants/url.js +++ b/app/scripts/constants/url.js @@ -2,6 +2,6 @@ module.exports = Object.freeze({ F95_BASE_URL: "https://f95zone.to", F95_SEARCH_URL: "https://f95zone.to/search/?type=post", F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_LOGIN_URL: "https://f95zone.to/login", + F95_LOGIN_URL: "https://f95zone.to/login/login", F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", }); diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index d2a160b..3ed132b 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -7,7 +7,8 @@ const ky = require("ky-universal").create({ throwHttpErrors: false, }); const cheerio = require("cheerio"); -const qs = require("querystring"); +const axiosCookieJarSupport = require("axios-cookiejar-support").default; +const tough = require("tough-cookie"); // Modules from file const shared = require("./shared.js"); @@ -17,6 +18,8 @@ const f95url = require("./constants/url.js"); const userAgent = "Mozilla/5.0 (X11; Linux x86_64)" + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; +axiosCookieJarSupport(axios); +const cookieJar = new tough.CookieJar(); /** * @protected @@ -30,6 +33,8 @@ module.exports.fetchHTML = async function (url) { headers: { "User-Agent": userAgent }, + withCredentials: true, + jar: cookieJar }); return response.data; } catch (e) { @@ -40,50 +45,44 @@ module.exports.fetchHTML = async function (url) { /** * @protected - * Gets the HTML code of a login-protected page. - * @param {String} url URL to fetch + * It authenticates to the platform using the credentials + * and token obtained previously. Save cookies on your + * device after authentication. * @param {Credentials} credentials Platform access credentials - * @returns {Promise} HTML code or `null` if an error arise + * @returns {Promise} Result of the operation */ -module.exports.fetchHTMLWithAuth = async function (url, credentials) { - shared.logger.trace(`Fetching ${url} with user ${credentials.username}`); +module.exports.autenticate = async function (credentials) { + shared.logger.info(`Authenticating with user ${credentials.username}`); + if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); - const data = { - "login": credentials.username, - "url": "", - "password": credentials.password, - "password_confirm": "", - "additional_security": "", - "remember": "1", - "_xfRedirect": "https://f95zone.to/", - "website_code": "", - "_xfToken": credentials.token, - }; + // Prepare the parameters to send to the platform to authenticate + const params = new URLSearchParams(); + params.append("login", credentials.username); + params.append("url", ""); + params.append("password", credentials.password); + params.append("password_confirm", ""); + params.append("additional_security", ""); + params.append("remember", "1"); + params.append("_xfRedirect", "https://f95zone.to/"); + params.append("website_code", ""); + params.append("_xfToken", credentials.token); const config = { headers: { "User-Agent": userAgent, - "Content-Type": "application/x-www-form-urlencoded" - } + "Content-Type": "application/x-www-form-urlencoded", + "Connection": "keep-alive" + }, + withCredentials: true, + jar: cookieJar // Retrieve the stored cookies! What a pain to understand that this is a MUST! }; try { - console.log(qs.stringify(data)); - const response = await axios({ - method: "post", - url: url, - data: qs.stringify(data), - headers: { - "user-agent": userAgent, - "content-type": "application/x-www-form-urlencoded;charset=utf-8" - }, - withCredentials: true - }); - //const response = await axios.post(url, qs.stringify(data), config); - return response.data; + await axios.post(f95url.F95_LOGIN_URL, params, config); + return true; } catch (e) { - shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`); - return null; + shared.logger.error(`Error ${e.message} occurred while authenticating to ${f95url.F95_LOGIN_URL}`); + return false; } }; @@ -93,12 +92,17 @@ module.exports.fetchHTMLWithAuth = async function (url, credentials) { */ module.exports.getF95Token = async function() { try { - // Fetch the response of the platform - const response = await axios.get(f95url.F95_LOGIN_URL, { + const config = { headers: { - "User-Agent": userAgent + "User-Agent": userAgent, + "Connection": "keep-alive" }, - }); + withCredentials: true, + jar: cookieJar // Used to store the token in the PC + }; + + // Fetch the response of the platform + const response = await axios.get(f95url.F95_LOGIN_URL, config); // The response is a HTML page, we need to find the with name "_xfToken" const $ = cheerio.load(response.data); diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js index a59eb7c..e0fd443 100644 --- a/app/scripts/scraper.js +++ b/app/scripts/scraper.js @@ -75,7 +75,7 @@ function extractInfoFromTitle(body) { // From the title we can extract: Name, author and version // TITLE [VERSION] [AUTHOR] - const matches = title.match(/\[(.*?)\]/); + const matches = title.match(/\[(.*?)\]/g); const endIndex = title.indexOf("["); // The open bracket of the version const name = title.substring(0, endIndex).trim(); const version = matches[0].trim(); diff --git a/app/scripts/searcher.js b/app/scripts/searcher.js index a7ba734..3285ccd 100644 --- a/app/scripts/searcher.js +++ b/app/scripts/searcher.js @@ -4,19 +4,19 @@ const cheerio = require("cheerio"); // Modules from file -const { fetchHTMLWithAuth } = require("./network-helper.js"); +const { fetchHTML } = require("./network-helper.js"); const shared = require("./shared.js"); const f95Selector = require("./constants/css-selector.js"); +const { F95_BASE_URL } = require("./constants/url.js"); //#region Public methods /** * @protected * Search for a game on F95Zone and return a list of URLs, one for each search result. * @param {String} name Game name - * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchGame = async function (name, credentials) { +module.exports.searchGame = async function (name) { shared.logger.info(`Searching games with name ${name}`); // Replace the whitespaces with + @@ -26,17 +26,16 @@ module.exports.searchGame = async function (name, credentials) { const url = `https://f95zone.to/search/83456043/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=2&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url, credentials); + return await fetchResultURLs(url); }; /** * @protected * Search for a mod on F95Zone and return a list of URLs, one for each search result. * @param {String} name Mod name - * @param {Credentials} credentials Platform access credentials * @returns {Promise} URLs of results */ -module.exports.searchMod = async function (name, credentials) { +module.exports.searchMod = async function (name) { shared.logger.info(`Searching mods with name ${name}`); // Replace the whitespaces with + @@ -46,7 +45,7 @@ module.exports.searchMod = async function (name, credentials) { const url = `https://f95zone.to/search/83459796/?q=${searchName}&t=post&c[child_nodes]=1&c[nodes][0]=41&c[title_only]=1&o=relevance`; // Fetch and parse the result URLs - return await fetchResultURLs(url, credentials); + return await fetchResultURLs(url); }; //#endregion Public methods @@ -55,14 +54,13 @@ module.exports.searchMod = async function (name, credentials) { * @private * Gets the URLs of the threads resulting from the F95Zone search. * @param {String} url Search URL - * @param {Credentials} credentials Platform access credentials * @return {Promise} List of URLs */ -async function fetchResultURLs(url, credentials) { +async function fetchResultURLs(url) { shared.logger.info(`Fetching ${url}...`); // Fetch HTML and prepare Cheerio - const html = await fetchHTMLWithAuth(url, credentials); + const html = await fetchHTML(url); const $ = cheerio.load(html); // Here we get all the DIV that are the body of the various query results @@ -84,11 +82,12 @@ async function fetchResultURLs(url, credentials) { * @returns {String} URL to thread */ function extractLinkFromResult(selector) { - const link = selector + const partialLink = selector .find(f95Selector.GS_RESULT_THREAD_TITLE) .attr("href") .trim(); - return link; + // Compose and return the URL + return new URL(partialLink, F95_BASE_URL).toString(); } //#endregion Private methods diff --git a/package-lock.json b/package-lock.json index 2029248..26492f3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -416,6 +416,15 @@ "follow-redirects": "^1.10.0" } }, + "axios-cookiejar-support": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/axios-cookiejar-support/-/axios-cookiejar-support-1.0.1.tgz", + "integrity": "sha512-IZJxnAJ99XxiLqNeMOqrPbfR7fRyIfaoSLdPUf4AMQEGkH8URs0ghJK/xtqBsD+KsSr3pKl4DEQjCn834pHMig==", + "requires": { + "is-redirect": "^1.0.0", + "pify": "^5.0.0" + } + }, "babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -1459,6 +1468,11 @@ "integrity": "sha1-caUMhCnfync8kqOQpKA7OfzVHT4=", "dev": true }, + "is-redirect": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-redirect/-/is-redirect-1.0.0.tgz", + "integrity": "sha1-HQPd7VO9jbDzDCbk+V02/HyH3CQ=" + }, "is-regex": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.1.1.tgz", @@ -2299,6 +2313,11 @@ "integrity": "sha512-q0M/9eZHzmr0AulXyPwNfZjtwZ/RBZlbN3K3CErVrk50T2ASYI7Bye0EvekFY3IP1Nt2DHu0re+V2ZHIpMkuWg==", "dev": true }, + "pify": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" + }, "pkg-dir": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", @@ -2342,11 +2361,15 @@ "iterate-value": "^1.0.0" } }, + "psl": { + "version": "1.8.0", + "resolved": "https://registry.npmjs.org/psl/-/psl-1.8.0.tgz", + "integrity": "sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ==" + }, "punycode": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", - "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", - "dev": true + "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, "randombytes": { "version": "2.1.0", @@ -2711,6 +2734,16 @@ "is-number": "^7.0.0" } }, + "tough-cookie": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-4.0.0.tgz", + "integrity": "sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg==", + "requires": { + "psl": "^1.1.33", + "punycode": "^2.1.1", + "universalify": "^0.1.2" + } + }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", diff --git a/package.json b/package.json index c57c96c..2fb3213 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,12 @@ }, "dependencies": { "axios": "^0.21.0", + "axios-cookiejar-support": "^1.0.1", "cheerio": "^1.0.0-rc.3", "ky": "^0.24.0", "ky-universal": "^0.8.2", - "log4js": "^6.3.0" + "log4js": "^6.3.0", + "tough-cookie": "^4.0.0" }, "devDependencies": { "babel-eslint": "^10.1.0", diff --git a/test/user-test.js b/test/user-test.js index afbdc94..71ebf4a 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -21,8 +21,9 @@ async function searchKOD() { await creds.fetchToken(); console.log(`Token obtained: ${creds.token}`); - const html = await networkHelper.fetchHTMLWithAuth("https://f95zone.to/login/login", creds); - console.log(html); + console.log("Authenticating..."); + const authenticated = await networkHelper.autenticate(creds); + console.log(`Authentication result: ${authenticated}`); console.log("Searching KOD..."); const urls = await searcher.searchGame("kingdom of deception", creds); From 1391f28ad180c96a3cea5ceb4e19f58e57ee8f27 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Sun, 1 Nov 2020 21:56:12 +0100 Subject: [PATCH 07/17] Better logging, completed game scraping, remove unused values --- app/scripts/classes/game-info.js | 25 ++- app/scripts/constants/css-selector.js | 39 ++--- app/scripts/network-helper.js | 150 ++++++++++------ app/scripts/scraper.js | 242 +++++++++++++++++++------- app/scripts/searcher.js | 4 +- app/scripts/shared.js | 204 +++++++--------------- test/user-test.js | 34 +++- 7 files changed, 414 insertions(+), 284 deletions(-) diff --git a/app/scripts/classes/game-info.js b/app/scripts/classes/game-info.js index 44e7137..9d9e564 100644 --- a/app/scripts/classes/game-info.js +++ b/app/scripts/classes/game-info.js @@ -24,9 +24,25 @@ class GameInfo { */ this.overview = null; /** - * List of tags associated with the game - * @type String[] - */ + * Game language. + * @type String + */ + this.language = null; + /** + * List of supported OS. + * @type + */ + this.supportedOS = []; + /** + * Specify whether the game has censorship + * measures regarding NSFW scenes. + * @type Boolean + */ + this.censored = null; + /** + * List of tags associated with the game + * @type String[] + */ this.tags = []; /** * Graphics engine used for game development @@ -86,6 +102,9 @@ class GameInfo { author: this.author, url: this.url, overview: this.overview, + language: this.language, + supportedOS: this.supportedOS, + censored: this.censored, engine: this.engine, status: this.status, previewSrc: this.previewSrc, diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index 3d27cea..3c624a0 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -1,31 +1,28 @@ module.exports = Object.freeze({ - AVATAR_INFO: "span.avatar", - AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", - ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", - FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", - GT_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", + BD_ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", + BD_STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", + GT_IMAGES: "img:not([title])[data-src^=\"https://attachments.f95zone.to\"][data-url=\"\"]", GT_TAGS: "a.tagItem", GT_TITLE: "h1.p-title-value", GT_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", - LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", - ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", - PASSWORD_INPUT: "input[name=\"password\"]", - SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", - SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", - STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", - GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", - GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", - TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", - WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", - USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", - USERNAME_INPUT: "input[name=\"login\"]", + GT_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", + GT_JSONLD: "script[type=\"application/ld+json\"]", WT_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", WT_NEXT_PAGE: "a.pageNav-jump--next", WT_URLS: "a[href^=\"/threads/\"][data-tp-primary]", - DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", + WT_UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", + GS_POSTS: "article.message-body:first-child > div.bbWrapper:first-of-type", + GS_RESULT_THREAD_TITLE: "h3.contentRow-title > a", GS_RESULT_BODY: "div.contentRow-main", GS_MEMBERSHIP: "li > a:not(.username)", - GT_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", + GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]", + + LOGIN_BUTTON: "button.button--icon--login", + LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", + PASSWORD_INPUT: "input[name=\"password\"]", + USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", + USERNAME_INPUT: "input[name=\"login\"]", + AVATAR_INFO: "span.avatar", + AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", + FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", }); diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 3ed132b..15434a7 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -13,14 +13,24 @@ const tough = require("tough-cookie"); // Modules from file const shared = require("./shared.js"); const f95url = require("./constants/url.js"); +const f95selector = require("./constants/css-selector.js"); +const LoginResult = require("./classes/login-result.js"); // Global variables -const userAgent = - "Mozilla/5.0 (X11; Linux x86_64)" + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; +const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15"; axiosCookieJarSupport(axios); const cookieJar = new tough.CookieJar(); +const commonConfig = { + headers: { + "User-Agent": userAgent, + "Connection": "keep-alive" + }, + withCredentials: true, + jar: cookieJar // Used to store the token in the PC +}; + /** * @protected * Gets the HTML code of a page. @@ -28,19 +38,13 @@ const cookieJar = new tough.CookieJar(); * @returns {Promise} HTML code or `null` if an error arise */ module.exports.fetchHTML = async function (url) { - try { - const response = await axios.get(url, { - headers: { - "User-Agent": userAgent - }, - withCredentials: true, - jar: cookieJar - }); - return response.data; - } catch (e) { - shared.logger.error(`Error ${e.message} occurred while trying to fetch ${url}`); + // Fetch the response of the platform + const response = await exports.fetchGETResponse(url); + if (!response) { + shared.logger.warn(`Unable to fetch HTML for ${url}`); return null; } + return response.data; }; /** @@ -49,12 +53,21 @@ module.exports.fetchHTML = async function (url) { * and token obtained previously. Save cookies on your * device after authentication. * @param {Credentials} credentials Platform access credentials - * @returns {Promise} Result of the operation + * @returns {Promise} Result of the operation */ module.exports.autenticate = async function (credentials) { shared.logger.info(`Authenticating with user ${credentials.username}`); if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); + // If the user is already logged, return + if(shared.isLogged) { + shared.logger.warn(`${credentials.username} already authenticated`); + return new LoginResult(true, "Already authenticated"); + } + + // Secure the URL + const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL); + // Prepare the parameters to send to the platform to authenticate const params = new URLSearchParams(); params.append("login", credentials.username); @@ -67,22 +80,23 @@ module.exports.autenticate = async function (credentials) { params.append("website_code", ""); params.append("_xfToken", credentials.token); - const config = { - headers: { - "User-Agent": userAgent, - "Content-Type": "application/x-www-form-urlencoded", - "Connection": "keep-alive" - }, - withCredentials: true, - jar: cookieJar // Retrieve the stored cookies! What a pain to understand that this is a MUST! - }; - try { - await axios.post(f95url.F95_LOGIN_URL, params, config); - return true; + // Try to log-in + const response = await axios.post(secureURL, params, commonConfig); + + // Parse the response HTML + const $ = cheerio.load(response.data); + + // Get the error message (if any) and remove the new line chars + const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, ""); + + // Return the result of the authentication + shared.isLogged = errorMessage === ""; + if (errorMessage === "") return new LoginResult(true, "Authentication successful"); + else return new LoginResult(false, errorMessage); } catch (e) { - shared.logger.error(`Error ${e.message} occurred while authenticating to ${f95url.F95_LOGIN_URL}`); - return false; + shared.logger.error(`Error ${e.message} occurred while authenticating to ${secureURL}`); + return new LoginResult(false, `Error ${e.message} while authenticating`); } }; @@ -91,25 +105,63 @@ module.exports.autenticate = async function (credentials) { * @returns {Promise} Token or `null` if an error arise */ module.exports.getF95Token = async function() { + // Fetch the response of the platform + const response = await exports.fetchGETResponse(f95url.F95_LOGIN_URL); + if (!response) { + shared.logger.warn("Unable to get the token for the session"); + return null; + } + + // The response is a HTML page, we need to find the with name "_xfToken" + const $ = cheerio.load(response.data); + const token = $("body").find(f95selector.GET_REQUEST_TOKEN).attr("value"); + return token; +}; + +/** + * @protected + * Gets the basic data used for game data processing + * (such as graphics engines and progress statuses) + * @deprecated + */ +module.exports.fetchPlatformData = async function() { + // Fetch the response of the platform + const response = await exports.fetchGETResponse(f95url.F95_LATEST_UPDATES); + if (!response) { + shared.logger.warn("Unable to get the token for the session"); + return; + } + + // The response is a HTML page, we need to find + // the base data, used when scraping the games + const $ = cheerio.load(response.data); + + // Extract the elements + const engineElements = $("body").find(f95selector.BD_ENGINE_ID_SELECTOR); + const statusesElements = $("body").find(f95selector.BD_STATUS_ID_SELECTOR); + + // Extract the raw text + engineElements.each(function extractEngineNames(idx, el) { + const engine = cheerio.load(el).text().trim(); + shared.engines.push(engine); + }); + + statusesElements.each(function extractEngineNames(idx, el) { + const status = cheerio.load(el).text().trim(); + shared.statuses.push(status); + }); +}; + +//#region Utility methods +module.exports.fetchGETResponse = async function(url) { + // Secure the URL + const secureURL = exports.enforceHttpsUrl(url); + try { - const config = { - headers: { - "User-Agent": userAgent, - "Connection": "keep-alive" - }, - withCredentials: true, - jar: cookieJar // Used to store the token in the PC - }; - - // Fetch the response of the platform - const response = await axios.get(f95url.F95_LOGIN_URL, config); - - // The response is a HTML page, we need to find the with name "_xfToken" - const $ = cheerio.load(response.data); - const token = $("body").find("input[name='_xfToken']").attr("value"); - return token; + // Fetch and return the response + return await axios.get(secureURL, commonConfig); } catch (e) { - shared.logger.error(`Error ${e.message} occurred while trying to fetch F95 token`); + shared.logger.error(`Error ${e.message} occurred while trying to fetch ${secureURL}`); return null; } }; @@ -121,8 +173,7 @@ module.exports.getF95Token = async function() { * @returns {String} */ module.exports.enforceHttpsUrl = function (url) { - const value = _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; - return value; + return _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; }; /** @@ -186,4 +237,5 @@ module.exports.urlExists = async function (url, checkRedirect) { module.exports.getUrlRedirect = async function (url) { const response = await ky.head(url); return response.url; -}; \ No newline at end of file +}; +//#endregion Utility methods \ No newline at end of file diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js index e0fd443..bfa823a 100644 --- a/app/scripts/scraper.js +++ b/app/scripts/scraper.js @@ -27,33 +27,33 @@ module.exports.getGameInfo = async function (url) { // Extract data const titleData = extractInfoFromTitle(body); - console.log(titleData); const tags = extractTags(body); - console.log(tags); - const mainPostData = extractInfoFromMainPost(mainPost); - console.log(mainPostData); + const prefixesData = parseGamePrefixes(body); + const src = extractPreviewSource(body); + const changelog = extractChangelog(mainPost); const structuredData = extractStructuredData(body); + const parsedInfos = parseMainPostText(structuredData["description"]); + const overview = getOverview(structuredData["description"], prefixesData.mod); // Obtain the updated URL const redirectUrl = await getUrlRedirect(url); - // TODO: Check to change - const parsedInfos = parseMainPostText(mainPost.text()); - const overview = getOverview(mainPost.text(), info.isMod); - // Fill in the GameInfo element with the information obtained const info = new GameInfo(); info.name = titleData.name; info.author = titleData.author; - info.isMod = titleData.mod; - info.engine = titleData.engine; - info.status = titleData.status; + info.isMod = prefixesData.mod; + info.engine = prefixesData.engine; + info.status = prefixesData.status; info.tags = tags; info.url = redirectUrl; + info.language = parsedInfos.Language; info.overview = overview; - info.lastUpdate = titleData.mod ? parsedInfos.UPDATED : parsedInfos.THREAD_UPDATED; - info.previewSource = mainPostData.previewSource; - info.changelog = mainPostData.changelog; + info.supportedOS = parsedInfos.SupportedOS; + info.censored = parsedInfos.Censored; + info.lastUpdate = parsedInfos.LastUpdate; + info.previewSrc = src; + info.changelog = changelog; info.version = titleData.version; shared.logger.info(`Founded data for ${info.name}`); @@ -63,41 +63,86 @@ module.exports.getGameInfo = async function (url) { //#region Private methods /** * @private - * Extracts all the possible informations from the title, including the prefixes. + * Parse the game prefixes obtaining the engine used, + * the advancement status and if the game is actually a game or a mod. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {Object} Dictionary of values + */ +function parseGamePrefixes(body) { + shared.logger.trace("Parsing prefixes..."); + + // Local variables + let mod = false, + engine = null, + status = null; + + // Obtain the title prefixes + const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES); + + prefixeElements.each(function parseGamePrefix(idx, el) { + // Obtain the prefix text + let prefix = cheerio.load(el).text().trim(); + + // Remove the square brackets + prefix = prefix.replace("[", "").replace("]", ""); + + // Check what the prefix indicates + if (isEngine(prefix)) engine = prefix; + else if (isStatus(prefix)) status = prefix; + else if (isMod(prefix)) mod = true; + }); + + // If the status is not set, then the game in in development (Ongoing) + if (!status) status = "Ongoing"; + + return { + engine, + status, + mod + }; +} + +/** + * @private + * Extracts all the possible informations from the title. * @param {cheerio.Cheerio} body Page `body` selector * @returns {Object} Dictionary of values */ function extractInfoFromTitle(body) { + shared.logger.trace("Extracting information from title..."); const title = body .find(f95Selector.GT_TITLE) .text() .trim(); // From the title we can extract: Name, author and version - // TITLE [VERSION] [AUTHOR] + // [PREFIXES] TITLE [VERSION] [AUTHOR] const matches = title.match(/\[(.*?)\]/g); - const endIndex = title.indexOf("["); // The open bracket of the version - const name = title.substring(0, endIndex).trim(); - const version = matches[0].trim(); - const author = matches[1].trim(); - // Parse the title prefixes - const prefixeElements = body.find(f95Selector.GT_TITLE_PREFIXES); - let mod = false, engine = null, status = null; - prefixeElements.each(function parseGamePrefixes(el) { - const prefix = el.text().trim(); - if(isEngine(prefix)) engine = prefix; - else if(isStatus(prefix)) status = prefix; - else if (isMod(prefix)) mod = true; + // Get the title name + let name = title; + matches.forEach(function replaceElementsInTitle(e) { + name = name.replace(e, ""); }); + name = name.trim(); + + // The regex [[\]]+ remove the square brackets + + // The version is the penultimate element. + // If the matches are less than 2, than the title + // is malformes and only the author is fetched + // (usually the author is always present) + let version = null; + if (matches.length >= 2) version = matches[matches.length - 2].replace(/[[\]]+/g, "").trim(); + else shared.logger.trace(`Malformed title: ${title}`); + + // Last element + const author = matches[matches.length - 1].replace(/[[\]]+/g, "").trim(); return { name, version, author, - engine, - status, - mod }; } @@ -108,32 +153,49 @@ function extractInfoFromTitle(body) { * @returns {String[]} List of tags */ function extractTags(body) { + shared.logger.trace("Extracting tags..."); + // Get the game tags const tagResults = body.find(f95Selector.GT_TAGS); - return tagResults.map((idx, el) => { - return el.text().trim(); + return tagResults.map(function parseGameTags(idx, el) { + return cheerio.load(el).text().trim(); }).get(); } /** * @private - * Extracts the name of the game, its author and its current version from the title of the page. - * @param {cheerio.Cheerio} mainPost Selector of the main post - * @returns {Object} Dictionary of values + * Gets the URL of the image used as a preview. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String} URL of the image */ -function extractInfoFromMainPost(mainPost) { - // Get the preview image - const previewElement = mainPost.find(f95Selector.GT_IMAGES); - const previewSource = previewElement ? previewElement.first().attr("src") : null; - - // Get the latest changelog - const changelogElement = mainPost.find(f95Selector.GT_LAST_CHANGELOG); - const changelog = changelogElement ? changelogElement.text().trim() : null; +function extractPreviewSource(body) { + shared.logger.trace("Extracting image preview source..."); + const image = body.find(f95Selector.GT_IMAGES); - return { - previewSource, - changelog - }; + // The "src" attribute is rendered only in a second moment, + // we need the "static" src value saved in the attribute "data-src" + const source = image ? image.attr("data-src") : null; + return source; +} + +/** + * @private + * Gets the changelog of the latest version. + * @param {cheerio.Cheerio} mainPost main post selector + * @returns {String} Changelog of the last version or `null` if no changelog is fetched + */ +function extractChangelog(mainPost) { + shared.logger.trace("Extracting last changelog..."); + + // Obtain changelog + let changelog = mainPost.find(f95Selector.GT_LAST_CHANGELOG).text().trim(); + + // Clean changelog + changelog = changelog.replace("Spoiler", ""); + changelog = changelog.replace(/\n+/g, "\n"); + + // Return changelog + return changelog ? changelog : null; } /** @@ -144,7 +206,9 @@ function extractInfoFromMainPost(mainPost) { * @returns {Object} Dictionary of information */ function parseMainPostText(text) { - const dataPairs = {}; + shared.logger.trace("Parsing main post raw text..."); + + const data = {}; // The information searched in the game post are one per line const splittedText = text.split("\n"); @@ -157,28 +221,80 @@ function parseMainPostText(text) { const value = splitted[1].trim(); // Add pair to the dict if valid - if (value !== "") dataPairs[key] = value; + if (value !== "") data[key] = value; } - return dataPairs; + // Parse the standard pairs + const parsedDict = {}; + + // Check if the game is censored + if (data.CENSORED) { + const censored = data.CENSORED.toUpperCase() === "NO" ? false : true; + parsedDict["Censored"] = censored; + delete data.CENSORED; + } + + // Last update of the main post + if (data.UPDATED) { + parsedDict["LastUpdate"] = new Date(data.UPDATED); + delete data.UPDATED; + } + else if (data.THREAD_UPDATED) { + parsedDict["LastUpdate"] = new Date(data.THREAD_UPDATED); + delete data.THREAD_UPDATED; + } + + // Parse the supported OS + if (data.OS) { + const listOS = []; + + // Usually the string is something like "Windows, Linux, Mac" + const splitted = data.OS.split(","); + splitted.forEach(function (os) { + listOS.push(os.trim()); + }); + + parsedDict["SupportedOS"] = listOS; + delete data.OS; + } + + // Rename the key for the language + if (data.LANGUAGE) { + parsedDict["Language"] = data.LANGUAGE; + delete data.LANGUAGE; + } + + // What remains is added to a sub dictionary + parsedDict["Various"] = data; + + return parsedDict; } /** * @private * Extracts and processes the JSON-LD values found at the bottom of the page. * @param {cheerio.Cheerio} body Page `body` selector - * @returns ??? + * @returns {Object} JSON-LD or `null` if no valid JSON is found */ function extractStructuredData(body) { - const structuredDataElements = body.find("..."); - for (const el in structuredDataElements) { - for (const child in structuredDataElements[el].children) { - const data = structuredDataElements[el].children[child].data; - console.log(data); - // TODO: The @type should be "Book" - // TODO: Test here - } - } + shared.logger.trace("Extracting JSON-LD data..."); + const structuredDataElements = body.find(f95Selector.GT_JSONLD); + const json = structuredDataElements.map(function parseScriptTag(idx, el) { + // Get the element HTML + const html = cheerio.load(el).html().trim(); + + // Obtain the JSON-LD + const data = html + .replace("", ""); + + // Convert the string to an object + const json = JSON.parse(data); + + // Return only the data of the game + if (json["@type"] === "Book") return json; + }).get(); + return json[0] ? json[0] : null; } /** @@ -190,6 +306,7 @@ function extractStructuredData(body) { * @returns {Promise} Game description */ function getOverview(text, mod) { + shared.logger.trace("Extracting game overview..."); // Get overview (different parsing for game and mod) const overviewEndIndex = mod ? text.indexOf("Updated") : text.indexOf("Thread Updated"); return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); @@ -235,6 +352,9 @@ function isMod(prefix) { * @returns {String[]} */ function toUpperCaseArray(a) { + // If the array is empty, return + if(a.length === 0) return []; + /** * Makes a string uppercase. * @param {String} s diff --git a/app/scripts/searcher.js b/app/scripts/searcher.js index 3285ccd..f310669 100644 --- a/app/scripts/searcher.js +++ b/app/scripts/searcher.js @@ -57,7 +57,7 @@ module.exports.searchMod = async function (name) { * @return {Promise} List of URLs */ async function fetchResultURLs(url) { - shared.logger.info(`Fetching ${url}...`); + shared.logger.trace(`Fetching ${url}...`); // Fetch HTML and prepare Cheerio const html = await fetchHTML(url); @@ -82,6 +82,8 @@ async function fetchResultURLs(url) { * @returns {String} URL to thread */ function extractLinkFromResult(selector) { + shared.logger.trace("Extracting thread link from result..."); + const partialLink = selector .find(f95Selector.GS_RESULT_THREAD_TITLE) .attr("href") diff --git a/app/scripts/shared.js b/app/scripts/shared.js index 85a5ba4..561441e 100644 --- a/app/scripts/shared.js +++ b/app/scripts/shared.js @@ -1,8 +1,6 @@ "use strict"; -// Core modules -const { join } = require("path"); - +// Public modules from npm const log4js = require("log4js"); /** @@ -10,164 +8,88 @@ const log4js = require("log4js"); */ class Shared { //#region Properties - /** - * Shows log messages and other useful functions for module debugging. - * @type Boolean - */ - static #_debug = false; - /** - * Indicates whether a user is logged in to the F95Zone platform or not. - * @type Boolean - */ - static #_isLogged = false; - /** - * List of cookies obtained from the F95Zone platform. - * @type Object[] - */ - static #_cookies = null; - /** - * List of possible game engines used for development. - * @type String[] - */ - static #_engines = null; - /** - * List of possible development statuses that a game can assume. - * @type String[] - */ - static #_statuses = null; - /** - * Wait instruction for the browser created by puppeteer. - * @type String - */ - static WAIT_STATEMENT = "domcontentloaded"; - /** - * Path to the directory to save the cache generated by the API. - * @type String - */ - static #_cacheDir = "./f95cache"; - /** - * If true, it opens a new browser for each request to - * the F95Zone platform, otherwise it reuses the same. - * @type Boolean - */ - static #_isolation = false; - /** - * Logger object used to write to both file and console. - * @type log4js.Logger - */ - static #_logger = log4js.getLogger(); - //#endregion Properties + /** + * Shows log messages and other useful functions for module debugging. + * @type Boolean + */ + static #_debug = false; + /** + * Indicates whether a user is logged in to the F95Zone platform or not. + * @type Boolean + */ + static #_isLogged = false; + /** + * List of possible game engines used for development. + * @type String[] + */ + static #_engines = ["ADRIFT", "Flash", "HTML", "Java", "Others", "QSP", "RAGS", "RPGM", "Ren'Py", "Tads", "Unity", "Unreal Engine", "WebGL", "Wolf RPG"]; + /** + * List of possible development statuses that a game can assume. + * @type String[] + */ + static #_statuses = ["Completed", "Onhold", "Abandoned"]; + /** + * Logger object used to write to both file and console. + * @type log4js.Logger + */ + static #_logger = log4js.getLogger(); + //#endregion Properties - //#region Getters - /** + //#region Getters + /** * Shows log messages and other useful functions for module debugging. * @returns {Boolean} */ - static get debug() { - return this.#_debug; - } - /** + static get debug() { + return this.#_debug; + } + /** * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {Boolean} */ - static get isLogged() { - return this.#_isLogged; - } - /** - * List of cookies obtained from the F95Zone platform. - * @returns {Object[]} - */ - static get cookies() { - return this.#_cookies; - } - /** + static get isLogged() { + return this.#_isLogged; + } + /** * List of possible game engines used for development. * @returns {String[]} */ - static get engines() { - return this.#_engines; - } - /** + static get engines() { + return this.#_engines; + } + /** * List of possible development states that a game can assume. * @returns {String[]} */ - static get statuses() { - return this.#_statuses; - } - /** - * Directory to save the API cache. - * @returns {String} - */ - static get cacheDir() { - return this.#_cacheDir; - } - /** - * Path to the F95 platform cache. - * @returns {String} - */ - static get cookiesCachePath() { - return join(this.#_cacheDir, "cookies.json"); - } - /** - * Path to the game engine cache. - * @returns {String} - */ - static get enginesCachePath() { - return join(this.#_cacheDir, "engines.json"); - } - /** - * Path to the cache of possible game states. - * @returns {String} - */ - static get statusesCachePath() { - return join(this.#_cacheDir, "statuses.json"); - } - /** - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {Boolean} - */ - static get isolation() { - return this.#_isolation; - } - /** + static get statuses() { + return this.#_statuses; + } + /** * Logger object used to write to both file and console. * @returns {log4js.Logger} */ - static get logger() { - return this.#_logger; - } - //#endregion Getters + static get logger() { + return this.#_logger; + } + //#endregion Getters - //#region Setters - static set cookies(val) { - this.#_cookies = val; - } + //#region Setters + static set engines(val) { + this.#_engines = val; + } - static set engines(val) { - this.#_engines = val; - } + static set statuses(val) { + this.#_statuses = val; + } - static set statuses(val) { - this.#_statuses = val; - } + static set debug(val) { + this.#_debug = val; + } - static set cacheDir(val) { - this.#_cacheDir = val; - } - - static set debug(val) { - this.#_debug = val; - } - - static set isLogged(val) { - this.#_isLogged = val; - } - - static set isolation(val) { - this.#_isolation = val; - } - //#endregion Setters + static set isLogged(val) { + this.#_isLogged = val; + } + //#endregion Setters } module.exports = Shared; diff --git a/test/user-test.js b/test/user-test.js index 71ebf4a..8569a15 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -12,27 +12,45 @@ const networkHelper = require("../app/scripts/network-helper.js"); // Configure the .env reader dotenv.config(); -// Search for Kingdom Of Deception data -searchKOD(); +// Login +auth().then(async function searchGames(result) { + if(!result) return; -async function searchKOD() { + // Search for Kingdom Of Deception data + await search("kingdom of deception"); + + // Search for Perverted Education data + await search("perverted education"); + + // Search for Corrupted Kingdoms data + await search("corrupted kingdoms"); + + // Search for Summertime Saga data + await search("summertime saga"); +}); + +async function auth() { console.log("Token fetch..."); const creds = new Credentials(process.env.F95_USERNAME, process.env.F95_PASSWORD); await creds.fetchToken(); console.log(`Token obtained: ${creds.token}`); console.log("Authenticating..."); - const authenticated = await networkHelper.autenticate(creds); - console.log(`Authentication result: ${authenticated}`); + const result = await networkHelper.autenticate(creds); + console.log(`Authentication result: ${result.message}`); + + return result.success; +} - console.log("Searching KOD..."); - const urls = await searcher.searchGame("kingdom of deception", creds); +async function search(gamename) { + console.log(`Searching '${gamename}'...`); + const urls = await searcher.searchGame(gamename); console.log(`Found: ${urls}`); console.log("Scraping data..."); for (const url of urls) { const gamedata = await scraper.getGameInfo(url); - console.log(gamedata); + console.log(`Found ${gamedata.name} (${gamedata.version}) by ${gamedata.author}`); } console.log("Scraping completed!"); } From 49712b23372411b459012bd03122b578c1842270 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Sun, 1 Nov 2020 22:23:03 +0100 Subject: [PATCH 08/17] Refactored main file, removed old files --- app/index.js | 496 +++--------------- app/scripts/network-helper.js | 7 - app/scripts/user-scraper.js | 1 + legacy/index.js | 609 ----------------------- legacy/scripts/classes/game-download.js | 111 ----- legacy/scripts/classes/game-info.js | 119 ----- legacy/scripts/classes/login-result.js | 20 - legacy/scripts/classes/user-data.js | 26 - legacy/scripts/constants/css-selector.js | 33 -- legacy/scripts/constants/url.js | 7 - legacy/scripts/game-scraper.js | 467 ----------------- legacy/scripts/game-searcher.js | 132 ----- legacy/scripts/puppeteer-helper.js | 60 --- legacy/scripts/shared.js | 173 ------- legacy/scripts/url-helper.js | 72 --- 15 files changed, 73 insertions(+), 2260 deletions(-) create mode 100644 app/scripts/user-scraper.js delete mode 100644 legacy/index.js delete mode 100644 legacy/scripts/classes/game-download.js delete mode 100644 legacy/scripts/classes/game-info.js delete mode 100644 legacy/scripts/classes/login-result.js delete mode 100644 legacy/scripts/classes/user-data.js delete mode 100644 legacy/scripts/constants/css-selector.js delete mode 100644 legacy/scripts/constants/url.js delete mode 100644 legacy/scripts/game-scraper.js delete mode 100644 legacy/scripts/game-searcher.js delete mode 100644 legacy/scripts/puppeteer-helper.js delete mode 100644 legacy/scripts/shared.js delete mode 100644 legacy/scripts/url-helper.js diff --git a/app/index.js b/app/index.js index cb26ad8..a8d62cf 100644 --- a/app/index.js +++ b/app/index.js @@ -1,24 +1,18 @@ "use strict"; -// Core modules -const fs = require("fs"); - // Modules from file -const shared = require("../app/scripts/shared.js"); -const urlK = require("../app/scripts/constants/url.js"); -const selectorK = require("../app/scripts/constants/css-selector.js"); -const urlHelper = require("../app/scripts/url-helper.js"); -const scraper = require("../app/scripts/game-scraper.js"); -const { - prepareBrowser, - preparePage, -} = require("../app/scripts/puppeteer-helper.js"); -const searcher = require("../app/scripts/game-searcher.js"); +const shared = require("./scripts/shared.js"); +const f95url = require("./scripts/constants/url.js"); +const f95selector = require("./scripts/constants/css-selector.js"); +const networkHelper = require("./scripts/network-helper.js"); +const scraper = require("./scripts/scraper.js"); +const searcher = require("./scripts/searcher.js"); // Classes from file -const GameInfo = require("../app/scripts/classes/game-info.js"); -const LoginResult = require("../app/scripts/classes/login-result.js"); -const UserData = require("../app/scripts/classes/user-data.js"); +const Credentials = require("./scripts/classes/credentials.js"); +const GameInfo = require("./scripts/classes/game-info.js"); +const LoginResult = require("./scripts/classes/login-result.js"); +const UserData = require("./scripts/classes/user-data.js"); //#region Export classes module.exports.GameInfo = GameInfo; @@ -35,7 +29,7 @@ module.exports.debug = function (value) { shared.debug = value; // Configure logger - shared.logger.level = value ? "debug" : "warn"; + shared.logger.level = value ? "trace" : "warn"; }; /** * @public @@ -45,46 +39,9 @@ module.exports.debug = function (value) { module.exports.isLogged = function () { return shared.isLogged; }; -/** - * @public - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {String} - */ -module.exports.setIsolation = function (value) { - shared.isolation = value; -}; -/** - * @public - * Path to the cache directory - * @returns {String} - */ -module.exports.getCacheDir = function () { - return shared.cacheDir; -}; -/** - * @public - * Set path to the cache directory - * @returns {String} - */ -module.exports.setCacheDir = function (value) { - shared.cacheDir = value; - - // Create directory if it doesn't exist - if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); -}; -/** - * @public - * Set local chromium path. - * @returns {String} - */ -module.exports.setChromiumPath = function (value) { - shared.chromiumLocalPath = value; -}; //#endregion Export properties //#region Global variables -var _browser = null; const USER_NOT_LOGGED = "User not authenticated, unable to continue"; //#endregion @@ -99,91 +56,24 @@ const USER_NOT_LOGGED = "User not authenticated, unable to continue"; */ module.exports.login = async function (username, password) { if (shared.isLogged) { - shared.logger.info("Already logged in"); - const result = new LoginResult(true, "Already logged in"); - return result; + shared.logger.info(`${username} already authenticated`); + return new LoginResult(true, `${username} already authenticated`); } - // If cookies are loaded, use them to authenticate - shared.cookies = loadCookies(); - if (shared.cookies !== null) { - shared.logger.info("Valid session, no need to re-authenticate"); - shared.isLogged = true; - const result = new LoginResult(true, "Logged with cookies"); - return result; - } + shared.logger.trace("Fetching token..."); + const creds = new Credentials(username, password); + await creds.fetchToken(); - // Else, log in throught browser - shared.logger.info( - "No saved sessions or expired session, login on the platform" - ); - - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const result = await loginF95(browser, username, password); + shared.logger.trace(`Authentication for ${username}`); + const result = await networkHelper.autenticate(creds); shared.isLogged = result.success; - if (result.success) { - // Reload cookies - shared.cookies = loadCookies(); - shared.logger.info("User logged in through the platform"); - } else { - shared.logger.warn("Error during authentication: " + result.message); - } - if (shared.isolation) await browser.close(); + if (result.success) shared.logger.info("User logged in through the platform"); + else shared.logger.warn(`Error during authentication: ${result.message}`); + return result; }; -/** - * @public - * This method loads the main data from the F95 portal - * used to provide game information. You **must** be logged - * in to the portal before calling this method. - * @returns {Promise} Result of the operation - */ -module.exports.loadF95BaseData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } - shared.logger.info("Loading base data..."); - - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - - // Go to latest update page and wait for it to load - await page.goto(urlK.F95_LATEST_UPDATES, { - waitUntil: shared.WAIT_STATEMENT, - }); - - // Obtain engines (disk/online) - await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); - shared.engines = await loadValuesFromLatestPage( - page, - shared.enginesCachePath, - selectorK.ENGINE_ID_SELECTOR, - "engines" - ); - - // Obtain statuses (disk/online) - await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); - shared.statuses = await loadValuesFromLatestPage( - page, - shared.statusesCachePath, - selectorK.STATUS_ID_SELECTOR, - "statuses" - ); - - await page.close(); - if (shared.isolation) await browser.close(); - shared.logger.info("Base data loaded"); - return true; -}; /** * @public * Chek if exists a new version of the game. @@ -191,68 +81,54 @@ module.exports.loadF95BaseData = async function () { * @param {GameInfo} info Information about the game to get the version for * @returns {Promise} true if an update is available, false otherwise */ -module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged || !shared.cookies) { +module.exports.checkIfGameHasUpdate = async function (info) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return false; } // F95 change URL at every game update, // so if the URL is different an update is available - const exists = await urlHelper.urlExists(info.f95url, true); + const exists = await networkHelper.urlExists(info.url, true); if (!exists) return true; // Parse version from title - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); - - if (shared.isolation) await browser.close(); - + const onlineVersion = await scraper.getGameInfo(info.url).version; + + // Compare the versions return onlineVersion.toUpperCase() !== info.version.toUpperCase(); }; + /** * @public * Starting from the name, it gets all the information about the game you are looking for. * You **must** be logged in to the portal before calling this method. * @param {String} name Name of the game searched - * @param {Boolean} includeMods Indicates whether to also take mods into account when searching + * @param {Boolean} mod Indicate if you are looking for mods or games * @returns {Promise} List of information obtained where each item corresponds to * an identified game (in the case of homonymy of titles) */ -module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged || !shared.cookies) { +module.exports.getGameData = async function (name, mod) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return null; } - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const urlList = await searcher.getSearchGameResults(browser, name); + // Gets the search results of the game/mod being searched for + let urls = []; + if(mod) urls = await searcher.searchMod(name); + else urls = await searcher.searchGame(name); // Process previous partial results - const promiseList = []; - for (const url of urlList) { - // Start looking for information - promiseList.push(scraper.getGameInfo(browser, url)); + const results = []; + for (const url of urls) { + // Start looking for information + const info = scraper.getGameInfo(url); + results.push(info); } - - // Filter for mods - const result = []; - for (const info of await Promise.all(promiseList)) { - // Ignore empty results - if (!info) continue; - // Skip mods if not required - if (info.isMod && !includeMods) continue; - // Else save data - result.push(info); - } - - if (shared.isolation) await browser.close(); - return result; + return results; }; + /** * @public * Starting from the url, it gets all the information about the game you are looking for. @@ -261,27 +137,20 @@ module.exports.getGameData = async function (name, includeMods) { * @returns {Promise} Information about the game. If no game was found, null is returned */ module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged || !shared.cookies) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return null; } - // Check URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); - if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); - - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - + // Check URL validity + const exists = await networkHelper.urlExists(url); + if (!exists) throw new URIError(`${url} is not a valid URL`); + if (!networkHelper.isF95URL(url)) throw new Error(`${url} is not a valid F95Zone URL`); + // Get game data - const result = await scraper.getGameInfo(browser, url); - - if (shared.isolation) await browser.close(); - return result; + return await scraper.getGameInfo(url); }; + /** * @public * Gets the data of the currently logged in user. @@ -289,297 +158,76 @@ module.exports.getGameDataFromURL = async function (url) { * @returns {Promise} Data of the user currently logged in */ module.exports.getUserData = async function () { - if (!shared.isLogged || !shared.cookies) { + if (!shared.isLogged) { shared.logger.warn(USER_NOT_LOGGED); return null; } - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_BASE_URL); // Go to base page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_ELEMENT), - page.waitForSelector(selectorK.AVATAR_PIC), - ]); - - const threads = getUserWatchedGameThreads(browser); + const threads = await getUserWatchedGameThreads(null); const username = await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector).innerText, - selectorK.USERNAME_ELEMENT + f95selector.USERNAME_ELEMENT ); const avatarSrc = await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector).getAttribute("src"), - selectorK.AVATAR_PIC + f95selector.AVATAR_PIC ); const ud = new UserData(); ud.username = username; - ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = await threads; - - await page.close(); - if (shared.isolation) await browser.close(); + ud.avatarSrc = networkHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; + ud.watchedThreads = threads; return ud; }; -/** - * @public - * Logout from the current user and gracefully close shared browser. - * You **must** be logged in to the portal before calling this method. - */ -module.exports.logout = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return; - } - - // Logout - shared.isLogged = false; - - // Gracefully close shared browser - if (!shared.isolation && _browser !== null) { - await _browser.close(); - _browser = null; - } -}; //#endregion //#region Private methods -//#region Cookies functions -/** - * @private - * Loads and verifies the expiration of previously stored cookies from disk - * if they exist, otherwise it returns null. - * @return {object[]} List of dictionaries or null if cookies don't exist - */ -function loadCookies() { - // Check the existence of the cookie file - if (fs.existsSync(shared.cookiesCachePath)) { - // Read cookies - const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); - const cookies = JSON.parse(cookiesJSON); - - // Check if the cookies have expired - for (const cookie of cookies) { - if (isCookieExpired(cookie)) return null; - } - - // Cookies loaded and verified - return cookies; - } else return null; -} -/** - * @private - * Check the validity of a cookie. - * @param {object} cookie Cookies to verify the validity. It's a dictionary - * @returns {Boolean} true if the cookie has expired, false otherwise - */ -function isCookieExpired(cookie) { - // Local variables - let expiredCookies = false; - - // Ignore cookies that never expire - const expirationUnixTimestamp = cookie.expire; - - if (expirationUnixTimestamp !== "-1") { - // Convert UNIX epoch timestamp to normal Date - const expirationDate = new Date(expirationUnixTimestamp * 1000); - - if (expirationDate < Date.now()) { - shared.logger.warn( - "Cookie " + cookie.name + " expired, you need to re-authenticate" - ); - expiredCookies = true; - } - } - - return expiredCookies; -} -//#endregion Cookies functions - -//#region Latest Updates page parserer -/** - * @private - * If present, it reads the file containing the searched values (engines or states) - * from the disk, otherwise it connects to the F95 portal (at the page - * https://f95zone.to/latest) and downloads them. - * @param {puppeteer.Page} page Page used to locate the required elements - * @param {String} path Path to disk of the JSON file containing the data to read / write - * @param {String} selector CSS selector of the required elements - * @param {String} elementRequested Required element (engines or states) used to detail log messages - * @returns {Promise} List of required values in uppercase - */ -async function loadValuesFromLatestPage( - page, - path, - selector, - elementRequested -) { - // If the values already exist they are loaded from disk without having to connect to F95 - shared.logger.info("Load " + elementRequested + " from disk..."); - if (fs.existsSync(path)) { - const valueJSON = fs.readFileSync(path); - return JSON.parse(valueJSON); - } - - // Otherwise, connect and download the data from the portal - shared.logger.info("No " + elementRequested + " cached, downloading..."); - const values = await getValuesFromLatestPage( - page, - selector, - "Getting " + elementRequested + " from page" - ); - fs.writeFileSync(path, JSON.stringify(values)); - return values; -} -/** - * @private - * Gets all the textual values of the elements present - * in the F95 portal page and identified by the selector - * passed by parameter - * @param {puppeteer.Page} page Page used to locate items specified by the selector - * @param {String} selector CSS selector - * @param {String} logMessage Log message indicating which items the selector is requesting - * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector - */ -async function getValuesFromLatestPage(page, selector, logMessage) { - shared.logger.info(logMessage); - - const result = []; - const elements = await page.$$(selector); - - for (const element of elements) { - const text = await element.evaluate( - /* istanbul ignore next */ (e) => e.innerText - ); - - // Save as upper text for better match if used in query - result.push(text.toUpperCase()); - } - return result; -} -//#endregion - //#region User -/** - * @private - * Log in to the F95Zone portal and, if successful, save the cookies. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} username Username to use during login - * @param {String} password Password to use during login - * @returns {Promise} Result of the operation - */ -async function loginF95(browser, username, password) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_LOGIN_URL); // Go to login page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_INPUT), - page.waitForSelector(selectorK.PASSWORD_INPUT), - page.waitForSelector(selectorK.LOGIN_BUTTON), - ]); - - await page.type(selectorK.USERNAME_INPUT, username); // Insert username - await page.type(selectorK.PASSWORD_INPUT, password); // Insert password - await Promise.all([ - page.click(selectorK.LOGIN_BUTTON), // Click on the login button - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Prepare result - let message = ""; - - // Check if the user is logged in - const success = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector) !== null, - selectorK.AVATAR_INFO - ); - - const errorMessageExists = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector) !== null, - selectorK.LOGIN_MESSAGE_ERROR - ); - - // Save cookies to avoid re-auth - if (success) { - const c = await page.cookies(); - fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); - message = "Authentication successful"; - } else if (errorMessageExists) { - const errorMessage = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.LOGIN_MESSAGE_ERROR - ); - - if (errorMessage === "Incorrect password. Please try again.") { - message = "Incorrect password"; - } else if ( - errorMessage === - "The requested user '" + username + "' could not be found." - ) { - // The escaped quotes are important! - message = "Incorrect username"; - } else message = errorMessage; - } else message = "Unknown error"; - - await page.close(); // Close the page - return new LoginResult(success, message); -} /** * @private * Gets the list of URLs of threads the user follows. * @param {puppeteer.Browser} browser Browser object used for navigation * @returns {Promise} URL list */ -async function getUserWatchedGameThreads(browser) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page +async function getUserWatchedGameThreads() { + const page = null; + await page.goto(f95url.F95_WATCHED_THREADS); // Go to the thread page // Explicitly wait for the required items to load - await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); + await page.waitForSelector(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON); // Show the popup await Promise.all([ - page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), + page.click(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON), + page.waitForSelector(f95selector.UNREAD_THREAD_CHECKBOX), + page.waitForSelector(f95selector.ONLY_GAMES_THREAD_OPTION), + page.waitForSelector(f95selector.FILTER_THREADS_BUTTON), ]); // Set the filters await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector).removeAttribute("checked"), - selectorK.UNREAD_THREAD_CHECKBOX + f95selector.UNREAD_THREAD_CHECKBOX ); // Also read the threads already read // Filter the threads - await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); - await page.click(selectorK.FILTER_THREADS_BUTTON); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + await page.click(f95selector.ONLY_GAMES_THREAD_OPTION); + await page.click(f95selector.FILTER_THREADS_BUTTON); + await page.waitForSelector(f95selector.WATCHED_THREAD_URLS); // Get the threads urls const urls = []; let nextPageExists = false; do { // Get all the URLs - for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { + for (const handle of await page.$$(f95selector.WATCHED_THREAD_URLS)) { const src = await page.evaluate( /* istanbul ignore next */ (element) => element.href, handle @@ -591,13 +239,13 @@ async function getUserWatchedGameThreads(browser) { nextPageExists = await page.evaluate( /* istanbul ignore next */ (selector) => document.querySelector(selector), - selectorK.WATCHED_THREAD_NEXT_PAGE + f95selector.WATCHED_THREAD_NEXT_PAGE ); // Click to next page if (nextPageExists) { - await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); + await page.click(f95selector.WATCHED_THREAD_NEXT_PAGE); + await page.waitForSelector(f95selector.WATCHED_THREAD_URLS); } } while (nextPageExists); diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 15434a7..19b609b 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -59,12 +59,6 @@ module.exports.autenticate = async function (credentials) { shared.logger.info(`Authenticating with user ${credentials.username}`); if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); - // If the user is already logged, return - if(shared.isLogged) { - shared.logger.warn(`${credentials.username} already authenticated`); - return new LoginResult(true, "Already authenticated"); - } - // Secure the URL const secureURL = exports.enforceHttpsUrl(f95url.F95_LOGIN_URL); @@ -91,7 +85,6 @@ module.exports.autenticate = async function (credentials) { const errorMessage = $("body").find(f95selector.LOGIN_MESSAGE_ERROR).text().replace(/\n/g, ""); // Return the result of the authentication - shared.isLogged = errorMessage === ""; if (errorMessage === "") return new LoginResult(true, "Authentication successful"); else return new LoginResult(false, errorMessage); } catch (e) { diff --git a/app/scripts/user-scraper.js b/app/scripts/user-scraper.js new file mode 100644 index 0000000..9a390c3 --- /dev/null +++ b/app/scripts/user-scraper.js @@ -0,0 +1 @@ +"use strict"; \ No newline at end of file diff --git a/legacy/index.js b/legacy/index.js deleted file mode 100644 index cb26ad8..0000000 --- a/legacy/index.js +++ /dev/null @@ -1,609 +0,0 @@ -"use strict"; - -// Core modules -const fs = require("fs"); - -// Modules from file -const shared = require("../app/scripts/shared.js"); -const urlK = require("../app/scripts/constants/url.js"); -const selectorK = require("../app/scripts/constants/css-selector.js"); -const urlHelper = require("../app/scripts/url-helper.js"); -const scraper = require("../app/scripts/game-scraper.js"); -const { - prepareBrowser, - preparePage, -} = require("../app/scripts/puppeteer-helper.js"); -const searcher = require("../app/scripts/game-searcher.js"); - -// Classes from file -const GameInfo = require("../app/scripts/classes/game-info.js"); -const LoginResult = require("../app/scripts/classes/login-result.js"); -const UserData = require("../app/scripts/classes/user-data.js"); - -//#region Export classes -module.exports.GameInfo = GameInfo; -module.exports.LoginResult = LoginResult; -module.exports.UserData = UserData; -//#endregion Export classes - -//#region Export properties -/** - * Shows log messages and other useful functions for module debugging. - * @param {Boolean} value - */ -module.exports.debug = function (value) { - shared.debug = value; - - // Configure logger - shared.logger.level = value ? "debug" : "warn"; -}; -/** - * @public - * Indicates whether a user is logged in to the F95Zone platform or not. - * @returns {String} - */ -module.exports.isLogged = function () { - return shared.isLogged; -}; -/** - * @public - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {String} - */ -module.exports.setIsolation = function (value) { - shared.isolation = value; -}; -/** - * @public - * Path to the cache directory - * @returns {String} - */ -module.exports.getCacheDir = function () { - return shared.cacheDir; -}; -/** - * @public - * Set path to the cache directory - * @returns {String} - */ -module.exports.setCacheDir = function (value) { - shared.cacheDir = value; - - // Create directory if it doesn't exist - if (!fs.existsSync(shared.cacheDir)) fs.mkdirSync(shared.cacheDir); -}; -/** - * @public - * Set local chromium path. - * @returns {String} - */ -module.exports.setChromiumPath = function (value) { - shared.chromiumLocalPath = value; -}; -//#endregion Export properties - -//#region Global variables -var _browser = null; -const USER_NOT_LOGGED = "User not authenticated, unable to continue"; -//#endregion - -//#region Export methods -/** - * @public - * Log in to the F95Zone platform. - * This **must** be the first operation performed before accessing any other script functions. - * @param {String} username Username used for login - * @param {String} password Password used for login - * @returns {Promise} Result of the operation - */ -module.exports.login = async function (username, password) { - if (shared.isLogged) { - shared.logger.info("Already logged in"); - const result = new LoginResult(true, "Already logged in"); - return result; - } - - // If cookies are loaded, use them to authenticate - shared.cookies = loadCookies(); - if (shared.cookies !== null) { - shared.logger.info("Valid session, no need to re-authenticate"); - shared.isLogged = true; - const result = new LoginResult(true, "Logged with cookies"); - return result; - } - - // Else, log in throught browser - shared.logger.info( - "No saved sessions or expired session, login on the platform" - ); - - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const result = await loginF95(browser, username, password); - shared.isLogged = result.success; - - if (result.success) { - // Reload cookies - shared.cookies = loadCookies(); - shared.logger.info("User logged in through the platform"); - } else { - shared.logger.warn("Error during authentication: " + result.message); - } - if (shared.isolation) await browser.close(); - return result; -}; -/** - * @public - * This method loads the main data from the F95 portal - * used to provide game information. You **must** be logged - * in to the portal before calling this method. - * @returns {Promise} Result of the operation - */ -module.exports.loadF95BaseData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } - - shared.logger.info("Loading base data..."); - - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - - // Go to latest update page and wait for it to load - await page.goto(urlK.F95_LATEST_UPDATES, { - waitUntil: shared.WAIT_STATEMENT, - }); - - // Obtain engines (disk/online) - await page.waitForSelector(selectorK.ENGINE_ID_SELECTOR); - shared.engines = await loadValuesFromLatestPage( - page, - shared.enginesCachePath, - selectorK.ENGINE_ID_SELECTOR, - "engines" - ); - - // Obtain statuses (disk/online) - await page.waitForSelector(selectorK.STATUS_ID_SELECTOR); - shared.statuses = await loadValuesFromLatestPage( - page, - shared.statusesCachePath, - selectorK.STATUS_ID_SELECTOR, - "statuses" - ); - - await page.close(); - if (shared.isolation) await browser.close(); - shared.logger.info("Base data loaded"); - return true; -}; -/** - * @public - * Chek if exists a new version of the game. - * You **must** be logged in to the portal before calling this method. - * @param {GameInfo} info Information about the game to get the version for - * @returns {Promise} true if an update is available, false otherwise - */ -module.exports.chekIfGameHasUpdate = async function (info) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return false; - } - - // F95 change URL at every game update, - // so if the URL is different an update is available - const exists = await urlHelper.urlExists(info.f95url, true); - if (!exists) return true; - - // Parse version from title - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - const onlineVersion = await scraper.getGameVersionFromTitle(browser, info); - - if (shared.isolation) await browser.close(); - - return onlineVersion.toUpperCase() !== info.version.toUpperCase(); -}; -/** - * @public - * Starting from the name, it gets all the information about the game you are looking for. - * You **must** be logged in to the portal before calling this method. - * @param {String} name Name of the game searched - * @param {Boolean} includeMods Indicates whether to also take mods into account when searching - * @returns {Promise} List of information obtained where each item corresponds to - * an identified game (in the case of homonymy of titles) - */ -module.exports.getGameData = async function (name, includeMods) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } - - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const urlList = await searcher.getSearchGameResults(browser, name); - - // Process previous partial results - const promiseList = []; - for (const url of urlList) { - // Start looking for information - promiseList.push(scraper.getGameInfo(browser, url)); - } - - // Filter for mods - const result = []; - for (const info of await Promise.all(promiseList)) { - // Ignore empty results - if (!info) continue; - // Skip mods if not required - if (info.isMod && !includeMods) continue; - // Else save data - result.push(info); - } - - if (shared.isolation) await browser.close(); - return result; -}; -/** - * @public - * Starting from the url, it gets all the information about the game you are looking for. - * You **must** be logged in to the portal before calling this method. - * @param {String} url URL of the game to obtain information of - * @returns {Promise} Information about the game. If no game was found, null is returned - */ -module.exports.getGameDataFromURL = async function (url) { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } - - // Check URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(url + " is not a valid URL"); - if (!urlHelper.isF95URL(url)) - throw new Error(url + " is not a valid F95Zone URL"); - - // Gets the search results of the game being searched for - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - - // Get game data - const result = await scraper.getGameInfo(browser, url); - - if (shared.isolation) await browser.close(); - return result; -}; -/** - * @public - * Gets the data of the currently logged in user. - * You **must** be logged in to the portal before calling this method. - * @returns {Promise} Data of the user currently logged in - */ -module.exports.getUserData = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return null; - } - - // Prepare a new web page - if (_browser === null && !shared.isolation) _browser = await prepareBrowser(); - const browser = shared.isolation ? await prepareBrowser() : _browser; - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_BASE_URL); // Go to base page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_ELEMENT), - page.waitForSelector(selectorK.AVATAR_PIC), - ]); - - const threads = getUserWatchedGameThreads(browser); - - const username = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.USERNAME_ELEMENT - ); - - const avatarSrc = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).getAttribute("src"), - selectorK.AVATAR_PIC - ); - - const ud = new UserData(); - ud.username = username; - ud.avatarSrc = urlHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = await threads; - - await page.close(); - if (shared.isolation) await browser.close(); - - return ud; -}; -/** - * @public - * Logout from the current user and gracefully close shared browser. - * You **must** be logged in to the portal before calling this method. - */ -module.exports.logout = async function () { - if (!shared.isLogged || !shared.cookies) { - shared.logger.warn(USER_NOT_LOGGED); - return; - } - - // Logout - shared.isLogged = false; - - // Gracefully close shared browser - if (!shared.isolation && _browser !== null) { - await _browser.close(); - _browser = null; - } -}; -//#endregion - -//#region Private methods - -//#region Cookies functions -/** - * @private - * Loads and verifies the expiration of previously stored cookies from disk - * if they exist, otherwise it returns null. - * @return {object[]} List of dictionaries or null if cookies don't exist - */ -function loadCookies() { - // Check the existence of the cookie file - if (fs.existsSync(shared.cookiesCachePath)) { - // Read cookies - const cookiesJSON = fs.readFileSync(shared.cookiesCachePath); - const cookies = JSON.parse(cookiesJSON); - - // Check if the cookies have expired - for (const cookie of cookies) { - if (isCookieExpired(cookie)) return null; - } - - // Cookies loaded and verified - return cookies; - } else return null; -} -/** - * @private - * Check the validity of a cookie. - * @param {object} cookie Cookies to verify the validity. It's a dictionary - * @returns {Boolean} true if the cookie has expired, false otherwise - */ -function isCookieExpired(cookie) { - // Local variables - let expiredCookies = false; - - // Ignore cookies that never expire - const expirationUnixTimestamp = cookie.expire; - - if (expirationUnixTimestamp !== "-1") { - // Convert UNIX epoch timestamp to normal Date - const expirationDate = new Date(expirationUnixTimestamp * 1000); - - if (expirationDate < Date.now()) { - shared.logger.warn( - "Cookie " + cookie.name + " expired, you need to re-authenticate" - ); - expiredCookies = true; - } - } - - return expiredCookies; -} -//#endregion Cookies functions - -//#region Latest Updates page parserer -/** - * @private - * If present, it reads the file containing the searched values (engines or states) - * from the disk, otherwise it connects to the F95 portal (at the page - * https://f95zone.to/latest) and downloads them. - * @param {puppeteer.Page} page Page used to locate the required elements - * @param {String} path Path to disk of the JSON file containing the data to read / write - * @param {String} selector CSS selector of the required elements - * @param {String} elementRequested Required element (engines or states) used to detail log messages - * @returns {Promise} List of required values in uppercase - */ -async function loadValuesFromLatestPage( - page, - path, - selector, - elementRequested -) { - // If the values already exist they are loaded from disk without having to connect to F95 - shared.logger.info("Load " + elementRequested + " from disk..."); - if (fs.existsSync(path)) { - const valueJSON = fs.readFileSync(path); - return JSON.parse(valueJSON); - } - - // Otherwise, connect and download the data from the portal - shared.logger.info("No " + elementRequested + " cached, downloading..."); - const values = await getValuesFromLatestPage( - page, - selector, - "Getting " + elementRequested + " from page" - ); - fs.writeFileSync(path, JSON.stringify(values)); - return values; -} -/** - * @private - * Gets all the textual values of the elements present - * in the F95 portal page and identified by the selector - * passed by parameter - * @param {puppeteer.Page} page Page used to locate items specified by the selector - * @param {String} selector CSS selector - * @param {String} logMessage Log message indicating which items the selector is requesting - * @return {Promise} List of uppercase strings indicating the textual values of the elements identified by the selector - */ -async function getValuesFromLatestPage(page, selector, logMessage) { - shared.logger.info(logMessage); - - const result = []; - const elements = await page.$$(selector); - - for (const element of elements) { - const text = await element.evaluate( - /* istanbul ignore next */ (e) => e.innerText - ); - - // Save as upper text for better match if used in query - result.push(text.toUpperCase()); - } - return result; -} -//#endregion - -//#region User -/** - * @private - * Log in to the F95Zone portal and, if successful, save the cookies. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} username Username to use during login - * @param {String} password Password to use during login - * @returns {Promise} Result of the operation - */ -async function loginF95(browser, username, password) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_LOGIN_URL); // Go to login page - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.USERNAME_INPUT), - page.waitForSelector(selectorK.PASSWORD_INPUT), - page.waitForSelector(selectorK.LOGIN_BUTTON), - ]); - - await page.type(selectorK.USERNAME_INPUT, username); // Insert username - await page.type(selectorK.PASSWORD_INPUT, password); // Insert password - await Promise.all([ - page.click(selectorK.LOGIN_BUTTON), // Click on the login button - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Prepare result - let message = ""; - - // Check if the user is logged in - const success = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector) !== null, - selectorK.AVATAR_INFO - ); - - const errorMessageExists = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector) !== null, - selectorK.LOGIN_MESSAGE_ERROR - ); - - // Save cookies to avoid re-auth - if (success) { - const c = await page.cookies(); - fs.writeFileSync(shared.cookiesCachePath, JSON.stringify(c)); - message = "Authentication successful"; - } else if (errorMessageExists) { - const errorMessage = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - selectorK.LOGIN_MESSAGE_ERROR - ); - - if (errorMessage === "Incorrect password. Please try again.") { - message = "Incorrect password"; - } else if ( - errorMessage === - "The requested user '" + username + "' could not be found." - ) { - // The escaped quotes are important! - message = "Incorrect username"; - } else message = errorMessage; - } else message = "Unknown error"; - - await page.close(); // Close the page - return new LoginResult(success, message); -} -/** - * @private - * Gets the list of URLs of threads the user follows. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @returns {Promise} URL list - */ -async function getUserWatchedGameThreads(browser) { - const page = await preparePage(browser); // Set new isolated page - await page.goto(urlK.F95_WATCHED_THREADS); // Go to the thread page - - // Explicitly wait for the required items to load - await page.waitForSelector(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON); - - // Show the popup - await Promise.all([ - page.click(selectorK.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(selectorK.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(selectorK.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(selectorK.FILTER_THREADS_BUTTON), - ]); - - // Set the filters - await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).removeAttribute("checked"), - selectorK.UNREAD_THREAD_CHECKBOX - ); // Also read the threads already read - - // Filter the threads - await page.click(selectorK.ONLY_GAMES_THREAD_OPTION); - await page.click(selectorK.FILTER_THREADS_BUTTON); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - - // Get the threads urls - const urls = []; - let nextPageExists = false; - do { - // Get all the URLs - for (const handle of await page.$$(selectorK.WATCHED_THREAD_URLS)) { - const src = await page.evaluate( - /* istanbul ignore next */ (element) => element.href, - handle - ); - // If 'unread' is left, it will redirect to the last unread post - const url = src.replace("/unread", ""); - urls.push(url); - } - - nextPageExists = await page.evaluate( - /* istanbul ignore next */ (selector) => document.querySelector(selector), - selectorK.WATCHED_THREAD_NEXT_PAGE - ); - - // Click to next page - if (nextPageExists) { - await page.click(selectorK.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(selectorK.WATCHED_THREAD_URLS); - } - } while (nextPageExists); - - await page.close(); - return urls; -} -//#endregion User - -//#endregion Private methods diff --git a/legacy/scripts/classes/game-download.js b/legacy/scripts/classes/game-download.js deleted file mode 100644 index e1240fb..0000000 --- a/legacy/scripts/classes/game-download.js +++ /dev/null @@ -1,111 +0,0 @@ -/* istanbul ignore file */ - -"use strict"; - -// Core modules -const fs = require("fs"); -const path = require("path"); - -// Public modules from npm -// const { File } = require('megajs'); - -// Modules from file -const { prepareBrowser, preparePage } = require("../puppeteer-helper.js"); -const shared = require("../shared.js"); - -class GameDownload { - constructor() { - /** - * @public - * Platform that hosts game files - * @type String - */ - this.hosting = ""; - /** - * @public - * Link to game files - * @type String - */ - this.link = null; - /** - * @public - * Operating systems supported by the game version indicated in this class. - * Can be *WINDOWS/LINUX/MACOS* - * @type String[] - */ - this.supportedOS = []; - } - - /** - * @public - * Download the game data in the indicated path. - * Supported hosting platforms: MEGA, NOPY - * @param {String} path Save path - * @return {Promise} Result of the operation - */ - async download(path) { - if (this.link.includes("mega.nz")) - return await downloadMEGA(this.link, path); - else if (this.link.includes("nopy.to")) - return await downloadNOPY(this.link, path); - } -} -module.exports = GameDownload; - -async function downloadMEGA(url, savepath) { - // The URL is masked - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url); - await page.waitForSelector("a.host_link"); - - // Obtain the link for the unmasked page and click it - const link = await page.$("a.host_link"); - await link.click(); - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // Obtain the URL after the redirect - const downloadURL = page.url(); - - // Close browser and page - await page.close(); - await browser.close(); - - const stream = fs.createWriteStream(savepath); - const file = File.fromURL(downloadURL); - file.download().pipe(stream); - return fs.existsSync(savepath); -} - -async function downloadNOPY(url, savepath) { - // Prepare browser - const browser = await prepareBrowser(); - const page = await preparePage(browser); - await page.goto(url); - await page.waitForSelector("#download"); - - // Set the save path - await page._client.send("Page.setDownloadBehavior", { - behavior: "allow", - downloadPath: path.basename(path.dirname(savepath)), // It's a directory - }); - - // Obtain the download button and click it - const downloadButton = await page.$("#download"); - await downloadButton.click(); - - // Await for all the connections to close - await page.waitForNavigation({ - waitUntil: "networkidle0", - timeout: 0, // Disable timeout - }); - - // Close browser and page - await page.close(); - await browser.close(); - - return fs.existsSync(savepath); -} diff --git a/legacy/scripts/classes/game-info.js b/legacy/scripts/classes/game-info.js deleted file mode 100644 index c9fcac5..0000000 --- a/legacy/scripts/classes/game-info.js +++ /dev/null @@ -1,119 +0,0 @@ -"use strict"; - -class GameInfo { - constructor() { - //#region Properties - /** - * Game name - * @type String - */ - this.name = null; - /** - * Game author - * @type String - */ - this.author = null; - /** - * URL to the game's official conversation on the F95Zone portal - * @type String - */ - this.f95url = null; - /** - * Game description - * @type String - */ - this.overview = null; - /** - * List of tags associated with the game - * @type String[] - */ - this.tags = []; - /** - * Graphics engine used for game development - * @type String - */ - this.engine = null; - /** - * Progress of the game - * @type String - */ - this.status = null; - /** - * Game description image URL - * @type String - */ - this.previewSource = null; - /** - * Game version - * @type String - */ - this.version = null; - /** - * Last time the game underwent updates - * @type String - */ - this.lastUpdate = null; - /** - * Last time the local copy of the game was run - * @type String - */ - this.lastPlayed = null; - /** - * Specifies if the game is original or a mod - * @type Boolean - */ - this.isMod = false; - /** - * Changelog for the last version. - * @type String - */ - this.changelog = null; - /** - * Directory containing the local copy of the game - * @type String - */ - this.gameDir = null; - /** - * Information on game file download links, - * including information on hosting platforms - * and operating system supported by the specific link - * @type GameDownload[] - */ - this.downloadInfo = []; - //#endregion Properties - } - - /** - * Converts the object to a dictionary used for JSON serialization - */ - /* istanbul ignore next */ - toJSON() { - return { - name: this.name, - author: this.author, - f95url: this.f95url, - overview: this.overview, - engine: this.engine, - status: this.status, - previewSource: this.previewSource, - version: this.version, - lastUpdate: this.lastUpdate, - lastPlayed: this.lastPlayed, - isMod: this.isMod, - changelog: this.changelog, - gameDir: this.gameDir, - downloadInfo: this.downloadInfo, - }; - } - - /** - * Return a new GameInfo from a JSON string - * @param {String} json JSON string used to create the new object - * @returns {GameInfo} - */ - /* istanbul ignore next */ - static fromJSON(json) { - return Object.assign(new GameInfo(), json); - } -} -module.exports = GameInfo; diff --git a/legacy/scripts/classes/login-result.js b/legacy/scripts/classes/login-result.js deleted file mode 100644 index 51fd9bc..0000000 --- a/legacy/scripts/classes/login-result.js +++ /dev/null @@ -1,20 +0,0 @@ -"use strict"; - -/** - * Object obtained in response to an attempt to login to the portal. - */ -class LoginResult { - constructor(success, message) { - /** - * Result of the login operation - * @type Boolean - */ - this.success = success; - /** - * Login response message - * @type String - */ - this.message = message; - } -} -module.exports = LoginResult; diff --git a/legacy/scripts/classes/user-data.js b/legacy/scripts/classes/user-data.js deleted file mode 100644 index 7edb904..0000000 --- a/legacy/scripts/classes/user-data.js +++ /dev/null @@ -1,26 +0,0 @@ -"use strict"; - -/** - * Class containing the data of the user currently connected to the F95Zone platform. - */ -class UserData { - constructor() { - /** - * User username. - * @type String - */ - this.username = ""; - /** - * Path to the user's profile picture. - * @type String - */ - this.avatarSrc = null; - /** - * List of followed thread URLs. - * @type URL[] - */ - this.watchedThreads = []; - } -} - -module.exports = UserData; diff --git a/legacy/scripts/constants/css-selector.js b/legacy/scripts/constants/css-selector.js deleted file mode 100644 index bf0f60b..0000000 --- a/legacy/scripts/constants/css-selector.js +++ /dev/null @@ -1,33 +0,0 @@ -module.exports = Object.freeze({ - AVATAR_INFO: "span.avatar", - AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", - ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", - FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", - GAME_IMAGES: "img[src^=\"https://attachments.f95zone.to\"]", - GAME_TAGS: "a.tagItem", - GAME_TITLE: "h1.p-title-value", - GAME_TITLE_PREFIXES: "h1.p-title-value > a.labelLink > span[dir=\"auto\"]", - LOGIN_BUTTON: "button.button--icon--login", - LOGIN_MESSAGE_ERROR: - "div.blockMessage.blockMessage--error.blockMessage--iconic", - ONLY_GAMES_THREAD_OPTION: "select[name=\"nodes[]\"] > option[value=\"2\"]", - PASSWORD_INPUT: "input[name=\"password\"]", - SEARCH_BUTTON: "form.block > * button.button--icon--search", - SEARCH_FORM_TEXTBOX: "input[name=\"keywords\"][type=\"search\"]", - SEARCH_ONLY_GAMES_OPTION: "select[name=\"c[nodes][]\"] > option[value=\"1\"]", - STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", - THREAD_POSTS: - "article.message-body:first-child > div.bbWrapper:first-of-type", - THREAD_TITLE: "h3.contentRow-title", - TITLE_ONLY_CHECKBOX: "form.block > * input[name=\"c[title_only]\"]", - UNREAD_THREAD_CHECKBOX: "input[type=\"checkbox\"][name=\"unread\"]", - USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", - USERNAME_INPUT: "input[name=\"login\"]", - WATCHED_THREAD_FILTER_POPUP_BUTTON: "a.filterBar-menuTrigger", - WATCHED_THREAD_NEXT_PAGE: "a.pageNav-jump--next", - WATCHED_THREAD_URLS: "a[href^=\"/threads/\"][data-tp-primary]", - DOWNLOAD_LINKS_CONTAINER: "span[style=\"font-size: 18px\"]", - SEARCH_THREADS_RESULTS_BODY: "div.contentRow-main", - SEARCH_THREADS_MEMBERSHIP: "li > a:not(.username)", - THREAD_LAST_CHANGELOG: "div.bbCodeBlock-content > div:first-of-type", -}); diff --git a/legacy/scripts/constants/url.js b/legacy/scripts/constants/url.js deleted file mode 100644 index fac63ba..0000000 --- a/legacy/scripts/constants/url.js +++ /dev/null @@ -1,7 +0,0 @@ -module.exports = Object.freeze({ - F95_BASE_URL: "https://f95zone.to", - F95_SEARCH_URL: "https://f95zone.to/search/?type=post", - F95_LATEST_UPDATES: "https://f95zone.to/latest", - F95_LOGIN_URL: "https://f95zone.to/login", - F95_WATCHED_THREADS: "https://f95zone.to/watched/threads", -}); diff --git a/legacy/scripts/game-scraper.js b/legacy/scripts/game-scraper.js deleted file mode 100644 index 7b20e47..0000000 --- a/legacy/scripts/game-scraper.js +++ /dev/null @@ -1,467 +0,0 @@ -"use strict"; - -// Public modules from npm -const HTMLParser = require("node-html-parser"); -const puppeteer = require("puppeteer"); // skipcq: JS-0128 - -// Modules from file -const shared = require("./shared.js"); -const selectorK = require("./constants/css-selector.js"); -const { preparePage } = require("./puppeteer-helper.js"); -const GameDownload = require("./classes/game-download.js"); -const GameInfo = require("./classes/game-info.js"); -const urlHelper = require("./url-helper.js"); - -/** - * @protected - * Get information from the game's main page. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} url URL (String) of the game/mod to extract data from - * @return {Promise} Complete information about the game you are - * looking for - */ -module.exports.getGameInfo = async function (browser, url) { - shared.logger.info("Obtaining game info"); - - // Verify the correctness of the URL - const exists = await urlHelper.urlExists(url); - if (!exists) throw new URIError(`${url} is not a valid URL`); - if (!urlHelper.isF95URL(url)) - throw new Error(`${url} is not a valid F95Zone URL`); - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // It asynchronously searches for the elements and - // then waits at the end to compile the object to be returned - let info = new GameInfo(); - const title = getGameTitle(page); - const author = getGameAuthor(page); - const tags = getGameTags(page); - const redirectUrl = urlHelper.getUrlRedirect(url); - info = await parsePrefixes(page, info); // Fill status/engines/isMod - const structuredText = await getMainPostStructuredText(page); - const overview = getOverview(structuredText, info.isMod); - const parsedInfos = parseConversationPage(structuredText); - const previewSource = getGamePreviewSource(page); - const changelog = getLastChangelog(page); - - // Fill in the GameInfo element with the information obtained - info.name = await title; - info.author = await author; - info.tags = await tags; - info.f95url = await redirectUrl; - info.overview = overview; - info.lastUpdate = info.isMod - ? parsedInfos.UPDATED - : parsedInfos.THREAD_UPDATED; - info.previewSource = await previewSource; - info.changelog = await changelog; - info.version = await exports.getGameVersionFromTitle(browser, info); - - //let downloadData = getGameDownloadLink(page); - //info.downloadInfo = await downloadData; - /* Downloading games without going directly to - * the platform appears to be prohibited by - * the guidelines. It is therefore useless to - * keep the links for downloading the games. */ - - await page.close(); // Close the page - shared.logger.info("Founded data for " + info.name); - return info; -}; - -/** - * Obtain the game version without parsing again all the data of the game. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {GameInfo} info Information about the game - * @returns {Promise} Online version of the game - */ -module.exports.getGameVersionFromTitle = async function (browser, info) { - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(info.f95url, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the game page and wait until it loads - - // Get the title - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const title = HTMLParser.parse(titleHTML).childNodes.pop().rawText; - - // The title is in the following format: [PREFIXES] NAME GAME [VERSION] [AUTHOR] - const startIndex = title.indexOf("[") + 1; - const endIndex = title.indexOf("]", startIndex); - let version = title.substring(startIndex, endIndex).trim().toUpperCase(); - if (version.startsWith("V")) version = version.replace("V", ""); // Replace only the first occurrence - await page.close(); - return cleanFSString(version); -}; - -//#region Private methods -/** - * Clean a string from invalid File System chars. - * @param {String} s - * @returns {String} - */ -function cleanFSString(s) { - const rx = /[/\\?%*:|"<>]/g; - return s.replace(rx, ""); -} - -/** - * @private - * Get the game description from its web page. - * Different processing depending on whether the game is a mod or not. - * @param {String} text Structured text extracted from the game's web page - * @param {Boolean} isMod Specify if it is a game or a mod - * @returns {Promise} Game description - */ -function getOverview(text, isMod) { - // Get overview (different parsing for game and mod) - let overviewEndIndex; - if (isMod) overviewEndIndex = text.indexOf("Updated"); - else overviewEndIndex = text.indexOf("Thread Updated"); - return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); -} - -/** - * @private - * Extrapolate the page structure by removing the element tags - * and leaving only the text and its spacing. - * @param {puppeteer.Page} page Page containing the text - * @returns {Promise} Structured text - */ -async function getMainPostStructuredText(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - - // The info are plain text so we need to parse the HTML code - const bodyHTML = await page.evaluate( - /* istanbul ignore next */ - (mainPost) => mainPost.innerHTML, - post - ); - return HTMLParser.parse(bodyHTML).structuredText; -} - -/** - * @private - * Extrapolates and cleans the author from the page passed by parameter. - * @param {puppeteer.Page} page Page containing the author to be extrapolated - * @returns {Promise} Game author - */ -async function getGameAuthor(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); - - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - - // The last square brackets contain the author - const startTitleIndex = gameTitle.lastIndexOf("[") + 1; - return gameTitle.substring(startTitleIndex, gameTitle.length - 1).trim(); -} - -/** - * @private - * Process the post text to get all the useful - * information in the format *DESCRIPTOR : VALUE*. - * @param {String} text Structured text of the post - * @returns {Object} Dictionary of information - */ -function parseConversationPage(text) { - const dataPairs = {}; - - // The information searched in the game post are one per line - const splittedText = text.split("\n"); - for (const line of splittedText) { - if (!line.includes(":")) continue; - - // Create pair key/value - const splitted = line.split(":"); - const key = splitted[0].trim().toUpperCase().replace(/ /g, "_"); // Uppercase to avoid mismatch - const value = splitted[1].trim(); - - // Add pair to the dict if valid - if (value !== "") dataPairs[key] = value; - } - - return dataPairs; -} - -/** - * @private - * Gets the URL of the image used as a preview for the game in the conversation. - * @param {puppeteer.Page} page Page containing the URL to be extrapolated - * @returns {Promise} URL (String) of the image or null if failed to get it - */ -async function getGamePreviewSource(page) { - // Wait for the selector or return an empty value - try { - await page.waitForSelector(selectorK.GAME_IMAGES); - } catch { - return null; - } - - const src = await page.evaluate( - /* istanbul ignore next */ - (selector) => { - // Get the firs image available - const img = document.querySelector(selector); - - if (img) return img.getAttribute("src"); - else return null; - }, - selectorK.GAME_IMAGES - ); - - // Check if the URL is valid - return urlHelper.isStringAValidURL(src) ? src : null; -} - -/** - * @private - * Extrapolates and cleans the title from the page passed by parameter. - * @param {puppeteer.Page} page Page containing the title to be extrapolated - * @returns {Promise} Game title - */ -async function getGameTitle(page) { - // Get the game/mod name (without square brackets) - const titleHTML = await page.evaluate( - /* istanbul ignore next */ - (selector) => document.querySelector(selector).innerHTML, - selectorK.GAME_TITLE - ); - const structuredTitle = HTMLParser.parse(titleHTML); - - // The last element **shoud be** the title without prefixes (engines, status, other...) - const gameTitle = structuredTitle.childNodes.pop().rawText; - const endTitleIndex = gameTitle.indexOf("["); - return gameTitle.substring(0, endTitleIndex).trim(); -} - -/** - * @private - * Get the alphabetically sorted list of tags associated with the game. - * @param {puppeteer.Page} page Page containing the tags to be extrapolated - * @returns {Promise} List of uppercase tags - */ -async function getGameTags(page) { - const tags = []; - - // Get the game tags - for (const handle of await page.$$(selectorK.GAME_TAGS)) { - const tag = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - tags.push(tag.toUpperCase()); - } - return tags.sort(); -} - -/** - * @private - * Process the game title prefixes to extract information such as game status, - * graphics engine used, and whether it is a mod or original game. - * @param {puppeteer.Page} page Page containing the prefixes to be extrapolated - * @param {GameInfo} info Object to assign the identified information to - * @returns {Promise} GameInfo object passed in to which the identified information has been added - */ -async function parsePrefixes(page, info) { - // The 'Ongoing' status is not specified, only 'Abandoned'/'OnHold'/'Complete' - info.status = "ONGOING"; - for (const handle of await page.$$(selectorK.GAME_TITLE_PREFIXES)) { - const value = await page.evaluate( - /* istanbul ignore next */ - (element) => element.innerText, - handle - ); - - // Clean the prefix - const prefix = value.toUpperCase().replace("[", "").replace("]", "").trim(); - - // Getting infos... - if (shared.statuses.includes(prefix)) info.status = prefix; - else if (shared.engines.includes(prefix)) info.engine = prefix; - // This is not a game but a mod - else if (prefix === "MOD" || prefix === "CHEAT MOD") info.isMod = true; - } - return info; -} - -/** - * @private - * Get the last changelog available for the game. - * @param {puppeteer.Page} page Page containing the changelog - * @returns {Promise} Changelog for the last version or a empty string if no changelog is found - */ -async function getLastChangelog(page) { - // Gets the first post, where are listed all the game's informations - const post = (await page.$$(selectorK.THREAD_POSTS))[0]; - - const spoiler = await post.$(selectorK.THREAD_LAST_CHANGELOG); - if (!spoiler) return ""; - - const changelogHTML = await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - spoiler - ); - let parsedText = HTMLParser.parse(changelogHTML).structuredText; - - // Clean the text - if (parsedText.startsWith("Spoiler")) - parsedText = parsedText.replace("Spoiler", ""); - if (parsedText.startsWith(":")) parsedText = parsedText.replace(":", ""); - return parsedText.trim(); -} - -/** - * @private - * Get game download links for different platforms. - * @param {puppeteer.Page} page Page containing the links to be extrapolated - * @returns {Promise} List of objects used for game download - * @deprecated - */ -/* istanbul ignore next */ -// skipcq: JS-0128 -async function getGameDownloadLink(page) { - // Most used hosting platforms - const hostingPlatforms = [ - "MEGA", - "NOPY", - "FILESUPLOAD", - "MIXDROP", - "UPLOADHAVEN", - "PIXELDRAIN", - "FILESFM", - ]; - - // Supported OS platforms - const platformOS = ["WIN", "LINUX", "MAC", "ALL"]; - - // Gets the which contains the download links - const temp = await page.$$(selectorK.DOWNLOAD_LINKS_CONTAINER); - if (temp.length === 0) return []; - - // Look for the container that contains the links - // It is necessary because the same css selector - // also identifies other elements on the page - let container = null; - for (const candidate of temp) { - if (container !== null) break; - const upperText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerText, - candidate - ) - ).toUpperCase(); - - // Search if the container contains the name of a hosting platform - for (const p of hostingPlatforms) { - if (upperText.includes(p)) { - container = candidate; - break; - } - } - } - if (container === null) return []; - - // Extract the HTML text from the container - const searchText = ( - await page.evaluate( - /* istanbul ignore next */ - (e) => e.innerHTML, - container - ) - ).toLowerCase(); - - // Parse the download links - const downloadData = []; - for (const platform of platformOS) { - const data = extractGameHostingData(platform, searchText); - downloadData.push(...data); - } - return downloadData; -} - -/** - * @private - * From the HTML text it extracts the game download links for the specified operating system. - * @param {String} platform Name of the operating system to look for a compatible link to. - * It can only be *WIN/LINUX/MAC/ALL* - * @param {String} text HTML string to extract links from - * @returns {GameDownload[]} List of game download links for the selected platform - * @deprecated - */ -/* istanbul ignore next */ -function extractGameHostingData(platform, text) { - const PLATFORM_BOLD_OPEN = ""; - const CONTAINER_SPAN_CLOSE = ""; - const LINK_OPEN = "platform - let endIndex = - text.indexOf(PLATFORM_BOLD_OPEN, startIndex) + PLATFORM_BOLD_OPEN.length; - - // Find the end of the container - if (endIndex === -1) - endIndex = - text.indexOf(CONTAINER_SPAN_CLOSE, startIndex) + - CONTAINER_SPAN_CLOSE.length; - - text = text.substring(startIndex, endIndex); - - const downloadData = []; - const linkTags = text.split(LINK_OPEN); - for (const tag of linkTags) { - // Ignore non-link string - if (!tag.includes(HREF_START)) continue; - - // Find the hosting platform name - startIndex = tag.indexOf(TAG_CLOSE) + TAG_CLOSE.length; - endIndex = tag.indexOf(LINK_CLOSE, startIndex); - const hosting = tag.substring(startIndex, endIndex); - - // Find the 'href' attribute - startIndex = tag.indexOf(HREF_START) + HREF_START.length; - endIndex = tag.indexOf(HREF_END, startIndex); - const link = tag.substring(startIndex, endIndex); - - if (urlHelper.isStringAValidURL(link)) { - const gd = new GameDownload(); - gd.hosting = hosting.toUpperCase(); - gd.link = link; - gd.supportedOS = platform.toUpperCase(); - - downloadData.push(gd); - } - } - return downloadData; -} - -//#endregion Private methods diff --git a/legacy/scripts/game-searcher.js b/legacy/scripts/game-searcher.js deleted file mode 100644 index 497edfe..0000000 --- a/legacy/scripts/game-searcher.js +++ /dev/null @@ -1,132 +0,0 @@ -"use strict"; - -// Public modules from npm -const puppeteer = require("puppeteer"); // skipcq: JS-0128 - -// Modules from file -const shared = require("./shared.js"); -const urlK = require("./constants/url.js"); -const selectorK = require("./constants/css-selector.js"); -const { preparePage } = require("./puppeteer-helper.js"); -const { isF95URL } = require("./url-helper.js"); - -/** - * @protected - * Search the F95Zone portal to find possible conversations regarding the game you are looking for. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @param {String} gamename Name of the game to search for - * @returns {Promise} List of URL of possible games obtained from the preliminary research on the F95 portal - */ -module.exports.getSearchGameResults = async function (browser, gamename) { - shared.logger.info(`Searching ${gamename} on F95Zone`); - - const page = await preparePage(browser); // Set new isolated page - await page.setCookie(...shared.cookies); // Set cookies to avoid login - await page.goto(urlK.F95_SEARCH_URL, { - waitUntil: shared.WAIT_STATEMENT, - }); // Go to the search form and wait for it - - // Explicitly wait for the required items to load - await Promise.all([ - page.waitForSelector(selectorK.SEARCH_FORM_TEXTBOX), - page.waitForSelector(selectorK.TITLE_ONLY_CHECKBOX), - page.waitForSelector(selectorK.SEARCH_ONLY_GAMES_OPTION), - page.waitForSelector(selectorK.SEARCH_BUTTON), - ]); - - await page.type(selectorK.SEARCH_FORM_TEXTBOX, gamename); // Type the game we desire - await page.click(selectorK.TITLE_ONLY_CHECKBOX); // Select only the thread with the game in the titles - await page.click(selectorK.SEARCH_ONLY_GAMES_OPTION); // Search only games and mod - await Promise.all([ - page.click(selectorK.SEARCH_BUTTON), // Execute search - page.waitForNavigation({ - waitUntil: shared.WAIT_STATEMENT, - }), // Wait for page to load - ]); - - // Select all conversation titles - const resultsThread = await page.$$(selectorK.SEARCH_THREADS_RESULTS_BODY); - - // For each element found extract the info about the conversation - shared.logger.info("Extracting info from conversations"); - const results = []; - for (const element of resultsThread) { - const gameUrl = await getOnlyGameThreads(page, element); - if (gameUrl !== null) results.push(gameUrl); - } - shared.logger.info(`Find ${results.length} conversations`); - await page.close(); // Close the page - - return results; -}; - -//#region Private methods -/** - * @private - * Return the link of a conversation if it is a game or a mod. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} divHandle Element of the conversation to be analyzed - * @return {Promise} URL of the game/mod or null if the URL is not of a game - */ -async function getOnlyGameThreads(page, divHandle) { - // Obtain the elements containing the basic information - const titleHandle = await divHandle.$(selectorK.THREAD_TITLE); - const forumHandle = await divHandle.$(selectorK.SEARCH_THREADS_MEMBERSHIP); - - // Get the forum where the thread was posted - const forum = await getMembershipForum(page, forumHandle); - if (forum !== "GAMES" && forum !== "MODS") return null; - - // Get the URL of the thread from the title - return await getThreadURL(page, titleHandle); -} - -/** - * @private - * Obtain the membership forum of the thread passed throught 'handle'. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} handle Handle containing the forum membership - * @returns {Promise} Uppercase membership category - */ -async function getMembershipForum(page, handle) { - // The link can be something like: - // + /forums/request.NUMBER/ - // + /forums/game-recommendations-identification.NUMBER/ - // + /forums/games.NUMBER/ <-- We need this - - let link = await page.evaluate( - /* istanbul ignore next */ - (e) => e.getAttribute("href"), - handle - ); - - // Parse link - link = link.replace("/forums/", ""); - const endIndex = link.indexOf("."); - const forum = link.substring(0, endIndex); - - return forum.toUpperCase(); -} - -/** - * @private - * Obtain the URL of the thread passed through 'handle'. - * @param {puppeteer.Page} page Page containing the conversation to be analyzed - * @param {puppeteer.ElementHandle} handle Handle containing the thread title - * @returns {Promise} URL of the thread - */ -async function getThreadURL(page, handle) { - const relativeURLThread = await page.evaluate( - /* istanbul ignore next */ - (e) => e.querySelector("a").href, - handle - ); - - // Some game already have a full URL... - if (isF95URL(relativeURLThread)) return relativeURLThread; - - // ... else compose the URL and return - const urlThread = new URL(relativeURLThread, urlK.F95_BASE_URL).toString(); - return urlThread; -} -//#endregion Private methods diff --git a/legacy/scripts/puppeteer-helper.js b/legacy/scripts/puppeteer-helper.js deleted file mode 100644 index 942da84..0000000 --- a/legacy/scripts/puppeteer-helper.js +++ /dev/null @@ -1,60 +0,0 @@ -"use strict"; - -// Public modules from npm -const puppeteer = require("puppeteer"); - -// Modules from file -const shared = require("./shared.js"); - -/** - * @protected - * Create a Chromium instance used to navigate with Puppeteer. - * By default the browser is headless. - * @returns {Promise} Created browser - */ -module.exports.prepareBrowser = async function () { - // Create a headless browser - let browser = null; - if (shared.chromiumLocalPath) { - browser = await puppeteer.launch({ - executablePath: shared.chromiumLocalPath, - headless: !shared.debug, // Use GUI when debug = true - }); - } else { - browser = await puppeteer.launch({ - headless: !shared.debug, // Use GUI when debug = true - }); - } - - return browser; -}; - -/** - * @protected - * Prepare a page used to navigate the browser. - * The page is set up to reject image download requests. The user agent is also changed. - * @param {puppeteer.Browser} browser Browser to use when navigating where the page will be created - * @returns {Promise} New page - */ -module.exports.preparePage = async function (browser) { - // Create new page in the browser argument - const page = await browser.newPage(); - - // Block image download - await page.setRequestInterception(true); - page.on("request", (request) => { - if (request.resourceType() === "image") request.abort(); - else if (request.resourceType === "font") request.abort(); - // else if (request.resourceType() == 'stylesheet') request.abort(); - // else if(request.resourceType == 'media') request.abort(); - else request.continue(); - }); - - // Set custom user-agent - const userAgent = - "Mozilla/5.0 (X11; Linux x86_64)" + - "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.39 Safari/537.36"; - await page.setUserAgent(userAgent); - - return page; -}; diff --git a/legacy/scripts/shared.js b/legacy/scripts/shared.js deleted file mode 100644 index 85a5ba4..0000000 --- a/legacy/scripts/shared.js +++ /dev/null @@ -1,173 +0,0 @@ -"use strict"; - -// Core modules -const { join } = require("path"); - -const log4js = require("log4js"); - -/** - * Class containing variables shared between modules. - */ -class Shared { - //#region Properties - /** - * Shows log messages and other useful functions for module debugging. - * @type Boolean - */ - static #_debug = false; - /** - * Indicates whether a user is logged in to the F95Zone platform or not. - * @type Boolean - */ - static #_isLogged = false; - /** - * List of cookies obtained from the F95Zone platform. - * @type Object[] - */ - static #_cookies = null; - /** - * List of possible game engines used for development. - * @type String[] - */ - static #_engines = null; - /** - * List of possible development statuses that a game can assume. - * @type String[] - */ - static #_statuses = null; - /** - * Wait instruction for the browser created by puppeteer. - * @type String - */ - static WAIT_STATEMENT = "domcontentloaded"; - /** - * Path to the directory to save the cache generated by the API. - * @type String - */ - static #_cacheDir = "./f95cache"; - /** - * If true, it opens a new browser for each request to - * the F95Zone platform, otherwise it reuses the same. - * @type Boolean - */ - static #_isolation = false; - /** - * Logger object used to write to both file and console. - * @type log4js.Logger - */ - static #_logger = log4js.getLogger(); - //#endregion Properties - - //#region Getters - /** - * Shows log messages and other useful functions for module debugging. - * @returns {Boolean} - */ - static get debug() { - return this.#_debug; - } - /** - * Indicates whether a user is logged in to the F95Zone platform or not. - * @returns {Boolean} - */ - static get isLogged() { - return this.#_isLogged; - } - /** - * List of cookies obtained from the F95Zone platform. - * @returns {Object[]} - */ - static get cookies() { - return this.#_cookies; - } - /** - * List of possible game engines used for development. - * @returns {String[]} - */ - static get engines() { - return this.#_engines; - } - /** - * List of possible development states that a game can assume. - * @returns {String[]} - */ - static get statuses() { - return this.#_statuses; - } - /** - * Directory to save the API cache. - * @returns {String} - */ - static get cacheDir() { - return this.#_cacheDir; - } - /** - * Path to the F95 platform cache. - * @returns {String} - */ - static get cookiesCachePath() { - return join(this.#_cacheDir, "cookies.json"); - } - /** - * Path to the game engine cache. - * @returns {String} - */ - static get enginesCachePath() { - return join(this.#_cacheDir, "engines.json"); - } - /** - * Path to the cache of possible game states. - * @returns {String} - */ - static get statusesCachePath() { - return join(this.#_cacheDir, "statuses.json"); - } - /** - * If true, it opens a new browser for each request - * to the F95Zone platform, otherwise it reuses the same. - * @returns {Boolean} - */ - static get isolation() { - return this.#_isolation; - } - /** - * Logger object used to write to both file and console. - * @returns {log4js.Logger} - */ - static get logger() { - return this.#_logger; - } - //#endregion Getters - - //#region Setters - static set cookies(val) { - this.#_cookies = val; - } - - static set engines(val) { - this.#_engines = val; - } - - static set statuses(val) { - this.#_statuses = val; - } - - static set cacheDir(val) { - this.#_cacheDir = val; - } - - static set debug(val) { - this.#_debug = val; - } - - static set isLogged(val) { - this.#_isLogged = val; - } - - static set isolation(val) { - this.#_isolation = val; - } - //#endregion Setters -} - -module.exports = Shared; diff --git a/legacy/scripts/url-helper.js b/legacy/scripts/url-helper.js deleted file mode 100644 index aae60a0..0000000 --- a/legacy/scripts/url-helper.js +++ /dev/null @@ -1,72 +0,0 @@ -"use strict"; - -// Public modules from npm -const ky = require("ky-universal").create({ - throwHttpErrors: false, -}); - -// Modules from file -const { F95_BASE_URL } = require("./constants/url.js"); - -/** - * @protected - * Check if the url belongs to the domain of the F95 platform. - * @param {String} url URL to check - * @returns {Boolean} true if the url belongs to the domain, false otherwise - */ -module.exports.isF95URL = function (url) { - if (url.toString().startsWith(F95_BASE_URL)) return true; - else return false; -}; - -/** - * @protected - * Checks if the string passed by parameter has a properly formatted and valid path to a URL. - * @param {String} url String to check for correctness - * @returns {Boolean} true if the string is a valid URL, false otherwise - */ -module.exports.isStringAValidURL = function (url) { - try { - new URL(url); // skipcq: JS-0078 - return true; - } catch (err) { - return false; - } -}; - -/** - * @protected - * Check if a particular URL is valid and reachable on the web. - * @param {String} url URL to check - * @param {Boolean} checkRedirect If true, the function will consider redirects a violation and return false - * @returns {Promise} true if the URL exists, false otherwise - */ -module.exports.urlExists = async function (url, checkRedirect) { - if (!exports.isStringAValidURL(url)) { - return false; - } - - const response = await ky.head(url); - let valid = response !== undefined && !/4\d\d/.test(response.status); - - if (!valid) return false; - - if (checkRedirect) { - const redirectUrl = await exports.getUrlRedirect(url); - if (redirectUrl === url) valid = true; - else valid = false; - } - - return valid; -}; - -/** - * @protected - * Check if the URL has a redirect to another page. - * @param {String} url URL to check for redirect - * @returns {Promise} Redirect URL or the passed URL - */ -module.exports.getUrlRedirect = async function (url) { - const response = await ky.head(url); - return response.url; -}; From 90100709badd6dd5a61249c575d52576f2bb8507 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 00:14:28 +0100 Subject: [PATCH 09/17] Addes user data fetching --- app/index.js | 94 +------------------------ app/scripts/constants/css-selector.js | 11 +-- app/scripts/network-helper.js | 4 +- app/scripts/user-scraper.js | 98 ++++++++++++++++++++++++++- test/user-test.js | 5 ++ 5 files changed, 109 insertions(+), 103 deletions(-) diff --git a/app/index.js b/app/index.js index a8d62cf..d1b16d8 100644 --- a/app/index.js +++ b/app/index.js @@ -2,11 +2,10 @@ // Modules from file const shared = require("./scripts/shared.js"); -const f95url = require("./scripts/constants/url.js"); -const f95selector = require("./scripts/constants/css-selector.js"); const networkHelper = require("./scripts/network-helper.js"); const scraper = require("./scripts/scraper.js"); const searcher = require("./scripts/searcher.js"); +const uScraper = require("./scripts/user-scraper.js"); // Classes from file const Credentials = require("./scripts/classes/credentials.js"); @@ -163,95 +162,6 @@ module.exports.getUserData = async function () { return null; } - const threads = await getUserWatchedGameThreads(null); - - const username = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).innerText, - f95selector.USERNAME_ELEMENT - ); - - const avatarSrc = await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).getAttribute("src"), - f95selector.AVATAR_PIC - ); - - const ud = new UserData(); - ud.username = username; - ud.avatarSrc = networkHelper.isStringAValidURL(avatarSrc) ? avatarSrc : null; - ud.watchedThreads = threads; - - return ud; + return await uScraper.getUserData(); }; //#endregion - -//#region Private methods - -//#region User -/** - * @private - * Gets the list of URLs of threads the user follows. - * @param {puppeteer.Browser} browser Browser object used for navigation - * @returns {Promise} URL list - */ -async function getUserWatchedGameThreads() { - const page = null; - await page.goto(f95url.F95_WATCHED_THREADS); // Go to the thread page - - // Explicitly wait for the required items to load - await page.waitForSelector(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON); - - // Show the popup - await Promise.all([ - page.click(f95selector.WATCHED_THREAD_FILTER_POPUP_BUTTON), - page.waitForSelector(f95selector.UNREAD_THREAD_CHECKBOX), - page.waitForSelector(f95selector.ONLY_GAMES_THREAD_OPTION), - page.waitForSelector(f95selector.FILTER_THREADS_BUTTON), - ]); - - // Set the filters - await page.evaluate( - /* istanbul ignore next */ (selector) => - document.querySelector(selector).removeAttribute("checked"), - f95selector.UNREAD_THREAD_CHECKBOX - ); // Also read the threads already read - - // Filter the threads - await page.click(f95selector.ONLY_GAMES_THREAD_OPTION); - await page.click(f95selector.FILTER_THREADS_BUTTON); - await page.waitForSelector(f95selector.WATCHED_THREAD_URLS); - - // Get the threads urls - const urls = []; - let nextPageExists = false; - do { - // Get all the URLs - for (const handle of await page.$$(f95selector.WATCHED_THREAD_URLS)) { - const src = await page.evaluate( - /* istanbul ignore next */ (element) => element.href, - handle - ); - // If 'unread' is left, it will redirect to the last unread post - const url = src.replace("/unread", ""); - urls.push(url); - } - - nextPageExists = await page.evaluate( - /* istanbul ignore next */ (selector) => document.querySelector(selector), - f95selector.WATCHED_THREAD_NEXT_PAGE - ); - - // Click to next page - if (nextPageExists) { - await page.click(f95selector.WATCHED_THREAD_NEXT_PAGE); - await page.waitForSelector(f95selector.WATCHED_THREAD_URLS); - } - } while (nextPageExists); - - await page.close(); - return urls; -} -//#endregion User - -//#endregion Private methods diff --git a/app/scripts/constants/css-selector.js b/app/scripts/constants/css-selector.js index 3c624a0..e985b2d 100644 --- a/app/scripts/constants/css-selector.js +++ b/app/scripts/constants/css-selector.js @@ -1,6 +1,7 @@ module.exports = Object.freeze({ BD_ENGINE_ID_SELECTOR: "div[id^=\"btn-prefix_1_\"]>span", BD_STATUS_ID_SELECTOR: "div[id^=\"btn-prefix_4_\"]>span", + GT_IMAGES: "img:not([title])[data-src^=\"https://attachments.f95zone.to\"][data-url=\"\"]", GT_TAGS: "a.tagItem", GT_TITLE: "h1.p-title-value", @@ -16,13 +17,7 @@ module.exports = Object.freeze({ GS_RESULT_BODY: "div.contentRow-main", GS_MEMBERSHIP: "li > a:not(.username)", GET_REQUEST_TOKEN: "input[name=\"_xfToken\"]", - - LOGIN_BUTTON: "button.button--icon--login", + UD_USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", + UD_AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", LOGIN_MESSAGE_ERROR: "div.blockMessage.blockMessage--error.blockMessage--iconic", - PASSWORD_INPUT: "input[name=\"password\"]", - USERNAME_ELEMENT: "a[href=\"/account/\"] > span.p-navgroup-linkText", - USERNAME_INPUT: "input[name=\"login\"]", - AVATAR_INFO: "span.avatar", - AVATAR_PIC: "a[href=\"/account/\"] > span.avatar > img[class^=\"avatar\"]", - FILTER_THREADS_BUTTON: "button[class=\"button--primary button\"]", }); diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 19b609b..16b1503 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -2,7 +2,7 @@ // Public modules from npm const axios = require("axios").default; -const _ = require("lodash"); +const { isString } = require("lodash"); const ky = require("ky-universal").create({ throwHttpErrors: false, }); @@ -166,7 +166,7 @@ module.exports.fetchGETResponse = async function(url) { * @returns {String} */ module.exports.enforceHttpsUrl = function (url) { - return _.isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; + return isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; }; /** diff --git a/app/scripts/user-scraper.js b/app/scripts/user-scraper.js index 9a390c3..0d577de 100644 --- a/app/scripts/user-scraper.js +++ b/app/scripts/user-scraper.js @@ -1 +1,97 @@ -"use strict"; \ No newline at end of file +"use strict"; + +// Public modules from npm +const cheerio = require("cheerio"); + +// Modules from file +const networkHelper = require("./network-helper.js"); +const f95Selector = require("./constants/css-selector.js"); +const f95url = require("./constants/url.js"); +const UserData = require("./classes/user-data.js"); + +module.exports.getUserData = async function() { + // Fetch data + const data = await fetchUsernameAndAvatar(); + const urls = await fetchWatchedThreadURLs(); + + // Create object + const ud = new UserData(); + ud.username = data.username; + ud.avatarSrc = data.source; + ud.watchedThreads = urls; + + return ud; +}; + +//#region Private methods +async function fetchUsernameAndAvatar() { + // Fetch page + const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL); + + // Load HTML response + const $ = cheerio.load(html); + const body = $("body"); + + // Fetch username + const username = body.find(f95Selector.UD_USERNAME_ELEMENT).first().text().trim(); + + // Fetch user avatar image source + const source = body.find(f95Selector.UD_AVATAR_PIC).first().attr("src"); + + return { + username, + source + }; +} + +async function fetchWatchedThreadURLs() { + // Local variables + let currentURL = f95url.F95_WATCHED_THREADS; + const wathcedThreadURLs = []; + + do { + // Fetch page + const html = await networkHelper.fetchHTML(currentURL); + + // Load HTML response + const $ = cheerio.load(html); + const body = $("body"); + + // Find the URLs + const urls = fetchPageURLs(body); + wathcedThreadURLs.push(...urls); + + // Find the next page (if any) + currentURL = fetchNextPageURL(body); + } + while (currentURL); + + return wathcedThreadURLs; +} + +function fetchPageURLs(body) { + const elements = body.find(f95Selector.WT_URLS); + + return elements.map(function extractURLs(idx, e) { + // Obtain the link (replace "unread" only for the unread threads) + const partialLink = e.attribs.href.replace("unread", ""); + + // Compose and return the URL + return new URL(partialLink, f95url.F95_BASE_URL).toString(); + }).get(); +} + +/** + * + * @param {cheerio.Cheerio} body + */ +function fetchNextPageURL(body) { + const element = body.find(f95Selector.WT_NEXT_PAGE).first(); + + // No element found + if(element.length === 0) return null; + + // Compose and return the URL + return new URL(element.attr("href"), f95url.F95_BASE_URL).toString(); +} +//#endregion Private methods \ No newline at end of file diff --git a/test/user-test.js b/test/user-test.js index 8569a15..5ec4fbc 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -8,6 +8,7 @@ const searcher = require("../app/scripts/searcher.js"); const scraper = require("../app/scripts/scraper.js"); const Credentials = require("../app/scripts/classes/credentials.js"); const networkHelper = require("../app/scripts/network-helper.js"); +const uScraper = require("../app/scripts/user-scraper.js"); // Configure the .env reader dotenv.config(); @@ -16,6 +17,10 @@ dotenv.config(); auth().then(async function searchGames(result) { if(!result) return; + console.log("Fetching user data..."); + const userdata = await uScraper.getUserData(); + console.log(`${userdata.username} follows ${userdata.watchedThreads.length} threads`); + // Search for Kingdom Of Deception data await search("kingdom of deception"); From f3b723624d7dbcd97a0c65fdfd9daf15b1ad211f Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 10:01:39 +0100 Subject: [PATCH 10/17] Added JSDOC --- app/scripts/network-helper.js | 1 - app/scripts/scraper.js | 10 ++++++---- app/scripts/user-scraper.js | 30 ++++++++++++++++++++++++++++-- 3 files changed, 34 insertions(+), 7 deletions(-) diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 16b1503..8c34bc9 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -21,7 +21,6 @@ const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15"; axiosCookieJarSupport(axios); const cookieJar = new tough.CookieJar(); - const commonConfig = { headers: { "User-Agent": userAgent, diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js index bfa823a..e5cd538 100644 --- a/app/scripts/scraper.js +++ b/app/scripts/scraper.js @@ -66,7 +66,7 @@ module.exports.getGameInfo = async function (url) { * Parse the game prefixes obtaining the engine used, * the advancement status and if the game is actually a game or a mod. * @param {cheerio.Cheerio} body Page `body` selector - * @returns {Object} Dictionary of values + * @returns {Object.} Dictionary of values with keys `engine`, `status`, `mod` */ function parseGamePrefixes(body) { shared.logger.trace("Parsing prefixes..."); @@ -106,7 +106,7 @@ function parseGamePrefixes(body) { * @private * Extracts all the possible informations from the title. * @param {cheerio.Cheerio} body Page `body` selector - * @returns {Object} Dictionary of values + * @returns {Object.} Dictionary of values with keys `name`, `author`, `version` */ function extractInfoFromTitle(body) { shared.logger.trace("Extracting information from title..."); @@ -202,8 +202,10 @@ function extractChangelog(mainPost) { * @private * Process the main post text to get all the useful * information in the format *DESCRIPTOR : VALUE*. + * Gets "standard" values such as: `Language`, `SupportedOS`, `Censored`, and `LastUpdate`. + * All non-canonical values are instead grouped together as a dictionary with the key `Various`. * @param {String} text Structured text of the post - * @returns {Object} Dictionary of information + * @returns {Object.} Dictionary of information */ function parseMainPostText(text) { shared.logger.trace("Parsing main post raw text..."); @@ -274,7 +276,7 @@ function parseMainPostText(text) { * @private * Extracts and processes the JSON-LD values found at the bottom of the page. * @param {cheerio.Cheerio} body Page `body` selector - * @returns {Object} JSON-LD or `null` if no valid JSON is found + * @returns {Object.} JSON-LD or `null` if no valid JSON is found */ function extractStructuredData(body) { shared.logger.trace("Extracting JSON-LD data..."); diff --git a/app/scripts/user-scraper.js b/app/scripts/user-scraper.js index 0d577de..05cff21 100644 --- a/app/scripts/user-scraper.js +++ b/app/scripts/user-scraper.js @@ -9,6 +9,11 @@ const f95Selector = require("./constants/css-selector.js"); const f95url = require("./constants/url.js"); const UserData = require("./classes/user-data.js"); +/** + * @protected + * Gets user data, such as username, url of watched threads, and profile picture url. + * @return {UserData} User data + */ module.exports.getUserData = async function() { // Fetch data const data = await fetchUsernameAndAvatar(); @@ -24,6 +29,13 @@ module.exports.getUserData = async function() { }; //#region Private methods +/** + * @private + * It connects to the page and extracts the name + * of the currently logged in user and the URL + * of their profile picture. + * @return {Object.} + */ async function fetchUsernameAndAvatar() { // Fetch page const html = await networkHelper.fetchHTML(f95url.F95_BASE_URL); @@ -44,6 +56,11 @@ async function fetchUsernameAndAvatar() { }; } +/** + * @private + * Gets the list of URLs of threads watched by the user. + * @returns {String[]} List of URLs + */ async function fetchWatchedThreadURLs() { // Local variables let currentURL = f95url.F95_WATCHED_THREADS; @@ -69,6 +86,12 @@ async function fetchWatchedThreadURLs() { return wathcedThreadURLs; } +/** + * @private + * Gets the URLs of the watched threads on the page. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String[]} + */ function fetchPageURLs(body) { const elements = body.find(f95Selector.WT_URLS); @@ -82,8 +105,11 @@ function fetchPageURLs(body) { } /** - * - * @param {cheerio.Cheerio} body + * @private + * Gets the URL of the next page containing the watched threads + * or `null` if that page does not exist. + * @param {cheerio.Cheerio} body Page `body` selector + * @returns {String} */ function fetchNextPageURL(body) { const element = body.find(f95Selector.WT_NEXT_PAGE).first(); From d335f192cdd1fb2f714916cac1543cb0ecb0a7d7 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 10:04:12 +0100 Subject: [PATCH 11/17] Fixed JSDOC --- app/scripts/user-scraper.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/scripts/user-scraper.js b/app/scripts/user-scraper.js index 05cff21..c9c96e3 100644 --- a/app/scripts/user-scraper.js +++ b/app/scripts/user-scraper.js @@ -12,7 +12,7 @@ const UserData = require("./classes/user-data.js"); /** * @protected * Gets user data, such as username, url of watched threads, and profile picture url. - * @return {UserData} User data + * @return {Promise} User data */ module.exports.getUserData = async function() { // Fetch data @@ -34,7 +34,7 @@ module.exports.getUserData = async function() { * It connects to the page and extracts the name * of the currently logged in user and the URL * of their profile picture. - * @return {Object.} + * @return {Promise>} */ async function fetchUsernameAndAvatar() { // Fetch page @@ -59,7 +59,7 @@ async function fetchUsernameAndAvatar() { /** * @private * Gets the list of URLs of threads watched by the user. - * @returns {String[]} List of URLs + * @returns {Promise} List of URLs */ async function fetchWatchedThreadURLs() { // Local variables From c88c9720a274097319d169354c736da367a59a30 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 10:21:36 +0100 Subject: [PATCH 12/17] Bugfix --- app/index.js | 2 +- app/scripts/scraper.js | 3 +- test/user-test.js | 72 ++++++++++++++++++------------------------ 3 files changed, 33 insertions(+), 44 deletions(-) diff --git a/app/index.js b/app/index.js index d1b16d8..4ceb06e 100644 --- a/app/index.js +++ b/app/index.js @@ -122,7 +122,7 @@ module.exports.getGameData = async function (name, mod) { const results = []; for (const url of urls) { // Start looking for information - const info = scraper.getGameInfo(url); + const info = await scraper.getGameInfo(url); results.push(info); } return results; diff --git a/app/scripts/scraper.js b/app/scripts/scraper.js index e5cd538..129c90c 100644 --- a/app/scripts/scraper.js +++ b/app/scripts/scraper.js @@ -305,10 +305,11 @@ function extractStructuredData(body) { * Different processing depending on whether the game is a mod or not. * @param {String} text Structured text extracted from the game's web page * @param {Boolean} mod Specify if it is a game or a mod - * @returns {Promise} Game description + * @returns {String} Game description */ function getOverview(text, mod) { shared.logger.trace("Extracting game overview..."); + // Get overview (different parsing for game and mod) const overviewEndIndex = mod ? text.indexOf("Updated") : text.indexOf("Thread Updated"); return text.substring(0, overviewEndIndex).replace("Overview:\n", "").trim(); diff --git a/test/user-test.js b/test/user-test.js index 5ec4fbc..804fa2d 100644 --- a/test/user-test.js +++ b/test/user-test.js @@ -4,58 +4,46 @@ const dotenv = require("dotenv"); // Modules from file -const searcher = require("../app/scripts/searcher.js"); -const scraper = require("../app/scripts/scraper.js"); -const Credentials = require("../app/scripts/classes/credentials.js"); -const networkHelper = require("../app/scripts/network-helper.js"); -const uScraper = require("../app/scripts/user-scraper.js"); +const F95API = require("../app/index.js"); // Configure the .env reader dotenv.config(); -// Login -auth().then(async function searchGames(result) { - if(!result) return; +main(); + +async function main() { + // Local variables + const gameList = [ + "kingdom of deception", + "perverted education", + "corrupted kingdoms", + "summertime saga", + "brothel king" + ]; + + // Log in the platform + console.log("Authenticating..."); + const result = await F95API.login(process.env.F95_USERNAME, process.env.F95_PASSWORD); + console.log(`Authentication result: ${result.message}`); + + // Get user data console.log("Fetching user data..."); - const userdata = await uScraper.getUserData(); + const userdata = await F95API.getUserData(); console.log(`${userdata.username} follows ${userdata.watchedThreads.length} threads`); - // Search for Kingdom Of Deception data - await search("kingdom of deception"); + for(const gamename of gameList) { + console.log(`Searching '${gamename}'...`); + const found = await F95API.getGameData(gamename, false); - // Search for Perverted Education data - await search("perverted education"); + // If no game is found + if (found.length === 0) { + console.log(`No data found for '${gamename}'`); + continue; + } - // Search for Corrupted Kingdoms data - await search("corrupted kingdoms"); - - // Search for Summertime Saga data - await search("summertime saga"); -}); - -async function auth() { - console.log("Token fetch..."); - const creds = new Credentials(process.env.F95_USERNAME, process.env.F95_PASSWORD); - await creds.fetchToken(); - console.log(`Token obtained: ${creds.token}`); - - console.log("Authenticating..."); - const result = await networkHelper.autenticate(creds); - console.log(`Authentication result: ${result.message}`); - - return result.success; -} - -async function search(gamename) { - console.log(`Searching '${gamename}'...`); - const urls = await searcher.searchGame(gamename); - console.log(`Found: ${urls}`); - - console.log("Scraping data..."); - for (const url of urls) { - const gamedata = await scraper.getGameInfo(url); + // Extract first game + const gamedata = found[0]; console.log(`Found ${gamedata.name} (${gamedata.version}) by ${gamedata.author}`); } - console.log("Scraping completed!"); } From af59febbb55b8c27cec6d1be4a90eadfbe46c880 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 15:06:09 +0100 Subject: [PATCH 13/17] Added unit tests --- test/user-test.js => app/example.js | 2 +- app/index.js | 4 +- app/scripts/classes/game-info.js | 9 +- app/scripts/network-helper.js | 36 ++-- app/scripts/shared.js | 1 + package-lock.json | 2 +- package.json | 7 +- test/index-test-legacy.js | 290 ++++++++++++++++++++++++++ test/index-test.js | 302 +++------------------------- test/suites/api-test.js | 64 ++++++ test/suites/credentials-test.js | 52 +++++ test/suites/network-helper-test.js | 107 ++++++++++ test/suites/scraper-test.js | 42 ++++ test/suites/searcher-test.js | 64 ++++++ test/suites/user-scraper-test.js | 54 +++++ 15 files changed, 738 insertions(+), 298 deletions(-) rename test/user-test.js => app/example.js (96%) create mode 100644 test/index-test-legacy.js create mode 100644 test/suites/api-test.js create mode 100644 test/suites/credentials-test.js create mode 100644 test/suites/network-helper-test.js create mode 100644 test/suites/scraper-test.js create mode 100644 test/suites/searcher-test.js create mode 100644 test/suites/user-scraper-test.js diff --git a/test/user-test.js b/app/example.js similarity index 96% rename from test/user-test.js rename to app/example.js index 804fa2d..286425e 100644 --- a/test/user-test.js +++ b/app/example.js @@ -4,7 +4,7 @@ const dotenv = require("dotenv"); // Modules from file -const F95API = require("../app/index.js"); +const F95API = require("./index.js"); // Configure the .env reader dotenv.config(); diff --git a/app/index.js b/app/index.js index 4ceb06e..3bff9ce 100644 --- a/app/index.js +++ b/app/index.js @@ -24,6 +24,7 @@ module.exports.UserData = UserData; * Shows log messages and other useful functions for module debugging. * @param {Boolean} value */ +/* istambul ignore next */ module.exports.debug = function (value) { shared.debug = value; @@ -35,6 +36,7 @@ module.exports.debug = function (value) { * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {String} */ +/* istambul ignore next */ module.exports.isLogged = function () { return shared.isLogged; }; @@ -64,7 +66,7 @@ module.exports.login = async function (username, password) { await creds.fetchToken(); shared.logger.trace(`Authentication for ${username}`); - const result = await networkHelper.autenticate(creds); + const result = await networkHelper.authenticate(creds); shared.isLogged = result.success; if (result.success) shared.logger.info("User logged in through the platform"); diff --git a/app/scripts/classes/game-info.js b/app/scripts/classes/game-info.js index 9d9e564..84a0aac 100644 --- a/app/scripts/classes/game-info.js +++ b/app/scripts/classes/game-info.js @@ -107,6 +107,7 @@ class GameInfo { censored: this.censored, engine: this.engine, status: this.status, + tags: this.tags, previewSrc: this.previewSrc, version: this.version, lastUpdate: this.lastUpdate, @@ -122,9 +123,13 @@ class GameInfo { * @param {String} json JSON string used to create the new object * @returns {GameInfo} */ - /* istanbul ignore next */ static fromJSON(json) { - return Object.assign(new GameInfo(), json); + // Convert string + const temp = Object.assign(new GameInfo(), JSON.parse(json)); + + // JSON cannot transform a string to a date implicitly + temp.lastUpdate = new Date(temp.lastUpdate); + return temp; } } module.exports = GameInfo; diff --git a/app/scripts/network-helper.js b/app/scripts/network-helper.js index 8c34bc9..7e3ae4a 100644 --- a/app/scripts/network-helper.js +++ b/app/scripts/network-helper.js @@ -2,7 +2,6 @@ // Public modules from npm const axios = require("axios").default; -const { isString } = require("lodash"); const ky = require("ky-universal").create({ throwHttpErrors: false, }); @@ -21,6 +20,7 @@ const userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.0 Safari/605.1.15"; axiosCookieJarSupport(axios); const cookieJar = new tough.CookieJar(); + const commonConfig = { headers: { "User-Agent": userAgent, @@ -52,9 +52,10 @@ module.exports.fetchHTML = async function (url) { * and token obtained previously. Save cookies on your * device after authentication. * @param {Credentials} credentials Platform access credentials - * @returns {Promise} Result of the operation + * @param {Boolea} force Specifies whether the request should be forced, ignoring any saved cookies + * @returns {Promise} Result of the operation */ -module.exports.autenticate = async function (credentials) { +module.exports.authenticate = async function (credentials, force) { shared.logger.info(`Authenticating with user ${credentials.username}`); if (!credentials.token) throw new Error(`Invalid token for auth: ${credentials.token}`); @@ -75,7 +76,9 @@ module.exports.autenticate = async function (credentials) { try { // Try to log-in - const response = await axios.post(secureURL, params, commonConfig); + let config = Object.assign({}, commonConfig); + if (force) delete config.jar; + const response = await axios.post(secureURL, params, config); // Parse the response HTML const $ = cheerio.load(response.data); @@ -116,6 +119,7 @@ module.exports.getF95Token = async function() { * (such as graphics engines and progress statuses) * @deprecated */ +/* istanbul ignore next */ module.exports.fetchPlatformData = async function() { // Fetch the response of the platform const response = await exports.fetchGETResponse(f95url.F95_LATEST_UPDATES); @@ -145,6 +149,12 @@ module.exports.fetchPlatformData = async function() { }; //#region Utility methods +/** + * @protected + * Performs a GET request to a specific URL and returns the response. + * If the request generates an error (for example 400) `null` is returned. + * @param {String} url + */ module.exports.fetchGETResponse = async function(url) { // Secure the URL const secureURL = exports.enforceHttpsUrl(url); @@ -162,10 +172,10 @@ module.exports.fetchGETResponse = async function(url) { * @protected * Enforces the scheme of the URL is https and returns the new URL. * @param {String} url - * @returns {String} + * @returns {String} Secure URL or `null` if the argument is not a string */ module.exports.enforceHttpsUrl = function (url) { - return isString(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; + return exports.isStringAValidURL(url) ? url.replace(/^(https?:)?\/\//, "https://") : null; }; /** @@ -181,17 +191,17 @@ module.exports.isF95URL = function (url) { /** * @protected - * Checks if the string passed by parameter has a properly formatted and valid path to a URL. + * Checks if the string passed by parameter has a + * properly formatted and valid path to a URL (HTTP/HTTPS). * @param {String} url String to check for correctness * @returns {Boolean} true if the string is a valid URL, false otherwise */ module.exports.isStringAValidURL = function (url) { - try { - new URL(url); // skipcq: JS-0078 - return true; - } catch (err) { - return false; - } + // Many thanks to Daveo at StackOverflow (https://preview.tinyurl.com/y2f2e2pc) + const expression = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&//=]*)/; + const regex = new RegExp(expression); + if (url.match(regex)) return true; + else return false; }; /** diff --git a/app/scripts/shared.js b/app/scripts/shared.js index 561441e..f04d700 100644 --- a/app/scripts/shared.js +++ b/app/scripts/shared.js @@ -1,3 +1,4 @@ +/* istanbul ignore file */ "use strict"; // Public modules from npm diff --git a/package-lock.json b/package-lock.json index 26492f3..9f2c127 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "f95api", - "version": "1.3.5", + "version": "1.5.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/package.json b/package.json index 2fb3213..30e5b5d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "main": "./app/index.js", "name": "f95api", - "version": "1.3.5", + "version": "1.5.0", "author": { "name": "Millennium Earl" }, @@ -19,7 +19,10 @@ "scraping", "login", "game", - "games" + "games", + "data", + "userdata", + "user data" ], "scripts": { "unit-test-mocha": "nyc --reporter=text mocha './test/index-test.js'", diff --git a/test/index-test-legacy.js b/test/index-test-legacy.js new file mode 100644 index 0000000..f2d9748 --- /dev/null +++ b/test/index-test-legacy.js @@ -0,0 +1,290 @@ +"use strict"; + +// Core modules +const fs = require("fs"); + +// Public modules from npm +const _ = require("lodash"); +const expect = require("chai").expect; +const sleep = require("sleep"); +const dotenv = require("dotenv"); + +// Modules from file +const urlHelper = require("../app/scripts/url-helper.js"); +const F95API = require("../app/index.js"); + +// Configure the .env reader +dotenv.config(); + +const COOKIES_SAVE_PATH = "./f95cache/cookies.json"; +const ENGINES_SAVE_PATH = "./f95cache/engines.json"; +const STATUSES_SAVE_PATH = "./f95cache/statuses.json"; +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; +const FAKE_USERNAME = "Fake_Username091276"; +const FAKE_PASSWORD = "fake_password"; + +//F95API.debug(false); + +function randomSleep() { + const random = Math.floor(Math.random() * 500) + 50; + sleep.msleep(500 + random); +} + +describe("Login without cookies", function () { + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + + before("Set isolation", function () { + F95API.setIsolation(true); + }); + + beforeEach("Remove all cookies", function () { + // Runs before each test in this block + if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up + + let testOrder = 0; + + it("Test with valid credentials", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 0) randomSleep(); + + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(result.message).equal("Authentication successful"); + + testOrder = 1; + }); + it("Test with invalid username", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 1) randomSleep(); + + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect username"); + + testOrder = 2; + }); + it("Test with invalid password", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 2) randomSleep(); + + const result = await F95API.login(USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect password"); + + testOrder = 3; + }); + it("Test with invalid credentials", async function () { + // Gain exclusive use of the cookies + while (testOrder !== 3) randomSleep(); + + const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); + expect(result.success).to.be.false; + expect(result.message).to.equal("Incorrect username"); // It should first check the username + + testOrder = 4; + }); +}); + +describe("Login with cookies", function () { + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + + before("Log in to create cookies then logout", async function () { + // Runs once before the first test in this block + if (!fs.existsSync(COOKIES_SAVE_PATH)) + await F95API.login(USERNAME, PASSWORD); // Download cookies + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up + + it("Test with valid credentials", async function () { + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(result.message).equal("Logged with cookies"); + }); +}); + +describe("Load base data without cookies", function () { + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + + before("Delete cache if exists", function () { + // Runs once before the first test in this block + if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); + if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); + }); + //#endregion Set-up + + it("With login", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; + + const result = await F95API.loadF95BaseData(); + + const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); + const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); + + expect(result).to.be.true; + expect(enginesCacheExists).to.be.true; + expect(statusesCacheExists).to.be.true; + }); + + it("Without login", async function () { + if (F95API.isLogged()) F95API.logout(); + const result = await F95API.loadF95BaseData(); + expect(result).to.be.false; + }); +}); + +describe("Search game data", function () { + //#region Set-up + this.timeout(60000); // All tests in this suite get 60 seconds before timeout + + beforeEach("Prepare API", function () { + // Runs once before the first test in this block + if (F95API.isLogged()) F95API.logout(); + }); + //#endregion Set-up + + let testGame = null; + + it("Search game when logged", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; + + const loadResult = await F95API.loadF95BaseData(); + expect(loadResult).to.be.true; + + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ + const gamesList = await F95API.getGameData("Kingdom of Deception", false); + expect(gamesList.length, "Should find only the game").to.equal(1); + const result = gamesList[0]; + const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; + + // Test only the main information + expect(result.name).to.equal("Kingdom of Deception"); + expect(result.author).to.equal("Hreinn Games"); + expect(result.isMod, "Should be false").to.be.false; + expect(result.engine).to.equal("REN'PY"); + expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? + testGame = Object.assign({}, result); + }); + it("Search game when not logged", async function () { + const result = await F95API.getGameData("Kingdom of Deception", false); + expect(result, "Without being logged should return null").to.be.null; + }); + it("Test game serialization", function () { + const json = JSON.stringify(testGame); + const parsedGameInfo = JSON.parse(json); + const result = _.isEqual(parsedGameInfo, testGame); + expect(result).to.be.true; + }); +}); + +describe("Load user data", function () { + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up + + it("Retrieve when logged", async function () { + // Login + await F95API.login(USERNAME, PASSWORD); + + // Then retrieve user data + const data = await F95API.getUserData(); + + expect(data).to.exist; + expect(data.username).to.equal(USERNAME); + }); + it("Retrieve when not logged", async function () { + // Logout + if (F95API.isLogged()) F95API.logout(); + + // Try to retrieve user data + const data = await F95API.getUserData(); + + expect(data).to.be.null; + }); +}); + +describe("Check game update", function () { + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up + + it("Get online game and verify that no update exists", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; + + const loadResult = await F95API.loadF95BaseData(); + expect(loadResult).to.be.true; + + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ + const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; + + const update = await F95API.chekIfGameHasUpdate(result); + expect(update).to.be.false; + }); + + it("Verify that update exists from old URL", async function () { + const loginResult = await F95API.login(USERNAME, PASSWORD); + expect(loginResult.success).to.be.true; + + // This test depend on the data on F95Zone at + // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ + const url = + "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; + const result = await F95API.getGameDataFromURL(url); + result.version = "0.9600"; + + const update = await F95API.chekIfGameHasUpdate(result); + expect(update).to.be.true; + }); +}); + +describe("Test url-helper", function () { + //#region Set-up + this.timeout(30000); // All tests in this suite get 30 seconds before timeout + //#endregion Set-up + + it("Check if URL exists", async function () { + // Check generic URLs... + let exists = await urlHelper.urlExists("https://www.google.com/"); + expect(exists, "Complete valid URL").to.be.true; + + exists = await urlHelper.urlExists("www.google.com"); + expect(exists, "URl without protocol prefix").to.be.false; + + exists = await urlHelper.urlExists("https://www.google/"); + expect(exists, "URL without third level domain").to.be.false; + + // Now check for more specific URLs (with redirect)... + exists = await urlHelper.urlExists( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" + ); + expect(exists, "URL with redirect without check").to.be.true; + + exists = await urlHelper.urlExists( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", + true + ); + expect(exists, "URL with redirect with check").to.be.false; + }); + + it("Check if URL belong to the platform", async function () { + let belong = urlHelper.isF95URL( + "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" + ); + expect(belong).to.be.true; + + belong = urlHelper.isF95URL("https://www.google/"); + expect(belong).to.be.false; + }); +}); diff --git a/test/index-test.js b/test/index-test.js index 7b69d4c..069c9bc 100644 --- a/test/index-test.js +++ b/test/index-test.js @@ -1,290 +1,36 @@ "use strict"; -// Core modules -const fs = require("fs"); +// Test suite +const api = require("./suites/api-test.js").suite; +const credentials = require("./suites/credentials-test.js").suite; +const network = require("./suites/network-helper-test.js").suite; +const scraper = require("./suites/scraper-test.js").suite; +const searcher = require("./suites/searcher-test.js").suite; +const uScraper = require("./suites/user-scraper-test.js").suite; -// Public modules from npm -const _ = require("lodash"); -const expect = require("chai").expect; -const sleep = require("sleep"); -const dotenv = require("dotenv"); - -// Modules from file -const urlHelper = require("../app/scripts/url-helper.js"); -const F95API = require("../app/index.js"); - -// Configure the .env reader -dotenv.config(); - -const COOKIES_SAVE_PATH = "./f95cache/cookies.json"; -const ENGINES_SAVE_PATH = "./f95cache/engines.json"; -const STATUSES_SAVE_PATH = "./f95cache/statuses.json"; -const USERNAME = process.env.F95_USERNAME; -const PASSWORD = process.env.F95_PASSWORD; -const FAKE_USERNAME = "FakeUsername091276"; -const FAKE_PASSWORD = "fake_password"; - -//F95API.debug(false); - -function randomSleep() { - const random = Math.floor(Math.random() * 500) + 50; - sleep.msleep(500 + random); -} - -describe("Login without cookies", function () { +describe("Test basic function", function testBasic() { //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - - before("Set isolation", function () { - F95API.setIsolation(true); - }); - - beforeEach("Remove all cookies", function () { - // Runs before each test in this block - if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); - if (F95API.isLogged()) F95API.logout(); - }); + this.timeout(15000); // All tests in this suite get 15 seconds before timeout //#endregion Set-up - let testOrder = 0; - - it("Test with valid credentials", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 0) randomSleep(); - - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Authentication successful"); - - testOrder = 1; - }); - it("Test with invalid username", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 1) randomSleep(); - - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); - - testOrder = 2; - }); - it("Test with invalid password", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 2) randomSleep(); - - const result = await F95API.login(USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect password"); - - testOrder = 3; - }); - it("Test with invalid credentials", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 3) randomSleep(); - - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); // It should first check the username - - testOrder = 4; - }); + describe("Test credentials class", credentials.bind(this)); + describe("Test network helper", network.bind(this)); }); -describe("Login with cookies", function () { +describe("Test F95 modules", function testF95Modules() { //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout + this.timeout(15000); // All tests in this suite get 15 seconds before timeout + //#endregion Set-up + + describe("Test scraper methods", scraper.bind(this)); + describe("Test searcher methods", searcher.bind(this)); + describe("Test user scraper methods", uScraper.bind(this)); +}); - before("Log in to create cookies then logout", async function () { - // Runs once before the first test in this block - if (!fs.existsSync(COOKIES_SAVE_PATH)) - await F95API.login(USERNAME, PASSWORD); // Download cookies - if (F95API.isLogged()) F95API.logout(); - }); +describe("Test complete API", function testAPI() { + //#region Set-up + this.timeout(15000); // All tests in this suite get 15 seconds before timeout //#endregion Set-up - it("Test with valid credentials", async function () { - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Logged with cookies"); - }); -}); - -describe("Load base data without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - - before("Delete cache if exists", function () { - // Runs once before the first test in this block - if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); - if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); - }); - //#endregion Set-up - - it("With login", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const result = await F95API.loadF95BaseData(); - - const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); - const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); - - expect(result).to.be.true; - expect(enginesCacheExists).to.be.true; - expect(statusesCacheExists).to.be.true; - }); - - it("Without login", async function () { - if (F95API.isLogged()) F95API.logout(); - const result = await F95API.loadF95BaseData(); - expect(result).to.be.false; - }); -}); - -describe("Search game data", function () { - //#region Set-up - this.timeout(60000); // All tests in this suite get 60 seconds before timeout - - beforeEach("Prepare API", function () { - // Runs once before the first test in this block - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up - - let testGame = null; - - it("Search game when logged", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const gamesList = await F95API.getGameData("Kingdom of Deception", false); - expect(gamesList.length, "Should find only the game").to.equal(1); - const result = gamesList[0]; - const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; - - // Test only the main information - expect(result.name).to.equal("Kingdom of Deception"); - expect(result.author).to.equal("Hreinn Games"); - expect(result.isMod, "Should be false").to.be.false; - expect(result.engine).to.equal("REN'PY"); - expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? - testGame = Object.assign({}, result); - }); - it("Search game when not logged", async function () { - const result = await F95API.getGameData("Kingdom of Deception", false); - expect(result, "Without being logged should return null").to.be.null; - }); - it("Test game serialization", function () { - const json = JSON.stringify(testGame); - const parsedGameInfo = JSON.parse(json); - const result = _.isEqual(parsedGameInfo, testGame); - expect(result).to.be.true; - }); -}); - -describe("Load user data", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Retrieve when logged", async function () { - // Login - await F95API.login(USERNAME, PASSWORD); - - // Then retrieve user data - const data = await F95API.getUserData(); - - expect(data).to.exist; - expect(data.username).to.equal(USERNAME); - }); - it("Retrieve when not logged", async function () { - // Logout - if (F95API.isLogged()) F95API.logout(); - - // Try to retrieve user data - const data = await F95API.getUserData(); - - expect(data).to.be.null; - }); -}); - -describe("Check game update", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Get online game and verify that no update exists", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; - - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.false; - }); - - it("Verify that update exists from old URL", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ - const url = - "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; - const result = await F95API.getGameDataFromURL(url); - result.version = "0.9600"; - - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.true; - }); -}); - -describe("Test url-helper", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Check if URL exists", async function () { - // Check generic URLs... - let exists = await urlHelper.urlExists("https://www.google.com/"); - expect(exists, "Complete valid URL").to.be.true; - - exists = await urlHelper.urlExists("www.google.com"); - expect(exists, "URl without protocol prefix").to.be.false; - - exists = await urlHelper.urlExists("https://www.google/"); - expect(exists, "URL without third level domain").to.be.false; - - // Now check for more specific URLs (with redirect)... - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(exists, "URL with redirect without check").to.be.true; - - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", - true - ); - expect(exists, "URL with redirect with check").to.be.false; - }); - - it("Check if URL belong to the platform", async function () { - let belong = urlHelper.isF95URL( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(belong).to.be.true; - - belong = urlHelper.isF95URL("https://www.google/"); - expect(belong).to.be.false; - }); -}); + describe("Test API", api.bind(this)); +}); \ No newline at end of file diff --git a/test/suites/api-test.js b/test/suites/api-test.js new file mode 100644 index 0000000..07e6016 --- /dev/null +++ b/test/suites/api-test.js @@ -0,0 +1,64 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); +const { + isEqual +} = require("lodash"); + +// Modules from file +const F95API = require("../../app/index.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; + +module.exports.suite = function suite() { + // Global suite variables + const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"; + + it("Test login", async function testLogin() { + const result = await F95API.login(USERNAME, PASSWORD); + expect(result.success).to.be.true; + expect(F95API.isLogged()).to.be.true; + }); + + it("Test user data fetching", async function testUserDataFetch() { + const userdata = await F95API.getUserData(); + expect(userdata.username).to.be.equal(USERNAME); + }); + + it("Test game update checking", async function testGameUpdateCheck() { + // We force the creation of a GameInfo object, + // knowing that the checkIfGameHasUpdate() function + // only needs the game URL + const info = new F95API.GameInfo(); + + // The gameURL identifies a game for which we know there is an update + info.url = gameURL; + + // Check for updates + const update = await F95API.checkIfGameHasUpdate(info); + expect(update).to.be.true; + }); + + it("Test game data fetching", async function testGameDataFetch() { + // Search a game by name + const gameList = await F95API.getGameData("perverted education", false); + + // We know that there is only one game with the selected name + expect(gameList.length).to.be.equal(1, `There should be only one game, not ${gameList.length}`); + const game = gameList[0]; + + // Than we fetch a game from URL + const gameFromURL = await F95API.getGameDataFromURL(game.url); + + // The two games must be equal + const equal = isEqual(game, gameFromURL); + expect(equal).to.be.true; + }); +}; \ No newline at end of file diff --git a/test/suites/credentials-test.js b/test/suites/credentials-test.js new file mode 100644 index 0000000..3bf60e2 --- /dev/null +++ b/test/suites/credentials-test.js @@ -0,0 +1,52 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); + +module.exports.suite = function suite() { + it("Check token formatting", async function testValidToken() { + // Token example: + // 1604309951,0338213c00fcbd894fd9415e6ba08403 + // 1604309986,ebdb75502337699381f0f55c86353555 + // 1604310008,2d50d55808e5ec3a157ec01953da9d26 + + // Fetch token (is a GET request, we don't need the credentials) + const cred = new Credentials(null, null); + await cred.fetchToken(); + + // Parse token for assert + const splitted = cred.token.split(","); + const unique = splitted[0]; + const hash = splitted[1]; + expect(splitted.length).to.be.equal(2, "The token consists of two parts"); + + // Check type of parts + expect(isNumeric(unique)).to.be.true; + expect(isNumeric(hash)).to.be.false; + + // The second part is most probably the MD5 hash of something + expect(hash.length).to.be.equal(32, "Hash should have 32 hex chars"); + }); +}; + +//#region Private methods +/** + * @private + * Check if a string is a number + * @param {String} str + * @author Dan, Ben Aston + * @see https://preview.tinyurl.com/y46jqwkt + */ +function isNumeric(str) { + // We only process strings! + if (typeof str != "string") return false; + + // Use type coercion to parse the _entirety_ of the string + // (`parseFloat` alone does not do this) and ensure strings + // of whitespace fail + return !isNaN(str) && !isNaN(parseFloat(str)); +} +//#endregion \ No newline at end of file diff --git a/test/suites/network-helper-test.js b/test/suites/network-helper-test.js new file mode 100644 index 0000000..a650c99 --- /dev/null +++ b/test/suites/network-helper-test.js @@ -0,0 +1,107 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); +const networkHelper = require("../../app/scripts/network-helper.js"); +const { + F95_SEARCH_URL +} = require("../../app/scripts/constants/url.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; +const FAKE_USERNAME = "Fake_Username091276"; +const FAKE_PASSWORD = "fake_password"; + +module.exports.suite = function suite() { + // Global suite variables + const gameURL = "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/"; + + it("Check if URL exists", async function checkURLExistence() { + // Check generic URLs... + let exists = await networkHelper.urlExists("https://www.google.com/"); + expect(exists, "Complete valid URL").to.be.true; + + exists = await networkHelper.urlExists("www.google.com"); + expect(exists, "URl without protocol prefix").to.be.false; + + exists = await networkHelper.urlExists("https://www.google/"); + expect(exists, "URL without third level domain").to.be.false; + + // Now check for more specific URLs (with redirect)... + exists = await networkHelper.urlExists(gameURL); + expect(exists, "URL with redirect without check").to.be.true; + + exists = await networkHelper.urlExists(gameURL, true); + expect(exists, "URL with redirect with check").to.be.false; + }); + + it("Check if URL belong to the platform", function checkIfURLIsF95() { + let belong = networkHelper.isF95URL(gameURL); + expect(belong).to.be.true; + + belong = networkHelper.isF95URL("https://www.google/"); + expect(belong).to.be.false; + }); + + it("Enforce secure URLs", function testSecureURLEnforcement() { + // This URL is already secure, should remain the same + let enforced = networkHelper.enforceHttpsUrl(gameURL); + expect(enforced).to.be.equal(gameURL, "The game URL is already secure"); + + // This URL is not secure + enforced = networkHelper.enforceHttpsUrl("http://www.google.com"); + expect(enforced).to.be.equal("https://www.google.com", "The URL was without SSL/TLS (HTTPs)"); + + // Finally, we check when we pass a invalid URL + enforced = networkHelper.enforceHttpsUrl("http://invalidurl"); + expect(enforced).to.be.null; + }); + + it("Check URL redirect", async function checkURLRedirect() { + // gameURL is an old URL it has been verified that it generates a redirect + const redirectURL = await networkHelper.getUrlRedirect(gameURL); + expect(redirectURL).to.not.be.equal(gameURL, "The original URL has redirect"); + + // If we recheck the new URL, we find that no redirect happens + const secondRedirectURL = await networkHelper.getUrlRedirect(redirectURL); + expect(secondRedirectURL).to.be.equal(redirectURL, "The URL has no redirect"); + }); + + it("Check response to GET request", async function testGETResponse() { + // We should be able to fetch a game page + let response = await networkHelper.fetchGETResponse(gameURL); + expect(response.status).to.be.equal(200, "The operation must be successful"); + + // We should NOT be able to fetch the search page (we must be logged) + response = await networkHelper.fetchGETResponse(F95_SEARCH_URL); + expect(response).to.be.null; + }); + + it("Test for authentication to platform", async function testAuthentication() { + // Try to authenticate with valid credentials + const creds = new Credentials(USERNAME, PASSWORD); + await creds.fetchToken(); + const validResult = await networkHelper.authenticate(creds); + expect(validResult.success).to.be.true; + + // Now we use fake credentials + const fakeCreds = new Credentials(FAKE_USERNAME, FAKE_PASSWORD); + await fakeCreds.fetchToken(); + const invalidResult = await networkHelper.authenticate(fakeCreds, true); + expect(invalidResult.success).to.be.false; + }); + + it("Test fetching HTML", async function testFetchHTML() { + // This should return the HTML code of the page + const html = await networkHelper.fetchHTML(gameURL); + expect(html.startsWith("")).to.be.true; + }); +}; \ No newline at end of file diff --git a/test/suites/scraper-test.js b/test/suites/scraper-test.js new file mode 100644 index 0000000..a68666b --- /dev/null +++ b/test/suites/scraper-test.js @@ -0,0 +1,42 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const { isEqual } = require("lodash"); +const GameInfo = require("../../app/scripts/classes/game-info.js"); + +// Modules from file +const scraper = require("../../app/scripts/scraper.js"); + +module.exports.suite = function suite() { + // Global suite variables + const gameURL = "https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/"; + const previewSrc = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; + + it("Search game", async function () { + // This test depend on the data on F95Zone at gameURL + const result = await scraper.getGameInfo(gameURL); + + // Test only the main information + expect(result.name).to.equal("Kingdom of Deception"); + expect(result.author).to.equal("Hreinn Games"); + expect(result.isMod, "Should be false").to.be.false; + expect(result.engine).to.equal("Ren'Py"); + expect(result.previewSrc).to.equal(previewSrc, "Preview not equals"); + }); + + it("Test game serialization", async function testGameSerialization() { + // This test depend on the data on F95Zone at gameURL + const testGame = await scraper.getGameInfo(gameURL); + + // Serialize... + const json = JSON.stringify(testGame); + + // Deserialize... + const parsedGameInfo = GameInfo.fromJSON(json); + + // Compare with lodash + const result = isEqual(parsedGameInfo, testGame); + expect(result).to.be.true; + }); +}; diff --git a/test/suites/searcher-test.js b/test/suites/searcher-test.js new file mode 100644 index 0000000..6b8aea4 --- /dev/null +++ b/test/suites/searcher-test.js @@ -0,0 +1,64 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); +const searcher = require("../../app/scripts/searcher.js"); +const { + authenticate +} = require("../../app/scripts/network-helper.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; + +module.exports.suite = function suite() { + // TODO: + // This method should delete the store F95Zone cookies, + // but what if the other tests require them? + + // it("Search game when not logged", async function searchGameWhenNotLogged() { + // // Search for a game that we know has only one result + // // but without logging in first + // const urls = await searcher.searchGame("kingdom of deception"); + // expect(urls.lenght).to.be.equal(0, "There should not be any URL"); + // }); + + it("Search game", async function searchGame() { + // Authenticate + const result = await auth(); + expect(result.success, "Authentication should be successful").to.be.true; + + // Search for a game that we know has only one result + const urls = await searcher.searchGame("kingdom of deception"); + expect(urls.length).to.be.equal(1, `There should be only one game result instead of ${urls.length}`); + }); + + it("Search mod", async function searchMod() { + // Authenticate + const result = await auth(); + expect(result.success, "Authentication should be successful").to.be.true; + + // Search for a mod that we know has only one result + const urls = await searcher.searchMod("kingdom of deception jdmod"); + expect(urls.length).to.be.equal(1, `There should be only one mod result instead of ${urls.length}`); + }); +}; + +//#region Private methods +/** + * @private + * Simple wrapper for authentication. + */ +async function auth() { + const creds = new Credentials(USERNAME, PASSWORD); + await creds.fetchToken(); + return await authenticate(creds); +} +//#endregion Private methods \ No newline at end of file diff --git a/test/suites/user-scraper-test.js b/test/suites/user-scraper-test.js new file mode 100644 index 0000000..e3abe0e --- /dev/null +++ b/test/suites/user-scraper-test.js @@ -0,0 +1,54 @@ +"use strict"; + +// Public module from npm +const expect = require("chai").expect; +const dotenv = require("dotenv"); + +// Modules from file +const Credentials = require("../../app/scripts/classes/credentials.js"); +const uScraper = require("../../app/scripts/user-scraper.js"); +const { + authenticate +} = require("../../app/scripts/network-helper.js"); + +// Configure the .env reader +dotenv.config(); + +// Global variables +const USERNAME = process.env.F95_USERNAME; +const PASSWORD = process.env.F95_PASSWORD; + +module.exports.suite = function suite() { + // TODO: + // This method should delete the store F95Zone cookies, + // but what if the other tests require them? + + // it("Fetch data when not logged", async function fetchUserDataWhenLogged() { + // const data = await uScraper.getUserData(); + // expect(data.username).to.be.equal(""); + // expect(data.avatarSrc).to.be.equal(""); + // expect(data.watchedThreads.length).to.be.equal(0); + // }); + + it("Fetch data when logged", async function fetchUserDataWhenNotLogged() { + // Authenticate + const result = await auth(); + expect(result.success, "Authentication should be successful").to.be.true; + + // We test only for the username, the other test data depends on the user logged + const data = await uScraper.getUserData(); + expect(data.username).to.be.equal(USERNAME); + }); +}; + +//#region Private methods +/** + * @private + * Simple wrapper for authentication. + */ +async function auth() { + const creds = new Credentials(USERNAME, PASSWORD); + await creds.fetchToken(); + return await authenticate(creds); +} +//#endregion Private methods \ No newline at end of file From 62408c55b8686bebf9a145d56f0141ea520628fc Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 15:26:06 +0100 Subject: [PATCH 14/17] Removed debug option and added logger level --- app/index.js | 14 +- app/scripts/shared.js | 16 --- test/index-test-legacy.js | 290 -------------------------------------- 3 files changed, 5 insertions(+), 315 deletions(-) delete mode 100644 test/index-test-legacy.js diff --git a/app/index.js b/app/index.js index 3bff9ce..1d59db9 100644 --- a/app/index.js +++ b/app/index.js @@ -21,23 +21,19 @@ module.exports.UserData = UserData; //#region Export properties /** - * Shows log messages and other useful functions for module debugging. - * @param {Boolean} value + * @public + * Set the logger level for module debugging. */ /* istambul ignore next */ -module.exports.debug = function (value) { - shared.debug = value; - - // Configure logger - shared.logger.level = value ? "trace" : "warn"; -}; +module.exports.loggerLevel = shared.logger.level; +exports.loggerLevel = "warn"; // By default log only the warn messages /** * @public * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {String} */ /* istambul ignore next */ -module.exports.isLogged = function () { +module.exports.isLogged = function isLogged() { return shared.isLogged; }; //#endregion Export properties diff --git a/app/scripts/shared.js b/app/scripts/shared.js index f04d700..cbefd62 100644 --- a/app/scripts/shared.js +++ b/app/scripts/shared.js @@ -9,11 +9,6 @@ const log4js = require("log4js"); */ class Shared { //#region Properties - /** - * Shows log messages and other useful functions for module debugging. - * @type Boolean - */ - static #_debug = false; /** * Indicates whether a user is logged in to the F95Zone platform or not. * @type Boolean @@ -38,13 +33,6 @@ class Shared { //#region Getters /** - * Shows log messages and other useful functions for module debugging. - * @returns {Boolean} - */ - static get debug() { - return this.#_debug; - } - /** * Indicates whether a user is logged in to the F95Zone platform or not. * @returns {Boolean} */ @@ -83,10 +71,6 @@ class Shared { this.#_statuses = val; } - static set debug(val) { - this.#_debug = val; - } - static set isLogged(val) { this.#_isLogged = val; } diff --git a/test/index-test-legacy.js b/test/index-test-legacy.js deleted file mode 100644 index f2d9748..0000000 --- a/test/index-test-legacy.js +++ /dev/null @@ -1,290 +0,0 @@ -"use strict"; - -// Core modules -const fs = require("fs"); - -// Public modules from npm -const _ = require("lodash"); -const expect = require("chai").expect; -const sleep = require("sleep"); -const dotenv = require("dotenv"); - -// Modules from file -const urlHelper = require("../app/scripts/url-helper.js"); -const F95API = require("../app/index.js"); - -// Configure the .env reader -dotenv.config(); - -const COOKIES_SAVE_PATH = "./f95cache/cookies.json"; -const ENGINES_SAVE_PATH = "./f95cache/engines.json"; -const STATUSES_SAVE_PATH = "./f95cache/statuses.json"; -const USERNAME = process.env.F95_USERNAME; -const PASSWORD = process.env.F95_PASSWORD; -const FAKE_USERNAME = "Fake_Username091276"; -const FAKE_PASSWORD = "fake_password"; - -//F95API.debug(false); - -function randomSleep() { - const random = Math.floor(Math.random() * 500) + 50; - sleep.msleep(500 + random); -} - -describe("Login without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - - before("Set isolation", function () { - F95API.setIsolation(true); - }); - - beforeEach("Remove all cookies", function () { - // Runs before each test in this block - if (fs.existsSync(COOKIES_SAVE_PATH)) fs.unlinkSync(COOKIES_SAVE_PATH); - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up - - let testOrder = 0; - - it("Test with valid credentials", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 0) randomSleep(); - - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Authentication successful"); - - testOrder = 1; - }); - it("Test with invalid username", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 1) randomSleep(); - - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); - - testOrder = 2; - }); - it("Test with invalid password", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 2) randomSleep(); - - const result = await F95API.login(USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect password"); - - testOrder = 3; - }); - it("Test with invalid credentials", async function () { - // Gain exclusive use of the cookies - while (testOrder !== 3) randomSleep(); - - const result = await F95API.login(FAKE_USERNAME, FAKE_PASSWORD); - expect(result.success).to.be.false; - expect(result.message).to.equal("Incorrect username"); // It should first check the username - - testOrder = 4; - }); -}); - -describe("Login with cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - - before("Log in to create cookies then logout", async function () { - // Runs once before the first test in this block - if (!fs.existsSync(COOKIES_SAVE_PATH)) - await F95API.login(USERNAME, PASSWORD); // Download cookies - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up - - it("Test with valid credentials", async function () { - const result = await F95API.login(USERNAME, PASSWORD); - expect(result.success).to.be.true; - expect(result.message).equal("Logged with cookies"); - }); -}); - -describe("Load base data without cookies", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - - before("Delete cache if exists", function () { - // Runs once before the first test in this block - if (fs.existsSync(ENGINES_SAVE_PATH)) fs.unlinkSync(ENGINES_SAVE_PATH); - if (fs.existsSync(STATUSES_SAVE_PATH)) fs.unlinkSync(STATUSES_SAVE_PATH); - }); - //#endregion Set-up - - it("With login", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const result = await F95API.loadF95BaseData(); - - const enginesCacheExists = fs.existsSync(ENGINES_SAVE_PATH); - const statusesCacheExists = fs.existsSync(STATUSES_SAVE_PATH); - - expect(result).to.be.true; - expect(enginesCacheExists).to.be.true; - expect(statusesCacheExists).to.be.true; - }); - - it("Without login", async function () { - if (F95API.isLogged()) F95API.logout(); - const result = await F95API.loadF95BaseData(); - expect(result).to.be.false; - }); -}); - -describe("Search game data", function () { - //#region Set-up - this.timeout(60000); // All tests in this suite get 60 seconds before timeout - - beforeEach("Prepare API", function () { - // Runs once before the first test in this block - if (F95API.isLogged()) F95API.logout(); - }); - //#endregion Set-up - - let testGame = null; - - it("Search game when logged", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const gamesList = await F95API.getGameData("Kingdom of Deception", false); - expect(gamesList.length, "Should find only the game").to.equal(1); - const result = gamesList[0]; - const src = "https://attachments.f95zone.to/2018/09/162821_f9nXfwF.png"; - - // Test only the main information - expect(result.name).to.equal("Kingdom of Deception"); - expect(result.author).to.equal("Hreinn Games"); - expect(result.isMod, "Should be false").to.be.false; - expect(result.engine).to.equal("REN'PY"); - expect(result.previewSource).to.equal(src); // Could be null -> Why sometimes doesn't get the image? - testGame = Object.assign({}, result); - }); - it("Search game when not logged", async function () { - const result = await F95API.getGameData("Kingdom of Deception", false); - expect(result, "Without being logged should return null").to.be.null; - }); - it("Test game serialization", function () { - const json = JSON.stringify(testGame); - const parsedGameInfo = JSON.parse(json); - const result = _.isEqual(parsedGameInfo, testGame); - expect(result).to.be.true; - }); -}); - -describe("Load user data", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Retrieve when logged", async function () { - // Login - await F95API.login(USERNAME, PASSWORD); - - // Then retrieve user data - const data = await F95API.getUserData(); - - expect(data).to.exist; - expect(data.username).to.equal(USERNAME); - }); - it("Retrieve when not logged", async function () { - // Logout - if (F95API.isLogged()) F95API.logout(); - - // Try to retrieve user data - const data = await F95API.getUserData(); - - expect(data).to.be.null; - }); -}); - -describe("Check game update", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Get online game and verify that no update exists", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - const loadResult = await F95API.loadF95BaseData(); - expect(loadResult).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/kingdom-of-deception-v0-10-8-hreinn-games.2733/ - const result = (await F95API.getGameData("Kingdom of Deception", false))[0]; - - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.false; - }); - - it("Verify that update exists from old URL", async function () { - const loginResult = await F95API.login(USERNAME, PASSWORD); - expect(loginResult.success).to.be.true; - - // This test depend on the data on F95Zone at - // https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/ - const url = - "https://f95zone.to/threads/perverted-education-v0-9701-april-ryan.1854/"; - const result = await F95API.getGameDataFromURL(url); - result.version = "0.9600"; - - const update = await F95API.chekIfGameHasUpdate(result); - expect(update).to.be.true; - }); -}); - -describe("Test url-helper", function () { - //#region Set-up - this.timeout(30000); // All tests in this suite get 30 seconds before timeout - //#endregion Set-up - - it("Check if URL exists", async function () { - // Check generic URLs... - let exists = await urlHelper.urlExists("https://www.google.com/"); - expect(exists, "Complete valid URL").to.be.true; - - exists = await urlHelper.urlExists("www.google.com"); - expect(exists, "URl without protocol prefix").to.be.false; - - exists = await urlHelper.urlExists("https://www.google/"); - expect(exists, "URL without third level domain").to.be.false; - - // Now check for more specific URLs (with redirect)... - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(exists, "URL with redirect without check").to.be.true; - - exists = await urlHelper.urlExists( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/", - true - ); - expect(exists, "URL with redirect with check").to.be.false; - }); - - it("Check if URL belong to the platform", async function () { - let belong = urlHelper.isF95URL( - "https://f95zone.to/threads/perverted-education-v0-9601-april-ryan.1854/" - ); - expect(belong).to.be.true; - - belong = urlHelper.isF95URL("https://www.google/"); - expect(belong).to.be.false; - }); -}); From 20524f75e65445817aa944ab2dd9a9f4a13380bc Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 15:46:07 +0100 Subject: [PATCH 15/17] Updated readme --- README.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++ app/example.js | 9 ++++++- 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 56ff3af..73ae63c 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,75 @@ Unofficial Node JS module for scraping F95Zone platform +These APIs have been developed to support this application and allow you to obtain data on games and mods on the platform [F95zone.to](www.f95zone.to) + +A simple usage example can be found in [app/example.js](https://github.com/MillenniumEarl/F95API/blob/master/app/example.js) + +**Attention**: Two-factor authentication is not supported + +# Data scraping +Games/mods can be obtained by name or URL + +```javascript +// The name is case insensitive +let listOfFoundGames = await F95API.getGameData("your game name"); +let listOfFoundMods = await F95API.getGameData("your mod name", true); + +let specificGame = await F95API.getGameDataFromURL("the URL of your game"); +``` + +While user data (after authenticating) with + +```javascript +let authResult = await F95API.login(username, password); + +let loggedUserData = await F95API.getUserData(); +``` + +# Classes +## Games and mods +Information about games and mods is stored in a GameInfo object with the following fields: + +``` +name: The game name +author: The game developer +url: The URL that leads to the game thread on F95Zone +overview: Description of the game +language: Main language of the game +supportedOS: List of supported OS (Windows/Linux/Mac/Android...) +censored: Are the NSFW parts censored? +engine: Game engine (Unity, Ren'Py, RPGM...) +status: Completed/Abandoned/Ongoing/Onhold +tags: List of tags +previewSrc: Source URL of the game description image +version: Version of the game +lastUpdate: Date of the last update (it's a Date object) +isMod: Is it a game or a mod? +changelog: Latest changelog available +``` + +The serialization in JSON format of this object is possible through `JSON.stringfy()` while the deserialization must happen through the static method `GameInfo.fromJSON()`. + +## User data +User data (after authentication) can be stored in a UserData object, consisting of the following fields: + +``` +username: Name of the logged in user +avatarSrc: Source URL of the user's profile picture +watchedThreads: List of URLs of threads followed by the user +``` + +## Login results +The outcome of the authentication process is represented by the LoginResult object: + +``` +success: Was the authentication successful?; +message: Possible error message (unrecognized user, wrong password ...) or authentication successful message +``` + +# Logging +To log the behavior of the application log4js is used with a default level of "warn". This option can be changed with the loggerLevel property. + # Guidelines for errors - If you can, return a meaningful value diff --git a/app/example.js b/app/example.js index 286425e..48eac22 100644 --- a/app/example.js +++ b/app/example.js @@ -1,3 +1,11 @@ +/* +to use this example, create an .env file +in the project root with the following values: + +F95_USERNAME = YOUR_USERNAME +F95_PASSWORD = YOUR_PASSWORD +*/ + "use strict"; // Public modules from npm @@ -9,7 +17,6 @@ const F95API = require("./index.js"); // Configure the .env reader dotenv.config(); - main(); async function main() { From 80c93fa90c72bffc70430b6a3fe5783d6bc9e4b8 Mon Sep 17 00:00:00 2001 From: Millennium Earl <22562005+MillenniumEarl@users.noreply.github.com> Date: Mon, 2 Nov 2020 16:02:02 +0100 Subject: [PATCH 16/17] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 73ae63c..565a308 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Unofficial Node JS module for scraping F95Zone platform -These APIs have been developed to support this application and allow you to obtain data on games and mods on the platform [F95zone.to](www.f95zone.to) +These APIs have been developed to support this application and allow you to obtain data on games and mods on the platform [F95zone](https://f95zone.to/) (**NSFW**) A simple usage example can be found in [app/example.js](https://github.com/MillenniumEarl/F95API/blob/master/app/example.js) @@ -75,7 +75,7 @@ message: Possible error message (unrecognized user, wrong password ...) or authe ``` # Logging -To log the behavior of the application log4js is used with a default level of "warn". This option can be changed with the loggerLevel property. +To log the behavior of the application [log4js](https://github.com/log4js-node/log4js-node) is used with a default level of "warn". This option can be changed with the `loggerLevel` property. # Guidelines for errors From 6d237cadf788e70976d4a0a1ce6c5815fde51c10 Mon Sep 17 00:00:00 2001 From: MillenniumEarl Date: Mon, 2 Nov 2020 16:05:19 +0100 Subject: [PATCH 17/17] Removed unused package --- package-lock.json | 15 --------------- package.json | 3 +-- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9f2c127..53b0e3a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2089,12 +2089,6 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, - "nan": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.1.tgz", - "integrity": "sha512-isWHgVjnFjh2x2yuJ/tj3JbwoHu3UC2dX5G/88Cm24yB6YopVgxvBObDY7n5xW6ExmFhJpSEQqFPvq9zaXc8Jw==", - "dev": true - }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -2503,15 +2497,6 @@ "integrity": "sha512-VUJ49FC8U1OxwZLxIbTTrDvLnf/6TDgxZcK8wxR8zs13xpx7xbG60ndBlhNrFi2EMuFRoeDoJO7wthSLq42EjA==", "dev": true }, - "sleep": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/sleep/-/sleep-6.3.0.tgz", - "integrity": "sha512-+WgYl951qdUlb1iS97UvQ01pkauoBK9ML9I/CMPg41v0Ze4EyMlTgFTDDo32iYj98IYqxIjDMRd+L71lawFfpQ==", - "dev": true, - "requires": { - "nan": "^2.14.1" - } - }, "slice-ansi": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-2.1.0.tgz", diff --git a/package.json b/package.json index 30e5b5d..621ee58 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,6 @@ "eslint": "^7.12.1", "lodash": "^4.17.20", "mocha": "^8.1.3", - "nyc": "^15.1.0", - "sleep": "^6.3.0" + "nyc": "^15.1.0" } }