diff options
Diffstat (limited to 'apps')
| -rw-r--r-- | apps/workers/metascraper-plugins/metascraper-amazon-improved.ts | 77 | ||||
| -rw-r--r-- | apps/workers/workers/crawlerWorker.ts | 2 |
2 files changed, 79 insertions, 0 deletions
diff --git a/apps/workers/metascraper-plugins/metascraper-amazon-improved.ts b/apps/workers/metascraper-plugins/metascraper-amazon-improved.ts new file mode 100644 index 00000000..ea9bf2e9 --- /dev/null +++ b/apps/workers/metascraper-plugins/metascraper-amazon-improved.ts @@ -0,0 +1,77 @@ +import type { Rules } from "metascraper"; + +/** + * Improved Amazon metascraper plugin that fixes image extraction. + * + * The default metascraper-amazon package uses `.a-dynamic-image` selector + * which matches the FIRST element with that class. On amazon.com pages, + * this is often the Prime logo instead of the product image. + * + * This plugin uses more specific selectors to target the actual product + * image: + * - #landingImage: The main product image ID + * - #imgTagWrapperId img: Fallback container for product images + * - #imageBlock img: Additional fallback for newer Amazon layouts + * + * By placing this plugin BEFORE metascraperAmazon() in the plugin chain, + * we ensure the correct image is extracted while keeping all other Amazon + * metadata (title, brand, description) from the original plugin. + */ + +const REGEX_AMAZON_URL = + /https?:\/\/(.*amazon\..*\/.*|.*amzn\..*\/.*|.*a\.co\/.*)/i; + +const test = ({ url }: { url: string }): boolean => REGEX_AMAZON_URL.test(url); + +const metascraperAmazonImproved = () => { + const rules: Rules = { + pkgName: "metascraper-amazon-improved", + test, + image: ({ htmlDom }) => { + // Try the main product image ID first (most reliable) + // Prefer data-old-hires attribute for high-resolution images + const landingImageHires = htmlDom("#landingImage").attr("data-old-hires"); + if (landingImageHires) { + return landingImageHires; + } + + const landingImageSrc = htmlDom("#landingImage").attr("src"); + if (landingImageSrc) { + return landingImageSrc; + } + + // Fallback to image block container + const imgTagHires = htmlDom("#imgTagWrapperId img").attr( + "data-old-hires", + ); + if (imgTagHires) { + return imgTagHires; + } + + const imgTagSrc = htmlDom("#imgTagWrapperId img").attr("src"); + if (imgTagSrc) { + return imgTagSrc; + } + + // Additional fallback for newer Amazon layouts + const imageBlockHires = htmlDom("#imageBlock img") + .first() + .attr("data-old-hires"); + if (imageBlockHires) { + return imageBlockHires; + } + + const imageBlockSrc = htmlDom("#imageBlock img").first().attr("src"); + if (imageBlockSrc) { + return imageBlockSrc; + } + + // Return undefined to allow next plugin to try + return undefined; + }, + }; + + return rules; +}; + +export default metascraperAmazonImproved; diff --git a/apps/workers/workers/crawlerWorker.ts b/apps/workers/workers/crawlerWorker.ts index 2c96338a..aedf4aa0 100644 --- a/apps/workers/workers/crawlerWorker.ts +++ b/apps/workers/workers/crawlerWorker.ts @@ -82,6 +82,7 @@ import { getRateLimitClient } from "@karakeep/shared/ratelimiting"; import { tryCatch } from "@karakeep/shared/tryCatch"; import { BookmarkTypes } from "@karakeep/shared/types/bookmarks"; +import metascraperAmazonImproved from "../metascraper-plugins/metascraper-amazon-improved"; import metascraperReddit from "../metascraper-plugins/metascraper-reddit"; function abortPromise(signal: AbortSignal): Promise<never> { @@ -125,6 +126,7 @@ const metascraperParser = metascraper([ dateModified: true, datePublished: true, }), + metascraperAmazonImproved(), // Fix image extraction bug - must come before metascraperAmazon() metascraperAmazon(), metascraperYoutube({ gotOpts: { |
