From d0faf3695dac9b1793a8a2a76c6b44bbef18a11e Mon Sep 17 00:00:00 2001
From: filesite <release@filesite.io>
Date: Tue, 1 Apr 2025 09:16:41 +0800
Subject: [PATCH] webcrawler bot support base64 image format

---
 bot/WebCrawler.mjs | 6 +++++-
 lib/common.mjs     | 9 +++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/bot/WebCrawler.mjs b/bot/WebCrawler.mjs
index 12be155..d20dcf3 100644
--- a/bot/WebCrawler.mjs
+++ b/bot/WebCrawler.mjs
@@ -90,7 +90,7 @@ class WebCrawler extends HeroBot {
 
 
             //get cover image's base64 data
-            if (typeof(data.cover) != 'undefined' && data.cover) {
+            if (typeof(data.cover) != 'undefined' && data.cover && /^data:image\/[a-z]+;base64,/i.test(data.cover) == false) {
                 data.cover = common.getAbsoluteUrl(data.cover);
 
                 const response = await hero.goto(data.cover);
@@ -100,6 +100,10 @@ class WebCrawler extends HeroBot {
                     data.cover_base64 = imgBuffer.toString('base64');
                     data.cover_type = common.getImageType(data.cover);
                 }
+            }else if ( typeof(data.cover) != 'undefined' && data.cover && /^data:image\/[a-z]+;base64,/i.test(data.cover) ) {
+                //support base64 image
+                data.cover_type = common.getImageType(data.cover);
+                data.cover_base64 = data.cover.replace(/^data:image\/[a-z]+;base64,/i, '');
             }
 
             await hero.close();
diff --git a/lib/common.mjs b/lib/common.mjs
index 7e75101..96a4ac5 100644
--- a/lib/common.mjs
+++ b/lib/common.mjs
@@ -51,16 +51,17 @@ export default {
         return url;
     },
 
+    //support base64 format
     getImageType: function(url) {
         let imgType = 'jpeg';
 
-        if (/\.jp(e)?g/ig.test(url)) {
+        if (/\.jp(e)?g/ig.test(url) || /image\/jp(e)?g/ig.test(url)) {
             imgType = 'jpeg';
-        }else if (/\.png/ig.test(url)) {
+        }else if (/\.png/ig.test(url) || /image\/png/ig.test(url)) {
             imgType = 'png';
-        }else if (/\.webp?/ig.test(url)) {
+        }else if (/\.webp?/ig.test(url) || /image\/webp/ig.test(url)) {
             imgType = 'webp';
-        }else if (/\.gif?/ig.test(url)) {
+        }else if (/\.gif?/ig.test(url) || /image\/gif/ig.test(url)) {
             imgType = 'gif';
         }