From 6f2c7a85755d7b4a05f87f7c96b1a03b413780b0 Mon Sep 17 00:00:00 2001
From: filesite <release@filesite.io>
Date: Sun, 24 Nov 2024 20:33:23 +0800
Subject: [PATCH] index data save improve

---
 .../beauty/controller/CommandController.php   | 73 ++++++++++++++++++-
 themes/beauty/controller/ListController.php   |  4 +-
 themes/beauty/controller/SiteController.php   |  4 +-
 3 files changed, 74 insertions(+), 7 deletions(-)

diff --git a/themes/beauty/controller/CommandController.php b/themes/beauty/controller/CommandController.php
index c19fde4..481a187 100644
--- a/themes/beauty/controller/CommandController.php
+++ b/themes/beauty/controller/CommandController.php
@@ -9,8 +9,9 @@ require_once __DIR__ . '/../../../plugins/Common.php';
 Class CommandController extends Controller {
     protected $logPrefix = '[MainBot]';
     protected $scanedDirCacheKey = 'MainBotScanedDirs';
-    protected $dateIndexCacheKey = 'MainBotDateIndex';
+    protected $dateIndexCacheKey = 'MainBotDateIndex';      //索引数据的key单独缓存，缓存key为此{cacheKey}_keys
     protected $noOriginalCtimeFilesCacheKey = 'MainBotNoOriginalCtimeFiles';
+    protected $allFilesCacheKey = 'MainBotAllFiles';
 
     public function actionIndex() {
         $commands = <<<eof
@@ -125,6 +126,8 @@ eof;
         $this->scanMediaFiles();
         $this->saveDateIndexIntoCacheFile();
         $this->saveNoOriginalCtimeFilesIntoFile();
+        //缓存所有文件id跟文件信息，便于根据id列表来渲染，并按id首字母分子目录存放，以支持大量文件的场景
+        $this->saveAllFilesIntoCacheFile();
 
         //while (true) {
             //$time = date('Y-m-d H:i:s');
@@ -200,6 +203,8 @@ eof;
                 $scanIndex ++;
 
                 if (!empty($item['filename'])) {
+                    //保存所有文件到索引
+                    $this->updateAllFilesCache($item);
                     //更新年份、月份时间索引
                     $this->updateDateIndex($item);
                     //更新没有拍摄时间的文件索引
@@ -292,7 +297,68 @@ eof;
             return false;
         }
 
-        return Common::saveCacheToFile($cacheKey, $cacheData);
+        $cacheDir = 'index';
+
+        //save index keys
+        $indexKeys = [];
+        foreach ($cacheData as $year => $item) {
+            $indexKeys[$year] = array_keys($item);
+        }
+        Common::saveCacheToFile("{$cacheKey}_keys", $indexKeys, $cacheDir);
+
+        return Common::saveCacheToFile($cacheKey, $cacheData, $cacheDir);
+    }
+
+    protected function updateAllFilesCache($file) {
+        $cacheKey = $this->allFilesCacheKey;
+        $cacheData = Common::getCache($cacheKey);
+        if (empty($cacheData)) {
+            $cacheData = array();
+        }
+
+        $cacheData[$file['id']] = $file;
+        return Common::setCache($cacheKey, $cacheData);
+    }
+
+    protected function getFilesByFirstCharcter($files, $dirNum) {
+        $byFirst = [];
+
+        foreach ($files as $id => $item) {
+            $firstChar = substr($id, 0, 1);
+            $ascNum = ord($firstChar);
+            $index = $ascNum % $dirNum;
+            if (empty($byFirst[$index])) {
+                $byFirst[$index] = [];
+            }
+            $byFirst[$index][$id] = $item;
+        }
+
+        return $byFirst;
+    }
+
+    protected function saveAllFilesIntoCacheFile() {
+        $cacheKey = $this->allFilesCacheKey;
+        $cacheData = Common::getCache($cacheKey);
+        if (empty($cacheData)) {
+            return false;
+        }
+
+        $total = count($cacheData);
+        $dirNum = 1;
+
+        if ($total > 1000 && $total <= 10000) {
+            $dirNum = 10;
+        }else if ($total > 10000){
+            $dirNum = 100;
+        }
+
+        $filesByFirstChar = $this->getFilesByFirstCharcter($cacheData, $dirNum);
+        $cacheDir = 'index';
+        for ($i=1;$i<=$dirNum;$i++) {
+            Common::saveCacheToFile("{$cacheKey}_{$i}", $filesByFirstChar[$i-1], $cacheDir);
+        }
+
+        return true;
     }
 
     //汇总每个目录下图片、视频、音乐文件数量
@@ -336,7 +402,8 @@ eof;
             return false;
         }
 
-        return Common::saveCacheToFile($cacheKey, $cacheData);
+        $cacheDir = 'index';
+        return Common::saveCacheToFile($cacheKey, $cacheData, $cacheDir);
     }
 
     public function actionTest() {
diff --git a/themes/beauty/controller/ListController.php b/themes/beauty/controller/ListController.php
index 35e9c07..4e44a51 100644
--- a/themes/beauty/controller/ListController.php
+++ b/themes/beauty/controller/ListController.php
@@ -8,7 +8,7 @@ require_once __DIR__ . '/../../../plugins/Common.php';
 require_once __DIR__ . '/../../../plugins/Html.php';
 
 Class ListController extends Controller {
-    protected $dateIndexCacheKey = 'MainBotDateIndex';
+    protected $dateIndexCacheKey = 'MainBotDateIndex';      //索引数据的key单独缓存，缓存key为此{cacheKey}_keys
     protected $noOriginalCtimeFilesCacheKey = 'MainBotNoOriginalCtimeFiles';
 
     public function actionIndex() {
@@ -303,7 +303,7 @@ Class ListController extends Controller {
         $isAdminIp = Common::isAdminIp($this->getUserIp());        //判断是否拥有管理权限
 
         //从缓存文件获取按年份、月份归类的索引数据
-        $cacheDataByDate = Common::getCacheFromFile($this->dateIndexCacheKey, 86400*365);
+        $cacheDataByDate = Common::getCacheFromFile($this->dateIndexCacheKey, 86400*365, 'index');
 
         $viewName = '//site/index';     //共享视图
         $params = compact(
diff --git a/themes/beauty/controller/SiteController.php b/themes/beauty/controller/SiteController.php
index 62f9088..d6ea0ae 100644
--- a/themes/beauty/controller/SiteController.php
+++ b/themes/beauty/controller/SiteController.php
@@ -8,7 +8,7 @@ require_once __DIR__ . '/../../../plugins/Common.php';
 require_once __DIR__ . '/../../../plugins/Html.php';
 
 Class SiteController extends Controller {
-    protected $dateIndexCacheKey = 'MainBotDateIndex';
+    protected $dateIndexCacheKey = 'MainBotDateIndex';      //索引数据的key单独缓存，缓存key为此{cacheKey}_keys
     protected $noOriginalCtimeFilesCacheKey = 'MainBotNoOriginalCtimeFiles';
 
     public function actionIndex() {
@@ -261,7 +261,7 @@ Class SiteController extends Controller {
 
 
         //从缓存文件获取按年份、月份归类的索引数据
-        $cacheDataByDate = Common::getCacheFromFile($this->dateIndexCacheKey, 86400*365);
+        $cacheDataByDate = Common::getCacheFromFile($this->dateIndexCacheKey, 86400*365, 'index');
 
         $viewName = 'index';
         $params = compact(