feat(file): index chat message attachments

This commit is contained in:
kuaifan 2026-08-09 13:00:23 +00:00
parent 6deec313f7
commit 380bfaaea6
12 changed files with 945 additions and 0 deletions

View File

@ -0,0 +1,131 @@
<?php
namespace App\Console\Commands;
use App\Models\WebSocketDialogMsg;
use App\Models\WebSocketDialogMsgAttachmentBackfill;
use App\Module\Base;
use App\Services\MessageAttachmentBackfillService;
use App\Services\MessageAttachmentService;
use Illuminate\Console\Command;
class BackfillMessageAttachments extends Command
{
protected $signature = 'collaboration-files:backfill-attachments
{--after=0 : 从此消息ID之后开始}
{--before=0 : 处理到此消息ID默认取启动时的最大ID}
{--batch=500 : 每批处理消息数量}
{--limit=0 : 本次最多处理消息数量0表示不限}
{--sleep=0 : 每批结束后的休眠毫秒数}
{--status : 查看自动回填状态,不处理消息}
{--dry-run : 只解析和统计,不写入附件表}';
protected $description = '分批回填协作文件所需的消息附件索引';
public function handle(): int
{
if ($this->option('status')) {
return $this->showStatus();
}
$after = max(0, intval($this->option('after')));
$before = max(0, intval($this->option('before')));
$batch = min(2000, max(1, intval($this->option('batch'))));
$limit = max(0, intval($this->option('limit')));
$sleep = min(60000, max(0, intval($this->option('sleep'))));
$dryRun = boolval($this->option('dry-run'));
if ($before === 0) {
$before = intval(WebSocketDialogMsg::withTrashed()->max('id'));
}
if ($before <= $after) {
$this->info("没有需要处理的消息after={$after}, before={$before}");
return self::SUCCESS;
}
$this->info(sprintf(
'开始回填消息附件:范围 (%d, %d],批量 %d%s',
$after,
$before,
$batch,
$dryRun ? ',仅预览' : ''
));
$lastId = $after;
$processed = 0;
$attachmentCount = 0;
$emptyCount = 0;
$failedIds = [];
while ($lastId < $before && ($limit === 0 || $processed < $limit)) {
$take = $limit > 0 ? min($batch, $limit - $processed) : $batch;
$messages = MessageAttachmentBackfillService::candidateQuery($lastId, $before)
->take($take)
->get();
if ($messages->isEmpty()) {
break;
}
foreach ($messages as $message) {
$lastId = intval($message->id);
$processed++;
try {
if ($dryRun) {
$data = Base::json2array($message->getRawOriginal('msg'));
$count = count(MessageAttachmentService::extract((string)$message->type, $data));
} else {
$count = MessageAttachmentService::sync($message);
}
$attachmentCount += $count;
if ($count === 0) {
$emptyCount++;
}
} catch (\Throwable $e) {
$failedIds[] = $lastId;
$this->warn("消息 {$lastId} 处理失败:{$e->getMessage()}");
}
}
$this->line("已处理 {$processed} 条消息,附件 {$attachmentCount}当前消息ID {$lastId}");
if ($sleep > 0 && $lastId < $before) {
usleep($sleep * 1000);
}
}
$this->newLine();
$this->info("回填结束:消息 {$processed} 条,附件 {$attachmentCount} 条,无有效附件 {$emptyCount}");
$this->info("续跑参数:--after={$lastId} --before={$before}");
if (!empty($failedIds)) {
$this->error('失败消息ID' . implode(',', $failedIds));
return self::FAILURE;
}
return self::SUCCESS;
}
private function showStatus(): int
{
$state = WebSocketDialogMsgAttachmentBackfill::query()->orderBy('id')->first();
if (!$state) {
$this->warn('自动回填任务尚未登记,请先执行数据库迁移');
return self::FAILURE;
}
$this->table(['项目', '值'], [
['状态', $state->status],
['消息快照上界', $state->before_msg_id],
['当前消息ID', $state->last_msg_id],
['成功处理消息', $state->processed_count],
['已索引附件', $state->attachment_count],
['无有效附件消息', $state->empty_count],
['失败尝试', $state->failure_count],
['当前连续重试', $state->retry_count],
['下次重试时间', $state->next_retry_at?->toDateTimeString() ?: '-'],
['最近错误', $state->last_error ?: '-'],
['首次开始时间', $state->started_at?->toDateTimeString() ?: '-'],
['完成时间', $state->completed_at?->toDateTimeString() ?: '-'],
]);
return self::SUCCESS;
}
}

View File

@ -25,6 +25,7 @@ use App\Tasks\ManticoreSyncTask;
use App\Tasks\UnclaimedTaskRemindTask;
use App\Tasks\TodoRemindTask;
use App\Tasks\AiTaskLoopTask;
use App\Tasks\MessageAttachmentBackfillTask;
use Hhxsv5\LaravelS\Swoole\Task\Task;
use App\Module\PatchedAvatar as Avatar;
@ -281,6 +282,8 @@ class IndexController extends InvokeController
Task::deliver(new ManticoreSyncTask());
// AI 任务建议
Task::deliver(new AiTaskLoopTask());
// 聊天附件历史索引回填
Task::deliver(new MessageAttachmentBackfillTask());
return "success";
}

View File

@ -11,6 +11,7 @@ use App\Tasks\PushTask;
use App\Models\ProjectTaskRelation;
use App\Exceptions\ApiException;
use App\Tasks\WebSocketDialogMsgTask;
use App\Services\MessageAttachmentService;
use Hhxsv5\LaravelS\Swoole\Task\Task;
use Illuminate\Database\Eloquent\SoftDeletes;
@ -1477,6 +1478,7 @@ class WebSocketDialogMsg extends AbstractModel
];
$dialogMsg->updateInstance($updateData);
$dialogMsg->generateKeyAndSave($search_key);
MessageAttachmentService::syncSafely($dialogMsg, true);
ProjectTaskRelation::recordMentionsFromMessage($dialogMsg);
//
WebSocketDialogUser::whereDialogId($dialog->id)->whereUserid($sender)->whereHide(1)->change([
@ -1544,6 +1546,7 @@ class WebSocketDialogMsg extends AbstractModel
'updated_at' => Carbon::now()->toDateTimeString('millisecond'),
]);
});
MessageAttachmentService::syncSafely($dialogMsg);
ProjectTaskRelation::recordMentionsFromMessage($dialogMsg);
//
$task = new WebSocketDialogMsgTask($dialogMsg->id);

View File

@ -0,0 +1,49 @@
<?php
namespace App\Models;
/**
* @property int $id
* @property int $msg_id 消息ID
* @property int $dialog_id 会话ID
* @property string $source_type 来源类型
* @property string $kind 附件类型
* @property int $position 消息内附件顺序
* @property string $name 附件名称
* @property string $ext 文件扩展名
* @property string|null $path 原文件地址
* @property string|null $thumb 缩略图地址
* @property int $size 文件大小(B)
* @property int $width 图片宽度
* @property int $height 图片高度
* @property \Illuminate\Support\Carbon|null $created_at
* @property \Illuminate\Support\Carbon|null $updated_at
* @method static \Illuminate\Database\Eloquent\Builder|WebSocketDialogMsgAttachment newModelQuery()
* @method static \Illuminate\Database\Eloquent\Builder|WebSocketDialogMsgAttachment newQuery()
* @method static \Illuminate\Database\Eloquent\Builder|WebSocketDialogMsgAttachment query()
* @method static \Illuminate\Database\Eloquent\Builder|WebSocketDialogMsgAttachment whereMsgId($value)
* @mixin \Eloquent
*/
class WebSocketDialogMsgAttachment extends AbstractModel
{
public const SOURCE_FILE_MESSAGE = 'file_message';
public const SOURCE_INLINE_IMAGE = 'inline_image';
public const KIND_FILE = 'file';
public const KIND_IMAGE = 'image';
protected $fillable = [
'msg_id',
'dialog_id',
'source_type',
'kind',
'position',
'name',
'ext',
'path',
'thumb',
'size',
'width',
'height',
];
}

View File

@ -0,0 +1,52 @@
<?php
namespace App\Models;
/**
* @property int $id
* @property string $status
* @property int $before_msg_id
* @property int $last_msg_id
* @property int $processed_count
* @property int $attachment_count
* @property int $empty_count
* @property int $failure_count
* @property int $retry_count
* @property string|null $last_error
* @property \Illuminate\Support\Carbon|null $started_at
* @property \Illuminate\Support\Carbon|null $completed_at
* @property \Illuminate\Support\Carbon|null $next_retry_at
* @property \Illuminate\Support\Carbon|null $created_at
* @property \Illuminate\Support\Carbon|null $updated_at
* @mixin \Eloquent
*/
class WebSocketDialogMsgAttachmentBackfill extends AbstractModel
{
public const STATUS_PENDING = 'pending';
public const STATUS_RUNNING = 'running';
public const STATUS_FAILED = 'failed';
public const STATUS_COMPLETED = 'completed';
protected $table = 'web_socket_dialog_msg_attachment_backfills';
protected $fillable = [
'status',
'before_msg_id',
'last_msg_id',
'processed_count',
'attachment_count',
'empty_count',
'failure_count',
'retry_count',
'last_error',
'started_at',
'completed_at',
'next_retry_at',
];
protected $casts = [
'started_at' => 'datetime',
'completed_at' => 'datetime',
'next_retry_at' => 'datetime',
];
}

View File

@ -0,0 +1,193 @@
<?php
namespace App\Services;
use App\Models\WebSocketDialogMsg;
use App\Models\WebSocketDialogMsgAttachmentBackfill;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Support\Facades\Cache;
class MessageAttachmentBackfillService
{
public const DEFAULT_BATCH_SIZE = 500;
public const DEFAULT_TIME_LIMIT = 40;
private const LOCK_KEY = 'collaboration-files:attachment-backfill';
private const LOCK_SECONDS = 120;
/**
* 自动回填与手工回填共用同一套候选消息筛选规则。
*/
public static function candidateQuery(int $after, int $before): Builder
{
return WebSocketDialogMsg::query()
->where('id', '>', $after)
->where('id', '<=', $before)
->where(function ($query) {
$query->where('type', 'file')
->orWhere(function ($text) {
$text->where('type', 'text')
->where(function ($images) {
$images->where('msg', 'like', '%browse%')
->orWhere('msg', 'like', '%[:IMAGE:browse:%');
});
});
})
->orderBy('id');
}
/**
* 在时间预算内推进自动回填;已有任务运行时直接跳过。
*/
public static function processAutomatic(
int $batchSize = self::DEFAULT_BATCH_SIZE,
int $timeLimit = self::DEFAULT_TIME_LIMIT
): array {
$batchSize = min(2000, max(1, $batchSize));
$timeLimit = min(50, max(1, $timeLimit));
$lockKey = config('app.env') . ':' . self::LOCK_KEY;
$lock = Cache::store('redis')->lock($lockKey, self::LOCK_SECONDS);
if (!$lock->get()) {
return ['status' => 'locked'];
}
try {
return self::processState($batchSize, $timeLimit);
} finally {
$lock->release();
}
}
private static function processState(int $batchSize, int $timeLimit): array
{
$state = WebSocketDialogMsgAttachmentBackfill::query()->orderBy('id')->first();
if (!$state) {
return ['status' => 'missing'];
}
if ($state->status === WebSocketDialogMsgAttachmentBackfill::STATUS_COMPLETED) {
return ['status' => WebSocketDialogMsgAttachmentBackfill::STATUS_COMPLETED];
}
if ($state->next_retry_at && $state->next_retry_at->isFuture()) {
return ['status' => 'waiting'];
}
$before = intval($state->before_msg_id);
if (!$state->started_at) {
// 迁移到新服务启动之间仍可能由旧进程写入消息,首次运行时补齐这段升级窗口。
$before = max($before, intval(WebSocketDialogMsg::withTrashed()->max('id')));
}
$state->update([
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_RUNNING,
'before_msg_id' => $before,
'started_at' => $state->started_at ?: now(),
'next_retry_at' => null,
]);
$startedAt = microtime(true);
$lastId = intval($state->last_msg_id);
$processed = intval($state->processed_count);
$attachmentCount = intval($state->attachment_count);
$emptyCount = intval($state->empty_count);
while ($lastId < $before && microtime(true) - $startedAt < $timeLimit) {
$messages = self::candidateQuery($lastId, $before)
->take($batchSize)
->get();
if ($messages->isEmpty()) {
return self::complete($state, $before, $processed, $attachmentCount, $emptyCount);
}
foreach ($messages as $message) {
try {
$count = MessageAttachmentService::sync($message);
} catch (\Throwable $e) {
self::fail($state, $lastId, $processed, $attachmentCount, $emptyCount, $e);
return [
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_FAILED,
'last_msg_id' => $lastId,
];
}
$lastId = intval($message->id);
$processed++;
$attachmentCount += $count;
if ($count === 0) {
$emptyCount++;
}
}
$state->update([
'last_msg_id' => $lastId,
'processed_count' => $processed,
'attachment_count' => $attachmentCount,
'empty_count' => $emptyCount,
'retry_count' => 0,
'last_error' => null,
]);
}
if ($lastId >= $before) {
return self::complete($state, $before, $processed, $attachmentCount, $emptyCount);
}
$state->update(['status' => WebSocketDialogMsgAttachmentBackfill::STATUS_PENDING]);
return [
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_PENDING,
'last_msg_id' => $lastId,
'processed_count' => $processed,
'attachment_count' => $attachmentCount,
];
}
private static function complete(
WebSocketDialogMsgAttachmentBackfill $state,
int $before,
int $processed,
int $attachmentCount,
int $emptyCount
): array {
$state->update([
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_COMPLETED,
'last_msg_id' => $before,
'processed_count' => $processed,
'attachment_count' => $attachmentCount,
'empty_count' => $emptyCount,
'retry_count' => 0,
'last_error' => null,
'next_retry_at' => null,
'completed_at' => now(),
]);
return [
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_COMPLETED,
'last_msg_id' => $before,
'processed_count' => $processed,
'attachment_count' => $attachmentCount,
];
}
private static function fail(
WebSocketDialogMsgAttachmentBackfill $state,
int $lastId,
int $processed,
int $attachmentCount,
int $emptyCount,
\Throwable $error
): void {
$retryCount = intval($state->retry_count) + 1;
$retryMinutes = min(30, 2 ** min(4, $retryCount - 1));
$state->update([
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_FAILED,
'last_msg_id' => $lastId,
'processed_count' => $processed,
'attachment_count' => $attachmentCount,
'empty_count' => $emptyCount,
'failure_count' => intval($state->failure_count) + 1,
'retry_count' => $retryCount,
'last_error' => mb_substr($error->getMessage(), 0, 2000),
'next_retry_at' => now()->addMinutes($retryMinutes),
]);
}
}

View File

@ -0,0 +1,236 @@
<?php
namespace App\Services;
use App\Models\File;
use App\Models\WebSocketDialogMsg;
use App\Models\WebSocketDialogMsgAttachment;
use App\Module\Base;
use DOMDocument;
use DOMElement;
use DOMXPath;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
class MessageAttachmentService
{
/**
* 从消息内容提取可检索附件。返回值不包含消息关联字段和时间字段。
*/
public static function extract(string $messageType, array $messageData): array
{
if ($messageType === 'file') {
$attachment = self::extractFileMessage($messageData);
return $attachment ? [$attachment] : [];
}
if ($messageType !== 'text') {
return [];
}
return self::extractInlineImages((string)($messageData['text'] ?? ''));
}
/**
* 将一条消息的附件索引同步为当前内容,重复执行不会产生重复记录。
*/
public static function sync(WebSocketDialogMsg $message, bool $removeMissing = true): int
{
$messageData = Base::json2array($message->getRawOriginal('msg'));
$attachments = self::extract((string)$message->type, $messageData);
if (empty($attachments) && !$removeMissing) {
return 0;
}
DB::transaction(function () use ($message, $attachments) {
if (empty($attachments)) {
WebSocketDialogMsgAttachment::whereMsgId($message->id)->delete();
return;
}
$now = now();
$sourceType = $attachments[0]['source_type'];
$positions = [];
$rows = [];
foreach ($attachments as $attachment) {
$positions[] = $attachment['position'];
$rows[] = array_merge($attachment, [
'msg_id' => intval($message->id),
'dialog_id' => intval($message->dialog_id),
'created_at' => $message->created_at ?: $now,
'updated_at' => $now,
]);
}
DB::table('web_socket_dialog_msg_attachments')->upsert(
$rows,
['msg_id', 'source_type', 'position'],
['dialog_id', 'kind', 'name', 'ext', 'path', 'thumb', 'size', 'width', 'height', 'updated_at']
);
WebSocketDialogMsgAttachment::whereMsgId($message->id)
->where(function ($query) use ($sourceType, $positions) {
$query->where('source_type', '!=', $sourceType)
->orWhereNotIn('position', $positions);
})
->delete();
});
return count($attachments);
}
/**
* 双写失败不影响聊天主链路,历史回填命令可再次同步失败消息。
*/
public static function syncSafely(WebSocketDialogMsg $message, bool $removeMissing = false): bool
{
if (!in_array($message->type, ['file', 'text'], true)) {
return true;
}
try {
self::sync($message, $removeMissing);
return true;
} catch (\Throwable $e) {
Log::warning('Message attachment sync failed', [
'msg_id' => intval($message->id),
'dialog_id' => intval($message->dialog_id),
'message' => $e->getMessage(),
]);
return false;
}
}
private static function extractFileMessage(array $messageData): ?array
{
$path = self::normalizeStoredPath((string)($messageData['path'] ?? ''));
if ($path === '') {
return null;
}
$ext = strtolower((string)($messageData['ext'] ?? self::pathExtension($path)));
$thumb = self::normalizeStoredPath((string)($messageData['thumb'] ?? ''));
return [
'source_type' => WebSocketDialogMsgAttachment::SOURCE_FILE_MESSAGE,
'kind' => in_array($ext, File::imageExt, true)
? WebSocketDialogMsgAttachment::KIND_IMAGE
: WebSocketDialogMsgAttachment::KIND_FILE,
'position' => 0,
'name' => (string)($messageData['name'] ?? ''),
'ext' => $ext,
'path' => $path,
'thumb' => $thumb,
'size' => max(0, intval($messageData['size'] ?? 0)),
'width' => max(0, intval($messageData['width'] ?? 0)),
'height' => max(0, intval($messageData['height'] ?? 0)),
];
}
private static function extractInlineImages(string $html): array
{
if ($html === '') {
return [];
}
$attachments = [];
if (str_contains($html, '<img')) {
$document = new DOMDocument('1.0', 'UTF-8');
$previous = libxml_use_internal_errors(true);
$loaded = $document->loadHTML(
'<?xml encoding="UTF-8"><div>' . $html . '</div>',
LIBXML_HTML_NOIMPLIED | LIBXML_HTML_NODEFDTD | LIBXML_NOERROR | LIBXML_NOWARNING
);
libxml_clear_errors();
libxml_use_internal_errors($previous);
if ($loaded) {
$nodes = (new DOMXPath($document))->query(
"//img[contains(concat(' ', normalize-space(@class), ' '), ' browse ')]"
);
foreach ($nodes ?: [] as $node) {
if (!$node instanceof DOMElement) {
continue;
}
self::appendInlineImage($attachments, [
'src' => $node->getAttribute('src'),
'width' => $node->getAttribute('width'),
'height' => $node->getAttribute('height'),
'name' => $node->getAttribute('alt'),
]);
}
}
}
if (empty($attachments)) {
preg_match_all('/\[:IMAGE:browse:([^:]*):([^:]*):(.*?):(.*?):\]/i', $html, $matches, PREG_SET_ORDER);
foreach ($matches as $match) {
self::appendInlineImage($attachments, [
'src' => $match[3] ?? '',
'width' => $match[1] ?? 0,
'height' => $match[2] ?? 0,
'name' => $match[4] ?? '',
]);
}
}
foreach ($attachments as $position => &$attachment) {
$attachment['position'] = $position;
}
unset($attachment);
return $attachments;
}
private static function appendInlineImage(array &$attachments, array $image): void
{
$thumb = self::normalizeStoredPath(html_entity_decode(trim((string)$image['src']), ENT_QUOTES | ENT_HTML5));
if ($thumb === '' || str_contains($thumb, 'images/other/imgerr.jpg')) {
return;
}
$path = Base::thumbRestore($thumb);
$attachments[] = [
'source_type' => WebSocketDialogMsgAttachment::SOURCE_INLINE_IMAGE,
'kind' => WebSocketDialogMsgAttachment::KIND_IMAGE,
'position' => 0,
'name' => trim(strip_tags((string)$image['name'])),
'ext' => self::pathExtension($path),
'path' => $path,
'thumb' => $thumb,
'size' => self::localFileSize($path),
'width' => max(0, intval($image['width'])),
'height' => max(0, intval($image['height'])),
];
}
private static function normalizeStoredPath(string $path): string
{
$path = trim(str_replace('{{RemoteURL}}', '', $path));
if ($path === '' || str_starts_with($path, 'data:')) {
return '';
}
$path = Base::unFillUrl($path);
if (str_starts_with($path, '/') && !str_starts_with($path, '//')) {
$path = ltrim($path, '/');
}
return $path;
}
private static function pathExtension(string $path): string
{
$urlPath = parse_url($path, PHP_URL_PATH);
return strtolower(pathinfo(urldecode((string)($urlPath ?: $path)), PATHINFO_EXTENSION));
}
private static function localFileSize(string $path): int
{
if ($path === '' || preg_match('/^[a-z][a-z0-9+.-]*:\/\//i', $path)) {
return 0;
}
$file = public_path(ltrim($path, '/'));
return is_file($file) ? max(0, intval(filesize($file))) : 0;
}
}

View File

@ -0,0 +1,22 @@
<?php
namespace App\Tasks;
use App\Services\MessageAttachmentBackfillService;
class MessageAttachmentBackfillTask extends AbstractTask
{
public function __construct()
{
parent::__construct();
}
public function start()
{
MessageAttachmentBackfillService::processAutomatic();
}
public function end()
{
}
}

View File

@ -0,0 +1,37 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::create('web_socket_dialog_msg_attachments', function (Blueprint $table) {
$table->bigIncrements('id');
$table->unsignedBigInteger('msg_id')->comment('消息ID');
$table->unsignedBigInteger('dialog_id')->comment('会话ID');
$table->string('source_type', 20)->comment('来源类型file_message、inline_image');
$table->string('kind', 20)->comment('附件类型file、image');
$table->unsignedSmallInteger('position')->default(0)->comment('消息内附件顺序');
$table->string('name')->default('')->comment('附件名称');
$table->string('ext', 20)->default('')->comment('文件扩展名');
$table->text('path')->nullable()->comment('原文件地址');
$table->text('thumb')->nullable()->comment('缩略图地址');
$table->unsignedBigInteger('size')->default(0)->comment('文件大小(B)');
$table->unsignedInteger('width')->default(0)->comment('图片宽度');
$table->unsignedInteger('height')->default(0)->comment('图片高度');
$table->timestamps();
$table->unique(['msg_id', 'source_type', 'position'], 'uniq_ws_msg_attachment_position');
$table->index(['dialog_id', 'id'], 'idx_ws_msg_attachment_dialog');
$table->index(['kind', 'id'], 'idx_ws_msg_attachment_kind');
});
}
public function down(): void
{
Schema::dropIfExists('web_socket_dialog_msg_attachments');
}
};

View File

@ -0,0 +1,47 @@
<?php
use Illuminate\Database\Migrations\Migration;
use Illuminate\Database\Schema\Blueprint;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
return new class extends Migration
{
public function up(): void
{
Schema::create('web_socket_dialog_msg_attachment_backfills', function (Blueprint $table) {
$table->bigIncrements('id');
$table->string('status', 20)->default('pending')->comment('状态pending、running、failed、completed');
$table->unsignedBigInteger('before_msg_id')->default(0)->comment('启动时消息快照上界');
$table->unsignedBigInteger('last_msg_id')->default(0)->comment('已处理到的消息ID');
$table->unsignedBigInteger('processed_count')->default(0)->comment('成功处理消息数');
$table->unsignedBigInteger('attachment_count')->default(0)->comment('已索引附件数');
$table->unsignedBigInteger('empty_count')->default(0)->comment('无有效附件消息数');
$table->unsignedBigInteger('failure_count')->default(0)->comment('失败尝试次数');
$table->unsignedInteger('retry_count')->default(0)->comment('当前消息连续重试次数');
$table->text('last_error')->nullable()->comment('最近一次错误');
$table->timestamp('started_at')->nullable()->comment('首次开始时间');
$table->timestamp('completed_at')->nullable()->comment('完成时间');
$table->timestamp('next_retry_at')->nullable()->comment('下次重试时间');
$table->timestamps();
$table->index(['status', 'next_retry_at'], 'idx_ws_msg_attachment_backfill_pending');
});
$beforeMsgId = intval(DB::table('web_socket_dialog_msgs')->max('id'));
$now = now();
DB::table('web_socket_dialog_msg_attachment_backfills')->insert([
'status' => $beforeMsgId > 0 ? 'pending' : 'completed',
'before_msg_id' => $beforeMsgId,
'last_msg_id' => 0,
'completed_at' => $beforeMsgId > 0 ? null : $now,
'created_at' => $now,
'updated_at' => $now,
]);
}
public function down(): void
{
Schema::dropIfExists('web_socket_dialog_msg_attachment_backfills');
}
};

View File

@ -0,0 +1,87 @@
<?php
namespace Tests\Feature;
use App\Models\WebSocketDialogMsg;
use App\Models\WebSocketDialogMsgAttachment;
use App\Models\WebSocketDialogMsgAttachmentBackfill;
use App\Services\MessageAttachmentBackfillService;
use Illuminate\Foundation\Testing\DatabaseTransactions;
use Tests\TestCase;
class MessageAttachmentBackfillServiceTest extends TestCase
{
use DatabaseTransactions;
public function test_automatic_backfill_advances_persisted_cursor_to_completion(): void
{
$fileMessage = $this->makeMessage('file', [
'name' => 'backfill.pdf',
'ext' => 'pdf',
'path' => 'uploads/test/backfill.pdf',
'size' => 1024,
]);
$imageMessage = $this->makeMessage('text', [
'text' => '<p><img class="browse" width="320" height="180" src="uploads/test/backfill.png_thumb.jpg"/></p>',
]);
$state = WebSocketDialogMsgAttachmentBackfill::query()->firstOrFail();
$state->update([
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_PENDING,
// 模拟迁移后、服务重启前又产生了一条图片消息。
'before_msg_id' => $fileMessage->id,
'last_msg_id' => $fileMessage->id - 1,
'processed_count' => 0,
'attachment_count' => 0,
'empty_count' => 0,
'failure_count' => 0,
'retry_count' => 0,
'last_error' => null,
'started_at' => null,
'completed_at' => null,
'next_retry_at' => null,
]);
$result = MessageAttachmentBackfillService::processAutomatic(1, 5);
$state->refresh();
$this->assertSame(WebSocketDialogMsgAttachmentBackfill::STATUS_COMPLETED, $result['status']);
$this->assertSame(WebSocketDialogMsgAttachmentBackfill::STATUS_COMPLETED, $state->status);
$this->assertSame($imageMessage->id, $state->before_msg_id);
$this->assertSame($imageMessage->id, $state->last_msg_id);
$this->assertSame(2, $state->processed_count);
$this->assertSame(2, $state->attachment_count);
$this->assertNotNull($state->completed_at);
$this->assertSame(2, WebSocketDialogMsgAttachment::whereIn('msg_id', [
$fileMessage->id,
$imageMessage->id,
])->count());
}
public function test_automatic_backfill_waits_until_retry_time(): void
{
$state = WebSocketDialogMsgAttachmentBackfill::query()->firstOrFail();
$state->update([
'status' => WebSocketDialogMsgAttachmentBackfill::STATUS_FAILED,
'next_retry_at' => now()->addMinutes(5),
]);
$result = MessageAttachmentBackfillService::processAutomatic(1, 1);
$this->assertSame('waiting', $result['status']);
$this->assertSame(WebSocketDialogMsgAttachmentBackfill::STATUS_FAILED, $state->fresh()->status);
}
private function makeMessage(string $type, array $data): WebSocketDialogMsg
{
$message = WebSocketDialogMsg::createInstance([
'dialog_id' => 0,
'userid' => 0,
'type' => $type,
'key' => '',
'msg' => json_encode($data, JSON_UNESCAPED_UNICODE),
]);
$message->save();
return $message;
}
}

View File

@ -0,0 +1,85 @@
<?php
namespace Tests\Unit;
use App\Models\WebSocketDialogMsgAttachment;
use App\Services\MessageAttachmentService;
use Tests\TestCase;
class MessageAttachmentServiceTest extends TestCase
{
public function test_extracts_file_message_metadata(): void
{
$attachments = MessageAttachmentService::extract('file', [
'name' => '方案.pdf',
'ext' => 'PDF',
'path' => 'uploads/chat/202608/1/plan.pdf',
'thumb' => 'images/ext/pdf.png',
'size' => 2048,
]);
$this->assertCount(1, $attachments);
$this->assertSame(WebSocketDialogMsgAttachment::SOURCE_FILE_MESSAGE, $attachments[0]['source_type']);
$this->assertSame(WebSocketDialogMsgAttachment::KIND_FILE, $attachments[0]['kind']);
$this->assertSame('pdf', $attachments[0]['ext']);
$this->assertSame(2048, $attachments[0]['size']);
}
public function test_classifies_uploaded_image_as_image_attachment(): void
{
$attachments = MessageAttachmentService::extract('file', [
'name' => 'design.png',
'ext' => 'png',
'path' => 'uploads/chat/202608/1/design.png',
'thumb' => 'uploads/chat/202608/1/design.png_thumb.jpg',
'size' => 4096,
'width' => 1200,
'height' => 800,
]);
$this->assertSame(WebSocketDialogMsgAttachment::KIND_IMAGE, $attachments[0]['kind']);
$this->assertSame(1200, $attachments[0]['width']);
$this->assertSame(800, $attachments[0]['height']);
}
public function test_extracts_each_browse_image_and_ignores_emoticons(): void
{
$html = <<<'HTML'
<p>
<img class="browse" width="640" height="480" src="{{RemoteURL}}uploads/chat/202608/1/design.png_thumb.jpg" alt="设计图"/>
<img class="emoticon" width="32" height="32" src="images/emoji/smile.png" alt="笑脸"/>
<img class="browse extra" width="320" height="240" src="{{RemoteURL}}uploads/chat/202608/1/design.png_thumb.jpg" alt="重复图片"/>
</p>
HTML;
$attachments = MessageAttachmentService::extract('text', ['text' => $html]);
$this->assertCount(2, $attachments);
$this->assertSame([0, 1], array_column($attachments, 'position'));
$this->assertSame('uploads/chat/202608/1/design.png', $attachments[0]['path']);
$this->assertSame('uploads/chat/202608/1/design.png_thumb.jpg', $attachments[0]['thumb']);
$this->assertSame('设计图', $attachments[0]['name']);
$this->assertSame('重复图片', $attachments[1]['name']);
}
public function test_extracts_legacy_image_token(): void
{
$attachments = MessageAttachmentService::extract('text', [
'text' => '<p>旧消息</p>[:IMAGE:browse:120:80:uploads/chat/202001/1/legacy.webp:旧截图:]',
]);
$this->assertCount(1, $attachments);
$this->assertSame('uploads/chat/202001/1/legacy.webp', $attachments[0]['path']);
$this->assertSame('webp', $attachments[0]['ext']);
$this->assertSame(120, $attachments[0]['width']);
$this->assertSame(80, $attachments[0]['height']);
}
public function test_ignores_broken_placeholder_and_non_attachment_messages(): void
{
$this->assertSame([], MessageAttachmentService::extract('text', [
'text' => '<img class="browse" width="90" height="90" src="images/other/imgerr.jpg"/>',
]));
$this->assertSame([], MessageAttachmentService::extract('meeting', ['name' => '周会']));
}
}