feat: queue imported attachments for indexing
This commit is contained in:
parent
3e9f6b11cc
commit
5d91eb4f5f
2 changed files with 36 additions and 2 deletions
|
|
@ -1 +1 @@
|
||||||
Subproject commit 38de2d0dc1ae233668cf546f3ab74046afa8986a
|
Subproject commit d658a0d6dc5110e11168e6a99f2db644a25af4e5
|
||||||
|
|
@ -16,6 +16,9 @@ import { unwrapFromParagraph } from '../utils/import-formatter';
|
||||||
import { resolveRelativeAttachmentPath } from '../utils/import.utils';
|
import { resolveRelativeAttachmentPath } from '../utils/import.utils';
|
||||||
import { load } from 'cheerio';
|
import { load } from 'cheerio';
|
||||||
import pLimit from 'p-limit';
|
import pLimit from 'p-limit';
|
||||||
|
import { InjectQueue } from '@nestjs/bullmq';
|
||||||
|
import { Queue } from 'bullmq';
|
||||||
|
import { QueueJob, QueueName } from '../../queue/constants';
|
||||||
|
|
||||||
interface AttachmentInfo {
|
interface AttachmentInfo {
|
||||||
href: string;
|
href: string;
|
||||||
|
|
@ -39,6 +42,7 @@ export class ImportAttachmentService {
|
||||||
constructor(
|
constructor(
|
||||||
private readonly storageService: StorageService,
|
private readonly storageService: StorageService,
|
||||||
@InjectKysely() private readonly db: KyselyDB,
|
@InjectKysely() private readonly db: KyselyDB,
|
||||||
|
@InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
|
||||||
) {}
|
) {}
|
||||||
|
|
||||||
async processAttachments(opts: {
|
async processAttachments(opts: {
|
||||||
|
|
@ -579,7 +583,7 @@ export class ImportAttachmentService {
|
||||||
if (!nonDrawioExtensions.has(ext)) {
|
if (!nonDrawioExtensions.has(ext)) {
|
||||||
drawioFiles.push(attachment);
|
drawioFiles.push(attachment);
|
||||||
} else {
|
} else {
|
||||||
//Skipped non-Draw.io file with mxfile MIME.}`,
|
//Skipped non-Draw.io file with mxfile MIME.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -792,6 +796,36 @@ export class ImportAttachmentService {
|
||||||
})
|
})
|
||||||
.execute();
|
.execute();
|
||||||
|
|
||||||
|
// Queue PDF and DOCX files for indexing
|
||||||
|
const supportedExtensions = ['.pdf', '.docx'];
|
||||||
|
if (supportedExtensions.includes(ext.toLowerCase())) {
|
||||||
|
try {
|
||||||
|
await this.attachmentQueue.add(
|
||||||
|
QueueJob.ATTACHMENT_INDEX_CONTENT,
|
||||||
|
{ attachmentId },
|
||||||
|
{
|
||||||
|
attempts: 2,
|
||||||
|
backoff: {
|
||||||
|
type: 'exponential',
|
||||||
|
delay: 30 * 1000,
|
||||||
|
},
|
||||||
|
deduplication: {
|
||||||
|
id: attachmentId,
|
||||||
|
},
|
||||||
|
removeOnComplete: true,
|
||||||
|
removeOnFail: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
this.logger.debug(
|
||||||
|
`Queued ${fileNameWithExt} for indexing (attachment ID: ${attachmentId})`,
|
||||||
|
);
|
||||||
|
} catch (err) {
|
||||||
|
this.logger.error(
|
||||||
|
`Failed to queue indexing for imported attachment ${attachmentId}: ${err}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
uploadStats.completed++;
|
uploadStats.completed++;
|
||||||
|
|
||||||
if (uploadStats.completed % 10 === 0) {
|
if (uploadStats.completed % 10 === 0) {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue