philc
1034 lines
37 KiB
JavaScript
1034 lines
37 KiB
JavaScript
/**
|
|
* @file models/mailconn.js
|
|
* @description Mail connector registry + provider adapters for mymail.
|
|
* Providers: imap (login/password), pop3 (login/password).
|
|
* This module is provider-side only: it never touches Odmdb nor routes. Mailboxes.js
|
|
* orchestrates persistence (upserts, counts, attachment cache) using the normalized
|
|
* items returned here. All connection helpers open their own session and always close
|
|
* it when done (a failed session is closed best-effort).
|
|
*/
|
|
|
|
const { ImapFlow } = require("imapflow");
|
|
const { simpleParser } = require("mailparser");
|
|
const POP3Client = require("poplib");
|
|
const nodemailer = require("nodemailer");
|
|
const crypto = require("crypto");
|
|
const dayjs = require("dayjs");
|
|
|
|
const PROVIDERS = {
|
|
imap: { authType: "password", label: "IMAP", needsConnector: true, needsSecrets: true, defaultConnector: { port: 993, tls: true, folder: "INBOX" } },
|
|
pop3: { authType: "password", label: "POP3", needsConnector: true, needsSecrets: true, defaultConnector: { port: 995, tls: true } },
|
|
};
|
|
|
|
const PASSWORD_PROVIDERS = ["imap", "pop3"];
|
|
|
|
// Safety nets, not windows: with a working high-watermark cursor the natural batch is the
|
|
// handful of messages received since the last sync, so the incremental path allows a much
|
|
// larger cap. The cursor-less full rescan keeps the historical 50 because it walks the
|
|
// whole mailbox result set.
|
|
const CURSOR_MAX_RESULTS = 500;
|
|
const FULL_SCAN_MAX_RESULTS = 50;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Metadata
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function providerMeta(provider) {
|
|
return PROVIDERS[provider] || null;
|
|
}
|
|
|
|
function supports(provider) {
|
|
return !!PROVIDERS[provider];
|
|
}
|
|
|
|
function authType(provider) {
|
|
return (PROVIDERS[provider] && PROVIDERS[provider].authType) || null;
|
|
}
|
|
|
|
function needsSecrets(provider) {
|
|
return !!(PROVIDERS[provider] && PROVIDERS[provider].needsSecrets);
|
|
}
|
|
|
|
function needsConnector(provider) {
|
|
return !!(PROVIDERS[provider] && PROVIDERS[provider].needsConnector);
|
|
}
|
|
|
|
function isPasswordProvider(provider) {
|
|
return authType(provider) === "password" || PASSWORD_PROVIDERS.includes(provider);
|
|
}
|
|
|
|
function emailIdSafe(id) {
|
|
return String(id).replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Address helpers (mailparser AddressObject or raw string)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function normalizeAddrString(str) {
|
|
if (!str) return { email: "", name: "" };
|
|
const s = String(str).trim();
|
|
const match = s.match(/^\s*"?([^"<]*)"?\s*<([^>]+)>\s*$/);
|
|
if (match) return { name: match[1].trim(), email: match[2].trim() };
|
|
return { name: "", email: s };
|
|
}
|
|
|
|
function addressesFromParsed(val) {
|
|
if (!val) return [];
|
|
if (Array.isArray(val.value)) {
|
|
return val.value
|
|
.map((a) => ({ name: (a.name || "").trim(), email: (a.address || "").trim() }))
|
|
.filter((a) => a.email);
|
|
}
|
|
if (typeof val === "string") {
|
|
return String(val)
|
|
.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean)
|
|
.map(normalizeAddrString);
|
|
}
|
|
return [];
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Generic message mapper (RFC822 parsed via mailparser) — IMAP eager + POP3
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build a full mails-schema item from a mailparser result.
|
|
* Attachment buffers are returned separately, never stored in the document.
|
|
* @param {object} mailbox - mailbox document
|
|
* @param {object} parsed - mailparser simpleParser result
|
|
* @param {object} meta - {emailId, dateReceived?, remoteUid?, folder?, seen?, size?}
|
|
*/
|
|
function mapParsedMessage(mailbox, parsed, meta) {
|
|
const subject = (parsed.subject && String(parsed.subject)) || "(no subject)";
|
|
const dateSent = parsed.date ? dayjs(parsed.date).toISOString() : meta.dateReceived || null;
|
|
const dateReceived = meta.dateReceived || dateSent || null;
|
|
const attachments = [];
|
|
const buffers = [];
|
|
(Array.isArray(parsed.attachments) ? parsed.attachments : []).forEach((a) => {
|
|
if (!a || typeof a !== "object") return;
|
|
const filename = a.filename && String(a.filename).trim()
|
|
? String(a.filename).trim()
|
|
: `attachment-${attachments.length + 1}`;
|
|
const att = {
|
|
filename,
|
|
mimeType: a.contentType || "application/octet-stream",
|
|
size: typeof a.size === "number" ? a.size : Buffer.isBuffer(a.content) ? a.content.length : 0,
|
|
contentId: a.contentId || a.cid || "",
|
|
storageName: "",
|
|
};
|
|
attachments.push(att);
|
|
buffers.push({ att, content: a.content });
|
|
});
|
|
const seen = meta.seen === true;
|
|
const flags = {
|
|
seen,
|
|
unread: !seen,
|
|
draft: false,
|
|
starred: false,
|
|
answered: false,
|
|
important: false,
|
|
};
|
|
return {
|
|
item: {
|
|
emailId: meta.emailId,
|
|
provider: mailbox.provider,
|
|
remoteUid: meta.remoteUid || "",
|
|
folder: meta.folder || "",
|
|
messageId: parsed.messageId || meta.emailId,
|
|
mailboxid: mailbox.mailboxid,
|
|
tribe: mailbox.tribe,
|
|
threadId: "",
|
|
inReplyTo: parsed.inReplyTo || "",
|
|
references: Array.isArray(parsed.references)
|
|
? parsed.references
|
|
: parsed.references
|
|
? String(parsed.references).split(/\s+/).filter(Boolean)
|
|
: [],
|
|
subject,
|
|
snippet: parsed.text ? String(parsed.text).replace(/\s+/g, " ").slice(0, 200) : "",
|
|
from: addressesFromParsed(parsed.from)[0] || { email: "", name: "" },
|
|
to: addressesFromParsed(parsed.to),
|
|
cc: addressesFromParsed(parsed.cc),
|
|
bcc: addressesFromParsed(parsed.bcc),
|
|
replyTo: addressesFromParsed(parsed.replyTo),
|
|
dateSent,
|
|
dateReceived,
|
|
size: meta.size || parsed.sizeOf || 0,
|
|
contentType: "message/rfc822",
|
|
flags,
|
|
hasAttachments: attachments.length > 0,
|
|
attachments,
|
|
status: seen ? "read" : "new",
|
|
bodyRef: "",
|
|
bodyHtml: parsed.html || parsed.textAsHtml || (parsed.text ? `<pre>${String(parsed.text)}</pre>` : ""),
|
|
actions: [],
|
|
},
|
|
buffers,
|
|
};
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// IMAP adapter
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function imapConfig(mailbox, secrets) {
|
|
const meta = providerMeta(mailbox.provider) || PROVIDERS.imap;
|
|
const conn = mailbox.connector && mailbox.connector.host ? mailbox.connector : meta.defaultConnector;
|
|
const port = parseInt(conn.port, 10) || 993;
|
|
const tls = conn.tls !== false && conn.tls !== 0;
|
|
const cfg = {
|
|
host: conn.host,
|
|
port: tls ? port : (port || 143),
|
|
secure: tls,
|
|
connectTimeout: 20000,
|
|
greetingTimeout: 15000,
|
|
authTimeout: 15000,
|
|
awaitResponseTimeout: 45000,
|
|
disableAutoIdle: true,
|
|
tlsOptions: {
|
|
rejectUnauthorized: conn.rejectUnauthorized !== false,
|
|
},
|
|
logger: false,
|
|
};
|
|
const user = (mailbox.connector && mailbox.connector.user) || "";
|
|
cfg.auth = { user, pass: (secrets && secrets.password) || "" };
|
|
return cfg;
|
|
}
|
|
|
|
function imapFolder(mailbox) {
|
|
const folder = mailbox.connector && mailbox.connector.folder;
|
|
return folder && folder.trim() ? folder.trim() : "INBOX";
|
|
}
|
|
|
|
/**
|
|
* Resolve the remote "Sent" folder of a mailbox. Overridable via connector.sentFolder,
|
|
* otherwise auto-detected: `[Gmail]/Sent Mail` on gmail.com hosts, `Sent` elsewhere.
|
|
*/
|
|
function imapSentFolder(mailbox) {
|
|
const c = mailbox.connector || {};
|
|
if (c.sentFolder && String(c.sentFolder).trim()) return String(c.sentFolder).trim();
|
|
const host = String(c.host || "").toLowerCase();
|
|
if (host.includes("imap.gmail.com")) return "[Gmail]/Sent Mail";
|
|
return "Sent";
|
|
}
|
|
|
|
/**
|
|
* Folders synced by a mailbox sync: inbox + sent (deduplicated).
|
|
*/
|
|
function imapFolders(mailbox) {
|
|
const folders = [imapFolder(mailbox)];
|
|
const sent = imapSentFolder(mailbox);
|
|
if (sent.trim() && sent.trim().toLowerCase() !== folders[0].toLowerCase()) folders.push(sent.trim());
|
|
return folders;
|
|
}
|
|
|
|
function isSentFolder(mailbox, folder) {
|
|
const f = String(folder || "").trim();
|
|
if (!f) return false;
|
|
return f.toLowerCase() === imapSentFolder(mailbox).toLowerCase();
|
|
}
|
|
|
|
function hashMessageId(messageId) {
|
|
return crypto.createHash("sha1").update(String(messageId || "")).digest("hex").slice(0, 24);
|
|
}
|
|
|
|
/**
|
|
* Stable emailId for messages sent from apxtri via SMTP (folder-independent, keyed on
|
|
* Message-ID so the Sent-folder sync upserts the very same document).
|
|
*/
|
|
function emailIdForSent(mailbox, messageId) {
|
|
return `sent-${mailbox.mailboxid}-${hashMessageId(messageId)}`;
|
|
}
|
|
|
|
function imapTrashFolder(mailbox) {
|
|
const trash = mailbox.connector && mailbox.connector.trashFolder;
|
|
return trash && trash.trim() ? trash.trim() : "Trash";
|
|
}
|
|
|
|
/**
|
|
* Walk an imapflow bodyStructure node and extract attachment metadata.
|
|
*/
|
|
function walkBodyStructure(node, out) {
|
|
if (!node || typeof node !== "object") return out;
|
|
if (node.childNodes) {
|
|
node.childNodes.forEach((child) => walkBodyStructure(child, out));
|
|
return out;
|
|
}
|
|
const type = String(node.type || "").toLowerCase();
|
|
if (!type.startsWith("multipart/")) {
|
|
const params = node.parameters || {};
|
|
const dispParams = node.dispositionParameters || {};
|
|
const filename = params.filename || dispParams.filename || "";
|
|
const isAttachment = filename || node.disposition === "attachment" || node.disposition === "inline";
|
|
if (isAttachment) {
|
|
out.push({
|
|
part: node.part || "",
|
|
filename,
|
|
mimeType: type,
|
|
size: node.size || 0,
|
|
contentId: node.id || "",
|
|
storageName: "",
|
|
});
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function flagsSeen(rec) {
|
|
if (!rec.flags) return false;
|
|
const arr = rec.flags && rec.flags.size ? [...rec.flags] : Array.isArray(rec.flags) ? rec.flags : [];
|
|
return arr.some((f) => String(f).toLowerCase().indexOf("seen") !== -1);
|
|
}
|
|
|
|
function mapImapMeta(mailbox, rec, folder) {
|
|
const env = rec.envelope || {};
|
|
const seen = flagsSeen(rec);
|
|
const from = (env.from && env.from[0]) || null;
|
|
const toArr = (env.to || []).map((a) => ({ name: (a.name || "").trim(), email: (a.address || "").trim() })).filter((a) => a.email);
|
|
const ccArr = (env.cc || []).map((a) => ({ name: (a.name || "").trim(), email: (a.address || "").trim() })).filter((a) => a.email);
|
|
const internalDate = rec.internalDate ? new Date(rec.internalDate).toISOString() : null;
|
|
const attachments = walkBodyStructure(rec.bodyStructure, []);
|
|
const prefix = "imap";
|
|
const f = folder || imapFolder(mailbox);
|
|
const sent = isSentFolder(mailbox, f);
|
|
const inboxId = `${prefix}-${mailbox.mailboxid}-${rec.uid}`;
|
|
const messageId = env.messageId || inboxId;
|
|
const emailId = sent
|
|
? emailIdSafe(emailIdForSent(mailbox, messageId).slice(0, 120))
|
|
: f.toLowerCase() === "inbox"
|
|
? emailIdSafe(inboxId.slice(0, 120))
|
|
: emailIdSafe(`${prefix}-${mailbox.mailboxid}-${emailIdSafe(f).slice(0, 20)}-${rec.uid}`.slice(0, 120));
|
|
return {
|
|
item: {
|
|
emailId,
|
|
provider: mailbox.provider || "imap",
|
|
remoteUid: String(rec.uid),
|
|
folder: f,
|
|
apxtriSent: sent,
|
|
messageId,
|
|
mailboxid: mailbox.mailboxid,
|
|
tribe: mailbox.tribe,
|
|
threadId: "",
|
|
inReplyTo: env.inReplyTo || "",
|
|
references: [],
|
|
subject: env.subject ? String(env.subject) : "(no subject)",
|
|
snippet: "",
|
|
from: { name: from ? (from.name || "").trim() : "", email: from ? (from.address || "").trim() : "" },
|
|
to: toArr,
|
|
cc: ccArr,
|
|
bcc: [],
|
|
replyTo: [],
|
|
dateSent: env.date ? dayjs(env.date).toISOString() : internalDate,
|
|
dateReceived: internalDate || (env.date ? dayjs(env.date).toISOString() : null),
|
|
size: rec.size || 0,
|
|
contentType: "message/rfc822",
|
|
flags: {
|
|
seen,
|
|
unread: !seen,
|
|
draft: rec.flags ? [...rec.flags].some((f) => /draft/i.test(f)) : false,
|
|
starred: rec.flags ? [...rec.flags].some((f) => /flagged/i.test(f)) : false,
|
|
answered: rec.flags ? [...rec.flags].some((f) => /answered/i.test(f)) : false,
|
|
important: false,
|
|
},
|
|
hasAttachments: attachments.length > 0,
|
|
attachments,
|
|
status: seen ? "read" : "new",
|
|
bodyRef: "",
|
|
actions: [],
|
|
},
|
|
};
|
|
}
|
|
|
|
async function connectImap(mailbox, secrets) {
|
|
const client = new ImapFlow(imapConfig(mailbox, secrets));
|
|
await client.connect();
|
|
return client;
|
|
}
|
|
|
|
async function closeImap(client) {
|
|
if (!client) return;
|
|
try {
|
|
await client.logout();
|
|
} catch (e) {
|
|
/* best effort */
|
|
}
|
|
}
|
|
|
|
async function downloadImapMessage(client, uid) {
|
|
const { meta, content } = await client.download(uid, undefined, { uid: true });
|
|
if (!meta || !content) {
|
|
throw new Error(`imapflow returned no content for UID ${uid}`);
|
|
}
|
|
const parsed = await simpleParser(content);
|
|
return { meta, parsed };
|
|
}
|
|
|
|
function mapImapEager(mailbox, rec, full, folder) {
|
|
const meta = mapImapMeta(mailbox, rec, folder);
|
|
const mapped = mapParsedMessage(mailbox, full.parsed, {
|
|
emailId: meta.item.emailId,
|
|
dateReceived: meta.item.dateReceived,
|
|
remoteUid: String(rec.uid),
|
|
folder: meta.item.folder,
|
|
size: rec.size,
|
|
seen: meta.item.flags.seen,
|
|
});
|
|
mapped.item.provider = mailbox.provider;
|
|
mapped.item.apxtriSent = meta.item.apxtriSent === true;
|
|
return { item: mapped.item, buffers: mapped.buffers };
|
|
}
|
|
|
|
/**
|
|
* Open the configured folder(s) — inbox and, unless disabled via opts.includeSent === false,
|
|
* the resolved Sent folder — list the most recent messages' metadata and, when eager, also
|
|
* download + parse every message. A missing Sent folder is skipped silently; inbox errors
|
|
* abort the sync. Returns {ok, folder, folders, messages, error, cursors} where each message is
|
|
* {item, buffers?}. Sent-folder items get apxtriSent:true and an emailId derived from their
|
|
* Message-ID (emailIdForSent). When cursors are provided (opts.cursors or mailbox.syncCursor),
|
|
* a per-folder cursor (uidValidity + lastUid) is used to fetch UIDs in the range lastUid+1:*;
|
|
* if uidValidity differs the cursor is reset to the current full scan behaviour. The default
|
|
* maxResults is 500 when a cursor applies, 50 for the cursor-less full-rescan path.
|
|
*/
|
|
async function fetchImapMessages(mailbox, secrets, opts = {}) {
|
|
const cursorsIn = opts.cursors || mailbox.syncCursor || {};
|
|
const eager = opts.eager === true;
|
|
const folders = opts.includeSent === false ? [imapFolder(mailbox)] : (opts.folders && opts.folders.length ? opts.folders : imapFolders(mailbox));
|
|
let client = null;
|
|
const cursorsOut = {};
|
|
try {
|
|
client = await connectImap(mailbox, secrets);
|
|
const messages = [];
|
|
for (const folder of folders) {
|
|
let lock = null;
|
|
try {
|
|
lock = await client.getMailboxLock(folder);
|
|
} catch (e) {
|
|
if (isSentFolder(mailbox, folder)) continue;
|
|
throw e;
|
|
}
|
|
try {
|
|
const status = await client.status(folder, { uidValidity: true, uidNext: true });
|
|
const rawUidValidity = status ? status.uidValidity : null;
|
|
const serverUidValidity = rawUidValidity == null ? null : Number(rawUidValidity);
|
|
const cursor = cursorsIn[folder] || null;
|
|
const prevUidValidity = cursor ? parseInt(cursor.uidValidity, 10) : NaN;
|
|
const prevLastUid = cursor && cursor.lastUid !== null && cursor.lastUid !== undefined
|
|
? parseInt(cursor.lastUid, 10)
|
|
: NaN;
|
|
// A cursor is only usable when the server UIDVALIDITY is unchanged: a different
|
|
// value means the folder was re-created and every stored UID is meaningless.
|
|
const useCursor =
|
|
!!cursor && !isNaN(prevUidValidity) && !isNaN(prevLastUid) && serverUidValidity === prevUidValidity;
|
|
const maxResults = Math.max(1, parseInt(opts.maxResults, 10) || (useCursor ? CURSOR_MAX_RESULTS : FULL_SCAN_MAX_RESULTS));
|
|
let recent = [];
|
|
if (useCursor) {
|
|
const uids = await client.search({ uid: `${prevLastUid + 1}:*` }, { uid: true });
|
|
// Defensive: never re-read a uid the cursor already covers.
|
|
recent = (uids || []).filter((u) => {
|
|
const n = parseInt(u, 10);
|
|
return !isNaN(n) && n > prevLastUid;
|
|
});
|
|
// Cap the incremental batch at maxResults, taking the LOWEST new UIDs first so a
|
|
// large backlog is drained in bounded chunks without ever skipping: the cursor
|
|
// lands on the highest uid actually retrieved and the next poll continues there.
|
|
if (recent.length > maxResults) recent = recent.slice(0, maxResults);
|
|
} else {
|
|
const uids = await client.search({ all: true }, { uid: true });
|
|
recent = (uids || []).slice(-maxResults);
|
|
}
|
|
let maxSeenUid = 0;
|
|
let records = [];
|
|
if (recent.length > 0) {
|
|
records =
|
|
(await client.fetchAll(
|
|
recent,
|
|
{ uid: true, flags: true, internalDate: true, size: true, envelope: true, bodyStructure: true },
|
|
{ uid: true }
|
|
)) || [];
|
|
for (const rec of records) {
|
|
let entry;
|
|
if (eager) {
|
|
try {
|
|
const full = await downloadImapMessage(client, rec.uid);
|
|
entry = mapImapEager(mailbox, rec, full, folder);
|
|
} catch (e) {
|
|
entry = mapImapMeta(mailbox, rec, folder);
|
|
}
|
|
} else {
|
|
entry = mapImapMeta(mailbox, rec, folder);
|
|
}
|
|
messages.push(entry);
|
|
const seenUid = parseInt(rec.uid, 10);
|
|
if (!isNaN(seenUid) && seenUid > maxSeenUid) maxSeenUid = seenUid;
|
|
}
|
|
}
|
|
// The cursor never goes backwards: keep the previous lastUid when this pass saw
|
|
// nothing (deleted or expunged messages must not rewind the watermark).
|
|
let lastUidOut = maxSeenUid;
|
|
if (lastUidOut <= 0) lastUidOut = useCursor ? prevLastUid : 0;
|
|
cursorsOut[folder] = {
|
|
uidValidity: serverUidValidity === null || serverUidValidity === undefined ? null : serverUidValidity,
|
|
lastUid: lastUidOut,
|
|
lastRemoteNum: null,
|
|
updatedAt: new Date().toISOString(),
|
|
};
|
|
} finally {
|
|
try { lock.release(); } catch (e) { /* noop */ }
|
|
}
|
|
}
|
|
return { ok: true, folder: imapFolder(mailbox), folders, messages, error: null, cursors: cursorsOut };
|
|
} catch (error) {
|
|
return { ok: false, folder: imapFolder(mailbox), folders, messages: [], error: error.message, cursors: cursorsIn || {} };
|
|
} finally {
|
|
await closeImap(client);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Download a single UID and parse it (used for lazy body/attachment fetch).
|
|
*/
|
|
async function imapFetchOne(mailbox, secrets, uid, folder) {
|
|
let client = null;
|
|
try {
|
|
client = await connectImap(mailbox, secrets);
|
|
const target = folder || imapFolder(mailbox);
|
|
const lock = await client.getMailboxLock(target);
|
|
try {
|
|
const { meta, parsed } = await downloadImapMessage(client, parseInt(uid, 10));
|
|
return { ok: true, meta, parsed, error: null };
|
|
} finally {
|
|
try { lock.release(); } catch (e) { /* noop */ }
|
|
}
|
|
} catch (error) {
|
|
return { ok: false, error: error.message };
|
|
} finally {
|
|
await closeImap(client);
|
|
}
|
|
}
|
|
|
|
async function extractHtmlFromParsed(parsed) {
|
|
return parsed.html || parsed.textAsHtml || (parsed.text ? `<pre>${String(parsed.text)}</pre>` : "");
|
|
}
|
|
|
|
/**
|
|
* The attachment ordering of a lazy metadata scan (walkBodyStructure) can differ from
|
|
* simpleParser's ordering on a full download. Locate the requested attachment by
|
|
* filename (+size) first, falling back to the raw index.
|
|
*/
|
|
function matchAttachmentIndex(mapped, item, index) {
|
|
const want = item.attachments && item.attachments[index];
|
|
if (want && want.filename) {
|
|
const hit = mapped.item.attachments.findIndex(
|
|
(a) => a.filename === want.filename && (a.size === want.size || !want.size)
|
|
);
|
|
if (hit !== -1) return hit;
|
|
}
|
|
return index;
|
|
}
|
|
|
|
async function imapGetBody(mailbox, secrets, item) {
|
|
const res = await imapFetchOne(mailbox, secrets, item.remoteUid, item.folder);
|
|
if (!res.ok) {
|
|
return { ok: false, html: "", error: res.error };
|
|
}
|
|
const html = await extractHtmlFromParsed(res.parsed);
|
|
return { ok: true, html, error: null };
|
|
}
|
|
|
|
async function imapGetAttachment(mailbox, secrets, item, index) {
|
|
const res = await imapFetchOne(mailbox, secrets, item.remoteUid, item.folder);
|
|
if (!res.ok) {
|
|
return { ok: false, error: res.error };
|
|
}
|
|
const mapped = mapParsedMessage(mailbox, res.parsed, {
|
|
emailId: item.emailId,
|
|
seen: item.flags && item.flags.seen,
|
|
});
|
|
const idx = matchAttachmentIndex(mapped, item, index);
|
|
const buffer = mapped.buffers[idx] && mapped.buffers[idx].content;
|
|
const att = mapped.item.attachments[idx];
|
|
if (!buffer || !att) {
|
|
return { ok: false, error: "attachment not found on server" };
|
|
}
|
|
return { ok: true, buffer, att, error: null };
|
|
}
|
|
|
|
async function imapSetAction(mailbox, secrets, item, action) {
|
|
let client = null;
|
|
try {
|
|
client = await connectImap(mailbox, secrets);
|
|
const folder = item.folder || imapFolder(mailbox);
|
|
const lock = await client.getMailboxLock(folder);
|
|
try {
|
|
const range = parseInt(item.remoteUid, 10);
|
|
if (!range) return { ok: false, error: "no remote uid on item" };
|
|
if (action === "read") {
|
|
await client.messageFlagsAdd(range, ["\\Seen"], { uid: true });
|
|
} else if (action === "unread") {
|
|
await client.messageFlagsRemove(range, ["\\Seen"], { uid: true });
|
|
} else if (action === "trash") {
|
|
await client.messageMove(range, imapTrashFolder(mailbox), { uid: true });
|
|
} else {
|
|
return { ok: false, error: `unsupported action ${action}` };
|
|
}
|
|
return { ok: true, error: null };
|
|
} finally {
|
|
try { lock.release(); } catch (e) { /* noop */ }
|
|
}
|
|
} catch (error) {
|
|
return { ok: false, error: error.message };
|
|
} finally {
|
|
await closeImap(client);
|
|
}
|
|
}
|
|
|
|
async function testImap(connector, secrets) {
|
|
const start = Date.now();
|
|
if (!connector || !connector.host) return { ok: false, latency: 0, error: "missing IMAP host" };
|
|
const config = imapConfig({ provider: "imap", connector }, { password: (secrets && secrets.password) || "" });
|
|
const client = new ImapFlow(config);
|
|
try {
|
|
await client.connect();
|
|
const folder = connector.folder && connector.folder.trim() ? connector.folder.trim() : "INBOX";
|
|
const lock = await client.getMailboxLock(folder);
|
|
try {
|
|
/* mailbox opened by getMailboxLock */
|
|
} finally {
|
|
try { lock.release(); } catch (e) { /* noop */ }
|
|
}
|
|
await client.logout();
|
|
return { ok: true, latency: Date.now() - start, error: null };
|
|
} catch (error) {
|
|
try { await client.logout(); } catch (e) { /* noop */ }
|
|
return { ok: false, latency: Date.now() - start, error: error.message };
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// POP3 adapter
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function pop3Config(mailbox, secrets) {
|
|
const meta = providerMeta(mailbox.provider) || PROVIDERS.pop3;
|
|
const conn = mailbox.connector && mailbox.connector.host ? mailbox.connector : meta.defaultConnector;
|
|
return {
|
|
host: conn.host,
|
|
port: parseInt(conn.port, 10) || 995,
|
|
tls: conn.tls !== false && conn.tls !== 0,
|
|
rejectUnauthorized: conn.rejectUnauthorized !== false,
|
|
user: (mailbox.connector && mailbox.connector.user) || "",
|
|
password: (secrets && secrets.password) || "",
|
|
deleteOnDownload: !!(conn.deleteOnDownload),
|
|
};
|
|
}
|
|
|
|
function connectPop(cfg) {
|
|
return new Promise((resolve, reject) => {
|
|
if (!cfg.host) return reject(new Error("missing POP3 host"));
|
|
const client = new POP3Client(cfg.port, cfg.host, {
|
|
enabletls: cfg.tls,
|
|
tlsopts: { rejectUnauthorized: cfg.rejectUnauthorized },
|
|
debug: false,
|
|
});
|
|
let settled = false;
|
|
const fail = (err) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timer);
|
|
reject(err instanceof Error ? err : new Error(String(err)));
|
|
};
|
|
const timer = setTimeout(() => fail("POP3 connection timeout"), 20000);
|
|
client.on("connect", () => {
|
|
client.login(cfg.user, cfg.password);
|
|
});
|
|
client.on("login", (status) => {
|
|
if (status) {
|
|
settled = true;
|
|
clearTimeout(timer);
|
|
resolve(client);
|
|
} else {
|
|
fail("POP3 login failed");
|
|
}
|
|
});
|
|
client.on("error", fail);
|
|
client.on("invalid-state", fail);
|
|
});
|
|
}
|
|
|
|
function disconnectPop(client) {
|
|
return new Promise((resolve) => {
|
|
if (!client) return resolve();
|
|
let done = false;
|
|
const fin = () => { if (!done) { done = true; resolve(); } };
|
|
client.on("close", fin);
|
|
client.on("error", fin);
|
|
try { client.quit(); } catch (e) { fin(); }
|
|
setTimeout(fin, 3000);
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Run a single poplib command (single-line response or full multiline data).
|
|
* @param {object} client - poplib client
|
|
* @param {string} method - poplib method to call (stat|uidl|retr|dele|top|...)
|
|
* @param {Array} args - method arguments
|
|
* @param {string} event - event name to await (defaults to method)
|
|
*/
|
|
function popCommand(client, method, args = [], event = method) {
|
|
return new Promise((resolve, reject) => {
|
|
const onEvent = (resp, a, b, data) => {
|
|
client.removeListener(event, onEvent);
|
|
client.removeListener("error", onErr);
|
|
if (resp) resolve({ resp, data, a, b });
|
|
else reject(new Error(String(data || `POP3 ${method} failed`)));
|
|
};
|
|
const onErr = (err) => {
|
|
client.removeListener(event, onEvent);
|
|
reject(err);
|
|
};
|
|
client.once(event, onEvent);
|
|
client.once("error", onErr);
|
|
try {
|
|
client[method].apply(client, args);
|
|
} catch (openErr) {
|
|
client.removeListener(event, onEvent);
|
|
client.removeListener("error", onErr);
|
|
reject(openErr);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Fetch the most recent messages. POP3 always downloads full messages (no header-only
|
|
* mode), so body and attachment buffers are always available on the returned items.
|
|
* Returns {ok, messages, error, cursors}. Notes: deleteOnDownload makes POP3 numbering
|
|
* unstable, so the cursor can drift; this is acceptable for POP3 and is why IMAP is
|
|
* the primary path.
|
|
*/
|
|
async function fetchPopMessages(mailbox, secrets, opts = {}) {
|
|
const cursorsIn = opts.cursors || mailbox.syncCursor || {};
|
|
const popCursor = cursorsIn.pop3 || null;
|
|
const maxResults = Math.max(1, parseInt(opts.maxResults, 10) || CURSOR_MAX_RESULTS);
|
|
const cfg = pop3Config(mailbox, secrets);
|
|
if (!cfg.host) return { ok: false, messages: [], error: "missing POP3 host", cursors: cursorsIn || {} };
|
|
let client = null;
|
|
try {
|
|
client = await connectPop(cfg);
|
|
const stat = await popCommand(client, "stat");
|
|
const count = parseInt(stat.a && stat.a.count !== undefined ? stat.a.count : stat.data[0], 10) || 0;
|
|
if (!count) {
|
|
const cursorsOut = { ...cursorsIn };
|
|
if (popCursor) {
|
|
cursorsOut.pop3 = { ...popCursor, updatedAt: new Date().toISOString() };
|
|
} else {
|
|
cursorsOut.pop3 = { lastRemoteNum: 0, updatedAt: new Date().toISOString() };
|
|
}
|
|
return { ok: true, messages: [], error: null, cursors: cursorsOut };
|
|
}
|
|
let uidlMap = {};
|
|
try {
|
|
const uidlRes = await popCommand(client, "uidl");
|
|
if (uidlRes.data && typeof uidlRes.data === "object" && !Array.isArray(uidlRes.data)) {
|
|
uidlMap = uidlRes.data;
|
|
} else {
|
|
const lines = Array.isArray(uidlRes.data) ? uidlRes.data.join("\n") : String(uidlRes.data || "");
|
|
for (const line of lines.split("\n")) {
|
|
const m = line.match(/^\s*(\d+)\s+(.+?)\s*$/);
|
|
if (m) uidlMap[parseInt(m[1], 10)] = m[2].trim();
|
|
}
|
|
}
|
|
} catch (e) {
|
|
// UIDL not available
|
|
}
|
|
// Incremental: retrieve only the messages numbered after the stored watermark.
|
|
const prevLastNum = popCursor && popCursor.lastRemoteNum !== null && popCursor.lastRemoteNum !== undefined
|
|
? parseInt(popCursor.lastRemoteNum, 10)
|
|
: NaN;
|
|
// A shrunk or reset remote mailbox (count < lastRemoteNum) invalidates the cursor.
|
|
const useCursor = !isNaN(prevLastNum) && prevLastNum <= count;
|
|
let numbers = [];
|
|
if (useCursor) {
|
|
for (let i = prevLastNum + 1; i <= count; i += 1) numbers.push(i);
|
|
} else {
|
|
numbers = Array.from({ length: count }, (_, i) => i + 1).slice(-FULL_SCAN_MAX_RESULTS);
|
|
}
|
|
if (useCursor && numbers.length > maxResults) {
|
|
// Cap the incremental batch, keeping the LOWEST unread numbers first so the cursor
|
|
// only ever lands on the highest message actually retrieved and the next sync
|
|
// continues from there (never advances past a message we did not retrieve).
|
|
numbers = numbers.slice(0, maxResults);
|
|
}
|
|
const messages = [];
|
|
for (const num of numbers) {
|
|
try {
|
|
const raw = await popCommand(client, "retr", [num]);
|
|
const buf = Buffer.isBuffer(raw.data) ? raw.data : Buffer.from(String(raw.data || ""), "utf8");
|
|
const parsed = await simpleParser(buf);
|
|
const uidlKey = uidlMap[num] || `msg-${num}`;
|
|
const emailId = emailIdSafe(`pop-${mailbox.mailboxid}-${uidlKey}`.slice(0, 120));
|
|
const mapped = mapParsedMessage(mailbox, parsed, {
|
|
emailId,
|
|
dateReceived: parsed.date ? dayjs(parsed.date).toISOString() : null,
|
|
remoteUid: String(num),
|
|
folder: "",
|
|
});
|
|
messages.push({ ...mapped, uidlKey });
|
|
} catch (e) {
|
|
// skip message on parse error
|
|
}
|
|
}
|
|
// The cursor never goes backwards: with an incremental pass and nothing retrieved the
|
|
// previous value is kept, otherwise it lands on the last retrieved message.
|
|
let lastRemoteNumOut = count;
|
|
if (useCursor && numbers.length === 0) lastRemoteNumOut = prevLastNum;
|
|
else if (numbers.length) lastRemoteNumOut = numbers[numbers.length - 1];
|
|
const cursorsOut = { ...cursorsIn };
|
|
cursorsOut.pop3 = {
|
|
lastRemoteNum: lastRemoteNumOut,
|
|
updatedAt: new Date().toISOString(),
|
|
};
|
|
return { ok: true, messages, error: null, cursors: cursorsOut };
|
|
} catch (error) {
|
|
return { ok: false, messages: [], error: error.message, cursors: cursorsIn || {} };
|
|
} finally {
|
|
if (client) await disconnectPop(client);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Re-download a single POP message (lazy body/attachment fetch) by remoteUid (msgnumber).
|
|
*/
|
|
async function popFetchOne(mailbox, secrets, remoteUid) {
|
|
const cfg = pop3Config(mailbox, secrets);
|
|
let client = null;
|
|
try {
|
|
client = await connectPop(cfg);
|
|
const num = parseInt(remoteUid, 10);
|
|
if (!num) return { ok: false, error: "no remote uid" };
|
|
const raw = await popCommand(client, "retr", [num]);
|
|
const buf = Buffer.isBuffer(raw.data) ? raw.data : Buffer.from(String(raw.data || ""), "utf8");
|
|
const parsed = await simpleParser(buf);
|
|
return { ok: true, parsed, error: null };
|
|
} catch (error) {
|
|
return { ok: false, error: error.message };
|
|
} finally {
|
|
if (client) await disconnectPop(client);
|
|
}
|
|
}
|
|
|
|
async function popGetBody(mailbox, secrets, item) {
|
|
const res = await popFetchOne(mailbox, secrets, item.remoteUid);
|
|
if (!res.ok) return { ok: false, html: "", error: res.error };
|
|
const html = await extractHtmlFromParsed(res.parsed);
|
|
return { ok: true, html, error: null };
|
|
}
|
|
|
|
async function popGetAttachment(mailbox, secrets, item, index) {
|
|
const res = await popFetchOne(mailbox, secrets, item.remoteUid);
|
|
if (!res.ok) return { ok: false, error: res.error };
|
|
const mapped = mapParsedMessage(mailbox, res.parsed, { emailId: item.emailId });
|
|
const idx = matchAttachmentIndex(mapped, item, index);
|
|
const buffer = mapped.buffers[idx] && mapped.buffers[idx].content;
|
|
const att = mapped.item.attachments[idx];
|
|
if (!buffer || !att) return { ok: false, error: "attachment not found on server" };
|
|
return { ok: true, buffer, att, error: null };
|
|
}
|
|
|
|
async function popDeleteRemote(mailbox, secrets, remoteUid) {
|
|
const cfg = pop3Config(mailbox, secrets);
|
|
let client = null;
|
|
try {
|
|
client = await connectPop(cfg);
|
|
const num = parseInt(remoteUid, 10);
|
|
if (!num) return { ok: false, error: "no remote uid" };
|
|
await popCommand(client, "dele", [num]);
|
|
return { ok: true, error: null };
|
|
} catch (error) {
|
|
return { ok: false, error: error.message };
|
|
} finally {
|
|
if (client) await disconnectPop(client);
|
|
}
|
|
}
|
|
|
|
async function popSetAction(mailbox, secrets, item, action) {
|
|
// POP3 has no server-side flags or folders: everything is local-only. Trash may
|
|
// delete the remote message when connector.deleteOnDownload is enabled.
|
|
if (action === "read" || action === "unread") {
|
|
return { ok: true, remote: "local-only", error: null };
|
|
}
|
|
if (action === "trash") {
|
|
const cfg = pop3Config(mailbox, secrets);
|
|
if (!cfg.deleteOnDownload) {
|
|
return { ok: true, remote: "local-only", error: null };
|
|
}
|
|
return await popDeleteRemote(mailbox, secrets, item.remoteUid);
|
|
}
|
|
return { ok: false, error: `unsupported action ${action}` };
|
|
}
|
|
|
|
async function testPop(connector, secrets) {
|
|
const start = Date.now();
|
|
if (!connector || !connector.host) return { ok: false, latency: 0, error: "missing POP3 host" };
|
|
const cfg = {
|
|
host: connector.host,
|
|
port: parseInt(connector.port, 10) || 995,
|
|
tls: connector.tls !== false && connector.tls !== 0,
|
|
rejectUnauthorized: connector.rejectUnauthorized !== false,
|
|
user: connector.user || "",
|
|
password: (secrets && secrets.password) || "",
|
|
};
|
|
let client = null;
|
|
try {
|
|
client = await connectPop(cfg);
|
|
await popCommand(client, "stat");
|
|
return { ok: true, latency: Date.now() - start, error: null };
|
|
} catch (error) {
|
|
return { ok: false, latency: Date.now() - start, error: error.message };
|
|
} finally {
|
|
if (client) await disconnectPop(client);
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// SMTP (outbox / sending)
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Build a nodemailer transport config for a mailbox.
|
|
* @param {object} mailbox - mailbox document ({provider, connector, email})
|
|
* @param {object} secrets - {password}
|
|
*/
|
|
function smtpConfig(mailbox, secrets) {
|
|
const mailFrom = (mailbox.connector && mailbox.connector.user) || mailbox.email;
|
|
const conn = (mailbox.connector && mailbox.connector.smtp) || {};
|
|
const config = {
|
|
host: conn.host,
|
|
port: parseInt(conn.port, 10) || 465,
|
|
secure: conn.tls !== false && conn.tls !== 0,
|
|
};
|
|
if (!config.host) return null;
|
|
config.auth = { user: mailFrom, pass: (secrets && secrets.password) || "" };
|
|
return config;
|
|
}
|
|
|
|
async function smtpVerify(mailbox, secrets) {
|
|
const config = smtpConfig(mailbox, secrets);
|
|
if (!config) return { ok: false, error: "no SMTP configuration for this mailbox" };
|
|
const transport = nodemailer.createTransport(config);
|
|
try {
|
|
await transport.verify();
|
|
await transport.close();
|
|
return { ok: true, error: null };
|
|
} catch (error) {
|
|
try { await transport.close(); } catch (e) { /* noop */ }
|
|
return { ok: false, error: error.message };
|
|
}
|
|
}
|
|
|
|
async function sendMessage(mailbox, secrets, msg) {
|
|
const config = smtpConfig(mailbox, secrets);
|
|
if (!config) {
|
|
return { ok: false, error: "no SMTP configuration for this mailbox", response: null };
|
|
}
|
|
const transport = nodemailer.createTransport(config);
|
|
try {
|
|
const info = await transport.sendMail({
|
|
from: { address: mailbox.email, name: mailbox.label || mailbox.email },
|
|
to: msg.to || [],
|
|
cc: msg.cc || [],
|
|
bcc: msg.bcc || [],
|
|
subject: msg.subject || "",
|
|
text: msg.text || "",
|
|
html: msg.html || undefined,
|
|
messageId: msg.messageId || undefined,
|
|
inReplyTo: msg.inReplyTo || undefined,
|
|
references: msg.references || undefined,
|
|
});
|
|
await transport.close();
|
|
return { ok: true, error: null, response: info.messageId || info.response || "" };
|
|
} catch (error) {
|
|
try { await transport.close(); } catch (e) { /* noop */ }
|
|
return { ok: false, error: error.message, response: null };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Test a provider connection end-to-end.
|
|
* imap/pop3: protocol login (+ SMTP verify when the mailbox declares SMTP settings).
|
|
*/
|
|
async function testProviderConnection(mailbox, secrets) {
|
|
if (mailbox.provider === "imap") {
|
|
const imap = await testImap(mailbox.connector, secrets);
|
|
let smtpRes = null;
|
|
if (mailbox.connector && mailbox.connector.smtp && mailbox.connector.smtp.host) {
|
|
smtpRes = await smtpVerify(mailbox, secrets);
|
|
}
|
|
return {
|
|
ok: imap.ok,
|
|
error: imap.ok ? null : imap.error,
|
|
latency: imap.latency,
|
|
smtp: smtpRes ? (smtpRes.ok ? "ok" : smtpRes.error) : null,
|
|
};
|
|
}
|
|
if (mailbox.provider === "pop3") {
|
|
const pop = await testPop(mailbox.connector, secrets);
|
|
let smtpRes = null;
|
|
if (mailbox.connector && mailbox.connector.smtp && mailbox.connector.smtp.host) {
|
|
smtpRes = await smtpVerify(mailbox, secrets);
|
|
}
|
|
return {
|
|
ok: pop.ok,
|
|
error: pop.ok ? null : pop.error,
|
|
latency: pop.latency,
|
|
smtp: smtpRes ? (smtpRes.ok ? "ok" : smtpRes.error) : null,
|
|
};
|
|
}
|
|
return { ok: false, error: `unsupported provider ${mailbox.provider}` };
|
|
}
|
|
|
|
module.exports = {
|
|
PROVIDERS,
|
|
PASSWORD_PROVIDERS,
|
|
providerMeta,
|
|
supports,
|
|
authType,
|
|
needsSecrets,
|
|
needsConnector,
|
|
isPasswordProvider,
|
|
emailIdSafe,
|
|
// IMAP
|
|
imapConfig,
|
|
imapFolder,
|
|
imapSentFolder,
|
|
imapFolders,
|
|
isSentFolder,
|
|
emailIdForSent,
|
|
hashMessageId,
|
|
imapTrashFolder,
|
|
mapImapMeta,
|
|
walkBodyStructure,
|
|
fetchImapMessages,
|
|
imapSetAction,
|
|
imapGetBody,
|
|
imapGetAttachment,
|
|
testImap,
|
|
// POP3
|
|
pop3Config,
|
|
fetchPopMessages,
|
|
popSetAction,
|
|
popGetBody,
|
|
popGetAttachment,
|
|
popDeleteRemote,
|
|
testPop,
|
|
// shared mapping
|
|
mapParsedMessage,
|
|
addressesFromParsed,
|
|
// SMTP / sending
|
|
smtpConfig,
|
|
smtpVerify,
|
|
sendMessage,
|
|
testProviderConnection,
|
|
}; |