feat(import): JSON / CSV vault import with heuristic column mapping

Round-trip companion to the existing doExport(). Supports two file
formats with auto-detection (extension + first-char sniff):

JSON
====
Native shape produced by doExport() AND a forgiving fallback for any
flat array of entry objects with site/url + password fields. Accepts:
  - { version, exported_at, entries: [...] }   (native)
  - [{ ... }, { ... }]                          (flat array)
  - mixed keys: site|url|name, username|user|login|email, etc.

CSV
===
RFC-4180-ish parser (~30 lines): quoted fields, escaped "", commas
inside quotes, CRLF line endings. No streaming since password-manager
imports are realistically MB-scale at most.

Heuristic column mapping (case + underscore tolerant) covers the
common exporters out of the box:

  Site/URL     : name, title, url, site, website, login_uri, login_url
  Username     : login_username, username, user, login, email
  Password     : login_password, password, pass, pwd
  Folder       : folder, group, category, path, collection
  Tags         : tags, labels (comma/semicolon-split)
  Notes        : notes, note, comment    (short notes joined into tags)
  TOTP         : login_totp, totp, otpauth, authenticator, two_factor

If the TOTP column holds a full otpauth:// URI it's parsed and only
the secret param is stored — same path used by the slide-over TOTP
field. Invalid base32 TOTP secrets are dropped silently rather than
failing the whole import.

Backend
=======
New endpoint: POST /entries/bulk-import
Body: { entries: [{ site, username, encrypted_password, iv, folder,
                    tags, totp_secret, totp_iv }, ... ] }
Caps at 10,000 entries per request as a sanity bound. Inserts inside
a single SQLite transaction — partial failure rolls back cleanly, the
user retries from the same source file. Returns { imported: N }.

Rows missing site or ciphertext are skipped within the transaction
(not failed) so one bad row in a 500-entry import doesn't blow up
the whole batch.

Client flow
===========
doImport():
  1. Hidden <input type="file" accept=".json,.csv"> picker
  2. Read text, detect format, route to parseEntriesFromJSON or CSV
  3. confirmDialog preview: count + first 3 sample sites + skipped rows
  4. On confirm: encryptImportEntry() each plaintext entry with the
     current vault key (reuses encryptPwd / base32Decode validation)
  5. Single POST to /entries/bulk-import
  6. Reload entries, refresh UI, trigger HIBP scan if enabled

UI
==
Two entry points (mirroring Export):
 - Sidebar "Import vault" nav item, next to "Export vault"
 - Settings panel "Import" section with descriptive blurb
Both call doImport(). New i-log-in icon added to the SVG sprite (mirror
of i-log-out used by Export).

Limitations
===========
 - No de-duplication: importing the same file twice yields duplicate
   entries. Trade-off to keep the v1 simple — the user can sort it
   out with the existing trash/multi-select UI.
 - No password-protected vault formats (Bitwarden encrypted JSON,
   KeePass kdbx). Only plaintext exports — same trade-off as
   doExport() which produces plaintext JSON.
This commit is contained in:
2026-05-23 05:30:08 +01:00
parent 60aa106a30
commit 4b15811221
3 changed files with 410 additions and 6 deletions
+111 -1
View File
@@ -480,9 +480,119 @@ begin
TJSONHelper.SendOK(AResponse, 'Trash emptied');
end;
// ===== POST /entries/bulk-import =============================================
// Accepts an array of already-encrypted entries (the client encrypts each
// entry with the vault key before posting). Inserts them all in a single
// transaction so a partial failure rolls back cleanly. Used by the JSON / CSV
// import flow — much faster than N sequential POST /entries for large vaults.
procedure HandleBulkImport(ARequest: TIdHTTPRequestInfo;
AResponse: TIdHTTPResponseInfo; const AParams: TArray<string>);
var
LUserId, I, LImported: Integer;
LBody, LObj, LEntry: TJSONObject;
LArr: TJSONArray;
LSite, LUser, LFolder, LEnc, LIV, LTags, LTotpSec, LTotpIv, LNow: string;
LQ: TFDQuery;
begin
try
LUserId := Authenticate(ARequest, AResponse);
RequireCSRF(ARequest, AResponse, LUserId);
except
on ESessionRejected do Exit;
end;
LBody := TJSONHelper.ReadBody(ARequest);
try
LArr := LBody.GetValue<TJSONArray>('entries');
if (LArr = nil) or (LArr.Count = 0) then
begin
TJSONHelper.SendError(AResponse, 400, 'Missing or empty entries array');
Exit;
end;
// Sanity cap. A real vault rarely has > 10k entries; if someone uploads
// a 100k-row CSV it's probably an attack or a mistake.
if LArr.Count > 10000 then
begin
TJSONHelper.SendError(AResponse, 413, 'Too many entries (max 10000 per request)');
Exit;
end;
LNow := FormatDateTime('yyyy-mm-dd hh:nn:ss', Now);
LImported := 0;
DB.Lock;
try
DB.Connection.StartTransaction;
try
LQ := TFDQuery.Create(nil);
try
LQ.Connection := DB.Connection;
LQ.SQL.Text :=
'INSERT INTO vault_entries ' +
'(user_id, site, username, encrypted_password, iv, encryption_method, ' +
' folder, tags, totp_secret, totp_iv, created_at, updated_at) ' +
'VALUES (:uid, :s, :u, :e, :i, ''client'', :f, :t, :ts, :tiv, :c, :c2)';
for I := 0 to LArr.Count - 1 do
begin
LEntry := LArr.Items[I] as TJSONObject;
LSite := Trim(LEntry.GetValue<string>('site', ''));
LUser := Trim(LEntry.GetValue<string>('username', ''));
LFolder := Trim(LEntry.GetValue<string>('folder', 'All'));
LEnc := LEntry.GetValue<string>('encrypted_password', '');
LIV := LEntry.GetValue<string>('iv', '');
LTags := Trim(LEntry.GetValue<string>('tags', ''));
LTotpSec := LEntry.GetValue<string>('totp_secret', '');
LTotpIv := LEntry.GetValue<string>('totp_iv', '');
// Skip silently if a row is missing the minimum required fields
// (site + ciphertext). Better than failing the whole batch on
// one bad row when the user is importing 500+ entries.
if (LSite = '') or (LEnc = '') or (LIV = '') then Continue;
LQ.ParamByName('uid').AsInteger := LUserId;
LQ.ParamByName('s').AsString := LSite;
LQ.ParamByName('u').AsString := LUser;
LQ.ParamByName('e').AsString := LEnc;
LQ.ParamByName('i').AsString := LIV;
LQ.ParamByName('f').AsString := LFolder;
LQ.ParamByName('t').AsString := LTags;
if LTotpSec = '' then LQ.ParamByName('ts').Clear
else LQ.ParamByName('ts').AsString := LTotpSec;
if LTotpIv = '' then LQ.ParamByName('tiv').Clear
else LQ.ParamByName('tiv').AsString := LTotpIv;
LQ.ParamByName('c').AsString := LNow;
LQ.ParamByName('c2').AsString := LNow;
LQ.ExecSQL;
Inc(LImported);
end;
finally
LQ.Free;
end;
DB.Connection.Commit;
except
DB.Connection.Rollback;
raise;
end;
finally
DB.Unlock;
end;
finally
LBody.Free;
end;
LogAudit(LUserId, Format('bulk_import %d entries', [LImported]), GetClientIP(ARequest));
LObj := TJSONObject.Create;
LObj.AddPair('imported', TJSONNumber.Create(LImported));
TJSONHelper.SendJSON(AResponse, LObj);
end;
initialization
// /entries/trash/empty must be registered BEFORE /entries/{id} to win the regex match
// /entries/trash/empty must be registered BEFORE /entries/{id} to win the regex match.
// Same logic for /entries/bulk-import — register before the catch-all /entries/{id}.
Router.Register('DELETE', '/entries/trash/empty', HandleEmptyTrash);
Router.Register('POST', '/entries/bulk-import', HandleBulkImport);
Router.Register('POST', '/entries/(\d+)/restore', HandleRestoreEntry);
Router.Register('POST', '/entries/(\d+)/favorite', HandleToggleFavorite);
Router.Register('GET', '/entries', HandleGetEntries);
+17
View File
@@ -31,6 +31,7 @@
<symbol id="i-dice" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="3" width="18" height="18" rx="2"/><circle cx="8" cy="8" r="1.2" fill="currentColor"/><circle cx="16" cy="8" r="1.2" fill="currentColor"/><circle cx="12" cy="12" r="1.2" fill="currentColor"/><circle cx="8" cy="16" r="1.2" fill="currentColor"/><circle cx="16" cy="16" r="1.2" fill="currentColor"/></symbol>
<symbol id="i-tag" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M20.59 13.41 13.42 20.58a2 2 0 0 1-2.83 0L2 12V2h10l8.59 8.59a2 2 0 0 1 0 2.82Z"/><line x1="7" y1="7" x2="7.01" y2="7"/></symbol>
<symbol id="i-alert" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M10.29 3.86 1.82 18a2 2 0 0 0 1.71 3h16.94a2 2 0 0 0 1.71-3L13.71 3.86a2 2 0 0 0-3.42 0Z"/><line x1="12" y1="9" x2="12" y2="13"/><line x1="12" y1="17" x2="12.01" y2="17"/></symbol>
<symbol id="i-log-in" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M15 3h4a2 2 0 0 1 2 2v14a2 2 0 0 1-2 2h-4"/><polyline points="10 17 15 12 10 7"/><line x1="15" y1="12" x2="3" y2="12"/></symbol>
<symbol id="i-rotate-ccw" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M3 12a9 9 0 1 0 3-6.7L3 8"/><path d="M3 3v5h5"/></symbol>
<symbol id="i-check" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M20 6 9 17l-5-5"/></symbol>
<symbol id="i-command" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M18 3a3 3 0 0 0-3 3v12a3 3 0 0 0 3 3 3 3 0 0 0 3-3 3 3 0 0 0-3-3H6a3 3 0 0 0-3 3 3 3 0 0 0 3 3 3 3 0 0 0 3-3V6a3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3h12a3 3 0 0 0 3-3 3 3 0 0 0-3-3Z"/></symbol>
@@ -175,6 +176,10 @@
<svg><use href="#i-dice"/></svg>
<span>Generator</span>
</button>
<button class="nav-item" id="sidebarImportBtn">
<svg><use href="#i-log-in"/></svg>
<span>Import vault</span>
</button>
<button class="nav-item" id="sidebarExportBtn">
<svg><use href="#i-log-out"/></svg>
<span>Export vault</span>
@@ -346,6 +351,18 @@
</button>
</div>
<div class="slideover-field">
<div class="slideover-field-label">Import</div>
<p style="font-size:12px;color:var(--text-dim);margin:0 0 8px;line-height:1.5">
Import a vault from a JSON export (this app) or a CSV file
(Bitwarden, KeePass, Chrome, 1Password…). Entries are added
to your current vault — duplicates are NOT removed.
</p>
<button class="btn btn-ghost btn-sm" id="importBtn">
<svg><use href="#i-log-in"/></svg> Import vault
</button>
</div>
<div class="slideover-field">
<div class="slideover-field-label">Account</div>
<p style="font-size:12px;color:var(--text-dim);margin:0 0 8px">
+277
View File
@@ -2407,6 +2407,281 @@ function closeReauth(ok) {
}
}
// ============================================================
// IMPORT — JSON (native round-trip) + CSV (universal)
// ============================================================
//
// Two supported input formats:
// 1. Native JSON: the same shape produced by doExport() above
// { version, exported_at, username, entries: [
// { site, username, password, folder, tags, favorite, ... }
// ]}
// 2. CSV: with a header row. Column names are mapped heuristically so
// exports from Bitwarden / KeePass / Chrome / 1Password generally
// "just work" without manual column mapping.
//
// Each parsed entry is encrypted client-side with the vault key (same
// flow as a single-entry add), then sent to /entries/bulk-import as one
// transactional batch.
// Minimal RFC 4180-ish CSV parser. Handles quoted fields, escaped quotes
// (""), commas inside quotes, and CRLF line endings. Returns an array of
// arrays (rows × columns). No streaming — fine for the ~MB-scale imports
// a password manager realistically deals with.
function parseCSV(text) {
const rows = [];
let row = [], field = '', inQuotes = false;
for (let i = 0; i < text.length; i++) {
const c = text[i];
if (inQuotes) {
if (c === '"') {
if (text[i + 1] === '"') { field += '"'; i++; } // escaped ""
else inQuotes = false;
} else field += c;
} else {
if (c === '"') inQuotes = true;
else if (c === ',') { row.push(field); field = ''; }
else if (c === '\n' || c === '\r') {
if (c === '\r' && text[i + 1] === '\n') i++; // CRLF
row.push(field); field = '';
if (row.length > 1 || (row.length === 1 && row[0] !== '')) rows.push(row);
row = [];
} else field += c;
}
}
// Flush trailing field/row (file without final newline)
if (field !== '' || row.length > 0) { row.push(field); rows.push(row); }
return rows;
}
// Header heuristics: pick the first matching column name (case-insensitive,
// underscore/space-tolerant). Returns null if no candidate header matches.
function findColumn(headers, candidates) {
const norm = s => String(s || '').toLowerCase().replace(/[\s_-]+/g, '');
const cand = candidates.map(norm);
for (let i = 0; i < headers.length; i++) {
if (cand.indexOf(norm(headers[i])) >= 0) return i;
}
return null;
}
// Parse a CSV text into an array of plaintext entries
// ({ site, username, password, folder, tags, totp_secret }). Returns
// { entries, skipped, columns } so the preview can show what was matched.
function parseEntriesFromCSV(text) {
const rows = parseCSV(text);
if (rows.length < 2) {
throw new Error('CSV needs a header row and at least one data row');
}
const headers = rows[0];
// Candidate names per format observed in real exports:
// Bitwarden CSV : folder, name, login_uri, login_username, login_password, login_totp, notes
// KeePass CSV : Title, URL, Username, Password, Group, Notes
// Chrome/Edge : name, url, username, password
// 1Password CSV : Title, URL, Username, Password, Notes
// findColumn normalizes (lowercase, strip _ and -) so 'login_uri' and
// 'loginuri' both match the same candidate.
const colSite = findColumn(headers, ['name', 'title', 'url', 'site', 'website', 'login_uri', 'login_url', 'web_site']);
const colUser = findColumn(headers, ['login_username', 'username', 'user', 'login', 'email', 'user_name']);
const colPwd = findColumn(headers, ['login_password', 'password', 'pass', 'pwd']);
const colFolder = findColumn(headers, ['folder', 'group', 'category', 'path', 'collection']);
const colTags = findColumn(headers, ['tags', 'labels']);
const colNotes = findColumn(headers, ['notes', 'note', 'comment', 'comments']);
const colTotp = findColumn(headers, ['login_totp', 'totp', 'totp_secret', 'otp', 'otpauth', 'authenticator', 'two_factor', 'twofa']);
if (colSite === null && colUser === null)
throw new Error('No recognizable site/url or username column in CSV header');
if (colPwd === null)
throw new Error('No recognizable password column in CSV header');
const entries = [];
let skipped = 0;
for (let i = 1; i < rows.length; i++) {
const r = rows[i];
const site = (colSite !== null ? r[colSite] : '').trim() ||
(colUser !== null ? r[colUser] : '').trim();
const pwd = (colPwd !== null ? r[colPwd] : '');
if (!site || !pwd) { skipped++; continue; }
// Tags: combine the tags column and any free-form notes into a
// comma-separated string. Notes often contain useful metadata we
// don't want to drop on the floor.
let tagsArr = [];
if (colTags !== null) {
String(r[colTags] || '').split(/[,;]/).forEach(t => {
t = t.trim();
if (t) tagsArr.push(t);
});
}
if (colNotes !== null) {
const n = String(r[colNotes] || '').trim();
if (n && n.length < 80) tagsArr.push(n); // long notes become noise as tags
}
// TOTP: support raw base32 OR full otpauth:// URI in the cell.
let totp = '';
if (colTotp !== null) {
const raw = String(r[colTotp] || '').trim();
totp = parseOtpAuthUri(raw) || raw;
}
entries.push({
site: site,
username: (colUser !== null ? r[colUser] : '').trim(),
password: pwd,
folder: (colFolder !== null ? r[colFolder] : '').trim() || 'All',
tags: tagsArr.join(','),
totp_secret: totp,
});
}
return { entries, skipped, columns: {
site: colSite, username: colUser, password: colPwd,
folder: colFolder, tags: colTags, notes: colNotes, totp: colTotp,
} };
}
// Parse a native JSON export. Forgiving: accepts both our own format and
// a flat array of entry objects.
function parseEntriesFromJSON(text) {
let data;
try { data = JSON.parse(text); }
catch (e) { throw new Error('Invalid JSON: ' + e.message); }
const raw = Array.isArray(data) ? data : (data.entries || []);
if (!Array.isArray(raw) || raw.length === 0)
throw new Error('No entries in JSON file');
const entries = [];
let skipped = 0;
for (const e of raw) {
if (!e || typeof e !== 'object') { skipped++; continue; }
const site = String(e.site || e.url || e.name || '').trim();
const pwd = String(e.password || '');
if (!site || !pwd) { skipped++; continue; }
const tagsVal = e.tags;
const tagsStr = Array.isArray(tagsVal) ? tagsVal.join(',')
: String(tagsVal || '');
entries.push({
site: site,
username: String(e.username || e.user || e.login || '').trim(),
password: pwd,
folder: String(e.folder || e.group || 'All').trim() || 'All',
tags: tagsStr,
totp_secret: String(e.totp || e.totp_secret || e.otpauth || '').trim(),
});
}
return { entries, skipped, columns: null }; // JSON: no column report
}
// Encrypt one parsed entry (plaintext password + optional TOTP) into the
// shape the bulk-import endpoint expects. Reuses encryptPwd which already
// generates a fresh IV per call.
async function encryptImportEntry(plain) {
const pw = await encryptPwd(plain.password);
let totpEnc = '', totpIv = '';
if (plain.totp_secret) {
try {
base32Decode(plain.totp_secret); // validate before encrypting
const t = await encryptPwd(plain.totp_secret);
totpEnc = t.encrypted;
totpIv = t.iv;
} catch (e) {
// Bad TOTP secret in source file — keep the entry but drop the
// 2FA silently. The user can fix it later via the slide-over.
}
}
return {
site: plain.site,
username: plain.username || '',
encrypted_password: pw.encrypted,
iv: pw.iv,
folder: plain.folder || 'All',
tags: plain.tags || '',
totp_secret: totpEnc,
totp_iv: totpIv,
};
}
// Open a hidden file picker, route the result through the right parser,
// show a preview confirmation, then bulk-encrypt + POST.
async function doImport() {
const fileInput = el('input', {
type: 'file',
accept: '.json,.csv,application/json,text/csv',
style: 'display:none',
});
document.body.appendChild(fileInput);
fileInput.addEventListener('change', async () => {
const file = fileInput.files && fileInput.files[0];
fileInput.remove();
if (!file) return;
let text;
try { text = await file.text(); }
catch (e) { return toast('Cannot read file: ' + e.message, 'error'); }
const isJSON = /\.json$/i.test(file.name) || text.trim().startsWith('{') || text.trim().startsWith('[');
let parsed;
try {
parsed = isJSON ? parseEntriesFromJSON(text) : parseEntriesFromCSV(text);
} catch (e) {
return toast('Parse error: ' + e.message, 'error');
}
if (parsed.entries.length === 0) {
return toast('No valid entries found in file', 'warning');
}
// Build the preview message (innerHTML target → escape user data)
const esc = s => String(s).replace(/[&<>"]/g,
c => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));
const parts = [];
parts.push('<b>' + parsed.entries.length + '</b> entries detected in <code>' +
esc(file.name) + '</code>');
if (parsed.skipped > 0)
parts.push('<span style="color:var(--text-dim)">' + parsed.skipped +
' rows skipped (missing site or password)</span>');
const sample = parsed.entries.slice(0, 3).map(e =>
'• ' + esc(e.site || '?') +
(e.username ? ' <span style="color:var(--text-dim)">(' + esc(e.username) + ')</span>' : '')
).join('<br>');
parts.push('<div style="margin-top:8px;font-size:12px">' + sample +
(parsed.entries.length > 3 ? '<br>…' : '') + '</div>');
parts.push('<div style="margin-top:10px">Import now? This adds the entries to your existing vault.</div>');
const confirmed = await confirmDialog({
title: 'Import vault',
message: parts.join('<br>'),
okText: 'Import',
});
if (!confirmed) return;
// Encrypt all entries client-side, then POST as one transaction.
toast('Encrypting ' + parsed.entries.length + ' entries…');
const encrypted = [];
for (const e of parsed.entries) {
encrypted.push(await encryptImportEntry(e));
}
try {
const r = await api('/entries/bulk-import', {
method: 'POST',
headers: authHeaders({ 'Content-Type': 'application/json' }),
body: JSON.stringify({ entries: encrypted }),
});
toast('Imported ' + r.imported + ' entries');
await loadEntries();
render();
if (state.hibpEnabled) hibpCheckAllEntries(); // scan the new entries too
} catch (err) {
toast('Import failed: ' + err.message, 'error');
}
});
fileInput.click();
}
async function doExport() {
const pwd = await askReauth('Enter your master password to export the vault as JSON. The file will be UNENCRYPTED.');
if (!pwd) return;
@@ -2712,6 +2987,7 @@ async function init() {
// Sidebar Generator tool
$('#sidebarGenBtn').addEventListener('click', () => openGen('standalone'));
$('#sidebarExportBtn').addEventListener('click', doExport);
$('#sidebarImportBtn').addEventListener('click', doImport);
// Idle warning "Stay unlocked"
$('#idleStayBtn').addEventListener('click', resetAutoLock);
@@ -2757,6 +3033,7 @@ async function init() {
toast('Open Windows Settings → System → Clipboard → turn off "Clipboard history"', 'warning');
});
$('#exportBtn').addEventListener('click', doExport);
$('#importBtn').addEventListener('click', doImport);
// Re-auth modal
$('#reauthForm').addEventListener('submit', e => { e.preventDefault(); closeReauth(true); });