From 4b15811221755038bc8237c69d4fcbf79a77c7c8 Mon Sep 17 00:00:00 2001 From: Zaki <18zaki18@gmail.com> Date: Sat, 23 May 2026 05:30:08 +0100 Subject: [PATCH] feat(import): JSON / CSV vault import with heuristic column mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-trip companion to the existing doExport(). Supports two file formats with auto-detection (extension + first-char sniff): JSON ==== Native shape produced by doExport() AND a forgiving fallback for any flat array of entry objects with site/url + password fields. Accepts: - { version, exported_at, entries: [...] } (native) - [{ ... }, { ... }] (flat array) - mixed keys: site|url|name, username|user|login|email, etc. CSV === RFC-4180-ish parser (~30 lines): quoted fields, escaped "", commas inside quotes, CRLF line endings. No streaming since password-manager imports are realistically MB-scale at most. Heuristic column mapping (case + underscore tolerant) covers the common exporters out of the box: Site/URL : name, title, url, site, website, login_uri, login_url Username : login_username, username, user, login, email Password : login_password, password, pass, pwd Folder : folder, group, category, path, collection Tags : tags, labels (comma/semicolon-split) Notes : notes, note, comment (short notes joined into tags) TOTP : login_totp, totp, otpauth, authenticator, two_factor If the TOTP column holds a full otpauth:// URI it's parsed and only the secret param is stored — same path used by the slide-over TOTP field. Invalid base32 TOTP secrets are dropped silently rather than failing the whole import. Backend ======= New endpoint: POST /entries/bulk-import Body: { entries: [{ site, username, encrypted_password, iv, folder, tags, totp_secret, totp_iv }, ... ] } Caps at 10,000 entries per request as a sanity bound. Inserts inside a single SQLite transaction — partial failure rolls back cleanly, the user retries from the same source file. Returns { imported: N }. Rows missing site or ciphertext are skipped within the transaction (not failed) so one bad row in a 500-entry import doesn't blow up the whole batch. Client flow =========== doImport(): 1. Hidden picker 2. Read text, detect format, route to parseEntriesFromJSON or CSV 3. confirmDialog preview: count + first 3 sample sites + skipped rows 4. On confirm: encryptImportEntry() each plaintext entry with the current vault key (reuses encryptPwd / base32Decode validation) 5. Single POST to /entries/bulk-import 6. Reload entries, refresh UI, trigger HIBP scan if enabled UI == Two entry points (mirroring Export): - Sidebar "Import vault" nav item, next to "Export vault" - Settings panel "Import" section with descriptive blurb Both call doImport(). New i-log-in icon added to the SVG sprite (mirror of i-log-out used by Export). Limitations =========== - No de-duplication: importing the same file twice yields duplicate entries. Trade-off to keep the v1 simple — the user can sort it out with the existing trash/multi-select UI. - No password-protected vault formats (Bitwarden encrypted JSON, KeePass kdbx). Only plaintext exports — same trade-off as doExport() which produces plaintext JSON. --- .../Handlers/PM.Handler.Entries.pas | 122 +++++++- index.html | 17 ++ js/app.js | 277 ++++++++++++++++++ 3 files changed, 410 insertions(+), 6 deletions(-) diff --git a/delphi-backend/Handlers/PM.Handler.Entries.pas b/delphi-backend/Handlers/PM.Handler.Entries.pas index d720434..363f7df 100644 --- a/delphi-backend/Handlers/PM.Handler.Entries.pas +++ b/delphi-backend/Handlers/PM.Handler.Entries.pas @@ -480,14 +480,124 @@ begin TJSONHelper.SendOK(AResponse, 'Trash emptied'); end; +// ===== POST /entries/bulk-import ============================================= +// Accepts an array of already-encrypted entries (the client encrypts each +// entry with the vault key before posting). Inserts them all in a single +// transaction so a partial failure rolls back cleanly. Used by the JSON / CSV +// import flow — much faster than N sequential POST /entries for large vaults. +procedure HandleBulkImport(ARequest: TIdHTTPRequestInfo; + AResponse: TIdHTTPResponseInfo; const AParams: TArray); +var + LUserId, I, LImported: Integer; + LBody, LObj, LEntry: TJSONObject; + LArr: TJSONArray; + LSite, LUser, LFolder, LEnc, LIV, LTags, LTotpSec, LTotpIv, LNow: string; + LQ: TFDQuery; +begin + try + LUserId := Authenticate(ARequest, AResponse); + RequireCSRF(ARequest, AResponse, LUserId); + except + on ESessionRejected do Exit; + end; + + LBody := TJSONHelper.ReadBody(ARequest); + try + LArr := LBody.GetValue('entries'); + if (LArr = nil) or (LArr.Count = 0) then + begin + TJSONHelper.SendError(AResponse, 400, 'Missing or empty entries array'); + Exit; + end; + + // Sanity cap. A real vault rarely has > 10k entries; if someone uploads + // a 100k-row CSV it's probably an attack or a mistake. + if LArr.Count > 10000 then + begin + TJSONHelper.SendError(AResponse, 413, 'Too many entries (max 10000 per request)'); + Exit; + end; + + LNow := FormatDateTime('yyyy-mm-dd hh:nn:ss', Now); + LImported := 0; + + DB.Lock; + try + DB.Connection.StartTransaction; + try + LQ := TFDQuery.Create(nil); + try + LQ.Connection := DB.Connection; + LQ.SQL.Text := + 'INSERT INTO vault_entries ' + + '(user_id, site, username, encrypted_password, iv, encryption_method, ' + + ' folder, tags, totp_secret, totp_iv, created_at, updated_at) ' + + 'VALUES (:uid, :s, :u, :e, :i, ''client'', :f, :t, :ts, :tiv, :c, :c2)'; + + for I := 0 to LArr.Count - 1 do + begin + LEntry := LArr.Items[I] as TJSONObject; + LSite := Trim(LEntry.GetValue('site', '')); + LUser := Trim(LEntry.GetValue('username', '')); + LFolder := Trim(LEntry.GetValue('folder', 'All')); + LEnc := LEntry.GetValue('encrypted_password', ''); + LIV := LEntry.GetValue('iv', ''); + LTags := Trim(LEntry.GetValue('tags', '')); + LTotpSec := LEntry.GetValue('totp_secret', ''); + LTotpIv := LEntry.GetValue('totp_iv', ''); + + // Skip silently if a row is missing the minimum required fields + // (site + ciphertext). Better than failing the whole batch on + // one bad row when the user is importing 500+ entries. + if (LSite = '') or (LEnc = '') or (LIV = '') then Continue; + + LQ.ParamByName('uid').AsInteger := LUserId; + LQ.ParamByName('s').AsString := LSite; + LQ.ParamByName('u').AsString := LUser; + LQ.ParamByName('e').AsString := LEnc; + LQ.ParamByName('i').AsString := LIV; + LQ.ParamByName('f').AsString := LFolder; + LQ.ParamByName('t').AsString := LTags; + if LTotpSec = '' then LQ.ParamByName('ts').Clear + else LQ.ParamByName('ts').AsString := LTotpSec; + if LTotpIv = '' then LQ.ParamByName('tiv').Clear + else LQ.ParamByName('tiv').AsString := LTotpIv; + LQ.ParamByName('c').AsString := LNow; + LQ.ParamByName('c2').AsString := LNow; + LQ.ExecSQL; + Inc(LImported); + end; + finally + LQ.Free; + end; + DB.Connection.Commit; + except + DB.Connection.Rollback; + raise; + end; + finally + DB.Unlock; + end; + finally + LBody.Free; + end; + + LogAudit(LUserId, Format('bulk_import %d entries', [LImported]), GetClientIP(ARequest)); + LObj := TJSONObject.Create; + LObj.AddPair('imported', TJSONNumber.Create(LImported)); + TJSONHelper.SendJSON(AResponse, LObj); +end; + initialization - // /entries/trash/empty must be registered BEFORE /entries/{id} to win the regex match - Router.Register('DELETE', '/entries/trash/empty', HandleEmptyTrash); + // /entries/trash/empty must be registered BEFORE /entries/{id} to win the regex match. + // Same logic for /entries/bulk-import — register before the catch-all /entries/{id}. + Router.Register('DELETE', '/entries/trash/empty', HandleEmptyTrash); + Router.Register('POST', '/entries/bulk-import', HandleBulkImport); Router.Register('POST', '/entries/(\d+)/restore', HandleRestoreEntry); Router.Register('POST', '/entries/(\d+)/favorite', HandleToggleFavorite); - Router.Register('GET', '/entries', HandleGetEntries); - Router.Register('POST', '/entries', HandleCreateEntry); - Router.Register('PUT', '/entries/(\d+)', HandleUpdateEntry); - Router.Register('DELETE', '/entries/(\d+)', HandleDeleteEntry); + Router.Register('GET', '/entries', HandleGetEntries); + Router.Register('POST', '/entries', HandleCreateEntry); + Router.Register('PUT', '/entries/(\d+)', HandleUpdateEntry); + Router.Register('DELETE', '/entries/(\d+)', HandleDeleteEntry); end. diff --git a/index.html b/index.html index 9b328d1..635a13e 100644 --- a/index.html +++ b/index.html @@ -31,6 +31,7 @@ + @@ -175,6 +176,10 @@ Generator + +
+
Import
+

+ Import a vault from a JSON export (this app) or a CSV file + (Bitwarden, KeePass, Chrome, 1Password…). Entries are added + to your current vault — duplicates are NOT removed. +

+ +
+
Account

diff --git a/js/app.js b/js/app.js index cb267bf..f348984 100644 --- a/js/app.js +++ b/js/app.js @@ -2407,6 +2407,281 @@ function closeReauth(ok) { } } +// ============================================================ +// IMPORT — JSON (native round-trip) + CSV (universal) +// ============================================================ +// +// Two supported input formats: +// 1. Native JSON: the same shape produced by doExport() above +// { version, exported_at, username, entries: [ +// { site, username, password, folder, tags, favorite, ... } +// ]} +// 2. CSV: with a header row. Column names are mapped heuristically so +// exports from Bitwarden / KeePass / Chrome / 1Password generally +// "just work" without manual column mapping. +// +// Each parsed entry is encrypted client-side with the vault key (same +// flow as a single-entry add), then sent to /entries/bulk-import as one +// transactional batch. + +// Minimal RFC 4180-ish CSV parser. Handles quoted fields, escaped quotes +// (""), commas inside quotes, and CRLF line endings. Returns an array of +// arrays (rows × columns). No streaming — fine for the ~MB-scale imports +// a password manager realistically deals with. +function parseCSV(text) { + const rows = []; + let row = [], field = '', inQuotes = false; + for (let i = 0; i < text.length; i++) { + const c = text[i]; + if (inQuotes) { + if (c === '"') { + if (text[i + 1] === '"') { field += '"'; i++; } // escaped "" + else inQuotes = false; + } else field += c; + } else { + if (c === '"') inQuotes = true; + else if (c === ',') { row.push(field); field = ''; } + else if (c === '\n' || c === '\r') { + if (c === '\r' && text[i + 1] === '\n') i++; // CRLF + row.push(field); field = ''; + if (row.length > 1 || (row.length === 1 && row[0] !== '')) rows.push(row); + row = []; + } else field += c; + } + } + // Flush trailing field/row (file without final newline) + if (field !== '' || row.length > 0) { row.push(field); rows.push(row); } + return rows; +} + +// Header heuristics: pick the first matching column name (case-insensitive, +// underscore/space-tolerant). Returns null if no candidate header matches. +function findColumn(headers, candidates) { + const norm = s => String(s || '').toLowerCase().replace(/[\s_-]+/g, ''); + const cand = candidates.map(norm); + for (let i = 0; i < headers.length; i++) { + if (cand.indexOf(norm(headers[i])) >= 0) return i; + } + return null; +} + +// Parse a CSV text into an array of plaintext entries +// ({ site, username, password, folder, tags, totp_secret }). Returns +// { entries, skipped, columns } so the preview can show what was matched. +function parseEntriesFromCSV(text) { + const rows = parseCSV(text); + if (rows.length < 2) { + throw new Error('CSV needs a header row and at least one data row'); + } + const headers = rows[0]; + // Candidate names per format observed in real exports: + // Bitwarden CSV : folder, name, login_uri, login_username, login_password, login_totp, notes + // KeePass CSV : Title, URL, Username, Password, Group, Notes + // Chrome/Edge : name, url, username, password + // 1Password CSV : Title, URL, Username, Password, Notes + // findColumn normalizes (lowercase, strip _ and -) so 'login_uri' and + // 'loginuri' both match the same candidate. + const colSite = findColumn(headers, ['name', 'title', 'url', 'site', 'website', 'login_uri', 'login_url', 'web_site']); + const colUser = findColumn(headers, ['login_username', 'username', 'user', 'login', 'email', 'user_name']); + const colPwd = findColumn(headers, ['login_password', 'password', 'pass', 'pwd']); + const colFolder = findColumn(headers, ['folder', 'group', 'category', 'path', 'collection']); + const colTags = findColumn(headers, ['tags', 'labels']); + const colNotes = findColumn(headers, ['notes', 'note', 'comment', 'comments']); + const colTotp = findColumn(headers, ['login_totp', 'totp', 'totp_secret', 'otp', 'otpauth', 'authenticator', 'two_factor', 'twofa']); + + if (colSite === null && colUser === null) + throw new Error('No recognizable site/url or username column in CSV header'); + if (colPwd === null) + throw new Error('No recognizable password column in CSV header'); + + const entries = []; + let skipped = 0; + for (let i = 1; i < rows.length; i++) { + const r = rows[i]; + const site = (colSite !== null ? r[colSite] : '').trim() || + (colUser !== null ? r[colUser] : '').trim(); + const pwd = (colPwd !== null ? r[colPwd] : ''); + if (!site || !pwd) { skipped++; continue; } + + // Tags: combine the tags column and any free-form notes into a + // comma-separated string. Notes often contain useful metadata we + // don't want to drop on the floor. + let tagsArr = []; + if (colTags !== null) { + String(r[colTags] || '').split(/[,;]/).forEach(t => { + t = t.trim(); + if (t) tagsArr.push(t); + }); + } + if (colNotes !== null) { + const n = String(r[colNotes] || '').trim(); + if (n && n.length < 80) tagsArr.push(n); // long notes become noise as tags + } + + // TOTP: support raw base32 OR full otpauth:// URI in the cell. + let totp = ''; + if (colTotp !== null) { + const raw = String(r[colTotp] || '').trim(); + totp = parseOtpAuthUri(raw) || raw; + } + + entries.push({ + site: site, + username: (colUser !== null ? r[colUser] : '').trim(), + password: pwd, + folder: (colFolder !== null ? r[colFolder] : '').trim() || 'All', + tags: tagsArr.join(','), + totp_secret: totp, + }); + } + return { entries, skipped, columns: { + site: colSite, username: colUser, password: colPwd, + folder: colFolder, tags: colTags, notes: colNotes, totp: colTotp, + } }; +} + +// Parse a native JSON export. Forgiving: accepts both our own format and +// a flat array of entry objects. +function parseEntriesFromJSON(text) { + let data; + try { data = JSON.parse(text); } + catch (e) { throw new Error('Invalid JSON: ' + e.message); } + const raw = Array.isArray(data) ? data : (data.entries || []); + if (!Array.isArray(raw) || raw.length === 0) + throw new Error('No entries in JSON file'); + + const entries = []; + let skipped = 0; + for (const e of raw) { + if (!e || typeof e !== 'object') { skipped++; continue; } + const site = String(e.site || e.url || e.name || '').trim(); + const pwd = String(e.password || ''); + if (!site || !pwd) { skipped++; continue; } + + const tagsVal = e.tags; + const tagsStr = Array.isArray(tagsVal) ? tagsVal.join(',') + : String(tagsVal || ''); + entries.push({ + site: site, + username: String(e.username || e.user || e.login || '').trim(), + password: pwd, + folder: String(e.folder || e.group || 'All').trim() || 'All', + tags: tagsStr, + totp_secret: String(e.totp || e.totp_secret || e.otpauth || '').trim(), + }); + } + return { entries, skipped, columns: null }; // JSON: no column report +} + +// Encrypt one parsed entry (plaintext password + optional TOTP) into the +// shape the bulk-import endpoint expects. Reuses encryptPwd which already +// generates a fresh IV per call. +async function encryptImportEntry(plain) { + const pw = await encryptPwd(plain.password); + let totpEnc = '', totpIv = ''; + if (plain.totp_secret) { + try { + base32Decode(plain.totp_secret); // validate before encrypting + const t = await encryptPwd(plain.totp_secret); + totpEnc = t.encrypted; + totpIv = t.iv; + } catch (e) { + // Bad TOTP secret in source file — keep the entry but drop the + // 2FA silently. The user can fix it later via the slide-over. + } + } + return { + site: plain.site, + username: plain.username || '', + encrypted_password: pw.encrypted, + iv: pw.iv, + folder: plain.folder || 'All', + tags: plain.tags || '', + totp_secret: totpEnc, + totp_iv: totpIv, + }; +} + +// Open a hidden file picker, route the result through the right parser, +// show a preview confirmation, then bulk-encrypt + POST. +async function doImport() { + const fileInput = el('input', { + type: 'file', + accept: '.json,.csv,application/json,text/csv', + style: 'display:none', + }); + document.body.appendChild(fileInput); + + fileInput.addEventListener('change', async () => { + const file = fileInput.files && fileInput.files[0]; + fileInput.remove(); + if (!file) return; + + let text; + try { text = await file.text(); } + catch (e) { return toast('Cannot read file: ' + e.message, 'error'); } + + const isJSON = /\.json$/i.test(file.name) || text.trim().startsWith('{') || text.trim().startsWith('['); + let parsed; + try { + parsed = isJSON ? parseEntriesFromJSON(text) : parseEntriesFromCSV(text); + } catch (e) { + return toast('Parse error: ' + e.message, 'error'); + } + + if (parsed.entries.length === 0) { + return toast('No valid entries found in file', 'warning'); + } + + // Build the preview message (innerHTML target → escape user data) + const esc = s => String(s).replace(/[&<>"]/g, + c => ({ '&': '&', '<': '<', '>': '>', '"': '"' }[c])); + const parts = []; + parts.push('' + parsed.entries.length + ' entries detected in ' + + esc(file.name) + ''); + if (parsed.skipped > 0) + parts.push('' + parsed.skipped + + ' rows skipped (missing site or password)'); + const sample = parsed.entries.slice(0, 3).map(e => + '• ' + esc(e.site || '?') + + (e.username ? ' (' + esc(e.username) + ')' : '') + ).join('
'); + parts.push('

' + sample + + (parsed.entries.length > 3 ? '
…' : '') + '
'); + parts.push('
Import now? This adds the entries to your existing vault.
'); + + const confirmed = await confirmDialog({ + title: 'Import vault', + message: parts.join('
'), + okText: 'Import', + }); + if (!confirmed) return; + + // Encrypt all entries client-side, then POST as one transaction. + toast('Encrypting ' + parsed.entries.length + ' entries…'); + const encrypted = []; + for (const e of parsed.entries) { + encrypted.push(await encryptImportEntry(e)); + } + + try { + const r = await api('/entries/bulk-import', { + method: 'POST', + headers: authHeaders({ 'Content-Type': 'application/json' }), + body: JSON.stringify({ entries: encrypted }), + }); + toast('Imported ' + r.imported + ' entries'); + await loadEntries(); + render(); + if (state.hibpEnabled) hibpCheckAllEntries(); // scan the new entries too + } catch (err) { + toast('Import failed: ' + err.message, 'error'); + } + }); + + fileInput.click(); +} + async function doExport() { const pwd = await askReauth('Enter your master password to export the vault as JSON. The file will be UNENCRYPTED.'); if (!pwd) return; @@ -2712,6 +2987,7 @@ async function init() { // Sidebar Generator tool $('#sidebarGenBtn').addEventListener('click', () => openGen('standalone')); $('#sidebarExportBtn').addEventListener('click', doExport); + $('#sidebarImportBtn').addEventListener('click', doImport); // Idle warning "Stay unlocked" $('#idleStayBtn').addEventListener('click', resetAutoLock); @@ -2757,6 +3033,7 @@ async function init() { toast('Open Windows Settings → System → Clipboard → turn off "Clipboard history"', 'warning'); }); $('#exportBtn').addEventListener('click', doExport); + $('#importBtn').addEventListener('click', doImport); // Re-auth modal $('#reauthForm').addEventListener('submit', e => { e.preventDefault(); closeReauth(true); });