openclaw/hermes-agent
1
0
Fork 0
Code Issues Pull Requests Actions 9 Packages Projects Releases Wiki Activity

feat: image pasting

Browse Source
This commit is contained in:
Brooklyn Nicholson
2026-04-09 13:45:23 -05:00
parent b2ea9b4176
commit 00e1d42b9e
7 changed files with 160 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
hermes_cli/clipboard.py
View File

@ -47,10 +47,11 @@ def has_clipboard_image() -> bool:
return _macos_has_image()
if sys.platform == "win32":
return _windows_has_image()
if _is_wsl():
return _wsl_has_image()
if os.environ.get("WAYLAND_DISPLAY"):
return _wayland_has_image()
# Match _linux_save fallthrough order: WSL → Wayland → X11
if _is_wsl() and _wsl_has_image():
return True
if os.environ.get("WAYLAND_DISPLAY") and _wayland_has_image():
return True
return _xclip_has_image()

12
tests/tools/test_clipboard.py
View File

@ -732,6 +732,18 @@ class TestHasClipboardImage:
assert has_clipboard_image() is True
m.assert_called_once()
def test_wsl_falls_through_to_wayland_when_windows_path_empty(self):
"""WSLg often bridges images to wl-paste even when powershell.exe check fails."""
with patch("hermes_cli.clipboard.sys") as mock_sys:
mock_sys.platform = "linux"
with patch("hermes_cli.clipboard._is_wsl", return_value=True):
with patch("hermes_cli.clipboard._wsl_has_image", return_value=False) as wsl:
with patch.dict(os.environ, {"WAYLAND_DISPLAY": "wayland-0"}):
with patch("hermes_cli.clipboard._wayland_has_image", return_value=True) as wl:
assert has_clipboard_image() is True
wsl.assert_called_once()
wl.assert_called_once()
def test_linux_wayland_dispatch(self):
with patch("hermes_cli.clipboard.sys") as mock_sys:
mock_sys.platform = "linux"

9
tui_gateway/server.py
View File

@ -691,16 +691,15 @@ def _(rid, params: dict) -> dict:
except Exception as e:
return _err(rid, 5027, f"clipboard unavailable: {e}")
if not has_clipboard_image():
return _ok(rid, {"attached": False, "message": "No image found in clipboard"})
session["image_counter"] = session.get("image_counter", 0) + 1
img_dir = _hermes_home / "images"
img_dir.mkdir(parents=True, exist_ok=True)
session["image_counter"] = session.get("image_counter", 0) + 1
img_path = img_dir / f"clip_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{session['image_counter']}.png"
# Save-first: mirrors CLI keybinding path; more robust than has_image() precheck
if not save_clipboard_image(img_path):
return _ok(rid, {"attached": False, "message": "Clipboard has image but extraction failed"})
msg = "Clipboard has image but extraction failed" if has_clipboard_image() else "No image found in clipboard"
return _ok(rid, {"attached": False, "message": msg})
session.setdefault("attached_images", []).append(str(img_path))
return _ok(rid, {"attached": True, "path": str(img_path), "count": len(session["attached_images"])})

2
ui-tui/README.md
View File

@ -94,7 +94,7 @@ Current input behavior is split across `app.tsx`, `components/textInput.tsx`, an
| `Ctrl+D` | Exit |
| `Ctrl+G` | Open `$EDITOR` with the current draft |
| `Ctrl+L` | New session (same as `/clear`) |
| `Ctrl+V` | Paste clipboard image (same as `/paste`) |
| `Ctrl+V` / `Alt+V` | Paste clipboard image (same as `/paste`) |
| `Tab` | Apply the active completion |
| `Up/Down` | Cycle completions if the completion list is open; otherwise edit queued messages first, then walk input history |
| `Left/Right` | Move the cursor |

155
ui-tui/src/app.tsx
View File

@ -229,15 +229,18 @@ export function App({ gw }: { gw: GatewayClient }) {
const [cols, setCols] = useState(stdout?.columns ?? 80)
useEffect(() => {
if (!stdout) {
return
}
if (!stdout) {return}
const sync = () => setCols(stdout.columns ?? 80)
stdout.on('resize', sync)
// Enable bracketed paste so image-only clipboard paste reaches the app
if (stdout.isTTY) {stdout.write('\x1b[?2004h')}
return () => {
stdout.off('resize', sync)
if (stdout.isTTY) {stdout.write('\x1b[?2004l')}
}
}, [stdout])
@ -499,12 +502,28 @@ export function App({ gw }: { gw: GatewayClient }) {
[pastes]
)
const paste = useCallback(
(quiet = false) =>
rpc('clipboard.paste', { session_id: sid }).then((r: any) =>
r?.attached
? sys(`📎 Image #${r.count} attached from clipboard`)
: quiet || sys(r?.message || 'No image found in clipboard')
),
[rpc, sid, sys]
)
const handleTextPaste = useCallback(
({ cursor, text, value }: { cursor: number; text: string; value: string }) => {
({ bracketed, cursor, hotkey, text, value }: import('./components/textInput.js').PasteEvent) => {
if (hotkey) { void paste(false);
return null }
if (bracketed) {void paste(true)}
if (!text) {return null}
const lineCount = text.split('\n').length
// Inline normal paste payloads exactly as typed. Only very large
// payloads are tokenized into attached snippets.
if (text.length < LARGE_PASTE.chars && lineCount < LARGE_PASTE.lines) {
return { cursor: cursor + text.length, value: value.slice(0, cursor) + text + value.slice(cursor) }
}
@ -536,7 +555,7 @@ export function App({ gw }: { gw: GatewayClient }) {
return { cursor: cursor + insert.length, value: value.slice(0, cursor) + insert + value.slice(cursor) }
},
[pushActivity]
[paste, pushActivity]
)
// ── Send ─────────────────────────────────────────────────────────
@ -599,11 +618,6 @@ export function App({ gw }: { gw: GatewayClient }) {
})
}
const paste = () =>
rpc('clipboard.paste', { session_id: sid }).then((r: any) =>
pushActivity(r.attached ? `image #${r.count} attached` : r.message || 'no image in clipboard')
)
const openEditor = () => {
const editor = process.env.EDITOR || process.env.VISUAL || 'vi'
const file = join(mkdtempSync(join(tmpdir(), 'hermes-')), 'prompt.md')
@ -756,37 +770,23 @@ export function App({ gw }: { gw: GatewayClient }) {
// ── Input handling ───────────────────────────────────────────────
const ctrl = (key: { ctrl: boolean }, ch: string, target: string) =>
key.ctrl && ch.toLowerCase() === target
useInput((ch, key) => {
if (isBlocked) {
if (pasteReview) {
if (key.return) {
const t = pasteReview.text
setPasteReview(null)
dispatchSubmission(t, true)
} else if (key.escape || (key.ctrl && ch === 'c')) {
setPasteReview(null)
setStatus('ready')
}
if (key.return) { setPasteReview(null); dispatchSubmission(pasteReview.text, true) }
else if (key.escape || ctrl(key, ch, 'c')) { setPasteReview(null); setStatus('ready') }
return
}
if (key.ctrl && ch === 'c') {
if (approval) {
gw.request('approval.respond', { choice: 'deny', session_id: sid }).catch(() => {})
setApproval(null)
sys('denied')
} else if (sudo) {
gw.request('sudo.respond', { request_id: sudo.requestId, password: '' }).catch(() => {})
setSudo(null)
sys('sudo cancelled')
} else if (secret) {
gw.request('secret.respond', { request_id: secret.requestId, value: '' }).catch(() => {})
setSecret(null)
sys('secret entry cancelled')
} else if (picker) {
setPicker(false)
}
if (ctrl(key, ch, 'c')) {
if (approval) { gw.request('approval.respond', { choice: 'deny', session_id: sid }).catch(() => {}); setApproval(null); sys('denied') }
else if (sudo) { gw.request('sudo.respond', { request_id: sudo.requestId, password: '' }).catch(() => {}); setSudo(null); sys('sudo cancelled') }
else if (secret) { gw.request('secret.respond', { request_id: secret.requestId, value: '' }).catch(() => {}); setSecret(null); sys('secret entry cancelled') }
else if (picker) {setPicker(false)}
} else if (key.escape && picker) {
setPicker(false)
}
@ -803,9 +803,7 @@ export function App({ gw }: { gw: GatewayClient }) {
if (!inputBuf.length && key.tab && completions.length) {
const row = completions[compIdx]
if (row) {
setInput(input.slice(0, compReplace) + row.text)
}
if (row) {setInput(input.slice(0, compReplace) + row.text)}
return
}
@ -813,19 +811,12 @@ export function App({ gw }: { gw: GatewayClient }) {
if (key.upArrow && !inputBuf.length) {
if (queueRef.current.length) {
const idx = queueEditIdx === null ? 0 : (queueEditIdx + 1) % queueRef.current.length
setQueueEdit(idx)
setHistoryIdx(null)
setInput(queueRef.current[idx] ?? '')
setQueueEdit(idx); setHistoryIdx(null); setInput(queueRef.current[idx] ?? '')
} else if (historyRef.current.length) {
const idx = historyIdx === null ? historyRef.current.length - 1 : Math.max(0, historyIdx - 1)
if (historyIdx === null) {
historyDraftRef.current = input
}
setHistoryIdx(idx)
setQueueEdit(null)
setInput(historyRef.current[idx] ?? '')
if (historyIdx === null) {historyDraftRef.current = input}
setHistoryIdx(idx); setQueueEdit(null); setInput(historyRef.current[idx] ?? '')
}
return
@ -833,55 +824,32 @@ export function App({ gw }: { gw: GatewayClient }) {
if (key.downArrow && !inputBuf.length) {
if (queueRef.current.length) {
const idx =
queueEditIdx === null
? queueRef.current.length - 1
: (queueEditIdx - 1 + queueRef.current.length) % queueRef.current.length
const idx = queueEditIdx === null
? queueRef.current.length - 1
: (queueEditIdx - 1 + queueRef.current.length) % queueRef.current.length
setQueueEdit(idx)
setHistoryIdx(null)
setInput(queueRef.current[idx] ?? '')
setQueueEdit(idx); setHistoryIdx(null); setInput(queueRef.current[idx] ?? '')
} else if (historyIdx !== null) {
const next = historyIdx + 1
if (next >= historyRef.current.length) {
setHistoryIdx(null)
setInput(historyDraftRef.current)
} else {
setHistoryIdx(next)
setInput(historyRef.current[next] ?? '')
}
if (next >= historyRef.current.length) { setHistoryIdx(null); setInput(historyDraftRef.current) }
else { setHistoryIdx(next); setInput(historyRef.current[next] ?? '') }
}
return
}
if (key.ctrl && ch === 'c') {
if (ctrl(key, ch, 'c')) {
if (busy && sid) {
interruptedRef.current = true
gw.request('session.interrupt', { session_id: sid }).catch(() => {})
const partial = (streaming || buf.current).trimStart()
partial ? appendMessage({ role: 'assistant', text: partial + '\n\n*[interrupted]*' }) : sys('interrupted')
if (partial) {
appendMessage({ role: 'assistant', text: partial + '\n\n*[interrupted]*' })
} else {
sys('interrupted')
}
idle(); setReasoning(''); setActivity([]); turnToolsRef.current = []; setStatus('interrupted')
idle()
setReasoning('')
setActivity([])
turnToolsRef.current = []
setStatus('interrupted')
if (statusTimerRef.current) {
clearTimeout(statusTimerRef.current)
}
statusTimerRef.current = setTimeout(() => {
statusTimerRef.current = null
setStatus('ready')
}, 1500)
if (statusTimerRef.current) {clearTimeout(statusTimerRef.current)}
statusTimerRef.current = setTimeout(() => { statusTimerRef.current = null; setStatus('ready') }, 1500)
} else if (input || inputBuf.length) {
clearIn()
} else {
@ -891,26 +859,13 @@ export function App({ gw }: { gw: GatewayClient }) {
return
}
if (key.ctrl && ch === 'd') {
die()
}
if (ctrl(key, ch, 'd')) {return die()}
if (key.ctrl && ch === 'l') {
setStatus('forging session…')
newSession()
if (ctrl(key, ch, 'l')) { setStatus('forging session…'); newSession();
return
}
return }
if (key.ctrl && ch === 'v') {
paste()
return
}
if (key.ctrl && ch === 'g') {
return openEditor()
}
if (ctrl(key, ch, 'g')) {return openEditor()}
})
// ── Gateway events ───────────────────────────────────────────────

208
ui-tui/src/components/textInput.tsx
View File

@ -4,13 +4,9 @@ import { useEffect, useRef, useState } from 'react'
function wordLeft(s: string, p: number) {
let i = p - 1
while (i > 0 && /\s/.test(s[i]!)) {
i--
}
while (i > 0 && /\s/.test(s[i]!)) {i--}
while (i > 0 && !/\s/.test(s[i - 1]!)) {
i--
}
while (i > 0 && !/\s/.test(s[i - 1]!)) {i--}
return Math.max(0, i)
}
@ -18,13 +14,9 @@ function wordLeft(s: string, p: number) {
function wordRight(s: string, p: number) {
let i = p
while (i < s.length && !/\s/.test(s[i]!)) {
i++
}
while (i < s.length && !/\s/.test(s[i]!)) {i++}
while (i < s.length && /\s/.test(s[i]!)) {
i++
}
while (i < s.length && /\s/.test(s[i]!)) {i++}
return i
}
@ -35,13 +27,21 @@ const INV_OFF = ESC + '[27m'
const DIM = ESC + '[2m'
const DIM_OFF = ESC + '[22m'
const PRINTABLE = /^[ -~\u00a0-\uffff]+$/
const BRACKET_PASTE = new RegExp(`${ESC}\\[20[01]~`, 'g')
const BRACKET_PASTE = new RegExp(`${ESC}?\\[20[01]~`, 'g')
export interface PasteEvent {
bracketed?: boolean
cursor: number
hotkey?: boolean
text: string
value: string
}
interface Props {
value: string
onChange: (v: string) => void
onSubmit?: (v: string) => void
onPaste?: (data: { cursor: number; text: string; value: string }) => { cursor: number; value: string } | null
onPaste?: (e: PasteEvent) => { cursor: number; value: string } | null
placeholder?: string
focus?: boolean
}
@ -77,16 +77,9 @@ export function TextInput({ value, onChange, onPaste, onSubmit, placeholder = ''
}
}, [value])
useEffect(
() => () => {
if (pasteTimer.current) {
clearTimeout(pasteTimer.current)
}
},
[]
)
useEffect(() => () => { if (pasteTimer.current) {clearTimeout(pasteTimer.current)} }, [])
// ── Buffer ops (synchronous, ref-based — no stale closures) ─────
// ── Buffer ops (synchronous, ref-based) ─────────────────────────
const commit = (next: string, nextCur: number, track = true) => {
const prev = vRef.current
@ -95,10 +88,7 @@ export function TextInput({ value, onChange, onPaste, onSubmit, placeholder = ''
if (track && next !== prev) {
undoStack.current.push({ cursor: curRef.current, value: prev })
if (undoStack.current.length > 200) {
undoStack.current.shift()
}
if (undoStack.current.length > 200) {undoStack.current.shift()}
redoStack.current = []
}
@ -115,59 +105,58 @@ export function TextInput({ value, onChange, onPaste, onSubmit, placeholder = ''
const swap = (from: typeof undoStack, to: typeof redoStack) => {
const entry = from.current.pop()
if (!entry) {
return
}
if (!entry) {return}
to.current.push({ cursor: curRef.current, value: vRef.current })
commit(entry.value, entry.cursor, false)
}
const emitPaste = (e: PasteEvent) => {
const handled = onPasteRef.current?.(e)
if (handled) {commit(handled.value, handled.cursor)}
return !!handled
}
const flushPaste = () => {
const pasted = pasteBuf.current
const text = pasteBuf.current
const at = pastePos.current
pasteBuf.current = ''
pasteTimer.current = null
if (!pasted) {
return
}
if (!text) {return}
const v = vRef.current
const handled = onPasteRef.current?.({ cursor: at, text: pasted, value: v })
if (handled) {
return commit(handled.value, handled.cursor)
}
if (PRINTABLE.test(pasted)) {
commit(v.slice(0, at) + pasted + v.slice(at), at + pasted.length)
if (!emitPaste({ cursor: at, text, value: vRef.current }) && PRINTABLE.test(text)) {
commit(vRef.current.slice(0, at) + text + vRef.current.slice(at), at + text.length)
}
}
// ── Input handler (reads only from refs) ────────────────────────
const insert = (v: string, c: number, s: string) => v.slice(0, c) + s + v.slice(c)
// ── Input handler ───────────────────────────────────────────────
useInput(
(inp, k) => {
if (
k.upArrow ||
k.downArrow ||
(k.ctrl && inp === 'c') ||
k.tab ||
(k.shift && k.tab) ||
k.pageUp ||
k.pageDown ||
k.escape
) {
// Paste hotkeys — single owner, no competing listeners in App
if ((k.ctrl || k.meta) && inp.toLowerCase() === 'v') {
emitPaste({ cursor: curRef.current, hotkey: true, text: '', value: vRef.current })
return
}
// Keys handled by App.useInput
if (
k.upArrow || k.downArrow ||
(k.ctrl && inp === 'c') ||
k.tab || (k.shift && k.tab) ||
k.pageUp || k.pageDown ||
k.escape
) {return}
if (k.return) {
if (k.shift || k.meta) {
commit(vRef.current.slice(0, curRef.current) + '\n' + vRef.current.slice(curRef.current), curRef.current + 1)
} else {
onSubmitRef.current?.(vRef.current)
}
;(k.shift || k.meta)
? commit(insert(vRef.current, curRef.current, '\n'), curRef.current + 1)
: onSubmitRef.current?.(vRef.current)
return
}
@ -176,78 +165,46 @@ export function TextInput({ value, onChange, onPaste, onSubmit, placeholder = ''
let v = vRef.current
const mod = k.ctrl || k.meta
if (k.ctrl && inp === 'z') {
return swap(undoStack, redoStack)
}
if (k.ctrl && inp === 'z') {return swap(undoStack, redoStack)}
if ((k.ctrl && inp === 'y') || (k.meta && k.shift && inp === 'z')) {
return swap(redoStack, undoStack)
}
if ((k.ctrl && inp === 'y') || (k.meta && k.shift && inp === 'z')) {return swap(redoStack, undoStack)}
if (k.home || (k.ctrl && inp === 'a')) {
c = 0
} else if (k.end || (k.ctrl && inp === 'e')) {
c = v.length
} else if (k.leftArrow) {
c = mod ? wordLeft(v, c) : Math.max(0, c - 1)
} else if (k.rightArrow) {
c = mod ? wordRight(v, c) : Math.min(v.length, c + 1)
} else if ((k.backspace || k.delete) && c > 0) {
if (mod) {
const t = wordLeft(v, c)
v = v.slice(0, t) + v.slice(c)
c = t
} else {
v = v.slice(0, c - 1) + v.slice(c)
c--
}
} else if (k.ctrl && inp === 'w' && c > 0) {
const t = wordLeft(v, c)
v = v.slice(0, t) + v.slice(c)
c = t
} else if (k.ctrl && inp === 'u') {
v = v.slice(c)
c = 0
} else if (k.ctrl && inp === 'k') {
v = v.slice(0, c)
} else if (k.meta && inp === 'b') {
c = wordLeft(v, c)
} else if (k.meta && inp === 'f') {
c = wordRight(v, c)
} else if (inp.length > 0) {
if (k.home || (k.ctrl && inp === 'a')) {c = 0}
else if (k.end || (k.ctrl && inp === 'e')) {c = v.length}
else if (k.leftArrow) {c = mod ? wordLeft(v, c) : Math.max(0, c - 1)}
else if (k.rightArrow) {c = mod ? wordRight(v, c) : Math.min(v.length, c + 1)}
else if ((k.backspace || k.delete) && c > 0) {
if (mod) { const t = wordLeft(v, c); v = v.slice(0, t) + v.slice(c); c = t }
else { v = v.slice(0, c - 1) + v.slice(c); c-- }
}
else if (k.ctrl && inp === 'w' && c > 0) { const t = wordLeft(v, c); v = v.slice(0, t) + v.slice(c); c = t }
else if (k.ctrl && inp === 'u') { v = v.slice(c); c = 0 }
else if (k.ctrl && inp === 'k') {v = v.slice(0, c)}
else if (k.meta && inp === 'b') {c = wordLeft(v, c)}
else if (k.meta && inp === 'f') {c = wordRight(v, c)}
else if (inp.length > 0) {
const bracketed = inp.includes('[200~')
const raw = inp.replace(BRACKET_PASTE, '').replace(/\r\n/g, '\n').replace(/\r/g, '\n')
if (!raw) {
return
}
if (bracketed && emitPaste({ bracketed: true, cursor: c, text: raw, value: v })) {return}
if (raw === '\n') {
return commit(v.slice(0, c) + '\n' + v.slice(c), c + 1)
}
if (!raw) {return}
if (raw === '\n') {return commit(insert(v, c, '\n'), c + 1)}
if (raw.length > 1 || raw.includes('\n')) {
if (!pasteBuf.current) {
pastePos.current = c
}
if (!pasteBuf.current) {pastePos.current = c}
pasteBuf.current += raw
if (pasteTimer.current) {
clearTimeout(pasteTimer.current)
}
if (pasteTimer.current) {clearTimeout(pasteTimer.current)}
pasteTimer.current = setTimeout(flushPaste, 50)
return
}
if (PRINTABLE.test(raw)) {
v = v.slice(0, c) + raw + v.slice(c)
c += raw.length
} else {
return
}
} else {
return
}
if (PRINTABLE.test(raw)) { v = v.slice(0, c) + raw + v.slice(c); c += raw.length }
else {return}
} else {return}
commit(v, c)
},
@ -256,17 +213,16 @@ export function TextInput({ value, onChange, onPaste, onSubmit, placeholder = ''
// ── Render ──────────────────────────────────────────────────────
if (!focus) {
return <Text>{value || (placeholder ? DIM + placeholder + DIM_OFF : '')}</Text>
}
if (!focus) {return <Text>{value || (placeholder ? DIM + placeholder + DIM_OFF : '')}</Text>}
if (!value && placeholder) {
return <Text>{INV + (placeholder[0] ?? ' ') + INV_OFF + DIM + placeholder.slice(1) + DIM_OFF}</Text>
}
const rendered =
[...value].map((ch, i) => (i === cur ? INV + ch + INV_OFF : ch)).join('') +
(cur === value.length ? INV + ' ' + INV_OFF : '')
return <Text>{rendered}</Text>
return (
<Text>
{[...value].map((ch, i) => (i === cur ? INV + ch + INV_OFF : ch)).join('') +
(cur === value.length ? INV + ' ' + INV_OFF : '')}
</Text>
)
}

2
ui-tui/src/constants.ts
View File

@ -24,7 +24,7 @@ export const HOTKEYS: [string, string][] = [
['Ctrl+D', 'exit'],
['Ctrl+G', 'open $EDITOR for prompt'],
['Ctrl+L', 'new session (clear)'],
['Ctrl+V', 'paste clipboard image'],
['Ctrl+V / Alt+V', 'paste clipboard image'],
['Tab', 'apply completion'],
['↑/↓', 'completions / queue edit / history'],
['Ctrl+A/E', 'home / end of line'],