需求分析

让AI可以通过指令控制键盘操作

技术选型

  1. Ollama
  2. Spring AI
  3. 键盘控制

实现

SpringBoot 项目初始化

Java 21 + SpringBoot 3.5.8
Lombok + Spring Boot Devtools + Ollama + Spring Web

环境配置

需要重启电脑
管理员身份打开powershell，运行下面代码
New-Item $PROFILE -ItemType File -Force
找到并编辑这ps1文件
$OutputEncoding = [console]::InputEncoding = [console]::OutputEncoding = New-Object System.Text.UTF8Encoding
然后运行命令
Set-ExecutionPolicy Unrestricted
输入chcp，查看代码活动页是否是65001

1
2
3
4
5
6
$text = "按up键"
$bytes = [System.Text.Encoding]::UTF8.GetBytes($text)
curl -Method POST `
     -Headers @{"Content-Type" = "text/plain; charset=utf-8"} `
     -Body $bytes `
     http://localhost:8080/keyboard

简化输入：
PowerShell 配置文件添加

1
2
3
4
function aikey($text) {
    $bytes = [System.Text.Encoding]::UTF8.GetBytes($text)
    curl -Method POST -Headers @{"Content-Type"="text/plain; charset=utf-8"} -Body $bytes http://localhost:8080/keyboard
}

aikey “按up键”
aikey “复制选中的内容”

代码

  1. 指令 –> Spring
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
@RestController
public class KeyboardApiController {

    private final KeyboardAssistant assistant;
    private final KeyboardController controller;

    public KeyboardApiController(KeyboardAssistant assistant, KeyboardController controller) {
        this.assistant = assistant;
        this.controller = controller;
    }

    @PostMapping(path = "/keyboard", consumes = "text/plain;charset=UTF-8")
    public ResponseEntity<String> executeKeyboardAction(@RequestBody String userRequest) {
        try {
            String command = assistant.generateKeyboardCommand(userRequest);
            System.out.println("🧠 模型原始输出: [" + command + "]");
            System.out.println("🧠 长度: " + command.length());
            System.out.println("🧠 是否以 KEY: 开头: " + command.startsWith("KEY: "));
            System.out.println("🧠 模型输出: " + command);
            controller.executeCommand(command);
            return ResponseEntity.ok("Executed: " + command);
        } catch (Exception e) {
            return ResponseEntity.badRequest().body("Error: " + e.getMessage());
        }
    }
  1. Spring –> AI
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
@Component
public class KeyboardAssistant {
    private static final String SYSTEM_PROMPT = """
    你是一个严格的键盘自动化助手。请根据用户请求，输出对应的键盘操作。
    必须遵守以下规则：
    1. 只输出一行，以 "KEY: " 开头
    2. 不要任何解释、标点、空格（除必要外）或额外文字
    3. 优先使用标准组合键（如 Ctrl+C），不要自由发挥
    
    常见操作映射：
    - 按键位 -> KEY: 相应键位
    - 按w键 / 按键w / 按w -> KEY: w
    - 按W键 / 按键W / 按W -> KEY: W
    - 按1键 / 按1 -> KEY: 1
    - 复制 / 复制选中的内容 / 复制文本 → KEY: Ctrl+C
    - 粘贴 / 粘贴内容 → KEY: Ctrl+V
    - 剪切 → KEY: Ctrl+X
    - 全选 → KEY: Ctrl+A
    - 撤销 → KEY: Ctrl+Z
    - 保存 → KEY: Ctrl+S
    - 关闭窗口 → KEY: Alt+F4
    - 切换输入法 → KEY: Shift
    - 回车 / 确认 → KEY: Enter
    - 删除 → KEY: Delete
    - 退格 → KEY: Backspace
    - 输入任意文字（如“Hello”）→ KEY: Hello
    
    如果请求不在上述列表中，且无法确定按键，则输出：KEY: UNKNOWN
    
    现在处理请求：
    """;

    private final ChatClient chatClient;

    public KeyboardAssistant(ChatClient.Builder chatClientBuilder) {
        this.chatClient = chatClientBuilder.build();
    }

    public String generateKeyboardCommand(String userRequest) {
        String raw = chatClient.prompt()
                .system(SYSTEM_PROMPT)
                .user(userRequest)
                .call()
                .content();

        System.out.println("🔍 原始模型响应: [" + raw + "]");
        // 使用正则查找 "KEY: ..." 行（忽略前后空白和多余内容）
        Pattern pattern = Pattern.compile("(?m)^\\s*KEY:\\s*(.*)$");
        Matcher matcher = pattern.matcher(raw);
        if (matcher.find()) {
            String keyPart = "KEY: " + matcher.group(1).trim();
            return keyPart;
        }

        // 如果完全找不到，fallback
        throw new RuntimeException("No valid KEY: line found in model response. Raw: " + raw);
    }
}

  1. AI –> Robot
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
@Service
public class KeyboardController {

    private final Robot robot;

    public KeyboardController() throws AWTException {
        System.out.println("=== KeyboardController 初始化 ===");
        System.out.println("OS: " + System.getProperty("os.name"));
        System.out.println("Java Home: " + System.getProperty("java.home"));
        System.out.println("Headless system property: " + System.getProperty("java.awt.headless"));
        System.out.println("GraphicsEnvironment.isHeadless(): " + GraphicsEnvironment.isHeadless());
        System.out.println("==================================");

        this.robot = new Robot(); // Line 14
    }

    public void executeCommand(String command) {
        if (!command.startsWith("KEY: ")) {
            throw new IllegalArgumentException("Invalid command format");
        }

        String action = command.substring(5).trim();

        // 情况1：普通文本输入
        if (!action.contains("+")) {
            typeString(action);
            return;
        }

        // 情况2：组合键（如 Ctrl+C）
        String[] keys = action.split("\\+");
        pressKeys(keys);
    }

    private void typeString(String text) {
        for (char c : text.toCharArray()) {
            int keyCode = KeyEvent.getExtendedKeyCodeForChar(c);
            if (keyCode != KeyEvent.VK_UNDEFINED) {
                robot.keyPress(keyCode);
                robot.keyRelease(keyCode);
            } else {
                // 处理特殊字符（简化版）
                System.out.println("⚠️ 无法输入字符: " + c);
            }
        }
    }

    private void pressKeys(String[] keyNames) {
        int[] keyCodes = new int[keyNames.length];

        for (int i = 0; i < keyNames.length; i++) {
            keyCodes[i] = getKeyCode(keyNames[i].trim());
            robot.keyPress(keyCodes[i]);
        }

        // 释放（反向）
        for (int i = keyNames.length - 1; i >= 0; i--) {
            robot.keyRelease(keyCodes[i]);
        }
    }

    private int getKeyCode(String key) {
        return switch (key.toLowerCase()) {
            case "ctrl" -> KeyEvent.VK_CONTROL;
            case "alt" -> KeyEvent.VK_ALT;
            case "shift" -> KeyEvent.VK_SHIFT;
            case "enter" -> KeyEvent.VK_ENTER;
            case "space" -> KeyEvent.VK_SPACE;
            case "backspace" -> KeyEvent.VK_BACK_SPACE;
            case "tab" -> KeyEvent.VK_TAB;
            case "esc" -> KeyEvent.VK_ESCAPE;
            case "up" -> KeyEvent.VK_UP;
            case "down" -> KeyEvent.VK_DOWN;
            case "left" -> KeyEvent.VK_LEFT;
            case "right" -> KeyEvent.VK_RIGHT;
            case "c" -> KeyEvent.VK_C;
            case "v" -> KeyEvent.VK_V;
            case "x" -> KeyEvent.VK_X;
            case "a" -> KeyEvent.VK_A;
            // 可继续扩展...
            default -> KeyEvent.getExtendedKeyCodeForChar(key.charAt(0));
        };
    }
}

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
- 按a键 / 按a / a -> KEY: a
- 按b键 / 按b / b -> KEY: b
- 按c键 / 按c / c -> KEY: c
- 按d键 / 按d / d -> KEY: d
- 按e键 / 按e / e -> KEY: e
- 按f键 / 按f / f -> KEY: f
- 按g键 / 按g / g -> KEY: g
- 按h键 / 按h / h -> KEY: h
- 按i键 / 按i / i -> KEY: i
- 按j键 / 按j / j -> KEY: j
- 按k键 / 按k / k -> KEY: k
- 按l键 / 按l / l -> KEY: l
- 按m键 / 按m / m -> KEY: m
- 按n键 / 按n / n -> KEY: n
- 按o键 / 按o / o -> KEY: o
- 按p键 / 按p / p -> KEY: p
- 按q键 / 按q / q -> KEY: q
- 按r键 / 按r / r -> KEY: r
- 按s键 / 按s / s -> KEY: s
- 按t键 / 按t / t -> KEY: t
- 按u键 / 按u / u -> KEY: u
- 按v键 / 按v / v -> KEY: v
- 按w键 / 按w / w -> KEY: w
- 按x键 / 按x / x -> KEY: x
- 按y键 / 按y / y -> KEY: y
- 按z键 / 按z / z -> KEY: z
- 按A键 / 按A / A -> KEY: A
- 按B键 / 按B / B -> KEY: B
- 按C键 / 按C / C -> KEY: C
- 按D键 / 按D / D -> KEY: D
- 按E键 / 按E / E -> KEY: E
- 按F键 / 按F / F -> KEY: F
- 按G键 / 按G / G -> KEY: G
- 按H键 / 按H / H -> KEY: H
- 按I键 / 按I / I -> KEY: I
- 按J键 / 按J / J -> KEY: J
- 按K键 / 按K / K -> KEY: K
- 按L键 / 按L / L -> KEY: L
- 按M键 / 按M / M -> KEY: M
- 按N键 / 按N / N -> KEY: N
- 按O键 / 按O / O -> KEY: O
- 按P键 / 按P / P -> KEY: P
- 按Q键 / 按Q / Q -> KEY: Q
- 按R键 / 按R / R -> KEY: R
- 按S键 / 按S / S -> KEY: S
- 按T键 / 按T / T -> KEY: T
- 按U键 / 按U / U -> KEY: U
- 按V键 / 按V / V -> KEY: V
- 按W键 / 按W / W -> KEY: W
- 按X键 / 按X / X -> KEY: X
- 按Y键 / 按Y / Y -> KEY: Y
- 按Z键 / 按Z / Z -> KEY: Z
- 复制 / 复制选中的内容 / 复制文本 → KEY: Ctrl+C
- 粘贴 / 粘贴内容 → KEY: Ctrl+V
- 剪切 → KEY: Ctrl+X
- 全选 → KEY: Ctrl+A
- 撤销 → KEY: Ctrl+Z
- 保存 → KEY: Ctrl+S
- 关闭窗口 → KEY: Alt+F4
- 切换输入法 → KEY: Shift
- 回车 / 确认 → KEY: Enter
- 删除 → KEY: Delete
- 退格 → KEY: Backspace
- 输入任意文字（如“Hello”）→ KEY: Hello