Theres this app, remotemouse, that displays a trackpad and keyboard, sends data from those over WiFi and moves the mouse or writes text on a pc. I want to implement something similar, but with an esp-32 instead of a phone and a joystick instead of a trackpad.
I already have a (kind of) working version.
I implemented communication using websockets. I transmitted the joystick data on the esp and processed it on the pc. I then used the robotjs library to use this data for mouse movement.
I expected the mouse to move smoothly and almost instantly in relation to my joystick inputs, but it was a bit choppy and delayed.
The esp code, programmed with C++ and platformIO and running as a server:
#include <Arduino.h>
#include <WiFi.h>
#include <WebSocketsServer.h>
#include <ArduinoJson.h>
#include "modules/joystick/rjc_joystick.h"
WiFiClient wifi_client;
WebSocketsServer websocket = WebSocketsServer(81);
static const char* SSID = "MY_SSID";
static const char* PASSWORD = "MY_PASSWORD";
void webSocketEvent(uint8_t num, WStype_t type, uint8_t * payload, size_t length) {
// Handle events here
}
void setup() {
Serial.begin(9600);
WiFi.mode(WIFI_STA);
WiFi.begin(SSID, PASSWORD);
Serial.println("Connecting to WiFi ");
while (WiFi.status() != WL_CONNECTED) {
delay(500);
Serial.print(".");
}
Serial.printf("WiFi connected, IP: %sn", WiFi.localIP().toString().c_str());
websocket.begin();
}
void loop() {
JoystickPosition joystick_position = RJC_JOYSTICK::get_mapped_position(-10, 10);
// Example: Send joystick position over WebSocket when the joystick is moved
if (joystick_position.x != 0 || joystick_position.y != 0) {
// Create a JSON document
StaticJsonDocument<100> jsonDocument;
// Populate the JSON document
jsonDocument["type"] = "cursor";
jsonDocument["data"]["x"] = joystick_position.x;
jsonDocument["data"]["y"] = joystick_position.y;
// Serialize the JSON document to a String
String message;
serializeJson(jsonDocument, message);
// Send the message to all connected clients
websocket.broadcastTXT(message);
Serial.println("Sent joystick position over WebSocket: " + message);
}
websocket.loop();
delay(35);
}
The back-end pc code programmed with nodejs and running as a client:
let WebSocketClient = require('websocket').client;
const robot = require('robotjs');
let client = new WebSocketClient();
client.on('connect', function(connection) {
console.log('WebSocket Connected');
connection.on('error', function(error) {
console.log("Connection Error: " + error.toString());
});
connection.on('close', function() {
console.log('Connection Closed.');
});
connection.on('message', function(message) {
if (typeof message.utf8Data === 'string') {
try {
const data = JSON.parse(message.utf8Data);
const type = data.type;
const x = data.data.x;
const y = data.data.y;
if(type == "cursor") {
moveMouseWithJoystick(x, y);
} else if (type == "scroll") {
robot.scrollMouse(x, y);
}
console.log(`[ws]: ${type}: ${x}, ${y}`);
} catch (error) {
console.error('Error parsing JSON:', error);
}
} else {
console.error('Received non-string data:', message.utf8Data);
}
});
});
client.connect('ws://192.168.1.104:81/');
function moveMouseWithJoystick(x, y) {
let pos = robot.getMousePos();
pos.x += x + 5;
pos.y += y + 5;
robot.moveMouse(pos.x, pos.y);
}
Later other group members will write a front end for this as well.
You can look at the complete code here.
Should I be using something other than websockets? I know BLE works (I have working version of this), but I would like to communicate over WiFi because I also plan on adding other WiFi related features later, and connecting the device to a client through multiple protocols doesn’t seem great for user experience.
I implemented communication using websockets. I transmitted the joystick data on the esp and processed it on the pc. I then used the robotjs library to use this data for mouse movement.
I expected the mouse to move smoothly and almost instantly in relation to my joystick inputs, but it was a bit choppy and delayed.
Thanks for any help!
