1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
##
# @file
# This includes the code for the server-side of Wireless-X
#
# The server running on laptop or PC is responsible for receiving the actions performed by user on the
# Wireless-X android app as well as receiving the camera frames of the user's smartphone (if the user has turned)
# on the camera). Such actions are transmitted to the server in encoded form, the server decodes the received
# message and instructs the laptop or PC to perform the action described in that message.
##
import socket
import time
import threading
import autopy
import cv2
import numpy as np
import binascii
import pyfakewebcam
import subprocess
from pynput.keyboard import Key, Controller as KeyboardController
from pynput.mouse import Button, Controller as MouseController
## Creates a virtual camera on the laptop/PC
virtualCamera = subprocess.run(["sudo", "modprobe", "v4l2loopback", "devices=1", "video_nr=20", "card_label='Wireless-X Camera'", "exclusive_caps=1"])
print('\n----------------Wireless-X Server----------------\n')
## @var width
# Stores the width of the screen
## @var height
# Stores the height of the screen
width, height = autopy.screen.size()
## @var curr_x
# Stores the x-coordinate of the current mouse location
## @var curr_y
# Stores the y-coordinate of the current mouse location
curr_x, curr_y = autopy.mouse.location()
## Stores the mid of x-coordinate of current mouse location
remote_x = curr_x/2
## Stores the mid of y-coordinate of current mouse location
remote_y = curr_y/2
## Socket used for receiving keyboard and mouse related actions
s=''
## Socket used for receiving the camera frames of user's smartphone
cameraSocket=''
## Width of the camera frame
img_width = 720
## Height of the camera frame
img_height = 480
## Virtual webcam device
camera = pyfakewebcam.FakeWebcam('/dev/video20', img_width, img_height)
## The camera and keyboard-mouse sockets receive user requests until this variable is set to 'True'
thread_run = True
## Initializing the KeyboardController object
keyboard = KeyboardController()
## Initializing the MouseController object
mouse = MouseController()
## Speed of mouse movement
mouse_speed = 2
## Screenshot counter to keep track of screenshots
screenshot_count = 0
##
# @brief This function establishes two sockets for receiving camera frames as well as mouse and keyboard actions
##
def bind_sockets():
"""This function creates a camera socket which is responsible for receiving the camera frames, and it also
creates another socket which is responsible for receiving the keyboard and mouse frames."""
try:
global s, cameraSocket
cameraSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
cameraSocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
cameraSocket.settimeout(1)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
cam_port=9998
cameraSocket.bind(("0.0.0.0", cam_port))
#print("Camera stream port: "+ str(cam_port))
cameraSocket.listen(10)
key_port=6666
s.bind(("0.0.0.0", key_port))
#print("Keyboard and Mouse Listening port: "+ str(key_port))
s.listen(10)
print("Wireless Server Up and Running........\n")
print("Enter the below IP address in your android app:\n")
ps = subprocess.Popen(('hostname', '-I'), stdout=subprocess.PIPE)
output = subprocess.check_output(('awk', '{print $1}'), stdin=ps.stdout)
ps.wait()
print(output.decode("utf-8"))
print('Press Ctrl+C to terminate the server')
except socket.error as msg:
print("Socket Binding error: "+str(msg))
time.sleep(5) # Time to sleep before reattempting the bind connection
bind_sockets()
## Maps the keys in keyboard layout to the actual keyboard keys
special_key_android_dictionary = {"F1": "F1", "F2":"F2", "F3":"F3", "F4":"F4", "F5":"F5", "F6":"F6", "F7":"F7", "F8":"F8", "F9":"F9", "F10":"F10", "F11":"F11", "F12":"F12", "Alt":"ALT", "Backspace":"BACKSPACE", "Caps\nLock":"CAPS_LOCK", "Ctrl":"CONTROL", "Delete":"DELETE", "↓":"DOWN_ARROW", "End":"END", "Esc":"ESCAPE", "Home":"HOME", "←":"LEFT_ARROW", "META":"META", "Page Down":"PAGE_DOWN", "Page Up":"PAGE_UP", "Enter":"RETURN", "→":"RIGHT_ARROW", "Shift":"SHIFT", "Space":"SPACE", "↑":"UP_ARROW", "Tab":"Tab"}
##
# @brief This function decodes the received mouse and keyboard actions and acts accordingly
##
def mouse_keyboard_connections():
"""This function checks if the message received corresponds to a mouse action (left-click, scroll, etc.) or a keyboard action
(such as keypress). It then instructs the laptop to perform these actions, using the 'autopy' and 'pynput' libraries. Some special
characters in keyboard are not supported by 'autopy' library, so the actions corresponding to these special characters are performed
by the 'pynput' library, other key actions are handled by the 'autopy' library. In case of mouse, the 'autopy' library was not that
efficient, so we used the 'pynput' library."""
global thread_run, curr_x, curr_y, remote_x, remote_y
global screenshot_count
thread_run=True
prev_x = int(width/2)
prev_y = int(height/2)
while thread_run:
try:
conn, address = s.accept()
#Mouse coordinates receiving function
#print("Mouse Accept")
peer_response=str(conn.recv(1024).decode("utf-8"))
if "!#Mouse#!" in peer_response:
peer_response=peer_response.split("!#Mouse#!")[1]
xy=peer_response.split(',')
if len(xy)==2:
mouse.position = (float(xy[0])*width, float(xy[1])*height)
elif "!#Keyboard#!" in peer_response:
peer_response=peer_response.split("!#Keyboard#!")[1]
if peer_response in special_key_android_dictionary.keys():
print("Special: "+ special_key_android_dictionary[peer_response])
if peer_response == "Backspace":
keyboard.press(Key.backspace)
keyboard.release(Key.backspace)
elif peer_response == "Tab":
keyboard.press(Key.tab)
keyboard.release(Key.tab)
else:
autopy.key.tap(eval("autopy.key.Code."+special_key_android_dictionary[peer_response]))
elif peer_response == "Cmd": # Windows key or Command Key in Mac
keyboard.press(Key.cmd)
keyboard.release(Key.cmd)
elif peer_response == "Print\nScreen":
print("Entered Print Screen: "+ str(screenshot_count))
autopy.bitmap.capture_screen().save(str("screenshot_"+str(screenshot_count)+".png"))
screenshot_count+=1
else:
print("Not Special :"+peer_response)
if peer_response.isalnum and peer_response != "'" and peer_response != "<":
print("Is Alpha Numeric")
autopy.key.tap(peer_response.lower())
elif peer_response == "'":
keyboard.press("'")
keyboard.release("'")
elif peer_response == "<":
keyboard.press("<")
keyboard.release("<")
elif "!#MouseClick#!" in peer_response:
if "LEFT" in peer_response:
mouse.click(Button.left, 1)
else:
mouse.click(Button.right, 1)
elif "!#MouseScroll#!" in peer_response:
if "SCROLL \nUP" in peer_response:
mouse.scroll(0, 2) # Scroll Up
else:
mouse.scroll(0, -2) # Scroll Down
elif "!#Test#!" in peer_response:
conn.send(bytes("Success "+str(int(width/mouse_speed))+" "+str(int(height/mouse_speed)),"utf-8"))
conn.close()
except Exception as msg:
pass
##
# @brief This function is responsible for handling the camera frames
##
def camera_stream_connections():
"""This function uses the 'OpenCV' library to decode and resize the camera frame. Then, this frame is scheduled on
the virtual webcam device created using 'pyfakewebcam' library."""
thread_run = True
image_window_open=False
while thread_run:
try:
conn, address = cameraSocket.accept()
client_response = str(conn.recv(1024).decode("utf-8"))
while "#$#$#$" not in client_response:
client_response += str(conn.recv(1024).decode("utf-8"))
pic = client_response.partition("#$#$#$")[0]
client_response = client_response.partition("#$#$#$")[2]
b = binascii.a2b_base64(pic)
nparr = np.frombuffer(b, dtype=np.uint8)
frame = cv2.imdecode(nparr, cv2.IMREAD_COLOR)
img = cv2.resize(frame, (img_width, img_height))
try:
camera.schedule_frame(img)
except Exception as msg:
print ("error: "+ str(msg))
conn.close()
except Exception as msg:
if "timed out" not in str(msg):
print("camera_stream_connections() Error: " + str(msg))
if image_window_open:
cv2.destroyWindow("Main")
image_window_open=False
pass
##
# @brief This function is responsible for listening to connections
##
def listening_connections():
"""This function creates two threads corresponding to the two sockets, one for handling mouse and keyboard events
and the other for handling camera frames received from the user's smartphone."""
bind_sockets()
mouse_thread=threading.Thread(target=mouse_keyboard_connections)
mouse_thread.daemon=True
camera_thread=threading.Thread(target=camera_stream_connections)
camera_thread.daemon=True
mouse_thread.start()
camera_thread.start()
mouse_thread.join()
camera_thread.join()
t2=threading.Thread(target=listening_connections)
t2.daemon=True
t2.start()
t2.join()