We want to help people who are unable to communicate verbally and who cannot use sign language by providing them with a portable “eye-to-speech” solution.
Devices and components
Arduino-Nano
TO-92 NPN Transistor (40V, 0.6A, 0.625W)
Insulated braided copper wire (3m)
USB 3.0 cable (male A > female A, 1.8m)
USB 2.0 cable (EASY male A > male mini-B, 0.5m)
Speaker 8 Ohms 2 watts, 36 mm
QTR-1RC reflectance sensor
Metal Film Resistor (20 Ohm, Axial, 2W, 5%)
3.5mm angled/straight jack cable (AUX, 1.5m)
3.5mm audio jack
USB 3.0 Type A connector “G46”
LED (generic)
Metal Film Resistor (220 Ohm)
Parallax Emic 2 speech synthesis module
Materials and tools
Solder wire, lead-free
Help tool, with magnifying glass
Soldering iron (generic)
Project description
1. Introduction
1.1 Motives
1.2 Objectives
1.3 Context
2. Design method
3. Solution design
3.1 Hardware
3.2 List of materials
3.3 Architecture
3.3.1 Cable
3.3.2 Glasses
3.3.3 Main device
3.4 User interface and implementation
3.4.1 Eye measurement logic
3.4.2 Profiles
3.4.3 Implementation
const char textdata_16[] PROGMEM = "I want to wash myself";
const char *const data_table[] PROGMEM = {textdata_0, textdata_1, …
3.5 Home page
3.6 Tutorials
$Demo Video: A short demo to display on the landing page, generating general interest and understanding of our project. $
$Step 1. What you need: A video on how to order 3D printed materials and the pre-machined PCB, alongside the list of materials needed. $
$Step 2. How to install the software: Explanations on setting up the integrated development environment and using the profile customization tool. $
$Step 3. Hardware Setup: A video on Arduinos hardware setup. $
$Step 4. Setting up the glasses: Covering the creation of the glasses, including topics such as aperture, USB cable, and sensor mounting. $
$Step 5. Upload the software to your Arduino: The process of uploading the code to the Arduino. $
$Step 6. Using the glasses: Brief explanations for calibrating and using the glasses $
4. Discussion
4.1 Evaluation
4.2 Theoretical and practical contribution
4.3 Perspectives and extensibility
Speak4Me GitHub repository
The repository contains the complete source of the project, including the web page with the customization tool.
Arduino code
arduino
Code required for the Arduino. In lines 72 to 232 you can customize your expressions. If you want to use a no-code solution to modify your expressions, you can use the customization tool on your project website: https://speak4me.github.io/speak4me/profile-customization-tool.html
1//sensor imports
2#include <QTRSensors.h>
3
4// For serial communication with the emic
5#include <SoftwareSerial.h>
6
7//To use timed actions -> protothreading
8#include <arduino-timer.h>
9
10// To save data to program space
11#include <avr/pgmspace.h>
12
13//Create timer to schedule the project parts
14auto timer = timer_create_default();
15
16//definitions
17#define rxPin 6 // Connect SOUT pin of the Emic 2 module to the RX pin
18#define txPin 7 // Connect SIN pin of the Emic 2 module to the TX pin
19#define keepAlivePin 8
20#define StatusLED 9
21
22
23//CUSTOM_SETTINGS_DECLARATION_START
24#define voiceStyle 1 // Male or female voice
25#define initialVolume 1 // Update Volume Funktion einbinden, Aus dem Customizationtool kommt 1-5
26#define closedDuration 2000 // How long to close the eyes to open the menu
27#define keepAliveOpt true
28#define statusLEDActive true // Learning LED which turns on when a direction is detected // Funktion to be implemented
29#define dirDuration 500 // Duration for which a direction has to be measured to be recognized
30#define dirPause 1000 // Cooldown after dir1 has been recognized
31#define timeOutDuration 4000 // Timelimit after which direction 1 gets rejected
32#define useAutoCalibration true
33//CUSTOM_SETTINGS_DECLARATION_END
34
35
36//variables: settings
37bool muted = false;
38uint8_t volume = initialVolume;
39uint8_t activeProfile = 1;
40bool inMenu = true;
41
42// Margins for eye tracking detection
43uint8_t factorL = 87;
44uint8_t factorR = 87;
45uint8_t factorUp = 90;
46uint8_t factorDown = 83;
47
48//variables: program logic
49//Sensors
50
51QTRSensors qtr;
52uint16_t sensorValue[4];
53uint16_t neutralSensorValue[4];
54
55//Emic
56SoftwareSerial emicSerial = SoftwareSerial(rxPin, txPin);
57bool emicReady;
58
59//Input treatment
60int8_t dir1 = -1; //0 1 2 3 for left Up right down; 4 for closed; -1 for null
61int8_t dir2 = -1; //0 1 2 3 for left Up right down; 4 for closed; -1 for null
62uint8_t status = 0; //0: Waiting for recognition; 1: timeout after dir1; 2: waiting for recognition 2
63unsigned long signalTracker; //Variable to track for how long a certain signal has been detected
64unsigned long timeoutTracker; //Variable to track the time when a timeout is reached
65
66
67
68char textdataBuffer[100]; // To buffer PROGMEM readings - Has to be large enough for the largest string in must hold
69
70//Loading Text Data into Progmem
71// Menu
72const char textdata_0[] PROGMEM = "Leave Menu";
73const char textdata_1[] PROGMEM = "Volume Up";
74const char textdata_2[] PROGMEM = "Toggle Mute"; // Muted - Unmuted mglich?
75const char textdata_3[] PROGMEM = "Volume Down";
76const char textdata_4[] PROGMEM = "No";
77const char textdata_5[] PROGMEM = "Okay";
78const char textdata_6[] PROGMEM = "Yes";
79const char textdata_7[] PROGMEM = "Help";
80const char textdata_8[] PROGMEM = "Profile 1";
81const char textdata_9[] PROGMEM = "Profile 2";
82const char textdata_10[] PROGMEM = "Profile 3";
83const char textdata_11[] PROGMEM = "Profile 4";
84const char textdata_12[] PROGMEM = "Profile 5";
85const char textdata_13[] PROGMEM = "Profile 6";
86const char textdata_14[] PROGMEM = "Profile 7";
87const char textdata_15[] PROGMEM = "Profile 8";
88
89// PLATZHALTER Profile 1 - Home
90//CUSTOM_EXPRESSION_DECLARATION_START
91const char textdata_16[] PROGMEM = "I want to wash myself";
92const char textdata_17[] PROGMEM = "I need to go to the toilet";
93const char textdata_18[] PROGMEM = "I would like to brush my teeth";
94const char textdata_19[] PROGMEM = "I would like to change my clothes";
95const char textdata_20[] PROGMEM = "Okay";
96const char textdata_21[] PROGMEM = "Yes";
97const char textdata_22[] PROGMEM = "Help";
98const char textdata_23[] PROGMEM = "No";
99const char textdata_24[] PROGMEM = "I want time for myself";
100const char textdata_25[] PROGMEM = "I want to sleep";
101const char textdata_26[] PROGMEM = "I need a break";
102const char textdata_27[] PROGMEM = "I would like to stop";
103const char textdata_28[] PROGMEM = "It is too hot";
104const char textdata_29[] PROGMEM = "I am hungry";
105const char textdata_30[] PROGMEM = "I am thirsty";
106const char textdata_31[] PROGMEM = "I want sweets";
107
108// Profile 2 - Friends
109const char textdata_32[] PROGMEM = "I dont like this";
110const char textdata_33[] PROGMEM = "I am good, thanks";
111const char textdata_34[] PROGMEM = "I am feeling not so good today";
112const char textdata_35[] PROGMEM = "I am tired";
113const char textdata_36[] PROGMEM = "Okay";
114const char textdata_37[] PROGMEM = "Yes";
115const char textdata_38[] PROGMEM = "Help";
116const char textdata_39[] PROGMEM = "No";
117const char textdata_40[] PROGMEM = "Do you want to hang out?";
118const char textdata_41[] PROGMEM = "How are you?";
119const char textdata_42[] PROGMEM = "What are your plans for the day??";
120const char textdata_43[] PROGMEM = "Want to go for a walk?";
121const char textdata_44[] PROGMEM = "Buz kz buz kz buz buz kschschsch";
122const char textdata_45[] PROGMEM = "That is wonderful";
123const char textdata_46[] PROGMEM = "eeeee";
124const char textdata_47[] PROGMEM = "Haahaahaa huuuhuuu looooooooooooool";
125
126// Profile 3 - Caretaker
127const char textdata_48[] PROGMEM = "Tighten my shoe";
128const char textdata_49[] PROGMEM = "I am freezing, I need warmer clothes";
129const char textdata_50[] PROGMEM = "I am warm, I need less clothes";
130const char textdata_51[] PROGMEM = "Change my clothes";
131const char textdata_52[] PROGMEM = "Okay";
132const char textdata_53[] PROGMEM = "Yes";
133const char textdata_54[] PROGMEM = "Help";
134const char textdata_55[] PROGMEM = "No";
135const char textdata_56[] PROGMEM = "I want to lay down";
136const char textdata_57[] PROGMEM = "Turn me";
137const char textdata_58[] PROGMEM = "Scratch me";
138const char textdata_59[] PROGMEM = "I want to sit";
139const char textdata_60[] PROGMEM = "I need a massage";
140const char textdata_61[] PROGMEM = "I need a medical treatment";
141const char textdata_62[] PROGMEM = "I am in pain";
142const char textdata_63[] PROGMEM = "Something is wrong";
143
144// Profile 4 - Doctor visit
145const char textdata_64[] PROGMEM = "I have a problem";
146const char textdata_65[] PROGMEM = "I dont feel well today";
147const char textdata_66[] PROGMEM = "I feel good today";
148const char textdata_67[] PROGMEM = "I am in pain";
149const char textdata_68[] PROGMEM = "Okay";
150const char textdata_69[] PROGMEM = "Yes";
151const char textdata_70[] PROGMEM = "Help";
152const char textdata_71[] PROGMEM = "No";
153const char textdata_72[] PROGMEM = "Ask me which part of the body is affected";
154const char textdata_73[] PROGMEM = "Something is wrong";
155const char textdata_74[] PROGMEM = "I need medication";
156const char textdata_75[] PROGMEM = "I need treatment";
157const char textdata_76[] PROGMEM = "Left";
158const char textdata_77[] PROGMEM = "Higher";
159const char textdata_78[] PROGMEM = "Right";
160const char textdata_79[] PROGMEM = "Lower";
161
162// Profile 5 - Chess
163const char textdata_80[] PROGMEM = "8";
164const char textdata_81[] PROGMEM = "5";
165const char textdata_82[] PROGMEM = "6";
166const char textdata_83[] PROGMEM = "7";
167const char textdata_84[] PROGMEM = "4";
168const char textdata_85[] PROGMEM = "1";
169const char textdata_86[] PROGMEM = "2";
170const char textdata_87[] PROGMEM = "3";
171const char textdata_88[] PROGMEM = "D";
172const char textdata_89[] PROGMEM = "A";
173const char textdata_90[] PROGMEM = "B";
174const char textdata_91[] PROGMEM = "C";
175const char textdata_92[] PROGMEM = "H";
176const char textdata_93[] PROGMEM = "E";
177const char textdata_94[] PROGMEM = "F";
178const char textdata_95[] PROGMEM = "G";
179
180// profile 6 - to be customized
181const char textdata_96[] PROGMEM = "Please use the customizer to create your own profile";
182const char textdata_97[] PROGMEM = "Please use the customizer to create your own profile";
183const char textdata_98[] PROGMEM = "Please use the customizer to create your own profile";
184const char textdata_99[] PROGMEM = "Please use the customizer to create your own profile";
185const char textdata_100[] PROGMEM = "Please use the customizer to create your own profile";
186const char textdata_101[] PROGMEM = "Please use the customizer to create your own profile";
187const char textdata_102[] PROGMEM = "Please use the customizer to create your own profile";
188const char textdata_103[] PROGMEM = "Please use the customizer to create your own profile";
189const char textdata_104[] PROGMEM = "Please use the customizer to create your own profile";
190const char textdata_105[] PROGMEM = "Please use the customizer to create your own profile";
191const char textdata_106[] PROGMEM = "Please use the customizer to create your own profile";
192const char textdata_107[] PROGMEM = "Please use the customizer to create your own profile";
193const char textdata_108[] PROGMEM = "Please use the customizer to create your own profile";
194const char textdata_109[] PROGMEM = "Please use the customizer to create your own profile";
195const char textdata_110[] PROGMEM = "Please use the customizer to create your own profile";
196const char textdata_111[] PROGMEM = "Please use the customizer to create your own profile";
197
198// profile 7 - to be customized
199const char textdata_112[] PROGMEM = "Please use the customizer to create your own profile";
200const char textdata_113[] PROGMEM = "Please use the customizer to create your own profile";
201const char textdata_114[] PROGMEM = "Please use the customizer to create your own profile";
202const char textdata_115[] PROGMEM = "Please use the customizer to create your own profile";
203const char textdata_116[] PROGMEM = "Please use the customizer to create your own profile";
204const char textdata_117[] PROGMEM = "Please use the customizer to create your own profile";
205const char textdata_118[] PROGMEM = "Please use the customizer to create your own profile";
206const char textdata_119[] PROGMEM = "Please use the customizer to create your own profile";
207const char textdata_120[] PROGMEM = "Please use the customizer to create your own profile";
208const char textdata_121[] PROGMEM = "Please use the customizer to create your own profile";
209const char textdata_122[] PROGMEM = "Please use the customizer to create your own profile";
210const char textdata_123[] PROGMEM = "Please use the customizer to create your own profile";
211const char textdata_124[] PROGMEM = "Please use the customizer to create your own profile";
212const char textdata_125[] PROGMEM = "Please use the customizer to create your own profile";
213const char textdata_126[] PROGMEM = "Please use the customizer to create your own profile";
214const char textdata_127[] PROGMEM = "Please use the customizer to create your own profile";
215
216// profile 8 - to be customized
217const char textdata_128[] PROGMEM = "Please use the customizer to create your own profile";
218const char textdata_129[] PROGMEM = "Please use the customizer to create your own profile";
219const char textdata_130[] PROGMEM = "Please use the customizer to create your own profile";
220const char textdata_131[] PROGMEM = "Please use the customizer to create your own profile";
221const char textdata_132[] PROGMEM = "Please use the customizer to create your own profile";
222const char textdata_133[] PROGMEM = "Please use the customizer to create your own profile";
223const char textdata_134[] PROGMEM = "Please use the customizer to create your own profile";
224const char textdata_135[] PROGMEM = "Please use the customizer to create your own profile";
225const char textdata_136[] PROGMEM = "Please use the customizer to create your own profile";
226const char textdata_137[] PROGMEM = "Please use the customizer to create your own profile";
227const char textdata_138[] PROGMEM = "Please use the customizer to create your own profile";
228const char textdata_139[] PROGMEM = "Please use the customizer to create your own profile";
229const char textdata_140[] PROGMEM = "Please use the customizer to create your own profile";
230const char textdata_141[] PROGMEM = "Please use the customizer to create your own profile";
231const char textdata_142[] PROGMEM = "Please use the customizer to create your own profile";
232const char textdata_143[] PROGMEM = "Please use the customizer to create your own profile";
233//CUSTOM_EXPRESSION_DECLARATION_END
234
235
236const char *const data_table[] PROGMEM = {textdata_0, textdata_1, textdata_2, textdata_3, textdata_4, textdata_5, textdata_6, textdata_7, textdata_8, textdata_9, textdata_10,
237 textdata_11, textdata_12, textdata_13, textdata_14, textdata_15, textdata_16, textdata_17, textdata_18, textdata_19, textdata_20,
238 textdata_21, textdata_22, textdata_23, textdata_24, textdata_25, textdata_26, textdata_27, textdata_28, textdata_29, textdata_30,
239 textdata_31, textdata_32, textdata_33, textdata_34, textdata_35, textdata_36, textdata_37, textdata_38, textdata_39, textdata_40,
240 textdata_41, textdata_42, textdata_43, textdata_44, textdata_45, textdata_46, textdata_47, textdata_48, textdata_49, textdata_50,
241 textdata_51, textdata_52, textdata_53, textdata_54, textdata_55, textdata_56, textdata_57, textdata_58, textdata_59, textdata_60,
242 textdata_61, textdata_62, textdata_63, textdata_64, textdata_65, textdata_66, textdata_67, textdata_68, textdata_69, textdata_70,
243 textdata_71, textdata_72, textdata_73, textdata_74, textdata_75, textdata_76, textdata_77, textdata_78, textdata_79, textdata_80,
244 textdata_81, textdata_82, textdata_83, textdata_84, textdata_85, textdata_86, textdata_87, textdata_88, textdata_89, textdata_90,
245 textdata_91, textdata_92, textdata_93, textdata_94, textdata_95, textdata_96, textdata_97, textdata_98, textdata_99, textdata_100,
246 textdata_101, textdata_102, textdata_103, textdata_104, textdata_105, textdata_106, textdata_107, textdata_108, textdata_109, textdata_110,
247 textdata_111, textdata_112, textdata_113, textdata_114, textdata_115, textdata_116, textdata_117, textdata_118, textdata_119, textdata_120,
248 textdata_121, textdata_122, textdata_123, textdata_124, textdata_125, textdata_126, textdata_127, textdata_128, textdata_129, textdata_130,
249 textdata_131, textdata_132, textdata_133, textdata_134, textdata_135, textdata_136, textdata_137, textdata_138, textdata_139, textdata_140,
250 textdata_141, textdata_142, textdata_143
251 }; //has to include all 144 strings in the end
252
253
254void setup()
255{
256 pinMode(keepAlivePin, OUTPUT);//LED for debug purposes at current stage of the project
257 pinMode(StatusLED, OUTPUT);//LED for debug purposes at current stage of the project
258
259 // set the data rate for the hardware serial port
260 Serial.begin(9600);
261
262 // configure the sensors
263 qtr.setTypeRC();
264 qtr.setSensorPins((const uint8_t[]) {
265 2, 3, 4, 5
266 }, 4);
267
268 qtr.setTimeout(5000);
269
270 // set the data rate for the SoftwareSerial port
271 pinMode(rxPin, INPUT);
272 pinMode(txPin, OUTPUT);
273 emicSerial.begin(9600);
274 emicSerial.print('\
275'); // Send a CR in case the system is already up
276 while (emicSerial.read() != ':'); // When the Emic 2 has initialized and is ready, it will send a single ':' character, so wait here until we receive it
277 emicSerial.flush(); // Flush the receive buffer
278
279 // set volume of emic
280 emicSerial.print('V');
281 emicSerial.print(String(volume));
282 emicSerial.print('\
283');
284
285 // set voicestyle of emic
286 emicSerial.print('N');
287 switch (voiceStyle) {
288 case 1: emicSerial.print('0'); break;
289 case 2: emicSerial.print('8'); break;
290 }
291 emicSerial.print('\
292');
293
294 emicReady = true;
295
296 if (useAutoCalibration) {
297 autocalibrateSensors();
298 } else {
299 calibrateSensors();
300 setupTimers();
301 }
302}
303
304void setupTimers() {
305 // Setting up the timers
306 timer.every(100, readSensors);
307 timer.every(100, updateEmicReady);
308 timer.every(3000, printDebugInfo);
309 if (keepAlive) {
310 timer.every(5000, keepAlive);
311 }
312}
313
314void calibrateSensors() {
315 qtr.read(neutralSensorValue);
316 neutralSensorValue[0] = uint32_t(neutralSensorValue[0]) * factorL / 100;
317 neutralSensorValue[1] = uint32_t(neutralSensorValue[1]) * factorUp / 100;
318 neutralSensorValue[2] = uint32_t(neutralSensorValue[2]) * factorL / 100;
319 neutralSensorValue[3] = uint32_t(neutralSensorValue[3]) * factorDown / 100;
320 Serial.println(neutralSensorValue[0]);
321 Serial.println(neutralSensorValue[1]);
322 Serial.println(neutralSensorValue[2]);
323 Serial.println(neutralSensorValue[3]);
324}
325
326void autocalibrateSensors() {
327 qtr.read(neutralSensorValue);
328
329 letEmicSpeak("Left");
330 while (emicSerial.read() != ':');
331 emicReady = true;
332 delay(2000);
333 autocalibrateSensor(2);
334
335 letEmicSpeak("Up");
336 while (emicSerial.read() != ':');
337 emicReady = true;
338 delay(2000);
339 autocalibrateSensor(3);
340
341 letEmicSpeak("Right");
342 while (emicSerial.read() != ':');
343 emicReady = true;
344 delay(2000);
345 autocalibrateSensor(0);
346
347 letEmicSpeak("Down");
348 while (emicSerial.read() != ':');
349 emicReady = true;
350 delay(2000);
351 autocalibrateSensor(1);
352
353 letEmicSpeak("Calibration completed");
354 while (emicSerial.read() != ':');
355 emicReady = true;
356 setupTimers();
357}
358
359void autocalibrateSensor(uint8_t sensornumber) {
360 qtr.read(sensorValue);
361 neutralSensorValue[sensornumber] = (4 * sensorValue[sensornumber] + neutralSensorValue[sensornumber]) / 5;
362}
363
364void setFactor(uint8_t sensornumber) {
365
366}
367bool updateEmicReady() {
368 if (emicSerial.read() == ':') {
369 emicReady = true;
370 }
371}
372
373void letEmicSpeak(char message[]) {
374 if (emicReady) { //Sollten wir bei jeder Sprachausgabe prfen, um dem emic erst eine neue Ausgabe zu senden, wenn er bereit ist
375 emicReady = false;
376 emicSerial.print('S');
377 emicSerial.print(message); // Send the desired string to convert to speech
378 emicSerial.print('\
379');
380 }
381}
382
383void toggleMute() {
384 muted = !muted;
385}
386
387void bufferTextdata(uint8_t entrynumber) {
388 strcpy_P(textdataBuffer, (char *)pgm_read_word(&(data_table[entrynumber])));
389}
390
391// Gets the detected directions as an input and performs the suitable actions depending on active profile, muted, etc.
392void performCommand(int8_t dir1, int8_t dir2) {
393
394 int8_t combinedDirs = dir1 * 10 + dir2; //combine two dirs into one variable
395
396 if (muted) {
397 if (combinedDirs == 2) {
398 toggleMute();
399 letEmicSpeak("Unmuted");
400 }
401 } else {
402
403 if (inMenu) { //menu commands
404 switch (combinedDirs) {
405
406 case 0: // leave menu
407 inMenu = false;
408 letEmicSpeak("Closing Menu");
409 break;
410
411 case 1: // Volume Up
412 raiseVolume();
413 break;
414
415 case 2: // Toggle Mute
416 toggleMute();
417 letEmicSpeak("Muted");
418 break;
419
420 case 3: // Volume Down
421 lowerVolume();
422 break;
423
424 case 10:
425 bufferTextdata(4);
426 letEmicSpeak(textdataBuffer);
427 break;
428
429 case 11:
430 bufferTextdata(5);
431 letEmicSpeak(textdataBuffer);
432 break;
433
434 case 12:
435 bufferTextdata(6);
436 letEmicSpeak(textdataBuffer);
437 break;
438
439 case 13:
440 bufferTextdata(7);
441 letEmicSpeak(textdataBuffer);
442 break;
443
444 case 20:
445 changeProfile(4);
446 break;
447
448 case 21:
449 changeProfile(1);
450 break;
451
452 case 22:
453 changeProfile(2);
454 break;
455
456 case 23:
457 changeProfile(3);
458 break;
459
460 case 30: // down
461 changeProfile(8);
462 break;
463
464 case 31:
465 changeProfile(5);
466 break;
467
468 case 32:
469 changeProfile(6);
470 break;
471
472 case 33:
473 changeProfile(7);
474 break;
475 }
476
477 } else { //profile commands
478 switch (combinedDirs) {
479
480 case 0:
481 bufferTextdata(16 * activeProfile);
482 letEmicSpeak(textdataBuffer);
483 break;
484
485 case 1:
486 bufferTextdata(16 * activeProfile + 1);
487 letEmicSpeak(textdataBuffer);
488 break;
489
490 case 2:
491 bufferTextdata(16 * activeProfile + 2);
492 letEmicSpeak(textdataBuffer);
493 break;
494
495 case 3:
496 bufferTextdata(16 * activeProfile + 3);
497 letEmicSpeak(textdataBuffer);
498 break;
499
500 case 10:
501 bufferTextdata(16 * activeProfile + 4);
502 letEmicSpeak(textdataBuffer);
503 break;
504
505 case 11:
506 bufferTextdata(16 * activeProfile + 5);
507 letEmicSpeak(textdataBuffer);
508 break;
509
510 case 12:
511 bufferTextdata(16 * activeProfile + 6);
512 letEmicSpeak(textdataBuffer);
513 break;
514
515 case 13:
516 bufferTextdata(16 * activeProfile + 7);
517 letEmicSpeak(textdataBuffer);
518 break;
519
520 case 20:
521 bufferTextdata(16 * activeProfile + 8);
522 letEmicSpeak(textdataBuffer);
523 break;
524
525 case 21:
526 bufferTextdata(16 * activeProfile + 9);
527 letEmicSpeak(textdataBuffer);
528 break;
529
530 case 22:
531 bufferTextdata(16 * activeProfile + 10);
532 letEmicSpeak(textdataBuffer);
533 break;
534
535 case 23:
536 bufferTextdata(16 * activeProfile + 11);
537 letEmicSpeak(textdataBuffer);
538 break;
539
540 case 30:
541 bufferTextdata(16 * activeProfile + 12);
542 letEmicSpeak(textdataBuffer);
543 break;
544
545 case 31:
546 bufferTextdata(16 * activeProfile + 13);
547 letEmicSpeak(textdataBuffer);
548 break;
549
550 case 32:
551 bufferTextdata(16 * activeProfile + 14);
552 letEmicSpeak(textdataBuffer);
553 break;
554
555 case 33:
556 bufferTextdata(16 * activeProfile + 15);
557 letEmicSpeak(textdataBuffer);
558 break;
559
560 case 40:
561 inMenu = true;
562 letEmicSpeak("Menu");
563 break;
564 }
565 }
566 }
567}
568
569// processes a recognized (or no recognized) eye detection direction and fires the according events depending on status, singalTracker and timeoutTracker
570void processDirection(int8_t recognizedDir) {
571 if (status == 0) {
572
573 if (recognizedDir == -1) {
574 dir1 = -1;
575
576 } else if (recognizedDir != dir1) {
577 dir1 = recognizedDir;
578 signalTracker = millis();
579
580 } else if (dir1 == 4) {
581 if (millis() - signalTracker > closedDuration) {
582 dir1 = -1;
583 performCommand(4, 0);
584 }
585 }
586
587 else if (millis() - signalTracker > dirDuration) {
588 status = 1;
589 timeoutTracker = millis();
590 if (!muted) {
591 letStatusLEDBlink();
592 }
593 }
594
595 } else if (status == 1) {
596
597 if (millis() - timeoutTracker > dirPause) {
598 status = 2;
599 timeoutTracker = millis();
600 processDirection(recognizedDir);
601 }
602
603 } else if (status == 2) {
604
605 if (millis() - timeoutTracker > timeOutDuration) {
606 status = 0;
607 dir1 = -1;
608 dir2 = -1;
609 processDirection(recognizedDir);
610
611 } else if (recognizedDir == -1) {
612 dir2 = -1;
613
614 } else if (recognizedDir != dir2) {
615 dir2 = recognizedDir;
616 signalTracker = millis();
617
618 } else if (millis() - signalTracker > dirDuration) {
619 performCommand(dir1, dir2);
620 status = 0;
621 dir1 = -1;
622 dir2 = -1;
623 timeoutTracker = millis();
624 }
625 }
626}
627
628// uses the hardware to read a set of sensor values and assumes in which direction the user was looking
629bool readSensors() {
630 qtr.read(sensorValue);
631
632 if (sensorValue[0] < neutralSensorValue[0] //if eyes closed
633 && sensorValue[2] < neutralSensorValue[2]
634 && sensorValue[1] < neutralSensorValue[1]
635 && sensorValue[3] < neutralSensorValue[3]) {
636 Serial.println("ReadSensors: Closed eyes");
637 processDirection(4);
638 } else if (sensorValue[3] < neutralSensorValue[3]) { //if looking up
639 Serial.println("ReadSensors: U ");
640 processDirection(1);
641 } else if (sensorValue[1] < neutralSensorValue[1]) { //if looking down
642 Serial.println("ReadSensors: D ");
643 processDirection(3);
644 } else if (sensorValue[0] < neutralSensorValue[0]) { //if looking right
645 Serial.println("ReadSensors: R ");
646 processDirection(2);
647 } else if (sensorValue[2] < neutralSensorValue[2]) { //if looking left
648 Serial.println("ReadSensors: L ");
649 processDirection(0);
650 } else {
651 Serial.println("ReadSensors: X ");
652 processDirection(-1);
653 }
654
655 return true;
656}
657
658void changeProfile(uint8_t profilenumber) {
659 activeProfile = profilenumber;
660 inMenu = false;
661 bufferTextdata(7 + profilenumber);
662 letEmicSpeak(textdataBuffer);
663}
664
665void raiseVolume() {
666 if (volume < 5) {
667 updateVolume(volume + 1);
668 bufferTextdata(1);
669 timer.in(150, [] { letEmicSpeak(textdataBuffer); });
670 } else {
671 letEmicSpeak("Maximal volume reached");
672 }
673}
674
675void lowerVolume() {
676 if (volume > 1) {
677 updateVolume(volume - 1);
678 bufferTextdata(3);
679 timer.in(150, [] { letEmicSpeak(textdataBuffer); });
680 } else {
681 letEmicSpeak("Minimal volume reached");
682 }
683}
684
685void updateVolume(int8_t newVolume) {
686 if (emicReady) { //Should be tested at every emic output to be sure its ready to receive commands
687 emicReady = false;
688 volume = newVolume;
689 emicSerial.print('V');
690 switch (volume) {
691 case 1: emicSerial.print("-40"); break;
692 case 2: emicSerial.print("-20"); break;
693 case 3: emicSerial.print("0"); break;
694 case 4: emicSerial.print("10"); break;
695 case 5: emicSerial.print("18"); break;
696 }
697 emicSerial.print('\
698');;
699 }
700}
701
702void letStatusLEDBlink() {
703 if (statusLEDActive) {
704 digitalWrite(StatusLED, HIGH);
705 timer.in(200, [] { digitalWrite(StatusLED, LOW); });
706 }
707}
708
709bool keepAlive() { //uses the keep alive wiring to consume some power so that powerbanks do not turn off
710 digitalWrite(keepAlivePin, HIGH);
711 timer.in(300, [] { digitalWrite(keepAlivePin, LOW); });
712 return true;
713}
714
715bool printDebugInfo() {
716 Serial.print(sensorValue[0]);
717 Serial.print('\ ');
718 Serial.print(sensorValue[1]);
719 Serial.print('\ ');
720 Serial.print(sensorValue[2]);
721 Serial.print('\ ');
722 Serial.print(sensorValue[3]);
723 Serial.print('\ ');
724 Serial.println();
725
726 return true;
727}
728
729void loop()
730{
731 timer.tick();
732}
733
Speak4Me GitHub repository
The repository contains the complete source of the project, including the web page with the customization tool.
Arduino code
arduino
Code required for the Arduino. In lines 72 to 232 you can customize your expressions. If you want to use a no-code solution to modify your expressions, you can use the customization tool on your project website: https://speak4me.github.io/speak4me/profile-customization-tool.html
1//sensor imports
2#include <QTRSensors.h>
3
4// For serial communication with the emic
5#include <SoftwareSerial.h>
6
7//To use timed actions -> protothreading
8#include <arduino-timer.h>
9
10// To save data to program space
11#include <avr/pgmspace.h>
12
13//Create timer to schedule the project parts
14auto timer = timer_create_default();
15
16//definitions
17#define rxPin 6 // Connect SOUT pin of the Emic 2 module to the RX pin
18#define txPin 7 // Connect SIN pin of the Emic 2 module to the TX pin
19#define keepAlivePin 8
20#define StatusLED 9
21
22
23//CUSTOM_SETTINGS_DECLARATION_START
24#define voiceStyle 1 // Male or female voice
25#define initialVolume 1 // Update Volume Funktion einbinden, Aus dem Customizationtool kommt 1-5
26#define closedDuration 2000 // How long to close the eyes to open the menu
27#define keepAliveOpt true
28#define statusLEDActive true // Learning LED which turns on when a direction is detected // Funktion to be implemented
29#define dirDuration 500 // Duration for which a direction has to be measured to be recognized
30#define dirPause 1000 // Cooldown after dir1 has been recognized
31#define timeOutDuration 4000 // Timelimit after which direction 1 gets rejected
32#define useAutoCalibration true
33//CUSTOM_SETTINGS_DECLARATION_END
34
35
36//variables: settings
37bool muted = false;
38uint8_t volume = initialVolume;
39uint8_t activeProfile = 1;
40bool inMenu = true;
41
42// Margins for eye tracking detection
43uint8_t factorL = 87;
44uint8_t factorR = 87;
45uint8_t factorUp = 90;
46uint8_t factorDown = 83;
47
48//variables: program logic
49//Sensors
50
51QTRSensors qtr;
52uint16_t sensorValue[4];
53uint16_t neutralSensorValue[4];
54
55//Emic
56SoftwareSerial emicSerial = SoftwareSerial(rxPin, txPin);
57bool emicReady;
58
59//Input treatment
60int8_t dir1 = -1; //0 1 2 3 for left Up right down; 4 for closed; -1 for null
61int8_t dir2 = -1; //0 1 2 3 for left Up right down; 4 for closed; -1 for null
62uint8_t status = 0; //0: Waiting for recognition; 1: timeout after dir1; 2: waiting for recognition 2
63unsigned long signalTracker; //Variable to track for how long a certain signal has been detected
64unsigned long timeoutTracker; //Variable to track the time when a timeout is reached
65
66
67
68char textdataBuffer[100]; // To buffer PROGMEM readings - Has to be large enough for the largest string in must hold
69
70//Loading Text Data into Progmem
71// Menu
72const char textdata_0[] PROGMEM = "Leave Menu";
73const char textdata_1[] PROGMEM = "Volume Up";
74const char textdata_2[] PROGMEM = "Toggle Mute"; // Muted - Unmuted mglich?
75const char textdata_3[] PROGMEM = "Volume Down";
76const char textdata_4[] PROGMEM = "No";
77const char textdata_5[] PROGMEM = "Okay";
78const char textdata_6[] PROGMEM = "Yes";
79const char textdata_7[] PROGMEM = "Help";
80const char textdata_8[] PROGMEM = "Profile 1";
81const char textdata_9[] PROGMEM = "Profile 2";
82const char textdata_10[] PROGMEM = "Profile 3";
83const char textdata_11[] PROGMEM = "Profile 4";
84const char textdata_12[] PROGMEM = "Profile 5";
85const char textdata_13[] PROGMEM = "Profile 6";
86const char textdata_14[] PROGMEM = "Profile 7";
87const char textdata_15[] PROGMEM = "Profile 8";
88
89// PLATZHALTER Profile 1 - Home
90//CUSTOM_EXPRESSION_DECLARATION_START
91const char textdata_16[] PROGMEM = "I want to wash myself";
92const char textdata_17[] PROGMEM = "I need to go to the toilet";
93const char textdata_18[] PROGMEM = "I would like to brush my teeth";
94const char textdata_19[] PROGMEM = "I would like to change my clothes";
95const char textdata_20[] PROGMEM = "Okay";
96const char textdata_21[] PROGMEM = "Yes";
97const char textdata_22[] PROGMEM = "Help";
98const char textdata_23[] PROGMEM = "No";
99const char textdata_24[] PROGMEM = "I want time for myself";
100const char textdata_25[] PROGMEM = "I want to sleep";
101const char textdata_26[] PROGMEM = "I need a break";
102const char textdata_27[] PROGMEM = "I would like to stop";
103const char textdata_28[] PROGMEM = "It is too hot";
104const char textdata_29[] PROGMEM = "I am hungry";
105const char textdata_30[] PROGMEM = "I am thirsty";
106const char textdata_31[] PROGMEM = "I want sweets";
107
108// Profile 2 - Friends
109const char textdata_32[] PROGMEM = "I dont like this";
110const char textdata_33[] PROGMEM = "I am good, thanks";
111const char textdata_34[] PROGMEM = "I am feeling not so good today";
112const char textdata_35[] PROGMEM = "I am tired";
113const char textdata_36[] PROGMEM = "Okay";
114const char textdata_37[] PROGMEM = "Yes";
115const char textdata_38[] PROGMEM = "Help";
116const char textdata_39[] PROGMEM = "No";
117const char textdata_40[] PROGMEM = "Do you want to hang out?";
118const char textdata_41[] PROGMEM = "How are you?";
119const char textdata_42[] PROGMEM = "What are your plans for the day??";
120const char textdata_43[] PROGMEM = "Want to go for a walk?";
121const char textdata_44[] PROGMEM = "Buz kz buz kz buz buz kschschsch";
122const char textdata_45[] PROGMEM = "That is wonderful";
123const char textdata_46[] PROGMEM = "eeeee";
124const char textdata_47[] PROGMEM = "Haahaahaa huuuhuuu looooooooooooool";
125
126// Profile 3 - Caretaker
127const char textdata_48[] PROGMEM = "Tighten my shoe";
128const char textdata_49[] PROGMEM = "I am freezing, I need warmer clothes";
129const char textdata_50[] PROGMEM = "I am warm, I need less clothes";
130const char textdata_51[] PROGMEM = "Change my clothes";
131const char textdata_52[] PROGMEM = "Okay";
132const char textdata_53[] PROGMEM = "Yes";
133const char textdata_54[] PROGMEM = "Help";
134const char textdata_55[] PROGMEM = "No";
135const char textdata_56[] PROGMEM = "I want to lay down";
136const char textdata_57[] PROGMEM = "Turn me";
137const char textdata_58[] PROGMEM = "Scratch me";
138const char textdata_59[] PROGMEM = "I want to sit";
139const char textdata_60[] PROGMEM = "I need a massage";
140const char textdata_61[] PROGMEM = "I need a medical treatment";
141const char textdata_62[] PROGMEM = "I am in pain";
142const char textdata_63[] PROGMEM = "Something is wrong";
143
144// Profile 4 - Doctor visit
145const char textdata_64[] PROGMEM = "I have a problem";
146const char textdata_65[] PROGMEM = "I dont feel well today";
147const char textdata_66[] PROGMEM = "I feel good today";
148const char textdata_67[] PROGMEM = "I am in pain";
149const char textdata_68[] PROGMEM = "Okay";
150const char textdata_69[] PROGMEM = "Yes";
151const char textdata_70[] PROGMEM = "Help";
152const char textdata_71[] PROGMEM = "No";
153const char textdata_72[] PROGMEM = "Ask me which part of the body is affected";
154const char textdata_73[] PROGMEM = "Something is wrong";
155const char textdata_74[] PROGMEM = "I need medication";
156const char textdata_75[] PROGMEM = "I need treatment";
157const char textdata_76[] PROGMEM = "Left";
158const char textdata_77[] PROGMEM = "Higher";
159const char textdata_78[] PROGMEM = "Right";
160const char textdata_79[] PROGMEM = "Lower";
161
162// Profile 5 - Chess
163const char textdata_80[] PROGMEM = "8";
164const char textdata_81[] PROGMEM = "5";
165const char textdata_82[] PROGMEM = "6";
166const char textdata_83[] PROGMEM = "7";
167const char textdata_84[] PROGMEM = "4";
168const char textdata_85[] PROGMEM = "1";
169const char textdata_86[] PROGMEM = "2";
170const char textdata_87[] PROGMEM = "3";
171const char textdata_88[] PROGMEM = "D";
172const char textdata_89[] PROGMEM = "A";
173const char textdata_90[] PROGMEM = "B";
174const char textdata_91[] PROGMEM = "C";
175const char textdata_92[] PROGMEM = "H";
176const char textdata_93[] PROGMEM = "E";
177const char textdata_94[] PROGMEM = "F";
178const char textdata_95[] PROGMEM = "G";
179
180// profile 6 - to be customized
181const char textdata_96[] PROGMEM = "Please use the customizer to create your own profile";
182const char textdata_97[] PROGMEM = "Please use the customizer to create your own profile";
183const char textdata_98[] PROGMEM = "Please use the customizer to create your own profile";
184const char textdata_99[] PROGMEM = "Please use the customizer to create your own profile";
185const char textdata_100[] PROGMEM = "Please use the customizer to create your own profile";
186const char textdata_101[] PROGMEM = "Please use the customizer to create your own profile";
187const char textdata_102[] PROGMEM = "Please use the customizer to create your own profile";
188const char textdata_103[] PROGMEM = "Please use the customizer to create your own profile";
189const char textdata_104[] PROGMEM = "Please use the customizer to create your own profile";
190const char textdata_105[] PROGMEM = "Please use the customizer to create your own profile";
191const char textdata_106[] PROGMEM = "Please use the customizer to create your own profile";
192const char textdata_107[] PROGMEM = "Please use the customizer to create your own profile";
193const char textdata_108[] PROGMEM = "Please use the customizer to create your own profile";
194const char textdata_109[] PROGMEM = "Please use the customizer to create your own profile";
195const char textdata_110[] PROGMEM = "Please use the customizer to create your own profile";
196const char textdata_111[] PROGMEM = "Please use the customizer to create your own profile";
197
198// profile 7 - to be customized
199const char textdata_112[] PROGMEM = "Please use the customizer to create your own profile";
200const char textdata_113[] PROGMEM = "Please use the customizer to create your own profile";
201const char textdata_114[] PROGMEM = "Please use the customizer to create your own profile";
202const char textdata_115[] PROGMEM = "Please use the customizer to create your own profile";
203const char textdata_116[] PROGMEM = "Please use the customizer to create your own profile";
204const char textdata_117[] PROGMEM = "Please use the customizer to create your own profile";
205const char textdata_118[] PROGMEM = "Please use the customizer to create your own profile";
206const char textdata_119[] PROGMEM = "Please use the customizer to create your own profile";
207const char textdata_120[] PROGMEM = "Please use the customizer to create your own profile";
208const char textdata_121[] PROGMEM = "Please use the customizer to create your own profile";
209const char textdata_122[] PROGMEM = "Please use the customizer to create your own profile";
210const char textdata_123[] PROGMEM = "Please use the customizer to create your own profile";
211const char textdata_124[] PROGMEM = "Please use the customizer to create your own profile";
212const char textdata_125[] PROGMEM = "Please use the customizer to create your own profile";
213const char textdata_126[] PROGMEM = "Please use the customizer to create your own profile";
214const char textdata_127[] PROGMEM = "Please use the customizer to create your own profile";
215
216// profile 8 - to be customized
217const char textdata_128[] PROGMEM = "Please use the customizer to create your own profile";
218const char textdata_129[] PROGMEM = "Please use the customizer to create your own profile";
219const char textdata_130[] PROGMEM = "Please use the customizer to create your own profile";
220const char textdata_131[] PROGMEM = "Please use the customizer to create your own profile";
221const char textdata_132[] PROGMEM = "Please use the customizer to create your own profile";
222const char textdata_133[] PROGMEM = "Please use the customizer to create your own profile";
223const char textdata_134[] PROGMEM = "Please use the customizer to create your own profile";
224const char textdata_135[] PROGMEM = "Please use the customizer to create your own profile";
225const char textdata_136[] PROGMEM = "Please use the customizer to create your own profile";
226const char textdata_137[] PROGMEM = "Please use the customizer to create your own profile";
227const char textdata_138[] PROGMEM = "Please use the customizer to create your own profile";
228const char textdata_139[] PROGMEM = "Please use the customizer to create your own profile";
229const char textdata_140[] PROGMEM = "Please use the customizer to create your own profile";
230const char textdata_141[] PROGMEM = "Please use the customizer to create your own profile";
231const char textdata_142[] PROGMEM = "Please use the customizer to create your own profile";
232const char textdata_143[] PROGMEM = "Please use the customizer to create your own profile";
233//CUSTOM_EXPRESSION_DECLARATION_END
234
235
236const char *const data_table[] PROGMEM = {textdata_0, textdata_1, textdata_2, textdata_3, textdata_4, textdata_5, textdata_6, textdata_7, textdata_8, textdata_9, textdata_10,
237 textdata_11, textdata_12, textdata_13, textdata_14, textdata_15, textdata_16, textdata_17, textdata_18, textdata_19, textdata_20,
238 textdata_21, textdata_22, textdata_23, textdata_24, textdata_25, textdata_26, textdata_27, textdata_28, textdata_29, textdata_30,
239 textdata_31, textdata_32, textdata_33, textdata_34, textdata_35, textdata_36, textdata_37, textdata_38, textdata_39, textdata_40,
240 textdata_41, textdata_42, textdata_43, textdata_44, textdata_45, textdata_46, textdata_47, textdata_48, textdata_49, textdata_50,
241 textdata_51, textdata_52, textdata_53, textdata_54, textdata_55, textdata_56, textdata_57, textdata_58, textdata_59, textdata_60,
242 textdata_61, textdata_62, textdata_63, textdata_64, textdata_65, textdata_66, textdata_67, textdata_68, textdata_69, textdata_70,
243 textdata_71, textdata_72, textdata_73, textdata_74, textdata_75, textdata_76, textdata_77, textdata_78, textdata_79, textdata_80,
244 textdata_81, textdata_82, textdata_83, textdata_84, textdata_85, textdata_86, textdata_87, textdata_88, textdata_89, textdata_90,
245 textdata_91, textdata_92, textdata_93, textdata_94, textdata_95, textdata_96, textdata_97, textdata_98, textdata_99, textdata_100,
246 textdata_101, textdata_102, textdata_103, textdata_104, textdata_105, textdata_106, textdata_107, textdata_108, textdata_109, textdata_110,
247 textdata_111, textdata_112, textdata_113, textdata_114, textdata_115, textdata_116, textdata_117, textdata_118, textdata_119, textdata_120,
248 textdata_121, textdata_122, textdata_123, textdata_124, textdata_125, textdata_126, textdata_127, textdata_128, textdata_129, textdata_130,
249 textdata_131, textdata_132, textdata_133, textdata_134, textdata_135, textdata_136, textdata_137, textdata_138, textdata_139, textdata_140,
250 textdata_141, textdata_142, textdata_143
251 }; //has to include all 144 strings in the end
252
253
254void setup()
255{
256 pinMode(keepAlivePin, OUTPUT);//LED for debug purposes at current stage of the project
257 pinMode(StatusLED, OUTPUT);//LED for debug purposes at current stage of the project
258
259 // set the data rate for the hardware serial port
260 Serial.begin(9600);
261
262 // configure the sensors
263 qtr.setTypeRC();
264 qtr.setSensorPins((const uint8_t[]) {
265 2, 3, 4, 5
266 }, 4);
267
268 qtr.setTimeout(5000);
269
270 // set the data rate for the SoftwareSerial port
271 pinMode(rxPin, INPUT);
272 pinMode(txPin, OUTPUT);
273 emicSerial.begin(9600);
274 emicSerial.print('\n'); // Send a CR in case the system is already up
275 while (emicSerial.read() != ':'); // When the Emic 2 has initialized and is ready, it will send a single ':' character, so wait here until we receive it
276 emicSerial.flush(); // Flush the receive buffer
277
278 // set volume of emic
279 emicSerial.print('V');
280 emicSerial.print(String(volume));
281 emicSerial.print('\n');
282
283 // set voicestyle of emic
284 emicSerial.print('N');
285 switch (voiceStyle) {
286 case 1: emicSerial.print('0'); break;
287 case 2: emicSerial.print('8'); break;
288 }
289 emicSerial.print('\n');
290
291 emicReady = true;
292
293 if (useAutoCalibration) {
294 autocalibrateSensors();
295 } else {
296 calibrateSensors();
297 setupTimers();
298 }
299}
300
301void setupTimers() {
302 // Setting up the timers
303 timer.every(100, readSensors);
304 timer.every(100, updateEmicReady);
305 timer.every(3000, printDebugInfo);
306 if (keepAlive) {
307 timer.every(5000, keepAlive);
308 }
309}
310
311void calibrateSensors() {
312 qtr.read(neutralSensorValue);
313 neutralSensorValue[0] = uint32_t(neutralSensorValue[0]) * factorL / 100;
314 neutralSensorValue[1] = uint32_t(neutralSensorValue[1]) * factorUp / 100;
315 neutralSensorValue[2] = uint32_t(neutralSensorValue[2]) * factorL / 100;
316 neutralSensorValue[3] = uint32_t(neutralSensorValue[3]) * factorDown / 100;
317 Serial.println(neutralSensorValue[0]);
318 Serial.println(neutralSensorValue[1]);
319 Serial.println(neutralSensorValue[2]);
320 Serial.println(neutralSensorValue[3]);
321}
322
323void autocalibrateSensors() {
324 qtr.read(neutralSensorValue);
325
326 letEmicSpeak("Left");
327 while (emicSerial.read() != ':');
328 emicReady = true;
329 delay(2000);
330 autocalibrateSensor(2);
331
332 letEmicSpeak("Up");
333 while (emicSerial.read() != ':');
334 emicReady = true;
335 delay(2000);
336 autocalibrateSensor(3);
337
338 letEmicSpeak("Right");
339 while (emicSerial.read() != ':');
340 emicReady = true;
341 delay(2000);
342 autocalibrateSensor(0);
343
344 letEmicSpeak("Down");
345 while (emicSerial.read() != ':');
346 emicReady = true;
347 delay(2000);
348 autocalibrateSensor(1);
349
350 letEmicSpeak("Calibration completed");
351 while (emicSerial.read() != ':');
352 emicReady = true;
353 setupTimers();
354}
355
356void autocalibrateSensor(uint8_t sensornumber) {
357 qtr.read(sensorValue);
358 neutralSensorValue[sensornumber] = (4 * sensorValue[sensornumber] + neutralSensorValue[sensornumber]) / 5;
359}
360
361void setFactor(uint8_t sensornumber) {
362
363}
364bool updateEmicReady() {
365 if (emicSerial.read() == ':') {
366 emicReady = true;
367 }
368}
369
370void letEmicSpeak(char message[]) {
371 if (emicReady) { //Sollten wir bei jeder Sprachausgabe prfen, um dem emic erst eine neue Ausgabe zu senden, wenn er bereit ist
372 emicReady = false;
373 emicSerial.print('S');
374 emicSerial.print(message); // Send the desired string to convert to speech
375 emicSerial.print('\n');
376 }
377}
378
379void toggleMute() {
380 muted = !muted;
381}
382
383void bufferTextdata(uint8_t entrynumber) {
384 strcpy_P(textdataBuffer, (char *)pgm_read_word(&(data_table[entrynumber])));
385}
386
387// Gets the detected directions as an input and performs the suitable actions depending on active profile, muted, etc.
388void performCommand(int8_t dir1, int8_t dir2) {
389
390 int8_t combinedDirs = dir1 * 10 + dir2; //combine two dirs into one variable
391
392 if (muted) {
393 if (combinedDirs == 2) {
394 toggleMute();
395 letEmicSpeak("Unmuted");
396 }
397 } else {
398
399 if (inMenu) { //menu commands
400 switch (combinedDirs) {
401
402 case 0: // leave menu
403 inMenu = false;
404 letEmicSpeak("Closing Menu");
405 break;
406
407 case 1: // Volume Up
408 raiseVolume();
409 break;
410
411 case 2: // Toggle Mute
412 toggleMute();
413 letEmicSpeak("Muted");
414 break;
415
416 case 3: // Volume Down
417 lowerVolume();
418 break;
419
420 case 10:
421 bufferTextdata(4);
422 letEmicSpeak(textdataBuffer);
423 break;
424
425 case 11:
426 bufferTextdata(5);
427 letEmicSpeak(textdataBuffer);
428 break;
429
430 case 12:
431 bufferTextdata(6);
432 letEmicSpeak(textdataBuffer);
433 break;
434
435 case 13:
436 bufferTextdata(7);
437 letEmicSpeak(textdataBuffer);
438 break;
439
440 case 20:
441 changeProfile(4);
442 break;
443
444 case 21:
445 changeProfile(1);
446 break;
447
448 case 22:
449 changeProfile(2);
450 break;
451
452 case 23:
453 changeProfile(3);
454 break;
455
456 case 30: // down
457 changeProfile(8);
458 break;
459
460 case 31:
461 changeProfile(5);
462 break;
463
464 case 32:
465 changeProfile(6);
466 break;
467
468 case 33:
469 changeProfile(7);
470 break;
471 }
472
473 } else { //profile commands
474 switch (combinedDirs) {
475
476 case 0:
477 bufferTextdata(16 * activeProfile);
478 letEmicSpeak(textdataBuffer);
479 break;
480
481 case 1:
482 bufferTextdata(16 * activeProfile + 1);
483 letEmicSpeak(textdataBuffer);
484 break;
485
486 case 2:
487 bufferTextdata(16 * activeProfile + 2);
488 letEmicSpeak(textdataBuffer);
489 break;
490
491 case 3:
492 bufferTextdata(16 * activeProfile + 3);
493 letEmicSpeak(textdataBuffer);
494 break;
495
496 case 10:
497 bufferTextdata(16 * activeProfile + 4);
498 letEmicSpeak(textdataBuffer);
499 break;
500
501 case 11:
502 bufferTextdata(16 * activeProfile + 5);
503 letEmicSpeak(textdataBuffer);
504 break;
505
506 case 12:
507 bufferTextdata(16 * activeProfile + 6);
508 letEmicSpeak(textdataBuffer);
509 break;
510
511 case 13:
512 bufferTextdata(16 * activeProfile + 7);
513 letEmicSpeak(textdataBuffer);
514 break;
515
516 case 20:
517 bufferTextdata(16 * activeProfile + 8);
518 letEmicSpeak(textdataBuffer);
519 break;
520
521 case 21:
522 bufferTextdata(16 * activeProfile + 9);
523 letEmicSpeak(textdataBuffer);
524 break;
525
526 case 22:
527 bufferTextdata(16 * activeProfile + 10);
528 letEmicSpeak(textdataBuffer);
529 break;
530
531 case 23:
532 bufferTextdata(16 * activeProfile + 11);
533 letEmicSpeak(textdataBuffer);
534 break;
535
536 case 30:
537 bufferTextdata(16 * activeProfile + 12);
538 letEmicSpeak(textdataBuffer);
539 break;
540
541 case 31:
542 bufferTextdata(16 * activeProfile + 13);
543 letEmicSpeak(textdataBuffer);
544 break;
545
546 case 32:
547 bufferTextdata(16 * activeProfile + 14);
548 letEmicSpeak(textdataBuffer);
549 break;
550
551 case 33:
552 bufferTextdata(16 * activeProfile + 15);
553 letEmicSpeak(textdataBuffer);
554 break;
555
556 case 40:
557 inMenu = true;
558 letEmicSpeak("Menu");
559 break;
560 }
561 }
562 }
563}
564
565// processes a recognized (or no recognized) eye detection direction and fires the according events depending on status, singalTracker and timeoutTracker
566void processDirection(int8_t recognizedDir) {
567 if (status == 0) {
568
569 if (recognizedDir == -1) {
570 dir1 = -1;
571
572 } else if (recognizedDir != dir1) {
573 dir1 = recognizedDir;
574 signalTracker = millis();
575
576 } else if (dir1 == 4) {
577 if (millis() - signalTracker > closedDuration) {
578 dir1 = -1;
579 performCommand(4, 0);
580 }
581 }
582
583 else if (millis() - signalTracker > dirDuration) {
584 status = 1;
585 timeoutTracker = millis();
586 if (!muted) {
587 letStatusLEDBlink();
588 }
589 }
590
591 } else if (status == 1) {
592
593 if (millis() - timeoutTracker > dirPause) {
594 status = 2;
595 timeoutTracker = millis();
596 processDirection(recognizedDir);
597 }
598
599 } else if (status == 2) {
600
601 if (millis() - timeoutTracker > timeOutDuration) {
602 status = 0;
603 dir1 = -1;
604 dir2 = -1;
605 processDirection(recognizedDir);
606
607 } else if (recognizedDir == -1) {
608 dir2 = -1;
609
610 } else if (recognizedDir != dir2) {
611 dir2 = recognizedDir;
612 signalTracker = millis();
613
614 } else if (millis() - signalTracker > dirDuration) {
615 performCommand(dir1, dir2);
616 status = 0;
617 dir1 = -1;
618 dir2 = -1;
619 timeoutTracker = millis();
620 }
621 }
622}
623
624// uses the hardware to read a set of sensor values and assumes in which direction the user was looking
625bool readSensors() {
626 qtr.read(sensorValue);
627
628 if (sensorValue[0] < neutralSensorValue[0] //if eyes closed
629 && sensorValue[2] < neutralSensorValue[2]
630 && sensorValue[1] < neutralSensorValue[1]
631 && sensorValue[3] < neutralSensorValue[3]) {
632 Serial.println("ReadSensors: Closed eyes");
633 processDirection(4);
634 } else if (sensorValue[3] < neutralSensorValue[3]) { //if looking up
635 Serial.println("ReadSensors: U ");
636 processDirection(1);
637 } else if (sensorValue[1] < neutralSensorValue[1]) { //if looking down
638 Serial.println("ReadSensors: D ");
639 processDirection(3);
640 } else if (sensorValue[0] < neutralSensorValue[0]) { //if looking right
641 Serial.println("ReadSensors: R ");
642 processDirection(2);
643 } else if (sensorValue[2] < neutralSensorValue[2]) { //if looking left
644 Serial.println("ReadSensors: L ");
645 processDirection(0);
646 } else {
647 Serial.println("ReadSensors: X ");
648 processDirection(-1);
649 }
650
651 return true;
652}
653
654void changeProfile(uint8_t profilenumber) {
655 activeProfile = profilenumber;
656 inMenu = false;
657 bufferTextdata(7 + profilenumber);
658 letEmicSpeak(textdataBuffer);
659}
660
661void raiseVolume() {
662 if (volume < 5) {
663 updateVolume(volume + 1);
664 bufferTextdata(1);
665 timer.in(150, [] { letEmicSpeak(textdataBuffer); });
666 } else {
667 letEmicSpeak("Maximal volume reached");
668 }
669}
670
671void lowerVolume() {
672 if (volume > 1) {
673 updateVolume(volume - 1);
674 bufferTextdata(3);
675 timer.in(150, [] { letEmicSpeak(textdataBuffer); });
676 } else {
677 letEmicSpeak("Minimal volume reached");
678 }
679}
680
681void updateVolume(int8_t newVolume) {
682 if (emicReady) { //Should be tested at every emic output to be sure its ready to receive commands
683 emicReady = false;
684 volume = newVolume;
685 emicSerial.print('V');
686 switch (volume) {
687 case 1: emicSerial.print("-40"); break;
688 case 2: emicSerial.print("-20"); break;
689 case 3: emicSerial.print("0"); break;
690 case 4: emicSerial.print("10"); break;
691 case 5: emicSerial.print("18"); break;
692 }
693 emicSerial.print('\n');;
694 }
695}
696
697void letStatusLEDBlink() {
698 if (statusLEDActive) {
699 digitalWrite(StatusLED, HIGH);
700 timer.in(200, [] { digitalWrite(StatusLED, LOW); });
701 }
702}
703
704bool keepAlive() { //uses the keep alive wiring to consume some power so that powerbanks do not turn off
705 digitalWrite(keepAlivePin, HIGH);
706 timer.in(300, [] { digitalWrite(keepAlivePin, LOW); });
707 return true;
708}
709
710bool printDebugInfo() {
711 Serial.print(sensorValue[0]);
712 Serial.print('\ ');
713 Serial.print(sensorValue[1]);
714 Serial.print('\ ');
715 Serial.print(sensorValue[2]);
716 Serial.print('\ ');
717 Serial.print(sensorValue[3]);
718 Serial.print('\ ');
719 Serial.println();
720
721 return true;
722}
723
724void loop()
725{
726 timer.tick();
727}
728
Downloadable files
Architecture overview
Architecture overview
Documentation
3D print .stl files for sensor housing and bracket
Includes files required to print the sensor housing and support.
https://github.com/speak4me/speak4me/blob/main/docs/assets/3d-printing-stl-files.zip
Gerber and drill files for printed circuit boards
Includes all files needed to order a professional PCB for the main device.
https://github.com/speak4me/speak4me/blob/main/docs/assets/Gerber.zip
3D print .stl files for sensor housing and bracket
Includes files required to print the sensor housing and support.
https://github.com/speak4me/speak4me/blob/main/docs/assets/3d-printing-stl-files.zip
Gerber and drill files for printed circuit boards
Includes all files needed to order a professional PCB for the main device.
https://github.com/speak4me/speak4me/blob/main/docs/assets/Gerber.zip
Note: Content and images are from: https://projecthub.arduino.cc/, with some modifications.
If you want it removed due to copyright reasons, please leave a comment. Thank you.
I want to share this article more widely so that everyone knows about Arduino and your project.
