Week 9: IoT
Are your friends cosntantly looking over your shoulder at your computer? Are you in need of a defense mechanism? Well, you're in luck!
Facial Recognition Rubber Band Shooter
This week, Bobby, Lane, and I worked together to create a facial recognition rubber band shooter that takes
in the input from a computer camera, writes to a database, and then shoots a rubber band with a servo motor controlled mechanism after reading
a value from the database.
Materials:
- 2 ESP32 S2 Microcontrollers
- Computer with camera
- Firebase Database
- 1 Servo Motor mounted in wood
- 1 Rubber Band
- Wires
Assembly
Rubber Band Shooting Mechanism
Lane assembled the rubber band mechanism. To create one that can be controlled by a microcontroller, he attached a servo motor with a spoked attachment to a piece of wood that would then stretch the rubber band out. The servo was also connected to an ESP32 that would control it. See images below for details on how the mechanism was created.
Camera Facial Recognition Mechanism
To create our input, which was a facial recognition system, Bobby first started out with the ESP CAM, but then switched to using the camera built into his computer, along with a python script.
Face-Detection.py
import cv2
import serial
face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')
cap = cv2.VideoCapture(0)
ser = serial.Serial('/dev/cu.usbserial-14120', 115200)
while True:
_,img = cap.read()
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
faces = face_cascade.detectMultiScale(gray, 1.1, 12)
for (x,y,w,h) in faces:
print(x,y,w,h)
x = int(x)
y = int(y)
w = int(w)
h = int(h)
cv2.rectangle(img, (x, y), (x+w, y+h), (255, 0, 0), 2)
cv2.imshow('img', img)
data = len(faces)
ser.write(str(data).encode())
k = cv2.waitKey(30) & 0xff
if k == 27:
break
cap.release()
Firebase and ESP communication
I worked on the communication of the two ESP32's through a Firebase Realtime Database. To set up the Firebase RTDB, I followed this tutorial from Random Nerd Tutorials. The ESP32 connected to the facial recognition system would write to the database a boolean value (either true or false) when it detected a face. The ESP32 connected to the servo motor would then read from the database and if the value was true, it would shoot the rubber band. Below is the code for writing and reading to the database.
Writing to Database
// WRITE TO DATABASE
#include
#if defined(ESP32)
#include
#elif defined(ESP8266)
#include
#endif
#include
//Provide the token generation process info.
#include "addons/TokenHelper.h"
//Provide the RTDB payload printing info and other helper functions.
#include "addons/RTDBHelper.h"
// Insert your network credentials
#define WIFI_SSID "MAKERSPACE"
#define WIFI_PASSWORD "12345678"
// Insert Firebase project API Key
#define API_KEY "AIzaSyBHwyaBVBIYPsUAPQkU7nkF34QYDD51bRA"
// Insert RTDB URLefine the RTDB URL */
#define DATABASE_URL "https://esp32-rubberband-default-rtdb.firebaseio.com"
//Define Firebase Data object
FirebaseData fbdo;
FirebaseAuth auth;
FirebaseConfig config;
unsigned long sendDataPrevMillis = 0;
int count = 0;
bool signupOK = false;
bool faceRecognized = false;
void setup(){
Serial.begin(115200);
WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
Serial.print("Connecting to Wi-Fi");
while (WiFi.status() != WL_CONNECTED){
Serial.print(".");
delay(300);
}
Serial.println();
Serial.print("Connected with IP: ");
Serial.println(WiFi.localIP());
Serial.println();
config.api_key = API_KEY;
config.database_url = DATABASE_URL;
if (Firebase.signUp(&config, &auth, "", "")){
Serial.println("ok");
signupOK = true;
}
else{
Serial.printf("%s\n", config.signer.signupError.message.c_str());
}
config.token_status_callback = tokenStatusCallback; //see addons/TokenHelper.h
Firebase.begin(&config, &auth);
Firebase.reconnectWiFi(true);
}
void loop(){
if (Firebase.ready() && signupOK && (millis() - sendDataPrevMillis > 3000 || sendDataPrevMillis == 0)){
sendDataPrevMillis = millis();
int sample = 0;
int count = 0;
while (count < 10)
{
if (Serial.available()) {
char data = Serial.read();
sample += (int) data - '0';
count += 1;
// Do something with the received data
}
}
Serial.println(sample / 10.0 > 0.6);
if ((sample / 10.0) > 0.3 != faceRecognized)
{
faceRecognized = !faceRecognized;
if (Firebase.RTDB.setBool(&fbdo, "test/bool", faceRecognized)){
Serial.println("PASSED");
Serial.println("PATH: " + fbdo.dataPath());
Serial.println("TYPE: " + fbdo.dataType());
}
else {
Serial.println("FAILED");
Serial.println("REASON: " + fbdo.errorReason());
}
}
while (Serial.read() >= 0);
}
}
Reading from Database
// READ FROM DATABASE
#include
#if defined(ESP32)
#include
#elif defined(ESP8266)
#include
#endif
#include
//Provide the token generation process info.
#include "addons/TokenHelper.h"
//Provide the RTDB payload printing info and other helper functions.
#include "addons/RTDBHelper.h"
// Insert your network credentials
#define WIFI_SSID "MAKERSPACE"
#define WIFI_PASSWORD "12345678"
// Insert Firebase project API Key
#define API_KEY "AIzaSyBHwyaBVBIYPsUAPQkU7nkF34QYDD51bRA"
// Insert RTDB URLefine the RTDB URL */
#define DATABASE_URL "https://esp32-rubberband-default-rtdb.firebaseio.com"
#include
//Define Firebase Data object
FirebaseData fbdo;
FirebaseAuth auth;
FirebaseConfig config;
unsigned long sendDataPrevMillis = 0;
int intValue;
float floatValue;
bool signupOK = false;
Servo myservo;
int pos = 0;
int servoPin = 17;
int buttonPin = 15;
unsigned long previousMillis = 0;
long OnTime = 100;
long OffTime = 100;
bool faceRecon;
void setup() {
Serial.begin(115200);
WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
Serial.print("Connecting to Wi-Fi");
while (WiFi.status() != WL_CONNECTED) {
Serial.print(".");
delay(300);
}
Serial.println();
Serial.print("Connected with IP: ");
Serial.println(WiFi.localIP());
Serial.println();
/* Assign the api key (required) */
config.api_key = API_KEY;
/* Assign the RTDB URL (required) */
config.database_url = DATABASE_URL;
/* Sign up */
if (Firebase.signUp(&config, &auth, "", "")) {
Serial.println("ok");
signupOK = true;
}
else {
Serial.printf("%s\n", config.signer.signupError.message.c_str());
}
/* Assign the callback function for the long running token generation task */
config.token_status_callback = tokenStatusCallback; //see addons/TokenHelper.h
Firebase.begin(&config, &auth);
Firebase.reconnectWiFi(true);
ESP32PWM::allocateTimer(0);
ESP32PWM::allocateTimer(1);
ESP32PWM::allocateTimer(2);
ESP32PWM::allocateTimer(3);
myservo.setPeriodHertz(50);
myservo.attach(servoPin, 1000, 2000);
pinMode(buttonPin, INPUT_PULLUP);
}
void loop() {
if (Firebase.ready() && signupOK && (millis() - sendDataPrevMillis > 2000 || sendDataPrevMillis == 0)) {
sendDataPrevMillis = millis();
Serial.println(sendDataPrevMillis);
if (Firebase.RTDB.getBool(&fbdo, "/test/bool")){
Serial.println(fbdo.dataType());
if (fbdo.dataType() == "boolean") {
faceRecon = fbdo.boolData();
Serial.println(faceRecon);
if (faceRecon == true){
for (pos = 0; pos <= 180; pos += 2){
myservo.write(pos);
delay(10);
}
for (pos = 180; pos >= 0; pos -= 2){
myservo.write(pos);
delay(10);
}
}
exit(0);
}
}
else {
Serial.println(fbdo.errorReason());
}
}
}
Here are screenshots of the database when it switches from false to true when detecting a face:
All together now!
When all connected, here's what the system does: