ESP32-CAM Indian Currency Recognition and AI Speech

/*
* ESP32-CAM Currency Detection + Audio Announcement
* WITH WEB PREVIEW SERVER
*
* Changes from original:
*  1. Camera clarity: UXGA (1600×1200) with quality=8, plus sharpness/denoise tuning.
*  2. Web server on port 80:
*       /         → Dashboard with live MJPEG stream + last detection result
*       /stream   → Raw MJPEG stream (open in any browser tab or VLC)
*       /capture  → Single JPEG snapshot download
*       /status   → JSON with last currency result and heap info
*  3. Web server runs in a FreeRTOS task on Core 0.
*     detectCurrency() still runs on Core 1 (loop).
*  4. During detectCurrency(), camera is deinit'd briefly for RAM —
*     the stream task detects this and returns a 503 while detection runs.
*/
#include "esp_camera.h"
#include 
#include 
#include "soc/soc.h"
#include "soc/rtc_cntl_reg.h"
// Audio libraries
#include "AudioFileSourcePROGMEM.h"
#include "AudioGeneratorMP3.h"
#include "AudioOutputI2SNoDAC.h"
// Web server
#include 
// ─── Config ────────────────────────────────────────────────────────
const char* WIFI_SSID  = "Yourssidname";
const char* WIFI_PASS  = "Yourwifiassword";
const char* API_KEY    = "Yourapikey";
const char* serverName = "www.circuitdigest.cloud";
const char* serverPath = "/api/v1/currency-detection/detect";
const int   serverPort = 443;
// ─── Pins ──────────────────────────────────────────────────────────
#define TRIGGER_BTN  12
#define SPEAKER_PIN  13
// ─── AI-Thinker ESP32-CAM Camera Pins ─────────────────────────────
#define PWDN_GPIO_NUM  32
#define RESET_GPIO_NUM -1
#define XCLK_GPIO_NUM   0
#define SIOD_GPIO_NUM  26
#define SIOC_GPIO_NUM  27
#define Y9_GPIO_NUM    35
#define Y8_GPIO_NUM    34
#define Y7_GPIO_NUM    39
#define Y6_GPIO_NUM    36
#define Y5_GPIO_NUM    21
#define Y4_GPIO_NUM    19
#define Y3_GPIO_NUM    18
#define Y2_GPIO_NUM     5
#define VSYNC_GPIO_NUM 25
#define HREF_GPIO_NUM  23
#define PCLK_GPIO_NUM  22
// ─── Globals ───────────────────────────────────────────────────────
static unsigned long  lastTrigger    = 0;
static volatile bool  cameraActive   = true;  // false while deinit'd for detection
static String         lastCurrency   = "None yet";
static AudioOutputI2SNoDAC* g_audioOut = nullptr;
static WebServer            webServer(80);
// ─── Utility ───────────────────────────────────────────────────────
static void printHeap(const char* tag) {
 Serial.printf("[HEAP] %s  free=%u  maxBlock=%u  psram=%u\n",
   tag,
   (unsigned)ESP.getFreeHeap(),
   (unsigned)ESP.getMaxAllocHeap(),
   (unsigned)ESP.getFreePsram());
}
// ─── Camera Init ───────────────────────────────────────────────────
void initCamera() {
 pinMode(PWDN_GPIO_NUM, OUTPUT);
 digitalWrite(PWDN_GPIO_NUM, HIGH); delay(100);
 digitalWrite(PWDN_GPIO_NUM, LOW);  delay(100);
 camera_config_t cfg = {};
 cfg.ledc_channel = LEDC_CHANNEL_0; cfg.ledc_timer = LEDC_TIMER_0;
 cfg.pin_d0=Y2_GPIO_NUM; cfg.pin_d1=Y3_GPIO_NUM; cfg.pin_d2=Y4_GPIO_NUM;
 cfg.pin_d3=Y5_GPIO_NUM; cfg.pin_d4=Y6_GPIO_NUM; cfg.pin_d5=Y7_GPIO_NUM;
 cfg.pin_d6=Y8_GPIO_NUM; cfg.pin_d7=Y9_GPIO_NUM;
 cfg.pin_xclk=XCLK_GPIO_NUM; cfg.pin_pclk=PCLK_GPIO_NUM;
 cfg.pin_vsync=VSYNC_GPIO_NUM; cfg.pin_href=HREF_GPIO_NUM;
 cfg.pin_sscb_sda=SIOD_GPIO_NUM; cfg.pin_sscb_scl=SIOC_GPIO_NUM;
 cfg.pin_pwdn=PWDN_GPIO_NUM; cfg.pin_reset=RESET_GPIO_NUM;
 // Higher clock = sharper image, less motion blur
 cfg.xclk_freq_hz = 20000000;
 cfg.pixel_format = PIXFORMAT_JPEG;
 if (psramFound()) {
   // ── CLARITY UPGRADE ──────────────────────────────────────────
   // UXGA = 1600×1200 (was VGA 640×480)
   // quality 8 = very high (was 12); lower number = more detail
   // fb_count 2 keeps a spare frame ready while one is being sent
   cfg.frame_size   = FRAMESIZE_UXGA;
   cfg.jpeg_quality = 8;
   cfg.fb_count     = 2;
   cfg.fb_location  = CAMERA_FB_IN_PSRAM;
 } else {
   // No PSRAM fallback — SVGA is the best DRAM can handle reliably
   cfg.frame_size   = FRAMESIZE_SVGA;
   cfg.jpeg_quality = 10;
   cfg.fb_count     = 1;
   cfg.fb_location  = CAMERA_FB_IN_DRAM;
 }
 esp_err_t err = esp_camera_init(&cfg);
 if (err != ESP_OK) {
   Serial.printf("Camera init failed: 0x%x — restarting\n", err);
   delay(1000); ESP.restart();
 }
 // ── SENSOR TUNING FOR CLARITY ────────────────────────────────
 sensor_t* s = esp_camera_sensor_get();
 if (s) {
   s->set_brightness(s, 1);      // +1 brightness (range -2..2)
   s->set_contrast(s, 2);        // max contrast for crisp edges
   s->set_saturation(s, 0);      // neutral saturation
   s->set_sharpness(s, 2);       // max sharpness  ← NEW
   s->set_denoise(s, 1);         // light denoise (reduces JPEG noise) ← NEW
   s->set_whitebal(s, 1);        // auto white balance on
   s->set_awb_gain(s, 1);        // AWB gain on      ← NEW
   s->set_exposure_ctrl(s, 1);   // auto exposure on
   s->set_gain_ctrl(s, 1);       // auto gain on
   s->set_aec2(s, 1);            // AEC DSP on (better exposure in dark)
   s->set_ae_level(s, 1);        // +1 AE bias (slightly brighter)
   s->set_aec_value(s, 300);     // initial exposure hint ← NEW
   s->set_agc_gain(s, 0);        // start at gain 0 (let auto take over)
   s->set_gainceiling(s, (gainceiling_t)6); // max gain 128× ← NEW
   s->set_bpc(s, 1);             // bad pixel correction on ← NEW
   s->set_wpc(s, 1);             // white pixel correction on ← NEW
   s->set_raw_gma(s, 1);         // gamma correction on ← NEW
   s->set_lenc(s, 1);            // lens correction (fixes vignetting) ← NEW
   s->set_hmirror(s, 0);
   s->set_vflip(s, 0);
   // Special: OV2640 DCW (down-sample) off gives sharper large frames
   s->set_dcw(s, 0);             // ← NEW
 }
 cameraActive = true;
 Serial.println("Camera initialized (UXGA, quality=8, full sensor tuning).");
}
// ─── HTTP body reader ──────────────────────────────────────────────
static size_t readBodyFixed(WiFiClientSecure& c, char* buf, size_t maxLen) {
 size_t total = 0;
 uint32_t t = millis();
 while ((c.connected() || c.available()) && total  8000) break;
   if (!c.available()) { delay(10); continue; }
   int n = c.read((uint8_t*)buf + total, maxLen - 1 - total);
   if (n > 0) { total += n; t = millis(); }
 }
 buf[total] = '\0';
 return total;
}
static void skipHeaders(WiFiClientSecure& c) {
 while (c.connected()) {
   String line = c.readStringUntil('\n');
   if (line == "\r" || line.length() = 3) return "Connection Error";
   delay(1500);
 }
 Serial.println("Connected!");
 const char* bnd = "----ESP32Boundary";
 char partHead[256];
 int phLen = snprintf(partHead, sizeof(partHead),
   "--%s\r\n"
   "Content-Disposition: form-data; name=\"imageFile\"; filename=\"snap.jpg\"\r\n"
   "Content-Type: image/jpeg\r\n\r\n", bnd);
 char partTail[64];
 int ptLen = snprintf(partTail, sizeof(partTail), "\r\n--%s--\r\n", bnd);
 size_t contentLen = phLen + len + ptLen;
 client.printf(
   "POST %s HTTP/1.1\r\n"
   "Host: %s\r\n"
   "X-API-Key: %s\r\n"
   "Content-Type: multipart/form-data; boundary=%s\r\n"
   "Content-Length: %u\r\n"
   "Connection: close\r\n\r\n",
   serverPath, serverName, API_KEY, bnd, (unsigned)contentLen);
 client.write((const uint8_t*)partHead, phLen);
 for (size_t off = 0; off  String {
   const char* p = strstr(respBuf, key);
   if (!p) return "";
   p += strlen(key);
   const char* e = strchr(p, '"');
   if (!e) return "";
   return String(p).substring(0, e - p);
 };
 currency = findField("\"label\":\"");
 if (currency == "") currency = findField("\"class\":\"");
 if (currency == "") currency = findField("\"class_name\":\"");
 if (currency == "") currency = findField("\"denomination\":\"");
 if (currency == "" && (strstr(respBuf,"no_currency_found") || strstr(respBuf,"no_detections")))
   currency = "No currency detected";
 if (currency == "") currency = "Unknown currency";
 const char* rupeeVals[] = {"10","20","50","100","200","500","2000"};
 for (auto v : rupeeVals) if (currency == v) { currency += " rupees"; break; }
 return currency;
}
// ─── Phase 3: TTS Download ─────────────────────────────────────────
static uint8_t* downloadTTS(const String& text, size_t* outLen) {
 *outLen = 0;
 String encoded = text;
 encoded.replace(" ", "+");
 String path = "/translate_tts?ie=UTF-8&q=" + encoded + "&tl=en&client=tw-ob";
 WiFiClient httpClient;
 if (!httpClient.connect("translate.google.com", 80)) {
   Serial.println("[TTS] HTTP connect failed"); return nullptr;
 }
 httpClient.printf(
   "GET %s HTTP/1.1\r\nHost: translate.google.com\r\nUser-Agent: Mozilla/5.0\r\nConnection: close\r\n\r\n",
   path.c_str());
 uint32_t t = millis();
 while (!httpClient.available() && millis() - t  0 ? contentLength + 64 : 131072;
 uint8_t* mp3buf = nullptr;
 if (psramFound() && ESP.getFreePsram() > bufCap + 65536)
   mp3buf = (uint8_t*)ps_malloc(bufCap);
 if (!mp3buf) {
   Serial.printf("[TTS] Cannot allocate %u bytes\n", (unsigned)bufCap);
   httpClient.stop(); return nullptr;
 }
 size_t total = 0;
 t = millis();
 while ((httpClient.connected() || httpClient.available()) && total  15000) break;
   if (!httpClient.available()) { delay(5); continue; }
   int n = httpClient.read(mp3buf + total, bufCap - total);
   if (n > 0) { total += n; t = millis(); }
 }
 httpClient.stop();
 if (total == 0) { Serial.println("[TTS] No MP3 data"); free(mp3buf); return nullptr; }
 Serial.printf("[TTS] Downloaded %u bytes\n", (unsigned)total);
 *outLen = total;
 return mp3buf;
}
// ─── Phase 4: Playback ─────────────────────────────────────────────
static void playMP3(const uint8_t* mp3data, size_t mp3len) {
 if (!mp3data || mp3len == 0) return;
 printHeap("before playMP3");
 AudioFileSourcePROGMEM* src = new AudioFileSourcePROGMEM(mp3data, mp3len);
 AudioGeneratorMP3* mp3 = new AudioGeneratorMP3();
 if (mp3->begin(src, g_audioOut)) {
   while (mp3->isRunning()) { if (!mp3->loop()) mp3->stop(); }
   Serial.println("[TTS] Playback complete.");
 } else {
   Serial.println("[TTS] MP3 begin() failed");
 }
 delete mp3; delete src;
}
// ═══════════════════════════════════════════════════════════════════
// WEB SERVER — handlers
// ═══════════════════════════════════════════════════════════════════
// ── Dashboard HTML ─────────────────────────────────────────────────
static const char DASHBOARD_HTML[] PROGMEM = R"rawhtml(










 
 


 

   
   
CAM
   

     
     
       Detecting…
     
   
 
 

   

     
Last Detection
     
—
   
   
 
 

   Heap: —
   PSRAM: —
   Uptime: —
 




)rawhtml";
// ── Handler: dashboard ─────────────────────────────────────────────
void handleRoot() {
 webServer.send_P(200, "text/html", DASHBOARD_HTML);
}
// ── Handler: MJPEG stream ──────────────────────────────────────────
void handleStream() {
 if (!cameraActive) {
   webServer.send(503, "text/plain", "Camera busy during detection");
   return;
 }
 WiFiClient client = webServer.client();
 client.print(
   "HTTP/1.1 200 OK\r\n"
   "Content-Type: multipart/x-mixed-replace; boundary=frame\r\n"
   "Cache-Control: no-cache\r\n"
   "Connection: close\r\n\r\n");
 while (client.connected()) {
   if (!cameraActive) break;           // detection started — bail out
   camera_fb_t* fb = esp_camera_fb_get();
   if (!fb) { delay(30); continue; }
   client.printf(
     "--frame\r\n"
     "Content-Type: image/jpeg\r\n"
     "Content-Length: %u\r\n\r\n", (unsigned)fb->len);
   client.write(fb->buf, fb->len);
   client.print("\r\n");
   esp_camera_fb_return(fb);
   delay(50);  // ~20 fps cap — raise to 33ms for ~30 fps if your network handles it
 }
}
// ── Handler: single snapshot ───────────────────────────────────────
void handleCapture() {
 if (!cameraActive) {
   webServer.send(503, "text/plain", "Camera busy");
   return;
 }
 camera_fb_t* fb = esp_camera_fb_get();
 if (!fb) { webServer.send(500, "text/plain", "Capture failed"); return; }
 webServer.sendHeader("Content-Disposition", "inline; filename=\"snap.jpg\"");
 webServer.sendHeader("Cache-Control", "no-cache");
 webServer.send_P(200, "image/jpeg", (const char*)fb->buf, fb->len);
 esp_camera_fb_return(fb);
}
// ── Handler: JSON status ───────────────────────────────────────────
void handleStatus() {
 char buf[256];
 snprintf(buf, sizeof(buf),
   "{\"currency\":\"%s\","
   "\"busy\":%s,"
   "\"heap_free\":%u,"
   "\"psram_free\":%u,"
   "\"uptime_s\":%lu}",
   lastCurrency.c_str(),
   cameraActive ? "false" : "true",
   (unsigned)ESP.getFreeHeap(),
   (unsigned)ESP.getFreePsram(),
   millis() / 1000UL);
 webServer.send(200, "application/json", buf);
}
// ── Web server task (Core 0) ───────────────────────────────────────
static void webServerTask(void*) {
 webServer.on("http://www.bing.com/",        handleRoot);
 webServer.on("/stream",  handleStream);
 webServer.on("http://www.bing.com/capture", handleCapture);
 webServer.on("/status",  handleStatus);
 webServer.begin();
 Serial.println("[WEB] Server started on port 80");
 for (;;) { webServer.handleClient(); delay(2); }
}
// ═══════════════════════════════════════════════════════════════════
// DETECTION FLOW
// ═══════════════════════════════════════════════════════════════════
void detectCurrency() {
 printHeap("detectCurrency start");
 // Phase 1: Capture
 camera_fb_t* fb = esp_camera_fb_get();
 if (fb) esp_camera_fb_return(fb);
 delay(250);
 fb = esp_camera_fb_get();
 if (!fb) { Serial.println("Capture failed!"); return; }
 size_t imgLen = fb->len;
 Serial.printf("Frame: %u bytes\n", (unsigned)imgLen);
 uint8_t* imgBuf = nullptr;
 if (psramFound() && ESP.getFreePsram() > imgLen + 65536)
   imgBuf = (uint8_t*)ps_malloc(imgLen);
 if (!imgBuf && ESP.getMaxAllocHeap() > imgLen + 32768)
   imgBuf = (uint8_t*)malloc(imgLen);
 if (!imgBuf) {
   Serial.println("No memory for frame!");
   esp_camera_fb_return(fb);
   return;
 }
 memcpy(imgBuf, fb->buf, imgLen);
 esp_camera_fb_return(fb);
 // Deinit camera — signal web task to stop streaming
 cameraActive = false;
 delay(50);             // let any in-flight stream handler finish its current frame
 esp_camera_deinit();
 delay(150);
 printHeap("after cam deinit");
 // Phase 2: API
 String currency = sendImageToAPI(imgBuf, imgLen);
 free(imgBuf);
 lastCurrency = currency;
 Serial.println("=== Detection Result ===");
 Serial.println(currency);
 Serial.println("========================");
 printHeap("after API, before TTS download");
 // Phase 3: Download TTS
 size_t mp3Len = 0;
 uint8_t* mp3Buf = downloadTTS(currency, &mp3Len);
 // Phase 4: Play
 if (mp3Buf) { playMP3(mp3Buf, mp3Len); free(mp3Buf); }
 else         { Serial.println("[TTS] Skipping audio (download failed)"); }
 // Reinit camera and re-enable stream
 initCamera();
 cameraActive = true;
 printHeap("detectCurrency end");
}
// ═══════════════════════════════════════════════════════════════════
// SETUP / LOOP
// ═══════════════════════════════════════════════════════════════════
void setup() {
 WRITE_PERI_REG(RTC_CNTL_BROWN_OUT_REG, 0);
 Serial.begin(115200);
 delay(500);
 pinMode(TRIGGER_BTN, INPUT_PULLUP);
 initCamera();
 g_audioOut = new AudioOutputI2SNoDAC();
 g_audioOut->SetPinout(14, 15, SPEAKER_PIN);
 WiFi.mode(WIFI_STA);
 WiFi.setSleep(false);
 WiFi.begin(WIFI_SSID, WIFI_PASS);
 Serial.print("Connecting to WiFi");
 while (!WiFi.isConnected()) { delay(500); Serial.print("."); }
 Serial.printf("\nConnected: %s\n", WiFi.localIP().toString().c_str());
 // Print the URL prominently
 Serial.println("─────────────────────────────────────────");
 Serial.printf("  WEB PREVIEW → http://%s\n", WiFi.localIP().toString().c_str());
 Serial.printf("  MJPEG STREAM → http://%s/stream\n", WiFi.localIP().toString().c_str());
 Serial.println("─────────────────────────────────────────");
 configTime(19800, 0, "pool.ntp.org", "time.nist.gov");
 Serial.print("Syncing time");
 for (int i = 0; i  100000 ? "\nTime synced!" : "\nTime sync timeout");
 // Start web server on Core 0 (loop runs on Core 1 by default)
 xTaskCreatePinnedToCore(webServerTask, "WebSrv", 8192, nullptr, 1, nullptr, 0);
 printHeap("setup complete");
 Serial.println("Ready — press button to detect currency.");
}
void loop() {
 if (digitalRead(TRIGGER_BTN) == LOW && millis() - lastTrigger > 2000) {
   lastTrigger = millis();
   delay(150);
   Serial.println("Button pressed! Detecting currency...");
   detectCurrency();
 }
 delay(10);
}