LeiaSR SDK 720218b2 v1.32.7.6322 2025-02-13T14:55:38Z
Stable
Example C++ & OpenGL

1
5// External dependencies
6#include <GL/glew.h>
7#include <GLFW/glfw3.h>
8GLFWwindow* window;
9#include <glm/glm.hpp>
10#include <glm/gtc/matrix_transform.hpp>
11#include <thread>
12#include <chrono>
13
14#define GLFW_EXPOSE_NATIVE_WIN32
15#define GLFW_EXPOSE_NATIVE_WGL
16#include "glfw3native.h"
17
18#include <shellscalingapi.h>
19#undef near
20#undef far
21
22// Internal dependencies
23#include "pyramid.h"
24#include "shader.h"
25#include "sr/weaver/glweaver.h"
26
27// Simulated Reality includes
28#include "sr/types.h"
36
37#define WEAVING_ENABLED
38
39using namespace std::chrono_literals;
40
41// Flag to indicate that context is no longer valid
42bool contextValid = false;
43
44bool initializeSrObjects(); // forward declaration
45
46// User implementation of the SystemEventListener interface
47class SystemEventMonitor : public SR::SystemEventListener {
48public:
49 // Ensures SystemEventStream is cleaned up when Listener object is out of scope
51
52 // The accept function can process the system event data as soon as it becomes available
53 virtual void accept(const SR::SystemEvent& frame) override {
54 switch (frame.eventType) {
56 {
57 std::cout
58 << "ContextInvalid event received: "
59 << frame.time << " "
60 << frame.message << "\n";
61
62 contextValid = false;
63
64 std::thread([]() {
65 initializeSrObjects();
66 }).detach();
67
68 return;
69 }
70 break;
71 default:
72 std::cout << "Unknown event type" << std::endl;
73 break;
74 }
75 }
76
77};
78
79// My SR::EyePairListener that stores the last tracked eye positions
80class MyEyes : public SR::EyePairListener {
81private:
83public:
84 glm::vec3 left, right;
85 MyEyes(SR::EyeTracker* tracker) : left(-30, 0, 600), right(30, 0, 600) {
86 // Open a stream between tracker and this class
87 stream.set(tracker->openEyePairStream(this));
88 }
89 // Called by the tracker for each tracked eye pair
90 virtual void accept(const SR_eyePair& eyePair) override
91 {
92 // Remember the eye positions
93 left = glm::vec3(eyePair.left.x, eyePair.left.y, eyePair.left.z);
94 right = glm::vec3(eyePair.right.x, eyePair.right.y, eyePair.right.z);
95 }
96};
97
98// My SR::HandPoseListener that stores the last tracked index finger position
99class MyFinger : public SR::HandPoseListener {
100private:
102public:
103 glm::vec3 position;
104 MyFinger(SR::HandTracker* tracker) : position(0, 0, 0) {
105 // Open a stream between tracker and this class
106 stream.set(tracker->openHandPoseStream(this));
107 }
108 // Called by the tracker for each tracked hand pose
109 virtual void accept(const SR_handPose& handPose) override
110 {
111 // Remember the fingertip position
112 position = glm::vec3(handPose.index.tip.x, handPose.index.tip.y, handPose.index.tip.z);
113 }
114};
115
116// Create SRContext
117SR::SRContext* context = nullptr;
118SR::PredictingGLWeaver* weaver = nullptr;
119std::mutex constructNewContextMutex;
120MyEyes* eyes = nullptr;
121SystemEventMonitor* listener = nullptr;
122
123bool createSrContext() {
124 if (context != nullptr) {
125 delete context;
126 context = nullptr;
127 }
128 while (context == nullptr && contextValid == false) {
129 try {
130 context = new SR::SRContext();
131 return true;
132 }
134 std::cout << "Server not available, trying again in 0.5 second" << std::endl;
135 std::this_thread::sleep_for(500ms);
136 }
137 }
138 return false;
139}
140
141bool initializeSrObjects() {
142 std::lock_guard<std::mutex> lock(constructNewContextMutex);
143 // weaver needs to be deleted before deleting SRContext as weaver is using the context, but this function does not reconstruct weaver. Construct weaver wherever is needed
144
145 if (eyes != nullptr) {
146 delete eyes;
147 eyes = nullptr;
148 }
149 if (listener != nullptr) {
150 delete listener;
151 listener = nullptr;
152 }
153
154 while (weaver != nullptr) {
155 //wait for it
156 std::this_thread::sleep_for(10ms);
157 }
158 // constructing context
159 if (createSrContext() == false) {
160 return false;
161 }
162 // constructing EyePairListener
163 eyes = new MyEyes(SR::EyeTracker::create(*context));
164
165 SR::SystemSense* systemSense = SR::SystemSense::create(*context);
166 listener = new SystemEventMonitor();
167 // set systemEvent listener to the newly constructed systemsense
168 listener->stream.set(systemSense->openSystemEventStream(listener));
169
170 context->initialize();
171 contextValid = true;
172 return true;
173}
174
175glm::mat4 CalculateModel()
176{
177 const float size = 20.0; // Size of the object in mm
178 using namespace std::chrono;
179 auto now = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
180 const int ms_per_revolution = 5000;
181 float angle = float((2 * 3.1415 / ms_per_revolution) * (now % ms_per_revolution));
182
183 glm::mat4 Model = glm::mat4();
184 Model = scale(Model, glm::vec3(size, size, size)); //scale by size
185 Model = translate(Model, glm::vec3(0, 1, 0)); //translate bottom face to origin
186 Model = rotate(Model, angle, glm::vec3(0, 1, 0)); //rotate by angle
187
188 return Model;
189}
190
191float screenWidth_mm = 697.0;
192float screenHeight_mm = 394.0;
193
194glm::mat4 CalculateProjection(glm::vec3 &eye)
195{
196 // Implementation of: Kooima, Robert. "Generalized perspective projection." J. Sch. Electron. Eng. Comput. Sci (2009).
197
198 const float near = 1.0;
199 const float far = 10000.0;
200 const glm::vec3 pa(-screenWidth_mm / 2, screenHeight_mm / 2, 0);
201 const glm::vec3 pb(screenWidth_mm / 2, screenHeight_mm / 2, 0);
202 const glm::vec3 pc(-screenWidth_mm / 2, -screenHeight_mm / 2, 0);
203
204 const glm::vec3 vr(1, 0, 0);
205 const glm::vec3 vu(0, 1, 0);
206 const glm::vec3 vn(0, 0, 1);
207
208 // Compute the screen corner vectors.
209 glm::vec3 va = pa - eye;
210 glm::vec3 vb = pb - eye;
211 glm::vec3 vc = pc - eye;
212
213 // Find the distance from the eye to screen plane.
214 float distance = -dot(va, vn);
215
216 // Find the extent of the perpendicular projection.
217 float l = dot(vr, va) * near / distance;
218 float r = dot(vr, vb) * near / distance;
219 float b = dot(vu, vc) * near / distance;
220 float t = dot(vu, va) * near / distance;
221
222 // Load the perpendicular projection
223 glm::mat4 _frustum = glm::frustum(l, r, b, t, near, far);
224
225 // Rotate the projection to be non-perpendicular
226 glm::mat4 M = {
227 vr[0], vu[0], vn[0], 0,
228 vr[1], vu[1], vn[1], 0,
229 vr[2], vu[2], vn[2], 0,
230 0, 0, 0, 1.0,
231 }; //(formatting is transposed from the paper!)
232
233 // Move the apex of the frustum to the origin.
234 glm::mat4 _translate = translate(glm::mat4(), -eye);
235
236 // Combine
237 return _frustum * M * _translate;
238}
239
240void RenderScene(GLuint MatrixID, const size_t renderWidth, const size_t renderHeight, glm::vec3 leftEye, glm::vec3 rightEye, Pyramid& pyramid) {
242 // Update Model to put object on the finger
243 glm::mat4 View = glm::mat4();
244 glm::mat4 Model = CalculateModel();
245
246 // Set projection for left rendering
247 glm::mat4 Projection = CalculateProjection(leftEye);
248 glm::mat4 MVP = Projection * View * Model;
249 glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP[0][0]);
250
251 // Set viewport to the left half
252 glViewport(0, 0, renderWidth, renderHeight);
253
254 // Render the scene for the left eye
255 pyramid.draw();
256
257 // Set projection for right rendering
258 Projection = CalculateProjection(rightEye);
259 MVP = Projection * View * Model;
260 glUniformMatrix4fv(MatrixID, 1, GL_FALSE, &MVP[0][0]);
261
262 // Set viewport to the right half
263 glViewport(renderWidth, 0, renderWidth, renderHeight);
264
265 // Render the scene for the right eye
266 pyramid.draw();
268}
269
270// Returns true if function completed succesfully, returns false otherwise
271bool ensureWindowFitsToMonitor() {
272 // Get monitor that the window is currently located on.
273 HMONITOR windowMonitor = MonitorFromWindow(glfwGetWin32Window(window), MONITOR_DEFAULTTONEAREST);
274
275 // If no monitor handle was returned (no monitor may be attached), don't change window and return false
276 if (!windowMonitor) {
277 return false;
278 }
279
280 // Get monitor rectangle in the virtual screen
281 MONITORINFO monitorInfo;
282 ZeroMemory(&monitorInfo, sizeof(monitorInfo));
283 monitorInfo.cbSize = sizeof(monitorInfo);
284 GetMonitorInfoA(windowMonitor, &monitorInfo);
285
286 // Get current window rectangle
287 RECT windowRect;
288 GetWindowRect(glfwGetWin32Window(window), &windowRect);
289
290 // If the monitor rectangle is different from the current window rectangle...
291 if (windowRect.left != monitorInfo.rcMonitor.left ||
292 windowRect.right != monitorInfo.rcMonitor.right ||
293 windowRect.top != monitorInfo.rcMonitor.top ||
294 windowRect.bottom != monitorInfo.rcMonitor.bottom) {
295 //...set the window rectangle to fit the monitor rectangle
296 glfwSetWindowPos(window,
297 monitorInfo.rcMonitor.left,
298 monitorInfo.rcMonitor.top
299 );
300 glfwSetWindowSize(window,
301 monitorInfo.rcMonitor.right - monitorInfo.rcMonitor.left,
302 monitorInfo.rcMonitor.bottom - monitorInfo.rcMonitor.top
303 );
304 }
305
306 // Return true, as function has completed succesfully
307 return true;
308}
309
310bool windowFitsToMonitor = false;
311
312void windowMoveCallback(GLFWwindow* window, int xPos, int yPos) {
313 windowFitsToMonitor = false;
314}
315
316void monitorConfigurationCallback(GLFWmonitor* monitor, int event) {
317 windowFitsToMonitor = false;
318}
319
320int main(void)
321{
322 // Ensure the application receives unscaled display metrics
323 SetProcessDpiAwareness(PROCESS_DPI_AWARENESS::PROCESS_PER_MONITOR_DPI_AWARE);
324
325 // Initialise GLFW
326 if (!glfwInit())
327 {
328 fprintf(stderr, "Failed to initialize GLFW\n");
329 return -1;
330 }
331
332 glfwWindowHint(GLFW_SAMPLES, 4);
333 glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 3);
334 glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 3);
335 glfwWindowHint(GLFW_OPENGL_PROFILE, GLFW_OPENGL_COMPAT_PROFILE);
336 glfwWindowHint(GLFW_AUTO_ICONIFY, GL_FALSE);
337 glfwWindowHint(GLFW_DECORATED, GLFW_FALSE);
338
339 // Open a window and create its OpenGL context
340 GLFWmonitor* monitor = glfwGetPrimaryMonitor();
341 const GLFWvidmode* mode = glfwGetVideoMode(monitor);
342 window = glfwCreateWindow(mode->width, mode->height, "Simulated Reality - Cube demo", NULL, NULL);
343 if (window == NULL) {
344 fprintf(stderr, "Failed to open GLFW window.\n");
345 glfwTerminate();
346 return -1;
347 }
348 windowFitsToMonitor = true;
349 glfwMakeContextCurrent(window);
350 glfwSwapInterval(1);
351
352 // Initialize GLEW
353 glewExperimental = true; // Needed for core profile
354 if (glewInit() != GLEW_OK) {
355 fprintf(stderr, "Failed to initialize GLEW\n");
356 glfwTerminate();
357 return -1;
358 }
359
360 // Ensure we can capture the escape key being pressed below
361 glfwSetInputMode(window, GLFW_STICKY_KEYS, GL_TRUE);
362 glfwSetWindowPosCallback(window, windowMoveCallback);
363 glfwSetMonitorCallback(monitorConfigurationCallback);
364
365 glClearColor(0.5f, 0.5f, 0.5f, 0.0);
366 glEnable(GL_DEPTH_TEST);
367 glDepthFunc(GL_LESS);
368 GLuint programID = loadBasicShaders();
369 GLuint MatrixID = glGetUniformLocation(programID, "MVP");
370
372 initializeSrObjects();
373 MyFinger finger(SR::HandTracker::create(*context));
374 SR::Screen* screen = SR::Screen::create(*context);
376
377 glUseProgram(programID);
378
379 // Get the screen width and height in millimeters
380 int screenWidth_mm_i, screenHeight_mm_i;
381 glfwGetMonitorPhysicalSize(monitor, &screenWidth_mm_i, &screenHeight_mm_i);
382 screenWidth_mm = screenWidth_mm_i;
383 screenHeight_mm = screenHeight_mm_i;
384
385 //Allocate VAO for pyramid object
386 Pyramid pyramid;
387
388#ifdef WEAVING_ENABLED
389 // Render to lower resolution than native display resolution because three-dimensional image won't convey any more detail and we can save processing power.
390 // The weaved image will be native display resolution.
391 // Full-HD resolution for each view is ideal for 4K displays.
392 const size_t renderWidth = screen->getPhysicalResolutionWidth() / 2;
393 const size_t renderHeight = screen->getPhysicalResolutionHeight() / 2;
394#else
395 // Render to half of the resolution that the display is presenting itself as.
396 // Devices like the SR Development Kit present themselves as a display with 7680 x 2160 resolution to allow for rendering at 3840 x 2160 for each view.
397 // This is ideal for 8K displays like it.
398 const size_t renderWidth = mode->width * 0.5f;
399 const size_t renderHeight = mode->height;
400#endif
401
402#ifdef WEAVING_ENABLED
404 weaver = new SR::PredictingGLWeaver(*context, renderWidth * 2, renderHeight, glfwGetWin32Window(window));
405
406 context->initialize();
408#endif
409
410 bool lKeyPressed = false;
411
412 do {
413
414 // If the user presses the L key, toggle late latching.
415 bool prevLKeyPressed = lKeyPressed;
416 lKeyPressed = glfwGetKey(window, GLFW_KEY_L) == GLFW_PRESS;
417 if (lKeyPressed && !prevLKeyPressed)
418 weaver->enableLateLatching(!weaver->isLateLatchingEnabled());
419
420#ifdef WEAVING_ENABLED
422 // Draw to weaver with application specific shaders
423 if (contextValid && weaver != nullptr) {
424 glBindFramebuffer(GL_FRAMEBUFFER, weaver->getFrameBuffer());
425 }
426 else {
427 glBindFramebuffer(GL_FRAMEBUFFER, 0); //Start rendering to the display
428 }
430#endif
431 glUseProgram(programID);
432 glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
433 if (contextValid) {
434 RenderScene(MatrixID, renderWidth, renderHeight, eyes->left, eyes->right, pyramid);
435 }
436 else {
437 RenderScene(MatrixID, renderWidth, renderHeight, { -30, 0, 600 }, { 30, 0, 600 }, pyramid);
438 }
439
440#ifdef WEAVING_ENABLED
441 int windowWidth, windowHeight;
442 glfwGetFramebufferSize(window, &windowWidth, &windowHeight);
444 glBindFramebuffer(GL_FRAMEBUFFER, 0); //Start rendering to the display
445 // glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT); does not need to be called because weaving overwrites each pixel on the backbuffer
446 glViewport(0, 0, windowWidth, windowHeight);
447
448 // For weaving, the center-point between both eyes is used.
449 // It should be converted from millimeters to centimeters
450 if (contextValid && weaver != nullptr) {
451 weaver->weave((unsigned int)windowWidth, (unsigned int)windowHeight, 0, 0);
452 }
454#endif
455
456 // Swap buffers
457 glfwSwapBuffers(window);
458 glfwPollEvents();
459
460 if (!windowFitsToMonitor) {
461 windowFitsToMonitor = ensureWindowFitsToMonitor();
462 }
463
464#ifdef WEAVING_ENABLED
465 // initialize weaver if it was deleted before
466 if (contextValid && weaver == nullptr) {
468 weaver = new SR::PredictingGLWeaver(*context, renderWidth * 2, renderHeight, glfwGetWin32Window(window));
469
470 context->initialize();
472 }
473
474 // delete weaver if the SRContext is not valid
475 if (contextValid == false && weaver != nullptr) {
476 delete weaver;
477 weaver = nullptr;
478 }
479#endif
480
481 } while (
482 glfwGetKey(window, GLFW_KEY_ESCAPE) != GLFW_PRESS
483 && glfwWindowShouldClose(window) == 0
484 );
485
486#ifdef WEAVING_ENABLED
487 // Delete weaver GL resources before OpenGL context is deleted
488 if (weaver != nullptr) {
489 delete weaver;
490 weaver = nullptr;
491 }
492#endif
493
494 glDeleteProgram(programID);
495
496 glfwTerminate();
497
498 if (contextValid == false) {
499 contextValid = true;
500 std::this_thread::sleep_for(3000ms);
501 }
502}
void * SRContext
Definition: admin_c.h:10
Interface for listening to SR_eyePair updates.
Definition: eyepairlistener.h:15
virtual void accept(const SR_eyePair &frame)=0
Accept an SR_eyePair frame.
Sense class which provides face tracking functionality to the SR system.
Definition: eyetracker.h:35
static EyeTracker * create(SRContext &context)
Creates a functional EyeTracker instance.
virtual std::shared_ptr< EyePairStream > openEyePairStream(EyePairListener *listener)=0
Creates a EyePairStream for listener to be connected to.
GLuint getFrameBuffer()
Returns the buffer that will be used to create a weaved imaged. This buffer expects a side-by-side im...
void enableLateLatching(bool enable)
Enables late latching. Note that late latching requires applications to call weave() once per frame,...
void weave(unsigned int width, unsigned int height)
Can be called to render a weaved image of a certain size to the currently bound framebuffer.
bool isLateLatchingEnabled() const
Determines if late latching is enabled.
Interface for listening to SR_handPose updates.
Definition: handposelistener.h:34
virtual void accept(const SR_handPose &handPose)=0
Accept an SR_handPose frame.
Sense class which provides hand tracking functionality to the SR system.
Definition: handtracker.h:41
virtual std::shared_ptr< HandPoseStream > openHandPoseStream(HandPoseListener *listener)=0
Creates a HandPoseStream for listener to be connected to.
static HandTracker * create(SRContext &context)
Creates a functional HandTracker instance.
Template class to wrap data stream to a listener object.
Definition: inputstream.h:20
void set(std::shared_ptr< StreamType > input)
Must be set to allow proper cleanup of internal stream.
Definition: inputstream.h:48
Definition: glweaver.h:216
Maintains WorldObject and Sense objects during the application lifetime.
Definition: srcontext.h:75
void initialize()
Initialize all senses.
Class of WorldObject representing the screen in real space.
Definition: screen.h:34
const int getPhysicalResolutionWidth()
Get the horizontal native resolution of the physical screen itself.
static Screen * create(SRContext &context)
Creates an instance of a Screen class.
const int getPhysicalResolutionHeight()
Get the vertical native resolution of the physical screen itself.
Class of Exception which indicates that a SR Service was not available to connect with.
Definition: exception.h:96
Interface for listening to SR_systemEvent updates.
Definition: systemeventlistener.h:25
virtual void accept(const SystemEvent &frame)=0
Accept an SR_systemEvent frame.
Class containing dynamic-length event messages.
Definition: systemevent.h:60
uint64_t time
Time since epoch in microseconds.
Definition: systemevent.h:62
SR_eventType eventType
Type of the event.
Definition: systemevent.h:63
std::string message
Full description of the event.
Definition: systemevent.h:64
Sense class which shares information about the SR system throughout the SR system and to applications...
Definition: systemsense.h:36
static SystemSense * create(SRContext &context)
Creates a functional SystemSense instance.
virtual std::shared_ptr< SystemEventStream > openSystemEventStream(SystemEventListener *listener)=0
Creates a SystemEventStream for listener to be connected to.
unsigned int GLuint
Definition: glweaver.h:23
C-compatible struct containing the position of two eyes.
Definition: eyepair.h:23
SR_point3d left
Absolute position of left eye in millimeter.
Definition: eyepair.h:29
SR_point3d right
Absolute position of right eye in millimeter.
Definition: eyepair.h:30
C-compatible struct containing the pose of a hand.
Definition: handpose.h:119
SR_finger index
Index finger.
Definition: handpose.h:131
@ ContextInvalid
Context needs to be re-initialized for the application to recover.
Definition: systemevent.h:14
SR_point3d tip
distal alias
Definition: handpose.h:59
double y
Second value in the 3d vector.
Definition: types.h:77
double z
Third value in the 3d vector.
Definition: types.h:78
double x
First value in the 3d vector.
Definition: types.h:76