XR_ML_world_mesh_detection
This API is still an experimental extension not included in the official OpenXR registry and is subject to change.
12.110. XR_ML_world_mesh_detection
- Name String
XR_ML_world_mesh_detection- Extension Type
Instance extension
- Registered Extension Number
475
- Revision
1
- Extension and Version Dependencies
- Last Modified Date
2023-08-29
- Contributors
Ron Bessems, Magic Leap
Karthik Kadappan, Magic Leap
12.110.2. Creating a world mesh detector
The XrWorldMeshDetectorML handle is defined as:
// Provided by XR_ML_world_mesh_detection
XR_DEFINE_HANDLE(XrWorldMeshDetectorML)XrWorldMeshDetectorML is created by xrCreateWorldMeshDetectorML.
The xrCreateWorldMeshDetectorML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrCreateWorldMeshDetectorML(
XrSession session,
const XrWorldMeshDetectorCreateInfoML* createInfo,
XrWorldMeshDetectorML* detector);Permissions Android applications must have the com.magicleap.permission.SPATIAL_MAPPING permission listed in their manifest to use this extension. (protection level: dangerous) |
The XrWorldMeshDetectorCreateInfoML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshDetectorCreateInfoML {
XrStructureType type;
const void* next;
} XrWorldMeshDetectorCreateInfoML;12.110.3. Destroying a world mesh detector
The xrDestroyWorldMeshDetectorML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrDestroyWorldMeshDetectorML(
XrWorldMeshDetectorML detector);12.110.4. Detecting the World Mesh
Detecting the world mesh is done by blocks. Instead of returning the whole world mesh as one entity this system returns the world mesh in chunks called blocks. To retrieve the currently detected chunks usexrRequestWorldMeshStateAsyncML.
The xrRequestWorldMeshStateAsyncML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrRequestWorldMeshStateAsyncML(
XrWorldMeshDetectorML detector,
const XrWorldMeshStateRequestInfoML* stateRequest,
XrFutureEXT* future);The XrWorldMeshStateRequestInfoML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshStateRequestInfoML {
XrStructureType type;
const void* next;
XrSpace baseSpace;
XrTime time;
XrPosef boundingBoxCenter;
XrExtent3DfEXT boundingBoxExtents;
} XrWorldMeshStateRequestInfoML;Each mesh block may have a 'skirt' ifXR_WORLD_MESH_DETECTOR_REMOVE_MESH_SKIRT_BIT_ML was not specified during the creation of the XrWorldMeshDetectorML. A skirt provides some overlap between adjacent mesh blocks. While a skirt improves coverage between blocks, it does not create a watertight mesh.
// Provided by XR_ML_world_mesh_detection
typedef XrFlags64 XrWorldMeshDetectorFlagsML;// Provided by XR_ML_world_mesh_detection
// Flag bits for XrWorldMeshDetectorFlagsML
static const XrWorldMeshDetectorFlagsML XR_WORLD_MESH_DETECTOR_POINT_CLOUD_BIT_ML = 0x00000001;
static const XrWorldMeshDetectorFlagsML XR_WORLD_MESH_DETECTOR_COMPUTE_NORMALS_BIT_ML = 0x00000002;
static const XrWorldMeshDetectorFlagsML XR_WORLD_MESH_DETECTOR_COMPUTE_CONFIDENCE_BIT_ML = 0x00000004;
static const XrWorldMeshDetectorFlagsML XR_WORLD_MESH_DETECTOR_PLANARIZE_BIT_ML = 0x00000008;
static const XrWorldMeshDetectorFlagsML XR_WORLD_MESH_DETECTOR_REMOVE_MESH_SKIRT_BIT_ML = 0x00000010;
static const XrWorldMeshDetectorFlagsML XR_WORLD_MESH_DETECTOR_INDEX_ORDER_CW_BIT_ML = 0x00000020;xrRequestWorldMeshStateAsyncML is an asynchronous function andxrRequestWorldMeshStateCompleteML retrieves the data once the future is ready.
The xrRequestWorldMeshStateCompleteML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrRequestWorldMeshStateCompleteML(
XrWorldMeshDetectorML detector,
XrFutureEXT future,
XrWorldMeshStateRequestCompletionML* completion);The XrWorldMeshStateRequestCompletionML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshStateRequestCompletionML {
XrStructureType type;
void* next;
XrResult futureResult;
XrTime timestamp;
uint32_t meshBlockStateCapacityInput;
uint32_t meshBlockStateCountOutput;
XrWorldMeshBlockStateML* meshBlockStates;
} XrWorldMeshStateRequestCompletionML;The XrWorldMeshBlockStateML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshBlockStateML {
XrStructureType type;
void* next;
XrUuidEXT uuid;
XrPosef meshBoundingBoxCenter;
XrExtent3DfEXT meshBoundingBoxExtents;
XrTime lastUpdatedTimestamp;
XrWorldMeshBlockStatusML status;
} XrWorldMeshBlockStateML;// Provided by XR_ML_world_mesh_detection
typedef enum XrWorldMeshBlockStatusML {
XR_WORLD_MESH_BLOCK_STATUS_NEW_ML = 0,
XR_WORLD_MESH_BLOCK_STATUS_UPDATED_ML = 1,
XR_WORLD_MESH_BLOCK_STATUS_DELETED_ML = 2,
XR_WORLD_MESH_BLOCK_STATUS_UNCHANGED_ML = 3,
XR_WORLD_MESH_BLOCK_STATUS_MAX_ENUM_ML = 0x7FFFFFFF
} XrWorldMeshBlockStatusML;| Enum | Description |
|---|---|
| The mesh block has been created. |
| The mesh block has been updated. |
| The mesh block has been deleted. |
| The mesh block is unchanged. |
12.110.5. Allocate Mesh Block Memory
The next step is to retrieve the actual vertex data. This operation will require a buffer to be available for the duration of the asynchronous operation and for as long as the application needs access toXrWorldMeshRequestCompletionML.
This buffer must be allocated by the application, the system provides recommended buffer size using the xrGetWorldMeshBufferRecommendSizeMLfunction.
The xrGetWorldMeshBufferRecommendSizeML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrGetWorldMeshBufferRecommendSizeML(
XrWorldMeshDetectorML detector,
const XrWorldMeshBufferRecommendedSizeInfoML* sizeInfo,
XrWorldMeshBufferSizeML* size);The XrWorldMeshBufferRecommendedSizeInfoML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshBufferRecommendedSizeInfoML {
XrStructureType type;
const void* next;
uint32_t maxBlockCount;
} XrWorldMeshBufferRecommendedSizeInfoML;The value for maxBlockCount should be populatedXrWorldMeshStateRequestCompletionML::meshBlockStateCountOutput. As long as the maxBlockCount is equal or larger to thisXrWorldMeshStateRequestCompletionML::meshBlockStateCountOutput, a memory block may be re-used for new requests.
The XrWorldMeshBufferSizeML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshBufferSizeML {
XrStructureType type;
void* next;
uint32_t size;
} XrWorldMeshBufferSizeML;Some devices have optimized memory available that avoids memory copies and provides the fastest way to get the vertex data. Applications should use the xrAllocateWorldMeshBufferML function to reserve memory for the vertex data. The application however may choose to allocate its own memory using alternative methods.
The xrAllocateWorldMeshBufferML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrAllocateWorldMeshBufferML(
XrWorldMeshDetectorML detector,
const XrWorldMeshBufferSizeML* size,
XrWorldMeshBufferML* buffer);The XrWorldMeshBufferML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshBufferML {
XrStructureType type;
void* next;
uint32_t bufferSize;
void* buffer;
} XrWorldMeshBufferML;Memory blocks allocated with xrAllocateWorldMeshBufferML must be freed using xrFreeWorldMeshBufferML.
The xrFreeWorldMeshBufferML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrFreeWorldMeshBufferML(
XrWorldMeshDetectorML detector,
const XrWorldMeshBufferML* buffer);12.110.6. Start Mesh Data Query
Once a buffer has been allocated the mesh data retrieval may be started using xrRequestWorldMeshAsyncML.
The xrRequestWorldMeshAsyncML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrRequestWorldMeshAsyncML(
XrWorldMeshDetectorML detector,
const XrWorldMeshGetInfoML* getInfo,
XrWorldMeshBufferML* buffer,
XrFutureEXT* future);The XrWorldMeshGetInfoML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshGetInfoML {
XrStructureType type;
const void* next;
XrWorldMeshDetectorFlagsML flags;
float fillHoleLength;
float disconnectedComponentArea;
uint32_t blockCount;
XrWorldMeshBlockRequestML* blocks;
} XrWorldMeshGetInfoML;The XrWorldMeshBlockRequestML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshBlockRequestML {
XrStructureType type;
void* next;
XrUuidEXT uuid;
XrWorldMeshDetectorLodML lod;
} XrWorldMeshBlockRequestML;// Provided by XR_ML_world_mesh_detection
typedef enum XrWorldMeshDetectorLodML {
XR_WORLD_MESH_DETECTOR_LOD_MINIMUM_ML = 0,
XR_WORLD_MESH_DETECTOR_LOD_MEDIUM_ML = 1,
XR_WORLD_MESH_DETECTOR_LOD_MAXIMUM_ML = 2,
XR_WORLD_MESH_DETECTOR_LOD_MAX_ENUM_ML = 0x7FFFFFFF
} XrWorldMeshDetectorLodML;| Enum | Description |
|---|---|
| Minimum Level of Detail (LOD) for the mesh. |
| Medium Level of Detail (LOD) for the mesh. |
| Maximum Level of Detail (LOD) for the mesh. |
12.110.7. Complete Mesh Data Query
To complete the previously started mesh data queryxrRequestWorldMeshCompleteML is used.
The xrRequestWorldMeshCompleteML function is defined as:
// Provided by XR_ML_world_mesh_detection
XrResult xrRequestWorldMeshCompleteML(
XrWorldMeshDetectorML detector,
const XrWorldMeshRequestCompletionInfoML* completionInfo,
XrFutureEXT future,
XrWorldMeshRequestCompletionML* completion);The XrWorldMeshRequestCompletionInfoML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshRequestCompletionInfoML {
XrStructureType type;
const void* next;
XrSpace meshSpace;
XrTime meshSpaceLocateTime;
} XrWorldMeshRequestCompletionInfoML;The XrWorldMeshRequestCompletionML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshRequestCompletionML {
XrStructureType type;
void* next;
XrResult futureResult;
uint32_t blockCount;
XrWorldMeshBlockML* blocks;
} XrWorldMeshRequestCompletionML;The XrWorldMeshBlockML structure is defined as:
// Provided by XR_ML_world_mesh_detection
typedef struct XrWorldMeshBlockML {
XrStructureType type;
void* next;
XrUuidEXT uuid;
XrWorldMeshBlockResultML blockResult;
XrWorldMeshDetectorLodML lod;
XrWorldMeshDetectorFlagsML flags;
uint32_t indexCount;
uint16_t* indexBuffer;
uint32_t vertexCount;
XrVector3f* vertexBuffer;
uint32_t normalCount;
XrVector3f* normalBuffer;
uint32_t confidenceCount;
float* confidenceBuffer;
} XrWorldMeshBlockML;normalCount must be equal to vertexCount ifXR_WORLD_MESH_DETECTOR_COMPUTE_NORMALS_BIT_ML was specified duringXrWorldMeshDetectorML creation, otherwise 0.
confidenceCount must be equal to vertexCount ifXR_WORLD_MESH_DETECTOR_COMPUTE_CONFIDENCE_BIT_ML was specified duringXrWorldMeshDetectorML creation, otherwise 0.
// Provided by XR_ML_world_mesh_detection
typedef enum XrWorldMeshBlockResultML {
XR_WORLD_MESH_BLOCK_RESULT_SUCCESS_ML = 0,
XR_WORLD_MESH_BLOCK_RESULT_FAILED_ML = 1,
XR_WORLD_MESH_BLOCK_RESULT_PENDING_ML = 2,
XR_WORLD_MESH_BLOCK_RESULT_PARTIAL_UPDATE_ML = 3,
XR_WORLD_MESH_BLOCK_RESULT_MAX_ENUM_ML = 0x7FFFFFFF
} XrWorldMeshBlockResultML;| Enum | Description |
|---|---|
| Mesh request has succeeded. |
| Mesh request has failed. |
| Mesh request is pending. |
| There are partial updates on the mesh request. |
12.110.8. Sample code
#include <vector>
class MeshDetector {
private:
enum State {
INFO_START, INFO_WAIT_COMPLETE, MESH_START, MESH_WAIT_COMPLETE, DONE
};
// previously initialized functions:
PFN_xrPollFutureEXT xrPollFutureEXT;
PFN_xrCancelFutureEXT xrCancelFutureEXT;
PFN_xrCreateWorldMeshDetectorML xrCreateWorldMeshDetectorML;
PFN_xrDestroyWorldMeshDetectorML xrDestroyWorldMeshDetectorML;
PFN_xrRequestWorldMeshStateAsyncML xrRequestWorldMeshStateAsyncML;
PFN_xrRequestWorldMeshStateCompleteML xrRequestWorldMeshStateCompleteML;
PFN_xrGetWorldMeshBufferRecommendSizeML xrGetWorldMeshBufferRecommendSizeML;
PFN_xrAllocateWorldMeshBufferML xrAllocateWorldMeshBufferML;
PFN_xrFreeWorldMeshBufferML xrFreeWorldMeshBufferML;
PFN_xrRequestWorldMeshAsyncML xrRequestWorldMeshAsyncML;
PFN_xrRequestWorldMeshCompleteML xrRequestWorldMeshCompleteML;
XrInstance m_Instance; // previously initialized.
XrSession m_Session; // previously initialized.
XrSpace m_ViewSpace; // previously initialized.
XrSpace m_LocalSpace; // previously initialized.
State m_State{INFO_START};
XrFutureEXT m_Future{XR_NULL_FUTURE_EXT};
XrWorldMeshDetectorML m_Detector;
std::vector<XrWorldMeshBlockStateML> m_MeshBlocks;
std::array<uint32_t,2> m_MaxBlockCounts{0};
std::array<XrWorldMeshBufferML,2> m_WorldMeshBuffers{XR_TYPE_WORLD_MESH_BUFFER_ML};
uint32_t m_QueryBuffer{0};
std::vector<XrWorldMeshBlockML> m_WorldMeshBlocks;
bool StartInfoQuery(XrTime displayTime) {
XrWorldMeshStateRequestInfoML requestInfo{XR_TYPE_WORLD_MESH_STATE_REQUEST_INFO_ML};
requestInfo.baseSpace = m_ViewSpace;
requestInfo.time = displayTime;
requestInfo.boundingBoxCenter.orientation.w = 0.0f;
requestInfo.boundingBoxExtents = {10.0f, 10.0f, 10.0f};
return xrRequestWorldMeshStateAsyncML(m_Detector, &requestInfo, &m_Future)==XR_SUCCESS;
}
bool CompleteInfoQuery() {
XrWorldMeshStateRequestCompletionML completion{XR_TYPE_WORLD_MESH_STATE_REQUEST_COMPLETION_ML};
if (xrRequestWorldMeshStateCompleteML(m_Detector, m_Future, &completion)!=XR_SUCCESS) {
return false;
}
if (completion.futureResult!=XR_SUCCESS) {
return false;
}
m_MeshBlocks.resize(completion.meshBlockStateCountOutput);
for (auto &mb:m_MeshBlocks) {
mb.type = XR_TYPE_WORLD_MESH_BLOCK_STATE_ML;
}
completion.meshBlockStateCapacityInput = completion.meshBlockStateCountOutput;
completion.meshBlockStates = m_MeshBlocks.data();
CHK_XR(xrRequestWorldMeshStateCompleteML(m_Detector, m_Future, &completion));
if (completion.meshBlockStateCountOutput==0) {
return false; // start a new query.
}
// switch to next buffer.
m_QueryBuffer = ( m_QueryBuffer + 1 ) % 2;
if (completion.meshBlockStateCountOutput>m_MaxBlockCounts[m_QueryBuffer]) {
m_MaxBlockCounts[m_QueryBuffer] = completion.meshBlockStateCountOutput;
XrWorldMeshBufferRecommendedSizeInfoML sizeInfo{XR_TYPE_WORLD_MESH_BUFFER_RECOMMENDED_SIZE_INFO_ML};
XrWorldMeshBufferSizeML bufferSize{XR_TYPE_WORLD_MESH_BUFFER_SIZE_ML};
sizeInfo.maxBlockCount = m_MaxBlockCounts[m_QueryBuffer];
CHK_XR(xrGetWorldMeshBufferRecommendSizeML(m_Detector, &sizeInfo, &bufferSize ));
CHK_XR(xrAllocateWorldMeshBufferML(m_Detector, &bufferSize, &m_WorldMeshBuffers[m_QueryBuffer]));
}
return true;
}
bool StartMeshQuery() {
std::vector<XrWorldMeshBlockRequestML> blockRequests;
blockRequests.resize(m_MeshBlocks.size());
for (size_t i = 0; i< m_MeshBlocks.size();i++) {
blockRequests[i].type = XR_TYPE_WORLD_MESH_BLOCK_REQUEST_ML;
blockRequests[i].uuid = m_MeshBlocks[i].uuid;
blockRequests[i].lod = XR_WORLD_MESH_DETECTOR_LOD_MEDIUM_ML;
}
XrWorldMeshGetInfoML getInfo{XR_TYPE_WORLD_MESH_GET_INFO_ML};
getInfo.flags = 0;
getInfo.fillHoleLength = 0.5f;
getInfo.disconnectedComponentArea = 1.0f;
getInfo.blockCount = static_cast<uint32_t>(blockRequests.size());
getInfo.blocks = blockRequests.data();
CHK_XR(xrRequestWorldMeshAsyncML(m_Detector, &getInfo, &m_WorldMeshBuffers[m_QueryBuffer], &m_Future));
return true;
}
bool CompleteMeshQuery(XrTime displayTime) {
XrWorldMeshRequestCompletionML completion{XR_TYPE_WORLD_MESH_REQUEST_COMPLETION_ML};
m_WorldMeshBlocks.resize(m_MeshBlocks.size());
completion.blockCount = static_cast<uint32_t>(m_WorldMeshBlocks.size());
completion.blocks = m_WorldMeshBlocks.data();
XrWorldMeshRequestCompletionInfoML completionInfo{XR_TYPE_WORLD_MESH_REQUEST_COMPLETION_INFO_ML};
completionInfo.meshSpace = m_LocalSpace;
completionInfo.meshSpaceLocateTime = displayTime;
CHK_XR(xrRequestWorldMeshCompleteML(m_Detector, &completionInfo, m_Future, &completion));
CHK_XR(completion.futureResult);
// The vertex data is now usable.
// the backing buffer double-buffered, so the vertex data remains valid
// even though a new request might be processing.
return true;
}
public:
MeshDetector() {
XrWorldMeshDetectorCreateInfoML createInfo{XR_TYPE_WORLD_MESH_DETECTOR_CREATE_INFO_ML};
CHK_XR(xrCreateWorldMeshDetectorML(m_Session,&createInfo, &m_Detector));
}
~MeshDetector() {
// Must ensure the future has finished before destroying.
// std::assert(IsDone());
for (auto &buffer : m_WorldMeshBuffers) {
if ( buffer.buffer != nullptr ) {
xrFreeWorldMeshBufferML(m_Detector, &buffer);
}
}
xrDestroyWorldMeshDetectorML(m_Detector);
m_Detector = XR_NULL_HANDLE;
}
void RequestShutdown() {
if ( m_Future != XR_NULL_FUTURE_EXT) {
XrFutureCancelInfoEXT cancelInfo{XR_TYPE_FUTURE_CANCEL_INFO_EXT};
cancelInfo.future = m_Future;
xrCancelFutureEXT(m_Instance, &cancelInfo);
m_State = DONE;
} else {
m_State = DONE;
}
}
bool IsDone() {
return m_State == DONE;
}
void FrameLoop(XrTime displayTime) {
if (m_Future == XR_NULL_FUTURE_EXT) {
return;
}
XrFuturePollResultEXT pollResult{XR_TYPE_FUTURE_POLL_RESULT_EXT};
pollResult.state = XR_FUTURE_STATE_PENDING_EXT;
XrFuturePollInfoEXT pollInfo{XR_TYPE_FUTURE_POLL_INFO_EXT};
pollInfo.future = m_Future;
CHK_XR(xrPollFutureEXT(m_Instance, &pollInfo, &pollResult));
switch (m_State) {
case INFO_START:
if (StartInfoQuery(displayTime)) {
m_State = INFO_WAIT_COMPLETE;
}
break;
case INFO_WAIT_COMPLETE:
if (pollResult.state==XR_FUTURE_STATE_READY_EXT) {
if (CompleteInfoQuery()) {
m_State = MESH_START;
} else {
m_State = INFO_START;
}
}
break;
case MESH_START:
if (StartMeshQuery()) {
m_State = MESH_WAIT_COMPLETE;
}
break;
case MESH_WAIT_COMPLETE:
if (pollResult.state==XR_FUTURE_STATE_READY_EXT) {
if (CompleteMeshQuery(displayTime)) {
m_State = INFO_START;
}
}
break;
case DONE:
break;
}
}
};12.110.14. New Enum Constants
XR_ML_WORLD_MESH_DETECTION_EXTENSION_NAMEXR_ML_world_mesh_detection_SPEC_VERSIONExtending XrObjectType:
XR_OBJECT_TYPE_WORLD_MESH_DETECTOR_ML
Extending XrResult:
XR_ERROR_WORLD_MESH_DETECTOR_PERMISSION_DENIED_MLXR_ERROR_WORLD_MESH_DETECTOR_SPACE_NOT_LOCATABLE_ML
Extending XrStructureType:
XR_TYPE_WORLD_MESH_BLOCK_MLXR_TYPE_WORLD_MESH_BLOCK_REQUEST_MLXR_TYPE_WORLD_MESH_BLOCK_STATE_MLXR_TYPE_WORLD_MESH_BUFFER_MLXR_TYPE_WORLD_MESH_BUFFER_RECOMMENDED_SIZE_INFO_MLXR_TYPE_WORLD_MESH_BUFFER_SIZE_MLXR_TYPE_WORLD_MESH_DETECTOR_CREATE_INFO_MLXR_TYPE_WORLD_MESH_GET_INFO_MLXR_TYPE_WORLD_MESH_REQUEST_COMPLETION_INFO_MLXR_TYPE_WORLD_MESH_REQUEST_COMPLETION_MLXR_TYPE_WORLD_MESH_STATE_REQUEST_COMPLETION_MLXR_TYPE_WORLD_MESH_STATE_REQUEST_INFO_ML
Issues
Version History
Revision 1, 2023-08-29
Initial Revision