GStreamer - ZED Metadata The stream of data injected in a GStreamer pipeline by the zedsrc element contains color and depth information plus metadata with sensors and object detection data. The ZED metadata are defined in the GStreamer library gstzedmeta. Metadata information GstZedSrcMeta: metadata global container ZedInfo: general information about the ZED camera that acquired the data. ZedPose: Positional Tracking information. ZedSensors: data from the camera sensors (if available). ZedImu: 6 DOF inertial data (if available). ZedMag: 3 DOF magnetic field data (if available). ZedEnv: environmental data, atmospheric pressure and internal temperature (if available). ZedCamTemp: camera CMOS temperatures (if available). ZedObjectData: object detection data (if available). Data Structures GstZedSrcMeta struct _GstZedSrcMeta { GstMeta meta; ZedInfo info; ZedPose pose; ZedSensors sens; gboolean od_enabled; guint8 obj_count; ZedObjectData objects[256]; }; meta: metadata identifier field. info: camera information. pose: camera pose. sens: sensors data. od_enabled: indicates if object detection data are available. obj_*count: the number of detected objects [max 256]. objects: array of detected objects. ZedInfo struct _ZedInfo { gint cam_model; gint stream_type; guint grab_single_frame_width; guint grab_single_frame_height; }; cam_model: camera model (0 : ZED, 1 : ZED Mini, 2 : ZED2). stream_type: type of stream (0 : Left Image, 1 : Right image, 2 : Stereo couple, 3 : 16 bit depth map, 4 : Left + Depth). grab_single_frame_width: original width of image data. grab_single_frame_height: original heigth of image data. ZedPose struct _ZedPose { gboolean pose_avail; gint pos_tracking_state; gfloat pos[3]; gfloat orient[3]; }; pose_avail: indicates if camera pose is available. pos_tracking_state: status of the Positional Tracking algorithm (0 : SEARCHING, 1 : OK, 2 : OFF, 3 : FPS_TOO_LOW) pos: camera position orient: camera orientation (Eurel Angles). ZedSensors struct _ZedSensors { gboolean sens_avail; ZedImu imu; ZedMag mag; ZedEnv env; ZedCamTemp temp; }; sens_avail: indicates if sensors data are available (only with ZED2 and ZED Mini). imu: IMU data. mag: Magnetometer data. env: environment data. temp: camera temperatures data Note: An example about how to retrieve Sensors data is provided with the gstzeddatacsvsink element source code ZedImu struct _ZedImu { gboolean imu_avail; gfloat acc[3]; gfloat gyro[3]; gfloat temp; }; imu_avail: indicates if IMU data are available (only with ZED2 and ZED Mini). acc: 3 DOF accelerometer data in [m/s²]. gyro: 3 DOF gyroscope data in [rad/sec]. temp: IMU temperature in [°C]. ZedMag struct _ZedMag { gboolean mag_avail; gfloat mag[3]; }; mag_avail: indicates if magnetometer data are available (only with ZED2). mag: 3 DOF magnetic field data in [µT]. ZedEnv struct _ZedEnv { gboolean env_avail; gfloat press; gfloat temp; }; env_avail: indicates if environment data are available (only with ZED2). press: atmospheric pressure in [hPa]. temp: internal camera temperature in [°C]. ZedCamTemp struct _ZedCamTemp { gboolean temp_avail; gfloat temp_cam_left; gfloat temp_cam_right; }; env_avail: indicates if CMOS temperatures data are available (only with ZED2). temp_cam_left: temperature of the left CMOS sensor in [°C]. temp_cam_right: temperature of the right CMOS sensor in [°C]. ZedObjectData struct _ZedObjectData { gint id; OBJECT_CLASS label; OBJECT_SUBCLASS sublabel; OBJECT_TRACKING_STATE tracking_state; OBJECT_ACTION_STATE action_state; gfloat confidence; gfloat position[3]; gfloat position_covariance[6]; gfloat velocity[3]; unsigned int bounding_box_2d[4][2]; /* 3D bounding box of the person represented as eight 3D points 1 ------ 2 / /| 0 ------ 3 | | Object | 6 | |/ 4 ------ 7 */ gfloat bounding_box_3d[8][3]; gfloat dimensions[3]; // 3D object dimensions: width, height, length gboolean skeletons_avail; gfloat keypoint_2d[18][2]; // Negative coordinates -> point not valid gfloat keypoint_3d[18][3]; // Nan coordinates -> point not valid gfloat head_bounding_box_2d[4][2]; gfloat head_bounding_box_3d[8][3]; gfloat head_position[3]; }; id: univoque identifier of the tracked object. label: class of the identified object. sublabel: subclass of the identified object [only for MULTICLASS models] tracking_state: tracking status of the object. confidence: confidence level of the detection [0, 100]. position: 3D position of the center of the object. position_covariance: covariance matrix of the position. velocity: velocity of the object bounding_box_2d: 2D image coordinates of the four corners of the bounding box bounding_box_3d: 3D world coordinates of the eight corners of the bounding box dimensions: 3D dimensions of the 3D bounding box skeletons_avail: indicates if a skeleton tracking detection was enabled and if skeleton data are available keypoint_2d: 2D image coordinates of the eighteen skeleton joints keypoint_3d: 3D world coordinates of the eighteen skeleton joints head_bounding_box_2d: 2D image coordinates of the four corners of the bounding box of the head head_bounding_box_3d: 3D world coordinates of the eight corners of the bounding box of the head head_position: 3D world coordinates of the position of the center of the head enum class OBJECT_CLASS { PERSON = 0, VEHICLE = 1, BAG = 2, ANIMAL = 3, ELECTRONICS = 4, FRUIT_VEGETABLE = 5, LAST }; enum class OBJECT_SUBCLASS { PERSON = 0, /**< PERSON */ BICYCLE = 1, /**< VEHICLE */ CAR = 2, /**< VEHICLE */ MOTORBIKE = 3, /**< VEHICLE */ BUS = 4, /**< VEHICLE */ TRUCK = 5, /**< VEHICLE */ BOAT = 6, /**< VEHICLE */ BACKPACK = 7, /**< BAG */ HANDBAG = 8, /**< BAG */ SUITCASE = 9, /**< BAG */ BIRD = 10, /**< ANIMAL */ CAT = 11, /**< ANIMAL */ DOG = 12, /**< ANIMAL */ HORSE = 13, /**< ANIMAL */ SHEEP = 14, /**< ANIMAL */ COW = 15, /**< ANIMAL */ CELLPHONE = 16, /**< ELECTRONIC */ LAPTOP = 17, /**< ELECTRONIC */ BANANA = 18, /**< FRUIT/VEGETABLE */ APPLE = 19, /**< FRUIT/VEGETABLE */ ORANGE = 20, /**< FRUIT/VEGETABLE */ CARROT = 21, /**< FRUIT/VEGETABLE */ LAST = 22 }; enum class OBJECT_TRACKING_STATE { OFF, /**< The tracking is not yet initialized, the object ID is not usable */ OK, /**< The object is tracked */ SEARCHING, /**< The object couldn't be detected in the image and is potentially occluded, the trajectory is estimated */ TERMINATE, /**< This is the last searching state of the track, the track will be deleted in the next retreiveObject */ LAST }; enum class OBJECT_ACTION_STATE { IDLE = 0, /**< The object is staying static. */ MOVING = 1, /**< The object is moving. */ LAST }; namespace skeleton { enum class BODY_PARTS { NOSE = 0, NECK = 1, RIGHT_SHOULDER = 2, RIGHT_ELBOW= 3, RIGHT_WRIST = 4, LEFT_SHOULDER = 5, LEFT_ELBOW = 6, LEFT_WRIST = 7, RIGHT_HIP = 8, RIGHT_KNEE = 9, RIGHT_ANKLE = 10, LEFT_HIP = 11, LEFT_KNEE = 12, LEFT_ANKLE = 13, RIGHT_EYE = 14, LEFT_EYE = 15, RIGHT_EAR = 16, LEFT_EAR = 17, LAST = 18 }; inline int getIdx(BODY_PARTS part) { return static_cast<int>(part); } static const std::vector<std::pair< BODY_PARTS, BODY_PARTS>> BODY_BONES{ {BODY_PARTS::NOSE, BODY_PARTS::NECK}, {BODY_PARTS::NECK, BODY_PARTS::RIGHT_SHOULDER}, {BODY_PARTS::RIGHT_SHOULDER, BODY_PARTS::RIGHT_ELBOW}, {BODY_PARTS::RIGHT_ELBOW, BODY_PARTS::RIGHT_WRIST}, {BODY_PARTS::NECK, BODY_PARTS::LEFT_SHOULDER}, {BODY_PARTS::LEFT_SHOULDER, BODY_PARTS::LEFT_ELBOW}, {BODY_PARTS::LEFT_ELBOW, BODY_PARTS::LEFT_WRIST}, {BODY_PARTS::RIGHT_SHOULDER, BODY_PARTS::RIGHT_HIP}, {BODY_PARTS::RIGHT_HIP, BODY_PARTS::RIGHT_KNEE}, {BODY_PARTS::RIGHT_KNEE, BODY_PARTS::RIGHT_ANKLE}, {BODY_PARTS::LEFT_SHOULDER, BODY_PARTS::LEFT_HIP}, {BODY_PARTS::LEFT_HIP, BODY_PARTS::LEFT_KNEE}, {BODY_PARTS::LEFT_KNEE, BODY_PARTS::LEFT_ANKLE}, {BODY_PARTS::RIGHT_SHOULDER, BODY_PARTS::LEFT_SHOULDER}, {BODY_PARTS::RIGHT_HIP, BODY_PARTS::LEFT_HIP}, {BODY_PARTS::NOSE, BODY_PARTS::RIGHT_EYE}, {BODY_PARTS::RIGHT_EYE, BODY_PARTS::RIGHT_EAR}, {BODY_PARTS::NOSE, BODY_PARTS::LEFT_EYE}, {BODY_PARTS::LEFT_EYE, BODY_PARTS::LEFT_EAR} }; } Note: An example about how to use Object Detection data is provided with the gstzedodoverlay element source code