Revert "visionipc: remove OpenCL support (#673)"

SConscript

@@ -17,25 +17,31 @@ msgq = env.Library('msgq', msgq_objects)
 msgq_python = envCython.Program('msgq/ipc_pyx.so', 'msgq/ipc_pyx.pyx', LIBS=envCython["LIBS"]+[msgq, common])
 
 # Build Vision IPC
-vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc']
+vipc_files = ['visionipc.cc', 'visionipc_server.cc', 'visionipc_client.cc', 'visionbuf.cc']
+vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files]
+
 if arch == "larch64":
-  vipc_files += ['visionbuf_ion.cc']
+  vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_ion.cc']
 else:
-  vipc_files += ['visionbuf.cc']
-vipc_sources = [f'{visionipc_dir.abspath}/{f}' for f in vipc_files]
+  vipc_sources += [f'{visionipc_dir.abspath}/visionbuf_cl.cc']
 
 vipc_objects = env.SharedObject(vipc_sources)
 visionipc = env.Library('visionipc', vipc_objects)
 
 
+vipc_frameworks = []
 vipc_libs = envCython["LIBS"] + [visionipc, msgq, common]
+if arch == "Darwin":
+  vipc_frameworks.append('OpenCL')
+else:
+  vipc_libs.append('OpenCL')
 envCython.Program(f'{visionipc_dir.abspath}/visionipc_pyx.so', f'{visionipc_dir.abspath}/visionipc_pyx.pyx',
-                  LIBS=vipc_libs)
+                  LIBS=vipc_libs, FRAMEWORKS=vipc_frameworks)
 
 if GetOption('extras'):
   env.Program('msgq/test_runner', ['msgq/test_runner.cc', 'msgq/msgq_tests.cc'], LIBS=[msgq, common])
   env.Program(f'{visionipc_dir.abspath}/test_runner',
              [f'{visionipc_dir.abspath}/test_runner.cc', f'{visionipc_dir.abspath}/visionipc_tests.cc'],
-              LIBS=['pthread'] + vipc_libs)
+              LIBS=['pthread'] + vipc_libs, FRAMEWORKS=vipc_frameworks)
 
 Export('visionipc', 'msgq', 'msgq_python')

msgq/visionipc/visionbuf.cc

@@ -1,52 +1,5 @@
 #include "msgq/visionipc/visionbuf.h"
 
-#include <atomic>
-#include <stdio.h>
-#include <fcntl.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include <sys/types.h>
-
-std::atomic<int> offset = 0;
-
-static void *malloc_with_fd(size_t len, int *fd) {
-  char full_path[0x100];
-
-#ifdef __APPLE__
-  snprintf(full_path, sizeof(full_path)-1, "/tmp/visionbuf_%d_%d", getpid(), offset++);
-#else
-  snprintf(full_path, sizeof(full_path)-1, "/dev/shm/msgq_visionbuf_%d_%d", getpid(), offset++);
-#endif
-
-  *fd = open(full_path, O_RDWR | O_CREAT, 0664);
-  assert(*fd >= 0);
-
-  unlink(full_path);
-
-  ftruncate(*fd, len);
-  void *addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
-  assert(addr != MAP_FAILED);
-
-  return addr;
-}
-
-void VisionBuf::allocate(size_t length) {
-  this->len = length;
-  this->mmap_len = this->len + sizeof(uint64_t);
-  this->addr = malloc_with_fd(this->mmap_len, &this->fd);
-  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
-}
-
-void VisionBuf::import(){
-  assert(this->fd >= 0);
-  this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
-  assert(this->addr != MAP_FAILED);
-
-  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
-}
-
 void VisionBuf::init_yuv(size_t init_width, size_t init_height, size_t init_stride, size_t init_uv_offset){
   this->width = init_width;
   this->height = init_height;
@@ -57,17 +10,6 @@ void VisionBuf::init_yuv(size_t init_width, size_t init_height, size_t init_stri
   this->uv = this->y + this->uv_offset;
 }
 
-int VisionBuf::sync(int dir) {
-  return 0;
-}
-
-int VisionBuf::free() {
-  int err = munmap(this->addr, this->mmap_len);
-  if (err != 0) return err;
-
-  err = close(this->fd);
-  return err;
-}
 
 uint64_t VisionBuf::get_frame_id() {
   return *frame_id;

msgq/visionipc/visionbuf.h

@@ -2,6 +2,13 @@
 
 #include "msgq/visionipc/visionipc.h"
 
+#define CL_USE_DEPRECATED_OPENCL_1_2_APIS
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
 #define VISIONBUF_SYNC_FROM_DEVICE 0
 #define VISIONBUF_SYNC_TO_DEVICE 1
 
@@ -36,11 +43,16 @@ class VisionBuf {
   size_t idx = 0;
   VisionStreamType type;
 
+  // OpenCL
+  cl_mem buf_cl = nullptr;
+  cl_command_queue copy_q = nullptr;
+
   // ion
   int handle = 0;
 
   void allocate(size_t len);
   void import();
+  void init_cl(cl_device_id device_id, cl_context ctx);
   void init_yuv(size_t width, size_t height, size_t stride, size_t uv_offset);
   int sync(int dir);
   int free();

msgq/visionipc/visionbuf_cl.cc

@@ -0,0 +1,94 @@
+#include "msgq/visionipc/visionbuf.h"
+
+#include <atomic>
+#include <stdio.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+
+std::atomic<int> offset = 0;
+
+static void *malloc_with_fd(size_t len, int *fd) {
+  char full_path[0x100];
+
+#ifdef __APPLE__
+  snprintf(full_path, sizeof(full_path)-1, "/tmp/visionbuf_%d_%d", getpid(), offset++);
+#else
+  snprintf(full_path, sizeof(full_path)-1, "/dev/shm/msgq_visionbuf_%d_%d", getpid(), offset++);
+#endif
+
+  *fd = open(full_path, O_RDWR | O_CREAT, 0664);
+  assert(*fd >= 0);
+
+  unlink(full_path);
+
+  ftruncate(*fd, len);
+  void *addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
+  assert(addr != MAP_FAILED);
+
+  return addr;
+}
+
+void VisionBuf::allocate(size_t length) {
+  this->len = length;
+  this->mmap_len = this->len + sizeof(uint64_t);
+  this->addr = malloc_with_fd(this->mmap_len, &this->fd);
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
+}
+
+void VisionBuf::init_cl(cl_device_id device_id, cl_context ctx){
+  int err;
+
+  this->copy_q = clCreateCommandQueue(ctx, device_id, 0, &err);
+  assert(err == 0);
+
+  this->buf_cl = clCreateBuffer(ctx, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, this->len, this->addr, &err);
+  assert(err == 0);
+}
+
+
+void VisionBuf::import(){
+  assert(this->fd >= 0);
+  this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
+  assert(this->addr != MAP_FAILED);
+
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
+}
+
+
+int VisionBuf::sync(int dir) {
+  int err = 0;
+  if (!this->buf_cl) return 0;
+
+  if (dir == VISIONBUF_SYNC_FROM_DEVICE) {
+    err = clEnqueueReadBuffer(this->copy_q, this->buf_cl, CL_FALSE, 0, this->len, this->addr, 0, NULL, NULL);
+  } else {
+    err = clEnqueueWriteBuffer(this->copy_q, this->buf_cl, CL_FALSE, 0, this->len, this->addr, 0, NULL, NULL);
+  }
+
+  if (err == 0){
+    err = clFinish(this->copy_q);
+  }
+
+  return err;
+}
+
+int VisionBuf::free() {
+  int err = 0;
+  if (this->buf_cl){
+    err = clReleaseMemObject(this->buf_cl);
+    if (err != 0) return err;
+
+    err = clReleaseCommandQueue(this->copy_q);
+    if (err != 0) return err;
+  }
+
+  err = munmap(this->addr, this->mmap_len);
+  if (err != 0) return err;
+
+  err = close(this->fd);
+  return err;
+}

msgq/visionipc/visionbuf_ion.cc

@@ -10,6 +10,7 @@
 #include <fcntl.h>
 #include <unistd.h>
 #include <linux/ion.h>
+#include <CL/cl_ext.h>
 
 #include <msm_ion.h>
 
@@ -26,6 +27,19 @@
     ret;                                                      \
   })
 
+// just hard-code these for convenience
+// size_t device_page_size = 0;
+// clGetDeviceInfo(device_id, CL_DEVICE_PAGE_SIZE_QCOM,
+//                 sizeof(device_page_size), &device_page_size,
+//                 NULL);
+
+// size_t padding_cl = 0;
+// clGetDeviceInfo(device_id, CL_DEVICE_EXT_MEM_PADDING_IN_BYTES_QCOM,
+//                 sizeof(padding_cl), &padding_cl,
+//                 NULL);
+#define DEVICE_PAGE_SIZE_CL 4096
+#define PADDING_CL 0
+
 struct IonFileHandle {
   IonFileHandle() {
     fd = open("/dev/ion", O_RDWR | O_NONBLOCK);
@@ -44,7 +58,7 @@ int ion_fd() {
 
 void VisionBuf::allocate(size_t length) {
   struct ion_allocation_data ion_alloc = {0};
-  ion_alloc.len = length + sizeof(uint64_t);
+  ion_alloc.len = length + PADDING_CL + sizeof(uint64_t);
   ion_alloc.align = 4096;
   ion_alloc.heap_id_mask = 1 << ION_IOMMU_HEAP_ID;
   ion_alloc.flags = ION_FLAG_CACHED;
@@ -69,7 +83,7 @@ void VisionBuf::allocate(size_t length) {
   this->addr = mmap_addr;
   this->handle = ion_alloc.handle;
   this->fd = ion_fd_data.fd;
-  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len + PADDING_CL);
 }
 
 void VisionBuf::import(){
@@ -86,19 +100,27 @@ void VisionBuf::import(){
   this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
   assert(this->addr != MAP_FAILED);
 
-  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len + PADDING_CL);
 }
 
-void VisionBuf::init_yuv(size_t init_width, size_t init_height, size_t init_stride, size_t init_uv_offset){
-  this->width = init_width;
-  this->height = init_height;
-  this->stride = init_stride;
-  this->uv_offset = init_uv_offset;
+void VisionBuf::init_cl(cl_device_id device_id, cl_context ctx) {
+  int err;
+
+  assert(((uintptr_t)this->addr % DEVICE_PAGE_SIZE_CL) == 0);
 
-  this->y = (uint8_t *)this->addr;
-  this->uv = this->y + this->uv_offset;
+  cl_mem_ion_host_ptr ion_cl = {0};
+  ion_cl.ext_host_ptr.allocation_type = CL_MEM_ION_HOST_PTR_QCOM;
+  ion_cl.ext_host_ptr.host_cache_policy = CL_MEM_HOST_UNCACHED_QCOM;
+  ion_cl.ion_filedesc = this->fd;
+  ion_cl.ion_hostptr = this->addr;
+
+  this->buf_cl = clCreateBuffer(ctx,
+                              CL_MEM_USE_HOST_PTR | CL_MEM_EXT_HOST_PTR_QCOM,
+                              this->len, &ion_cl, &err);
+  assert(err == 0);
 }
 
+
 int VisionBuf::sync(int dir) {
   struct ion_flush_data flush_data = {0};
   flush_data.handle = this->handle;
@@ -121,7 +143,14 @@ int VisionBuf::sync(int dir) {
 }
 
 int VisionBuf::free() {
-  int err = munmap(this->addr, this->mmap_len);
+  int err = 0;
+
+  if (this->buf_cl){
+    err = clReleaseMemObject(this->buf_cl);
+    if (err != 0) return err;
+  }
+
+  err = munmap(this->addr, this->mmap_len);
   if (err != 0) return err;
 
   err = close(this->fd);
@@ -130,11 +159,3 @@ int VisionBuf::free() {
   struct ion_handle_data handle_data = {.handle = this->handle};
   return HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_FREE, &handle_data));
 }
-
-uint64_t VisionBuf::get_frame_id() {
-  return *frame_id;
-}
-
-void VisionBuf::set_frame_id(uint64_t id) {
-  *frame_id = id;
-}

msgq/visionipc/visionipc.pxd

@@ -8,6 +8,14 @@ from libc.stdint cimport uint32_t, uint64_t
 from libcpp cimport bool, int
 
 cdef extern from "msgq/visionipc/visionbuf.h":
+  struct _cl_device_id
+  struct _cl_context
+  struct _cl_mem
+
+  ctypedef _cl_device_id * cl_device_id
+  ctypedef _cl_context * cl_context
+  ctypedef _cl_mem * cl_mem
+
   cdef enum VisionStreamType:
     pass
 
@@ -20,6 +28,7 @@ cdef extern from "msgq/visionipc/visionbuf.h":
     size_t stride
     size_t uv_offset
     size_t idx
+    cl_mem buf_cl
     void set_frame_id(uint64_t id)
 
 cdef extern from "msgq/visionipc/visionipc.h":
@@ -33,7 +42,7 @@ cdef extern from "msgq/visionipc/visionipc_server.h":
   string get_endpoint_name(string, VisionStreamType)
 
   cdef cppclass VisionIpcServer:
-    VisionIpcServer(string)
+    VisionIpcServer(string, void*, void*)
     void create_buffers(VisionStreamType, size_t, size_t, size_t)
     void create_buffers_with_sizes(VisionStreamType, size_t, size_t, size_t, size_t, size_t, size_t)
     VisionBuf * get_buffer(VisionStreamType)
@@ -44,7 +53,7 @@ cdef extern from "msgq/visionipc/visionipc_client.h":
   cdef cppclass VisionIpcClient:
     int num_buffers
     VisionBuf buffers[1]
-    VisionIpcClient(string, VisionStreamType, bool)
+    VisionIpcClient(string, VisionStreamType, bool, void*, void*)
     VisionBuf * recv(VisionIpcBufExtra *, int)
     bool connect(bool)
     bool is_connected()