vrkin
Differences
This shows you the differences between two versions of the page.
Both sides previous revisionPrevious revisionNext revision | Previous revision | ||
vrkin [2018/04/23 20:51] – [Installing OpenCV and Kinect V2 package] keitaronishimura | vrkin [2018/04/24 11:26] (current) – [Importing Video Feed into HMD] keitaronishimura | ||
---|---|---|---|
Line 37: | Line 37: | ||
\\ | \\ | ||
{{ vive_tut: | {{ vive_tut: | ||
+ | \\ | ||
+ | <fc red> | ||
+ | Make sure to do [[http:// | ||
+ | {{ vive_tut: | ||
+ | </fc> | ||
+ | | ||
+ | \\ | ||
+ | {{ vive_tut: | ||
+ | \\ | ||
| | ||
\\ | \\ | ||
Line 46: | Line 55: | ||
Note: Though the installation uses Ubuntu 14.04 the 16.04 version will work just fine | Note: Though the installation uses Ubuntu 14.04 the 16.04 version will work just fine | ||
</fc> | </fc> | ||
- | ==== OpenVR | + | ==== OpenCV |
- | You will now need to | + | This tutorial |
+ | \\ | ||
+ | 1. Go to your catkin workspace (catkin_ws) and create a new package to house the OpenCV code. | ||
+ | \\ | ||
+ | 2. In the src directory make a main.cpp file and add the following code to it: | ||
+ | <code cpp> | ||
+ | //! [headers] | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | |||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | |||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | #include < | ||
+ | |||
+ | //! [headers] | ||
+ | |||
+ | |||
+ | bool protonect_shutdown = false; // Whether the running application should shut down. | ||
+ | |||
+ | void sigint_handler(int s) | ||
+ | { | ||
+ | protonect_shutdown = true; | ||
+ | } | ||
+ | |||
+ | void HSVOnly(int, | ||
+ | void Location(); | ||
+ | |||
+ | cv::Mat imgCircles; | ||
+ | cv::Mat imgThresholded; | ||
+ | cv::Mat rgbd_flip; | ||
+ | cv::Mat test; | ||
+ | cv::Mat rgbmat_flip; | ||
+ | |||
+ | int iLowH = 22; | ||
+ | int iHighH = 34; | ||
+ | int iLowS = 38; | ||
+ | int iHighS = 244; | ||
+ | int iLowV = 160; | ||
+ | int iHighV = 255; | ||
+ | int posX; | ||
+ | int posY; | ||
+ | int posY_test; | ||
+ | int cenX; | ||
+ | int cenY; | ||
+ | float hand_x; | ||
+ | float hand_y; | ||
+ | float hand_z; | ||
+ | float dd; | ||
+ | int something = 0; | ||
+ | |||
+ | using namespace cv; | ||
+ | using namespace std; | ||
+ | |||
+ | void hand_fk(const geometry_msgs:: | ||
+ | { | ||
+ | |||
+ | hand_x = 1000*ts-> | ||
+ | hand_y = 1000*ts-> | ||
+ | hand_z = 1000*ts-> | ||
+ | |||
+ | |||
+ | } | ||
+ | |||
+ | |||
+ | int main(int argc, char** argv) | ||
+ | { | ||
+ | |||
+ | ros:: | ||
+ | ros:: | ||
+ | image_transport:: | ||
+ | image_transport:: | ||
+ | image_transport:: | ||
+ | ros:: | ||
+ | |||
+ | |||
+ | std::cout << " | ||
+ | |||
+ | //! [context] | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | //! [context] | ||
+ | |||
+ | //! [discovery] | ||
+ | if(freenect2.enumerateDevices() == 0) | ||
+ | { | ||
+ | std::cout << "no device connected!" | ||
+ | return -1; | ||
+ | } | ||
+ | |||
+ | string serial = freenect2.getDefaultDeviceSerialNumber(); | ||
+ | |||
+ | std::cout << " | ||
+ | |||
+ | if(pipeline) | ||
+ | { | ||
+ | //! [open] | ||
+ | dev = freenect2.openDevice(serial, | ||
+ | //! [open] | ||
+ | } else { | ||
+ | dev = freenect2.openDevice(serial); | ||
+ | } | ||
+ | |||
+ | if(dev == 0) | ||
+ | { | ||
+ | std::cout << " | ||
+ | return -1; | ||
+ | } | ||
+ | |||
+ | signal(SIGINT, | ||
+ | protonect_shutdown = false; | ||
+ | |||
+ | //! [listeners] | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | |||
+ | dev-> | ||
+ | dev-> | ||
+ | //! [listeners] | ||
+ | |||
+ | //! [start] | ||
+ | dev-> | ||
+ | |||
+ | std::cout << " | ||
+ | std::cout << " | ||
+ | //! [start] | ||
+ | |||
+ | //! [registration setup] | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | //! [registration setup] | ||
+ | |||
+ | Mat rgbmat, depthmat, depthmatUndistorted, | ||
+ | |||
+ | |||
+ | \ | ||
+ | //! [loop start] | ||
+ | while(!protonect_shutdown) | ||
+ | { | ||
+ | ros:: | ||
+ | |||
+ | listener.waitForNewFrame(frames); | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | libfreenect2:: | ||
+ | //! [loop start] | ||
+ | |||
+ | //! [converting libfreenect2 to opencv mat] | ||
+ | cv:: | ||
+ | cv:: | ||
+ | cv:: | ||
+ | cv:: | ||
+ | cv:: | ||
+ | //! [converting libfreenect2 to opencv mat] | ||
+ | |||
+ | //! [registration] | ||
+ | registration-> | ||
+ | //! [registration] | ||
+ | |||
+ | //! [converting libfreenect2 to opencv mat] | ||
+ | cv:: | ||
+ | cv:: | ||
+ | //! [converting libfreenect2 to opencv mat] | ||
+ | |||
+ | //! [finding yellow objects] | ||
+ | HSVOnly(iLowH, | ||
+ | Location(); | ||
+ | //! [finding yellow objects] | ||
+ | |||
+ | int posY_comp = posY - 150; | ||
+ | |||
+ | if(something <= 10) | ||
+ | { | ||
+ | posY_test = posY; | ||
+ | } | ||
+ | |||
+ | |||
+ | |||
+ | |||
+ | |||
+ | |||
+ | while(something < 1) | ||
+ | { | ||
+ | dd = test.at< | ||
+ | printf(" | ||
+ | |||
+ | if(something == 0 && dd != 0 && dd < 10000) | ||
+ | { | ||
+ | posY_test = posY_test - 1; | ||
+ | |||
+ | } | ||
+ | if(something == 0 && dd == 0) | ||
+ | { | ||
+ | something = 1; | ||
+ | posY_test = posY_test + 7; | ||
+ | } | ||
+ | if(something == 1 && dd == 0) | ||
+ | { | ||
+ | posY_test = posY_test - 1; | ||
+ | } | ||
+ | if(something == 1 && dd != 0 ) | ||
+ | { | ||
+ | something = 2; | ||
+ | } | ||
+ | if(something == 2 && dd < 10000 && dd != 0) | ||
+ | { | ||
+ | posY_test = posY_test - 1; | ||
+ | |||
+ | } | ||
+ | if(something == 2 && dd == 0) | ||
+ | { | ||
+ | something = 3; | ||
+ | } | ||
+ | if(something == 3 && dd == 0) | ||
+ | { | ||
+ | something = 11; | ||
+ | posY_test = posY_test + 4; | ||
+ | dd = test.at< | ||
+ | printf(" | ||
+ | } | ||
+ | if(dd > 10000) | ||
+ | { | ||
+ | something = 11; | ||
+ | } | ||
+ | |||
+ | |||
+ | } | ||
+ | |||
+ | //! [adding distance data to image] | ||
+ | dd = test.at< | ||
+ | float dd_comp = dd+160; | ||
+ | float diff = dd_comp - hand_x; | ||
+ | cv::Point org; | ||
+ | cv::Point rgb_org; | ||
+ | org.x = posX-100; | ||
+ | org.y = posY_comp-50; | ||
+ | rgb_org.x = 3.5*posX; | ||
+ | rgb_org.y = posY_test; | ||
+ | char str[200]; | ||
+ | |||
+ | sprintf(str, | ||
+ | cv:: | ||
+ | cv:: | ||
+ | //! [adding distance data to image] | ||
+ | |||
+ | |||
+ | sensor_msgs:: | ||
+ | sensor_msgs:: | ||
+ | pub.publish(msg); | ||
+ | rgb_pub.publish(rgb_msg); | ||
+ | |||
+ | if (posX >= 0 && posY >= 0) | ||
+ | { | ||
+ | cenX = 2*posX; | ||
+ | cenY = 2*posY_test; | ||
+ | //cenY = 2*posY_comp; | ||
+ | //Draw a red circle encompassing object | ||
+ | circle(rgbd_flip, | ||
+ | circle(rgbd_flip, | ||
+ | circle(rgbd_flip, | ||
+ | circle(rgbd_flip, | ||
+ | circle(rgbd_flip, | ||
+ | |||
+ | } | ||
+ | |||
+ | cv:: | ||
+ | |||
+ | cv:: | ||
+ | cv:: | ||
+ | |||
+ | something = 0; | ||
+ | |||
+ | |||
+ | int key = cv:: | ||
+ | protonect_shutdown = protonect_shutdown || (key > 0 && ((key & 0xFF) == 27)); // shutdown on escape | ||
+ | |||
+ | //! [loop end] | ||
+ | listener.release(frames); | ||
+ | } | ||
+ | //! [loop end] | ||
+ | |||
+ | //! [stop] | ||
+ | dev-> | ||
+ | dev-> | ||
+ | //! [stop] | ||
+ | |||
+ | delete registration; | ||
+ | |||
+ | std::cout << " | ||
+ | return 0; | ||
+ | } | ||
+ | |||
+ | |||
+ | void HSVOnly(int iLowH, int iHighH, int iLowS, int iHighS, int iLowV, int iHighV) | ||
+ | { | ||
+ | cv::Mat imgHSV; | ||
+ | |||
+ | //Convert the captured frame from BGR to HSV | ||
+ | cvtColor(rgbd_flip, | ||
+ | |||
+ | //Threshold the image | ||
+ | inRange(imgHSV, | ||
+ | |||
+ | // | ||
+ | erode(imgThresholded, | ||
+ | dilate( imgThresholded, | ||
+ | |||
+ | } | ||
+ | |||
+ | |||
+ | void Location() | ||
+ | { | ||
+ | imgCircles = Mat::zeros( rgbd_flip.size(), | ||
+ | Moments oMoments = moments(imgThresholded); | ||
+ | |||
+ | double dM01 = oMoments.m01; | ||
+ | double dM10 = oMoments.m10; | ||
+ | double dArea = oMoments.m00; | ||
+ | |||
+ | if (dArea > 10000) | ||
+ | { | ||
+ | //calculate the position of the ball | ||
+ | posX = dM10 / dArea; | ||
+ | posY = dM01 / dArea; | ||
+ | |||
+ | } | ||
+ | |||
+ | |||
+ | } | ||
+ | </ | ||
+ | This code will take the data from the kinect and find the distance between the sensor and a yellow object. Then taking the FK data from Hubo's right hand will display the distance between the yellow object and Hubo's right hand above the detected object. How to run the code will be explained later on. | ||
+ | \\ | ||
+ | Then make a CMakeLists.txt file in the package and add the following code into it: | ||
+ | <code txt> | ||
+ | cmake_minimum_required(VERSION 2.8.3) | ||
+ | project(kinect2hubo) | ||
+ | |||
+ | set(CMAKE_CXX_FLAGS " | ||
+ | # Unused warnings | ||
+ | set(CMAKE_CXX_FLAGS " | ||
+ | # Additional warnings | ||
+ | set(CMAKE_CXX_FLAGS " | ||
+ | |||
+ | # Check for c++11 support | ||
+ | INCLUDE(CheckCXXCompilerFlag) | ||
+ | CHECK_CXX_COMPILER_FLAG(" | ||
+ | CHECK_CXX_COMPILER_FLAG(" | ||
+ | IF(COMPILER_SUPPORTS_CXX11) | ||
+ | SET(CMAKE_CXX_FLAGS " | ||
+ | ELSEIF(COMPILER_SUPPORTS_CXX0X) | ||
+ | SET(CMAKE_CXX_FLAGS " | ||
+ | ELSE() | ||
+ | MESSAGE(ERROR "The compiler ${CMAKE_CXX_COMPILER} has no C++11 support. Please use a different C++ compiler." | ||
+ | ENDIF() | ||
+ | |||
+ | find_package(catkin REQUIRED COMPONENTS | ||
+ | cv_bridge | ||
+ | geometry_msgs | ||
+ | message_runtime | ||
+ | roscpp | ||
+ | sensor_msgs | ||
+ | std_msgs | ||
+ | image_transport | ||
+ | ) | ||
+ | find_package(freenect2 REQUIRED HINTS " | ||
+ | find_package(OpenCV REQUIRED) | ||
+ | |||
+ | include_directories( | ||
+ | ${catkin_INCLUDE_DIRS} | ||
+ | ${OpenCV_INCLUDE_DIRS} | ||
+ | ${freenect2_INCLUDE_DIRS} | ||
+ | ) | ||
+ | |||
+ | add_executable(${PROJECT_NAME}_node src/ | ||
+ | |||
+ | | ||
+ | | ||
+ | | ||
+ | | ||
+ | ) | ||
+ | </ | ||
+ | Then make a package.xml file and add the following code: | ||
+ | <code xml> | ||
+ | <?xml version=" | ||
+ | <package format=" | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | |||
+ | < | ||
+ | |||
+ | < | ||
+ | |||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | < | ||
+ | |||
+ | </ | ||
+ | </ | ||
+ | Then you should be ready to make the package using catkin_make. | ||
==== Importing Video Feed into HMD ==== | ==== Importing Video Feed into HMD ==== | ||
+ | \\ | ||
+ | To insert the video feed into the HMD we will use the UniversalVROverlay package on the windows 10 computer running the HTC Vive. Please download the [[https:// | ||
+ | \\ | ||
+ | {{ vive_tut: | ||
+ | \\ | ||
+ | The following steps depict how to stream the video feed from the OpenCV code into the the Vive HMD. | ||
+ | 1. Connect all computers by router | ||
- | First introduce how this tutorial will show the reader how to get image and depth data into the openVR exe file using pre-existing package for ros, and opencv API. | + | 2. Run the iai kinect code |
- | First go through the kinect computer and setting it up. Going through | + | 3. Run the Opencv code |
- | Then go into how to create | + | 4. Run the web video server |
- | Then how to use the ros package to send the stream | + | 5. View the stream |
- | Finally, talk about how to get the stream | + | 6. Use the UniversalVRoverlay |
vrkin.1524541888.txt.gz · Last modified: by keitaronishimura