// This file is part of OpenCV project. // It is subject to the license terms in the LICENSE file found in the top-level directory // of this distribution and at http://opencv.org/license.html #include "../precomp.hpp" #include #include namespace cv { namespace aruco { using namespace std; struct Board::BoardImpl { std::vector > objPoints; Dictionary dictionary; Point3f rightBottomBorder; std::vector ids; BoardImpl() { dictionary = Dictionary(getPredefinedDictionary(PredefinedDictionaryType::DICT_4X4_50)); } }; Board::Board(): boardImpl(makePtr()) {} Board::~Board() {} Ptr Board::create(InputArrayOfArrays objPoints, const Dictionary &dictionary, InputArray ids) { CV_Assert(objPoints.total() == ids.total()); CV_Assert(objPoints.type() == CV_32FC3 || objPoints.type() == CV_32FC1); vector > obj_points_vector; Point3f rightBottomBorder = Point3f(0.f, 0.f, 0.f); for (unsigned int i = 0; i < objPoints.total(); i++) { vector corners; Mat corners_mat = objPoints.getMat(i); if (corners_mat.type() == CV_32FC1) corners_mat = corners_mat.reshape(3); CV_Assert(corners_mat.total() == 4); for (int j = 0; j < 4; j++) { const Point3f &corner = corners_mat.at(j); corners.push_back(corner); rightBottomBorder.x = std::max(rightBottomBorder.x, corner.x); rightBottomBorder.y = std::max(rightBottomBorder.y, corner.y); rightBottomBorder.z = std::max(rightBottomBorder.z, corner.z); } obj_points_vector.push_back(corners); } Board board; Ptr res = makePtr(board); ids.copyTo(res->boardImpl->ids); res->boardImpl->objPoints = obj_points_vector; res->boardImpl->dictionary = dictionary; res->boardImpl->rightBottomBorder = rightBottomBorder; return res; } const Dictionary& Board::getDictionary() const { return this->boardImpl->dictionary; } const vector >& Board::getObjPoints() const { return this->boardImpl->objPoints; } const Point3f& Board::getRightBottomCorner() const { return this->boardImpl->rightBottomBorder; } const vector& Board::getIds() const { return this->boardImpl->ids; } /** @brief Implementation of draw planar board that accepts a raw Board pointer. */ void Board::generateImage(Size outSize, OutputArray img, int marginSize, int borderBits) const { CV_Assert(!outSize.empty()); CV_Assert(marginSize >= 0); img.create(outSize, CV_8UC1); Mat out = img.getMat(); out.setTo(Scalar::all(255)); out.adjustROI(-marginSize, -marginSize, -marginSize, -marginSize); // calculate max and min values in XY plane CV_Assert(this->getObjPoints().size() > 0); float minX, maxX, minY, maxY; minX = maxX = this->getObjPoints()[0][0].x; minY = maxY = this->getObjPoints()[0][0].y; for(unsigned int i = 0; i < this->getObjPoints().size(); i++) { for(int j = 0; j < 4; j++) { minX = min(minX, this->getObjPoints()[i][j].x); maxX = max(maxX, this->getObjPoints()[i][j].x); minY = min(minY, this->getObjPoints()[i][j].y); maxY = max(maxY, this->getObjPoints()[i][j].y); } } float sizeX = maxX - minX; float sizeY = maxY - minY; // proportion transformations float xReduction = sizeX / float(out.cols); float yReduction = sizeY / float(out.rows); // determine the zone where the markers are placed if(xReduction > yReduction) { int nRows = int(sizeY / xReduction); int rowsMargins = (out.rows - nRows) / 2; out.adjustROI(-rowsMargins, -rowsMargins, 0, 0); } else { int nCols = int(sizeX / yReduction); int colsMargins = (out.cols - nCols) / 2; out.adjustROI(0, 0, -colsMargins, -colsMargins); } // now paint each marker Mat marker; Point2f outCorners[3]; Point2f inCorners[3]; for(unsigned int m = 0; m < this->getObjPoints().size(); m++) { // transform corners to markerZone coordinates for(int j = 0; j < 3; j++) { Point2f pf = Point2f(this->getObjPoints()[m][j].x, this->getObjPoints()[m][j].y); // move top left to 0, 0 pf -= Point2f(minX, minY); pf.x = pf.x / sizeX * float(out.cols); pf.y = pf.y / sizeY * float(out.rows); outCorners[j] = pf; } // get marker Size dst_sz(outCorners[2] - outCorners[0]); // assuming CCW order dst_sz.width = dst_sz.height = std::min(dst_sz.width, dst_sz.height); //marker should be square getDictionary().generateImageMarker(this->getIds()[m], dst_sz.width, marker, borderBits); if((outCorners[0].y == outCorners[1].y) && (outCorners[1].x == outCorners[2].x)) { // marker is aligned to image axes marker.copyTo(out(Rect(outCorners[0], dst_sz))); continue; } // interpolate tiny marker to marker position in markerZone inCorners[0] = Point2f(-0.5f, -0.5f); inCorners[1] = Point2f(marker.cols - 0.5f, -0.5f); inCorners[2] = Point2f(marker.cols - 0.5f, marker.rows - 0.5f); // remove perspective Mat transformation = getAffineTransform(inCorners, outCorners); warpAffine(marker, out, transformation, out.size(), INTER_LINEAR, BORDER_TRANSPARENT); } } void Board::matchImagePoints(InputArray detectedCorners, InputArray detectedIds, OutputArray _objPoints, OutputArray imgPoints) const { CV_Assert(getIds().size() == getObjPoints().size()); CV_Assert(detectedIds.total() == detectedCorners.total()); size_t nDetectedMarkers = detectedIds.total(); vector objPnts; objPnts.reserve(nDetectedMarkers); vector imgPnts; imgPnts.reserve(nDetectedMarkers); // look for detected markers that belong to the board and get their information for(unsigned int i = 0; i < nDetectedMarkers; i++) { int currentId = detectedIds.getMat().ptr< int >(0)[i]; for(unsigned int j = 0; j < getIds().size(); j++) { if(currentId == getIds()[j]) { for(int p = 0; p < 4; p++) { objPnts.push_back(getObjPoints()[j][p]); imgPnts.push_back(detectedCorners.getMat(i).ptr(0)[p]); } } } } // create output Mat(objPnts).copyTo(_objPoints); Mat(imgPnts).copyTo(imgPoints); } struct GridBoard::GridImpl { GridImpl(){}; // number of markers in X and Y directions int sizeX = 3, sizeY = 3; // marker side length (normally in meters) float markerLength = 1.f; // separation between markers in the grid float markerSeparation = .5f; }; GridBoard::GridBoard(): gridImpl(makePtr()) {} Ptr GridBoard::create(int markersX, int markersY, float markerLength, float markerSeparation, const Dictionary &dictionary, InputArray ids) { CV_Assert(markersX > 0 && markersY > 0 && markerLength > 0 && markerSeparation > 0); GridBoard board; Ptr res = makePtr(board); res->gridImpl->sizeX = markersX; res->gridImpl->sizeY = markersY; res->gridImpl->markerLength = markerLength; res->gridImpl->markerSeparation = markerSeparation; res->boardImpl->dictionary = dictionary; size_t totalMarkers = (size_t) markersX * markersY; CV_Assert(totalMarkers == ids.total()); vector > objPoints; objPoints.reserve(totalMarkers); ids.copyTo(res->boardImpl->ids); // calculate Board objPoints for (int y = 0; y < markersY; y++) { for (int x = 0; x < markersX; x++) { vector corners(4); corners[0] = Point3f(x * (markerLength + markerSeparation), y * (markerLength + markerSeparation), 0); corners[1] = corners[0] + Point3f(markerLength, 0, 0); corners[2] = corners[0] + Point3f(markerLength, markerLength, 0); corners[3] = corners[0] + Point3f(0, markerLength, 0); objPoints.push_back(corners); } } res->boardImpl->objPoints = objPoints; res->boardImpl->rightBottomBorder = Point3f(markersX * markerLength + markerSeparation * (markersX - 1), markersY * markerLength + markerSeparation * (markersY - 1), 0.f); return res; } Ptr GridBoard::create(int markersX, int markersY, float markerLength, float markerSeparation, const Dictionary &dictionary, int firstMarker) { vector ids(markersX*markersY); std::iota(ids.begin(), ids.end(), firstMarker); return GridBoard::create(markersX, markersY, markerLength, markerSeparation, dictionary, ids); } void GridBoard::generateImage(Size outSize, OutputArray _img, int marginSize, int borderBits) const { Board::generateImage(outSize, _img, marginSize, borderBits); } Size GridBoard::getGridSize() const { return Size(gridImpl->sizeX, gridImpl->sizeY); } float GridBoard::getMarkerLength() const { return gridImpl->markerLength; } float GridBoard::getMarkerSeparation() const { return gridImpl->markerSeparation; } struct CharucoBoard::CharucoImpl : GridBoard::GridImpl { // size of chessboard squares side (normally in meters) float squareLength; // marker side length (normally in meters) float markerLength; static void _getNearestMarkerCorners(CharucoBoard &board, float squareLength); // vector of chessboard 3D corners precalculated std::vector chessboardCorners; // for each charuco corner, nearest marker id and nearest marker corner id of each marker std::vector > nearestMarkerIdx; std::vector > nearestMarkerCorners; }; CharucoBoard::CharucoBoard(): charucoImpl(makePtr()) {} void CharucoBoard::generateImage(Size outSize, OutputArray _img, int marginSize, int borderBits) const { CV_Assert(!outSize.empty()); CV_Assert(marginSize >= 0); _img.create(outSize, CV_8UC1); _img.setTo(255); Mat out = _img.getMat(); Mat noMarginsImg = out.colRange(marginSize, out.cols - marginSize).rowRange(marginSize, out.rows - marginSize); double totalLengthX, totalLengthY; totalLengthX = charucoImpl->squareLength * charucoImpl->sizeX; totalLengthY = charucoImpl->squareLength * charucoImpl->sizeY; // proportional transformation double xReduction = totalLengthX / double(noMarginsImg.cols); double yReduction = totalLengthY / double(noMarginsImg.rows); // determine the zone where the chessboard is placed Mat chessboardZoneImg; if(xReduction > yReduction) { int nRows = int(totalLengthY / xReduction); int rowsMargins = (noMarginsImg.rows - nRows) / 2; chessboardZoneImg = noMarginsImg.rowRange(rowsMargins, noMarginsImg.rows - rowsMargins); } else { int nCols = int(totalLengthX / yReduction); int colsMargins = (noMarginsImg.cols - nCols) / 2; chessboardZoneImg = noMarginsImg.colRange(colsMargins, noMarginsImg.cols - colsMargins); } // determine the margins to draw only the markers // take the minimum just to be sure double squareSizePixels = min(double(chessboardZoneImg.cols) / double(charucoImpl->sizeX), double(chessboardZoneImg.rows) / double(charucoImpl->sizeY)); double diffSquareMarkerLength = (charucoImpl->squareLength - charucoImpl->markerLength) / 2; int diffSquareMarkerLengthPixels = int(diffSquareMarkerLength * squareSizePixels / charucoImpl->squareLength); // draw markers Mat markersImg; Board::generateImage(chessboardZoneImg.size(), markersImg, diffSquareMarkerLengthPixels, borderBits); markersImg.copyTo(chessboardZoneImg); // now draw black squares for(int y = 0; y < charucoImpl->sizeY; y++) { for(int x = 0; x < charucoImpl->sizeX; x++) { if(y % 2 != x % 2) continue; // white corner, dont do anything double startX, startY; startX = squareSizePixels * double(x); startY = squareSizePixels * double(y); Mat squareZone = chessboardZoneImg.rowRange(int(startY), int(startY + squareSizePixels)) .colRange(int(startX), int(startX + squareSizePixels)); squareZone.setTo(0); } } } /** * Fill nearestMarkerIdx and nearestMarkerCorners arrays */ void CharucoBoard::CharucoImpl::_getNearestMarkerCorners(CharucoBoard &board, float squareLength) { board.charucoImpl->nearestMarkerIdx.resize(board.charucoImpl->chessboardCorners.size()); board.charucoImpl->nearestMarkerCorners.resize(board.charucoImpl->chessboardCorners.size()); unsigned int nMarkers = (unsigned int)board.getIds().size(); unsigned int nCharucoCorners = (unsigned int)board.charucoImpl->chessboardCorners.size(); for(unsigned int i = 0; i < nCharucoCorners; i++) { double minDist = -1; // distance of closest markers Point3f charucoCorner = board.charucoImpl->chessboardCorners[i]; for(unsigned int j = 0; j < nMarkers; j++) { // calculate distance from marker center to charuco corner Point3f center = Point3f(0, 0, 0); for(unsigned int k = 0; k < 4; k++) center += board.getObjPoints()[j][k]; center /= 4.; double sqDistance; Point3f distVector = charucoCorner - center; sqDistance = distVector.x * distVector.x + distVector.y * distVector.y; if(j == 0 || fabs(sqDistance - minDist) < cv::pow(0.01 * squareLength, 2)) { // if same minimum distance (or first iteration), add to nearestMarkerIdx vector board.charucoImpl->nearestMarkerIdx[i].push_back(j); minDist = sqDistance; } else if(sqDistance < minDist) { // if finding a closest marker to the charuco corner board.charucoImpl->nearestMarkerIdx[i].clear(); // remove any previous added marker board.charucoImpl->nearestMarkerIdx[i].push_back(j); // add the new closest marker index minDist = sqDistance; } } // for each of the closest markers, search the marker corner index closer // to the charuco corner for(unsigned int j = 0; j < board.charucoImpl->nearestMarkerIdx[i].size(); j++) { board.charucoImpl->nearestMarkerCorners[i].resize(board.charucoImpl->nearestMarkerIdx[i].size()); double minDistCorner = -1; for(unsigned int k = 0; k < 4; k++) { double sqDistance; Point3f distVector = charucoCorner - board.getObjPoints()[board.charucoImpl->nearestMarkerIdx[i][j]][k]; sqDistance = distVector.x * distVector.x + distVector.y * distVector.y; if(k == 0 || sqDistance < minDistCorner) { // if this corner is closer to the charuco corner, assing its index // to nearestMarkerCorners minDistCorner = sqDistance; board.charucoImpl->nearestMarkerCorners[i][j] = k; } } } } } Ptr CharucoBoard::create(int squaresX, int squaresY, float squareLength, float markerLength, const Dictionary &dictionary, InputArray ids) { CV_Assert(squaresX > 1 && squaresY > 1 && markerLength > 0 && squareLength > markerLength); CharucoBoard board; Ptr res = makePtr(board); res->charucoImpl->sizeX = squaresX; res->charucoImpl->sizeY = squaresY; res->charucoImpl->squareLength = squareLength; res->charucoImpl->markerLength = markerLength; res->boardImpl->dictionary = dictionary; vector > objPoints; float diffSquareMarkerLength = (squareLength - markerLength) / 2; int totalMarkers = (int)(ids.total()); ids.copyTo(res->boardImpl->ids); // calculate Board objPoints int nextId = 0; for(int y = 0; y < squaresY; y++) { for(int x = 0; x < squaresX; x++) { if(y % 2 == x % 2) continue; // black corner, no marker here vector corners(4); corners[0] = Point3f(x * squareLength + diffSquareMarkerLength, y * squareLength + diffSquareMarkerLength, 0); corners[1] = corners[0] + Point3f(markerLength, 0, 0); corners[2] = corners[0] + Point3f(markerLength, markerLength, 0); corners[3] = corners[0] + Point3f(0, markerLength, 0); objPoints.push_back(corners); // first ids in dictionary if (totalMarkers == 0) res->boardImpl->ids.push_back(nextId); nextId++; } } if (totalMarkers > 0 && nextId != totalMarkers) CV_Error(cv::Error::StsBadSize, "Size of ids must be equal to the number of markers: "+std::to_string(nextId)); res->boardImpl->objPoints = objPoints; // now fill chessboardCorners for(int y = 0; y < squaresY - 1; y++) { for(int x = 0; x < squaresX - 1; x++) { Point3f corner; corner.x = (x + 1) * squareLength; corner.y = (y + 1) * squareLength; corner.z = 0; res->charucoImpl->chessboardCorners.push_back(corner); } } res->boardImpl->rightBottomBorder = Point3f(squaresX * squareLength, squaresY * squareLength, 0.f); CharucoBoard::CharucoImpl::_getNearestMarkerCorners(*res, res->charucoImpl->squareLength); return res; } Size CharucoBoard::getChessboardSize() const { return Size(charucoImpl->sizeX, charucoImpl->sizeY); } float CharucoBoard::getSquareLength() const { return charucoImpl->squareLength; } float CharucoBoard::getMarkerLength() const { return charucoImpl->markerLength; } bool CharucoBoard::checkCharucoCornersCollinear(InputArray charucoIds) const { unsigned int nCharucoCorners = (unsigned int)charucoIds.getMat().total(); if (nCharucoCorners <= 2) return true; // only test if there are 3 or more corners CV_Assert(charucoImpl->chessboardCorners.size() >= charucoIds.getMat().total()); Vec point0(charucoImpl->chessboardCorners[charucoIds.getMat().at(0)].x, charucoImpl->chessboardCorners[charucoIds.getMat().at(0)].y, 1); Vec point1(charucoImpl->chessboardCorners[charucoIds.getMat().at(1)].x, charucoImpl->chessboardCorners[charucoIds.getMat().at(1)].y, 1); // create a line from the first two points. Vec testLine = point0.cross(point1); Vec testPoint(0, 0, 1); double divisor = sqrt(testLine[0]*testLine[0] + testLine[1]*testLine[1]); CV_Assert(divisor != 0.0); // normalize the line with normal testLine /= divisor; double dotProduct; for (unsigned int i = 2; i < nCharucoCorners; i++){ testPoint(0) = charucoImpl->chessboardCorners[charucoIds.getMat().at(i)].x; testPoint(1) = charucoImpl->chessboardCorners[charucoIds.getMat().at(i)].y; // if testPoint is on testLine, dotProduct will be zero (or very, very close) dotProduct = testPoint.dot(testLine); if (std::abs(dotProduct) > 1e-6){ return false; } } // no points found that were off of testLine, return true that all points collinear. return true; } std::vector CharucoBoard::getChessboardCorners() const { return charucoImpl->chessboardCorners; } std::vector > CharucoBoard::getNearestMarkerIdx() const { return charucoImpl->nearestMarkerIdx; } std::vector > CharucoBoard::getNearestMarkerCorners() const { return charucoImpl->nearestMarkerCorners; } } }