diff --git a/modules/dnn/src/layers/permute_layer.cpp b/modules/dnn/src/layers/permute_layer.cpp
index e08bfa0c5c..e3129556ba 100644
--- a/modules/dnn/src/layers/permute_layer.cpp
+++ b/modules/dnn/src/layers/permute_layer.cpp
@@ -174,21 +174,9 @@ public:
         computeStrides(shape(inputs[0]), shape(outputs[0]));
 
 #ifdef HAVE_OPENCL
-        if (uorder.empty())
-        {
-            std::vector<int> orderVec(_order.begin(), _order.end());;
-            Mat morder(1, orderVec.size(), CV_32SC1, &orderVec[0]);
-
-            std::vector<int> oldStrideVec(_oldStride.begin(), _oldStride.end());
-            Mat mold_stride(1, _oldStride.size(), CV_32SC1, &oldStrideVec[0]);
-
-            std::vector<int> newStrideVec(_newStride.begin(), _newStride.end());
-            Mat mnew_stride(1, newStrideVec.size(), CV_32SC1, &newStrideVec[0]);
-
-            morder.copyTo(uorder);
-            mold_stride.copyTo(uold_stride);
-            mnew_stride.copyTo(unew_stride);
-        }
+        uorder.release();
+        uold_stride.release();
+        unew_stride.release();
 #endif
     }
 
@@ -276,6 +264,22 @@ public:
         if (!_needsPermute)
             return false;
 
+        if (uorder.empty())
+        {
+            std::vector<int> orderVec(_order.begin(), _order.end());;
+            Mat morder(1, orderVec.size(), CV_32SC1, &orderVec[0]);
+
+            std::vector<int> oldStrideVec(_oldStride.begin(), _oldStride.end());
+            Mat mold_stride(1, _oldStride.size(), CV_32SC1, &oldStrideVec[0]);
+
+            std::vector<int> newStrideVec(_newStride.begin(), _newStride.end());
+            Mat mnew_stride(1, newStrideVec.size(), CV_32SC1, &newStrideVec[0]);
+
+            morder.copyTo(uorder);
+            mold_stride.copyTo(uold_stride);
+            mnew_stride.copyTo(unew_stride);
+        }
+
         bool use_half = (inps.depth() == CV_16S);
         String opts = format("-DDtype=%s", use_half ? "half" : "float");
         for (size_t i = 0; i < inputs.size(); i++)