From f49ddb6e81b847069ef6565cb5c346a4521a1300 Mon Sep 17 00:00:00 2001
From: Michael Utama <13521137@std.stei.itb.ac.id>
Date: Sun, 19 May 2024 09:10:52 +0000
Subject: [PATCH] improve corner detection technique
---
.../data/view_models/ImageDataViewModel.kt | 11 +-
.../java/com/k2_9/omrekap/utils/CropHelper.kt | 95 ++++++++++++++++--
.../omrekap/utils/omr/ContourOMRHelper.kt | 11 +-
.../views/activities/PreviewActivity.kt | 2 +-
app/src/main/res/raw/mark16.png | Bin 0 -> 203 bytes
app/src/main/res/raw/mark20.png | Bin 0 -> 167 bytes
6 files changed, 103 insertions(+), 16 deletions(-)
create mode 100644 app/src/main/res/raw/mark16.png
create mode 100644 app/src/main/res/raw/mark20.png
diff --git a/app/src/main/java/com/k2_9/omrekap/data/view_models/ImageDataViewModel.kt b/app/src/main/java/com/k2_9/omrekap/data/view_models/ImageDataViewModel.kt
index 1ba7b58..bfa8ed2 100644
--- a/app/src/main/java/com/k2_9/omrekap/data/view_models/ImageDataViewModel.kt
+++ b/app/src/main/java/com/k2_9/omrekap/data/view_models/ImageDataViewModel.kt
@@ -94,11 +94,12 @@ class ImageDataViewModel : ViewModel() {
for ((section, value) in it) {
stringKeyResult[pageContent[section]!!] = value
- annotatedImage = ImageAnnotationHelper.annotateOMR(
- annotatedImage,
- contourOMRHelper.getSectionPosition(section),
- value
- )
+ annotatedImage =
+ ImageAnnotationHelper.annotateOMR(
+ annotatedImage,
+ contourOMRHelper.getSectionPosition(section),
+ value,
+ )
Log.d("Result", "${pageContent[section]}: $value")
}
}
diff --git a/app/src/main/java/com/k2_9/omrekap/utils/CropHelper.kt b/app/src/main/java/com/k2_9/omrekap/utils/CropHelper.kt
index 03324a2..57b8e46 100644
--- a/app/src/main/java/com/k2_9/omrekap/utils/CropHelper.kt
+++ b/app/src/main/java/com/k2_9/omrekap/utils/CropHelper.kt
@@ -4,15 +4,18 @@ import android.graphics.Bitmap
import android.util.Log
import com.k2_9.omrekap.data.models.CornerPoints
import org.opencv.android.Utils
+import org.opencv.core.Core
import org.opencv.core.CvType
import org.opencv.core.Mat
import org.opencv.core.MatOfPoint2f
import org.opencv.core.Point
+import org.opencv.core.Size
import org.opencv.imgproc.Imgproc
import org.opencv.imgproc.Imgproc.COLOR_BGR2GRAY
import org.opencv.imgproc.Imgproc.cvtColor
import org.opencv.imgproc.Imgproc.getPerspectiveTransform
import org.opencv.imgproc.Imgproc.warpPerspective
+import kotlin.math.min
import kotlin.math.pow
import kotlin.math.sqrt
@@ -54,9 +57,12 @@ object CropHelper {
throw Exception("Pattern not loaded!")
}
- val imgGray = img.clone()
+ var imgGray = img.clone()
cvtColor(img, imgGray, COLOR_BGR2GRAY)
+ // do local normalization here
+ imgGray = localNormalize(imgGray)
+
val resultMatrix =
Mat(
img.height() - pattern.height() + 1,
@@ -81,10 +87,22 @@ object CropHelper {
)
val pointsList: MutableList<PointsAndWeight> = mutableListOf()
-
- for (i in 0 until resultMatrix.height() step 4) {
- for (j in 0 until resultMatrix.width() step 4) {
- pointsList.add(PointsAndWeight(i, j, resultMatrix.get(i, j)[0]))
+ val diagonalLength = (resultMatrix.height().toDouble().pow(2) * resultMatrix.width().toDouble().pow(2)).pow(1 / 2)
+
+ for (i in 0 until resultMatrix.height() step 2) {
+ for (j in 0 until resultMatrix.width() step 2) {
+ pointsList.add(
+ PointsAndWeight(
+ i,
+ j,
+ resultMatrix.get(i, j)[0] *
+ getWeight(
+ (
+ min(i, resultMatrix.height() - i).toDouble().pow(2) * min(j, resultMatrix.width() - j).toDouble().pow(2)
+ ).pow(1 / 2) / diagonalLength,
+ ),
+ ),
+ )
}
}
@@ -93,9 +111,22 @@ object CropHelper {
pointsList.forEach {
if (needChange == 0) return@forEach
- val corner = nearWhichCorner(it.x, it.y, resultMatrix.height(), resultMatrix.width(), limFrac = 0.1F)
+ val corner = nearWhichCorner(it.x, it.y, resultMatrix.height(), resultMatrix.width(), limFrac = 0.6F)
if (corner == -1) return@forEach
+ if (it.weight > 0.45) {
+ // Corner not found, throw exception
+ val exceptionMessage =
+ "Not all corners found: {" +
+ (if (needed[0]) "Upper left," else "") +
+ (if (needed[1]) "Upper right," else "") +
+ (if (needed[2]) "Lower right," else "") +
+ (if (needed[3]) "Lower left," else "") +
+ "}"
+ // throw NotFoundException(exceptionMessage);
+ Log.e("Corner", exceptionMessage)
+ }
+
if (needed[corner]) {
needed[corner] = false
needChange--
@@ -238,4 +269,56 @@ object CropHelper {
else -> -1
}
}
+
+ /**
+ * Apply local normalization to an image
+ * source: https://stackoverflow.com/questions/43240604/python-local-normalization-in-opencv
+ * @see - https://bigwww.epfl.ch/demo/ip/demos/local-normalization/
+ *
+ * @param img input matrix
+ * @return local normalized img
+ */
+ private fun localNormalize(img: Mat): Mat {
+ // convert img to CV_32F
+ val gray = Mat()
+ img.convertTo(gray, CvType.CV_32F, 1.0 / 255.0)
+
+ val blur = Mat()
+ Imgproc.GaussianBlur(gray, blur, Size(0.0, 0.0), 2.0, 2.0)
+
+ val num = Mat()
+ Core.subtract(gray, blur, num)
+
+ val numSquared = Mat()
+ Core.multiply(num, num, numSquared)
+ val blur2 = Mat()
+ Imgproc.GaussianBlur(numSquared, blur2, Size(0.0, 0.0), 20.0, 20.0)
+
+ val den = Mat()
+ Core.sqrt(blur2, den)
+
+ val div = Mat()
+ Core.divide(num, den, div)
+
+ Core.normalize(div, div, 0.0, 1.0, Core.NORM_MINMAX)
+
+ // Convert back to uint8
+ val result = Mat()
+ div.convertTo(result, CvType.CV_8U, 255.0)
+
+ return result
+ }
+
+ /**
+ * Weight for a point
+ * Far from corner means bigger weight
+ *
+ * Smaller weight are more likely to be chosen as corner
+ *
+ * @param normDistance distance from nearest corner divided by diagonal
+ * @return weight
+ */
+ private fun getWeight(normDistance: Double): Double {
+ return 1 + 1 * normDistance.pow(1.5)
+ }
}
diff --git a/app/src/main/java/com/k2_9/omrekap/utils/omr/ContourOMRHelper.kt b/app/src/main/java/com/k2_9/omrekap/utils/omr/ContourOMRHelper.kt
index 43fdba7..c60aaea 100644
--- a/app/src/main/java/com/k2_9/omrekap/utils/omr/ContourOMRHelper.kt
+++ b/app/src/main/java/com/k2_9/omrekap/utils/omr/ContourOMRHelper.kt
@@ -170,15 +170,19 @@ class ContourOMRHelper(private val config: ContourOMRHelperConfig) : OMRHelper(c
* @param radius radius of the circle
* @return perfect circle contour
*/
- private fun getPerfectCircle(x: Double, y: Double, radius: Double): MatOfPoint {
- val numPoints = 100 // Adjust as needed
+ private fun getPerfectCircle(
+ x: Double,
+ y: Double,
+ radius: Double,
+ ): MatOfPoint {
+ val numPoints = 100 // Adjust as needed
val theta = DoubleArray(numPoints) { it * 2 * Math.PI / numPoints }
val circleX = DoubleArray(numPoints) { x + radius * cos(theta[it]) }
val circleY = DoubleArray(numPoints) { y + radius * sin(theta[it]) }
val circleContour = MatOfPoint()
for (i in 0 until numPoints) {
- circleContour.push_back(MatOfPoint(Point(circleX[i], circleY[i])));
+ circleContour.push_back(MatOfPoint(Point(circleX[i], circleY[i])))
}
return circleContour
@@ -388,7 +392,6 @@ class ContourOMRHelper(private val config: ContourOMRHelperConfig) : OMRHelper(c
Utils.matToBitmap(display, bitmap)
SaveHelper.saveImage(appContext!!, bitmap, "test", "lol.png")
-
}
return filteredContours
diff --git a/app/src/main/java/com/k2_9/omrekap/views/activities/PreviewActivity.kt b/app/src/main/java/com/k2_9/omrekap/views/activities/PreviewActivity.kt
index 240353e..bd37f1d 100644
--- a/app/src/main/java/com/k2_9/omrekap/views/activities/PreviewActivity.kt
+++ b/app/src/main/java/com/k2_9/omrekap/views/activities/PreviewActivity.kt
@@ -50,7 +50,7 @@ class PreviewActivity : AppCompatActivity() {
val bitmapOptions = BitmapFactory.Options()
bitmapOptions.inPreferredConfig = Bitmap.Config.ALPHA_8
bitmapOptions.inScaled = false
- val cornerPatternBitmap: Bitmap = BitmapFactory.decodeResource(resources, R.raw.corner_pattern, bitmapOptions)
+ val cornerPatternBitmap: Bitmap = BitmapFactory.decodeResource(resources, R.raw.mark20, bitmapOptions)
CropHelper.loadPattern(cornerPatternBitmap)
diff --git a/app/src/main/res/raw/mark16.png b/app/src/main/res/raw/mark16.png
new file mode 100644
index 0000000000000000000000000000000000000000..e34e5106f50567a9baecca15a4ba92bd1c58670f
GIT binary patch
literal 203
zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`jKx9jP7LeL$-D$|SkfJR9T^xl
z_H+M9WCij$3p^r=85o30K$!7fntTONFu~KsF~q{Zwa1a`fP=uHZ~y1tt137plEnK&
z%=Xd0n*oOpZ(P{cp6txaf4ogfu8-H$Mn*cT!ouvzhk~3dO@8*fx*ue`)@xe4b~i`g
jws%Svw~z(m&et;3EUKM9ZSglPpj`}}u6{1-oD!M<W!6Wf
literal 0
HcmV?d00001
diff --git a/app/src/main/res/raw/mark20.png b/app/src/main/res/raw/mark20.png
new file mode 100644
index 0000000000000000000000000000000000000000..7c65cbf6ea9c87669d1c11dec0d6d510b7ac3561
GIT binary patch
literal 167
zcmeAS@N?(olHy`uVBq!ia0vp^A|TAc1|)ksWqE-VV{wqX6T`Z5GB1G~mUKs7M+SzC
z{oH>NS%G}c0*}aI1_q%L5N5oWCSL&*wDELt46*P}PDx1k!GDD1;Q#+Fn#l=GmqM5q
zvjy?F@h*`uU@B$^jBJ=FIAtYA^F@z^GCfirmJ2pD^6)TRuH;av(rLL5G>5^{)z4*}
HQ$iB}hD<F3
literal 0
HcmV?d00001
--
GitLab