screen: update tresholds on text

This commit is contained in:
2025-11-07 19:19:48 +01:00
parent 53cf344cc0
commit 7c4281ef05

View File

@@ -67,21 +67,17 @@ func (d *Device) GetScreenResolution() (int, int, error) {
return d.Screenx, d.Screeny, nil
}
// Get the text on screen in a certain area using gotesseract
func (d *Device) GetScreenText(xStart int, xEnd int, yStart int, yEnd int, whitelist string, multiline bool) (string, error) {
// Take screenshot in memory and get bytes
data, err := d.RunCommand("exec-out screencap -p")
if err != nil {
return "", fmt.Errorf("failed to take screenshot: %w", err)
}
// Decode image from bytes
img, err := png.Decode(bytes.NewReader(data))
if err != nil {
return "", fmt.Errorf("failed to decode PNG: %w", err)
}
// Crop region
rect := image.Rect(0, 0, xEnd-xStart, yEnd-yStart)
cropped := image.NewRGBA(rect)
for y := yStart; y < yEnd; y++ {
@@ -90,27 +86,45 @@ func (d *Device) GetScreenText(xStart int, xEnd int, yStart int, yEnd int, white
}
}
// Convert to grayscale and apply threshold
threshold := uint8(160)
binarized := image.NewGray(rect)
// Convert to grayscale
gray := image.NewGray(rect)
for y := 0; y < rect.Dy(); y++ {
for x := 0; x < rect.Dx(); x++ {
gray := color.GrayModel.Convert(cropped.At(x, y)).(color.Gray)
if gray.Y > threshold {
binarized.SetGray(x, y, color.Gray{Y: 255})
} else {
gray.Set(x, y, color.GrayModel.Convert(cropped.At(x, y)))
}
}
// Apply simple mean-based adaptive thresholding
binarized := image.NewGray(rect)
window := 15 // size of local region
for y := 0; y < rect.Dy(); y++ {
for x := 0; x < rect.Dx(); x++ {
var sum int
var count int
for dy := -window / 2; dy <= window/2; dy++ {
for dx := -window / 2; dx <= window/2; dx++ {
xx := x + dx
yy := y + dy
if xx >= 0 && xx < rect.Dx() && yy >= 0 && yy < rect.Dy() {
sum += int(gray.GrayAt(xx, yy).Y)
count++
}
}
}
mean := sum / count
if int(gray.GrayAt(x, y).Y) < mean-10 {
binarized.SetGray(x, y, color.Gray{Y: 0})
} else {
binarized.SetGray(x, y, color.Gray{Y: 255})
}
}
}
// Encode binarized image to PNG
var buf bytes.Buffer
if err := png.Encode(&buf, binarized); err != nil {
return "", fmt.Errorf("failed to encode image: %w", err)
}
// Call tesseract
client := gosseract.NewClient()
defer client.Close()
client.SetImageFromBytes(buf.Bytes())