GoADB/screen.go

package goadb

import (
	"bytes"
	"fmt"
	"image"
	"image/color"
	"image/png"
	"strconv"
	"strings"

	"github.com/otiai10/gosseract/v2"
)

// Get the state of the screen (Asleep/Awake)
func (d *Device) GetScreenState() (string, error) {
	data, err := d.RunCommand("shell dumpsys power | grep mWakefulness | head -n1")
	if err != nil {
		return "", err
	}
	state := strings.Split(string(data), "=")[1]
	return state, err
}

// Toggle the screen by simulating a press of the power button
func (d *Device) ToggleScreen() error {
	_, err := d.RunCommand("shell input keyevent 26")
	return err
}

// Tap on a location on the screen
func (d *Device) TapScreen(x int, y int) error {
	_, err := d.RunCommand(fmt.Sprintf("shell input tap %d %d", x, y))
	return err
}

// Get the resolution of the screen
func (d *Device) GetScreenResolution() (int, int, error) {

	//Not yet set on the device so use adb to retrieve it
	if d.Screenx == 0 || d.Screeny == 0 {

		data, err := d.RunCommand("shell wm size")
		res := strings.Split(string(data), ":")[1]
		dimensions := strings.Split(res, "x")

		if len(dimensions) != 2 {
			return -1, -1, fmt.Errorf("invalid dimensions for screen")
		}

		x, err := strconv.Atoi(strings.TrimSpace(dimensions[0]))
		if err != nil {
			return -1, -1, err
		}

		y, err := strconv.Atoi(strings.TrimSpace(dimensions[1]))
		if err != nil {
			return -1, -1, err
		}

		d.Screenx = x
		d.Screeny = y

		return x, y, err
	}

	return d.Screenx, d.Screeny, nil
}

func (d *Device) GetScreenText(xStart int, xEnd int, yStart int, yEnd int, whitelist string, multiline bool, textPolarity string) (string, error) {
	data, err := d.RunCommand("exec-out screencap -p")
	if err != nil {
		return "", fmt.Errorf("failed to take screenshot: %w", err)
	}

	img, err := png.Decode(bytes.NewReader(data))
	if err != nil {
		return "", fmt.Errorf("failed to decode PNG: %w", err)
	}

	rect := image.Rect(0, 0, xEnd-xStart, yEnd-yStart)
	cropped := image.NewRGBA(rect)
	for y := yStart; y < yEnd; y++ {
		for x := xStart; x < xEnd; x++ {
			cropped.Set(x-xStart, y-yStart, img.At(x, y))
		}
	}

	// Convert to grayscale and compute average brightness
	gray := image.NewGray(rect)
	var totalBrightness int
	for y := 0; y < rect.Dy(); y++ {
		for x := 0; x < rect.Dx(); x++ {
			grayVal := color.GrayModel.Convert(cropped.At(x, y)).(color.Gray)
			gray.Set(x, y, grayVal)
			totalBrightness += int(grayVal.Y)
		}
	}
	avgBrightness := totalBrightness / (rect.Dx() * rect.Dy())

	// Decide polarity
	useLightText := false
	switch textPolarity {
	case "light":
		useLightText = true
	case "dark":
		useLightText = false
	case "auto":
		useLightText = avgBrightness < 128
	default:
		return "", fmt.Errorf("invalid textPolarity: must be 'dark', 'light', or 'auto'")
	}

	// Binarize based on polarity
	binarized := image.NewGray(rect)
	for y := 0; y < rect.Dy(); y++ {
		for x := 0; x < rect.Dx(); x++ {
			grayVal := gray.GrayAt(x, y).Y
			if useLightText {
				if int(grayVal) > avgBrightness+10 {
					binarized.SetGray(x, y, color.Gray{Y: 0})
				} else {
					binarized.SetGray(x, y, color.Gray{Y: 255})
				}
			} else {
				if int(grayVal) < avgBrightness-10 {
					binarized.SetGray(x, y, color.Gray{Y: 0})
				} else {
					binarized.SetGray(x, y, color.Gray{Y: 255})
				}
			}
		}
	}

	var buf bytes.Buffer
	if err := png.Encode(&buf, binarized); err != nil {
		return "", fmt.Errorf("failed to encode image: %w", err)
	}

	client := gosseract.NewClient()
	defer client.Close()
	client.SetImageFromBytes(buf.Bytes())
	client.SetWhitelist(whitelist)
	if multiline {
		client.SetPageSegMode(gosseract.PSM_AUTO)
	} else {
		client.SetPageSegMode(gosseract.PSM_SINGLE_BLOCK)
	}

	text, err := client.Text()
	if err != nil {
		return "", fmt.Errorf("tesseract error: %w", err)
	}
	return text, nil
}