package goadb import ( "bytes" "fmt" "image" "image/color" "image/png" "strconv" "strings" "github.com/otiai10/gosseract/v2" ) // Get the state of the screen (Asleep/Awake) func (d *Device) GetScreenState() (string, error) { data, err := d.RunCommand("shell dumpsys power | grep mWakefulness | head -n1") if err != nil { return "", err } state := strings.Split(string(data), "=")[1] return state, err } // Toggle the screen by simulating a press of the power button func (d *Device) ToggleScreen() error { _, err := d.RunCommand("shell input keyevent 26") return err } // Tap on a location on the screen func (d *Device) TapScreen(x int, y int) error { _, err := d.RunCommand(fmt.Sprintf("shell input tap %d %d", x, y)) return err } // Get the resolution of the screen func (d *Device) GetScreenResolution() (int, int, error) { //Not yet set on the device so use adb to retrieve it if d.Screenx == 0 || d.Screeny == 0 { data, err := d.RunCommand("shell wm size") res := strings.Split(string(data), ":")[1] dimensions := strings.Split(res, "x") if len(dimensions) != 2 { return -1, -1, fmt.Errorf("invalid dimensions for screen") } x, err := strconv.Atoi(strings.TrimSpace(dimensions[0])) if err != nil { return -1, -1, err } y, err := strconv.Atoi(strings.TrimSpace(dimensions[1])) if err != nil { return -1, -1, err } d.Screenx = x d.Screeny = y return x, y, err } return d.Screenx, d.Screeny, nil } func (d *Device) GetScreenText(xStart int, xEnd int, yStart int, yEnd int, whitelist string, multiline bool) (string, error) { data, err := d.RunCommand("exec-out screencap -p") if err != nil { return "", fmt.Errorf("failed to take screenshot: %w", err) } img, err := png.Decode(bytes.NewReader(data)) if err != nil { return "", fmt.Errorf("failed to decode PNG: %w", err) } rect := image.Rect(0, 0, xEnd-xStart, yEnd-yStart) cropped := image.NewRGBA(rect) for y := yStart; y < yEnd; y++ { for x := xStart; x < xEnd; x++ { cropped.Set(x-xStart, y-yStart, img.At(x, y)) } } // Convert to grayscale gray := image.NewGray(rect) for y := 0; y < rect.Dy(); y++ { for x := 0; x < rect.Dx(); x++ { gray.Set(x, y, color.GrayModel.Convert(cropped.At(x, y))) } } // Apply simple mean-based adaptive thresholding binarized := image.NewGray(rect) window := 15 // size of local region for y := 0; y < rect.Dy(); y++ { for x := 0; x < rect.Dx(); x++ { var sum int var count int for dy := -window / 2; dy <= window/2; dy++ { for dx := -window / 2; dx <= window/2; dx++ { xx := x + dx yy := y + dy if xx >= 0 && xx < rect.Dx() && yy >= 0 && yy < rect.Dy() { sum += int(gray.GrayAt(xx, yy).Y) count++ } } } mean := sum / count if int(gray.GrayAt(x, y).Y) < mean-10 { binarized.SetGray(x, y, color.Gray{Y: 0}) } else { binarized.SetGray(x, y, color.Gray{Y: 255}) } } } var buf bytes.Buffer if err := png.Encode(&buf, binarized); err != nil { return "", fmt.Errorf("failed to encode image: %w", err) } client := gosseract.NewClient() defer client.Close() client.SetImageFromBytes(buf.Bytes()) client.SetWhitelist(whitelist) if multiline { client.SetPageSegMode(gosseract.PSM_AUTO) } else { client.SetPageSegMode(gosseract.PSM_SINGLE_BLOCK) } text, err := client.Text() if err != nil { return "", fmt.Errorf("tesseract error: %w", err) } return text, nil }