package goadb import ( "bytes" "fmt" "image" "image/color" "image/png" "strconv" "strings" "github.com/otiai10/gosseract/v2" ) // Get the state of the screen (Asleep/Awake) func (d *Device) GetScreenState() (string, error) { data, err := d.RunCommand("shell dumpsys power | grep mWakefulness | head -n1") if err != nil { return "", err } state := strings.Split(string(data), "=")[1] return state, err } // Toggle the screen by simulating a press of the power button func (d *Device) ToggleScreen() error { _, err := d.RunCommand("shell input keyevent 26") return err } // Get the resolution of the screen func (d *Device) GetScreenResolution() (int, int, error) { //Not yet set on the device so use adb to retrieve it if d.Screenx == 0 || d.Screeny == 0 { data, err := d.RunCommand("shell wm size") res := strings.Split(string(data), ":")[1] dimensions := strings.Split(res, "x") if len(dimensions) != 2 { return -1, -1, fmt.Errorf("invalid dimensions for screen") } x, err := strconv.Atoi(strings.TrimSpace(dimensions[0])) if err != nil { return -1, -1, err } y, err := strconv.Atoi(strings.TrimSpace(dimensions[1])) if err != nil { return -1, -1, err } d.Screenx = x d.Screeny = y return x, y, err } return d.Screenx, d.Screeny, nil } // Get the text on screen in a certain area using gotesseract func (d *Device) GetScreenText(xStart int, xEnd int, yStart int, yEnd int, whitelist string, multiline bool) (string, error) { // Take screenshot in memory and get bytes data, err := d.RunCommand("exec-out screencap -p") if err != nil { return "", fmt.Errorf("failed to take screenshot: %w", err) } // Decode image from bytes img, err := png.Decode(bytes.NewReader(data)) if err != nil { return "", fmt.Errorf("failed to decode PNG: %w", err) } // Crop region rect := image.Rect(0, 0, xEnd-xStart, yEnd-yStart) cropped := image.NewRGBA(rect) for y := yStart; y < yEnd; y++ { for x := xStart; x < xEnd; x++ { cropped.Set(x-xStart, y-yStart, img.At(x, y)) } } // Convert to grayscale and apply threshold threshold := uint8(160) binarized := image.NewGray(rect) for y := 0; y < rect.Dy(); y++ { for x := 0; x < rect.Dx(); x++ { gray := color.GrayModel.Convert(cropped.At(x, y)).(color.Gray) if gray.Y > threshold { binarized.SetGray(x, y, color.Gray{Y: 255}) } else { binarized.SetGray(x, y, color.Gray{Y: 0}) } } } // Encode binarized image to PNG var buf bytes.Buffer if err := png.Encode(&buf, binarized); err != nil { return "", fmt.Errorf("failed to encode image: %w", err) } // Call tesseract client := gosseract.NewClient() defer client.Close() client.SetImageFromBytes(buf.Bytes()) client.SetWhitelist(whitelist) if multiline { client.SetPageSegMode(gosseract.PSM_AUTO) } else { client.SetPageSegMode(gosseract.PSM_SINGLE_BLOCK) } text, err := client.Text() if err != nil { return "", fmt.Errorf("tesseract error: %w", err) } return text, nil }