package goadb import ( "bytes" "fmt" "image" "image/color" "image/png" "strconv" "strings" "github.com/otiai10/gosseract/v2" ) // Get the state of the screen (Asleep/Awake) func (d *Device) GetScreenState() (string, error) { data, err := d.RunCommand("shell dumpsys power | grep mWakefulness | head -n1") if err != nil { return "", err } state := strings.Split(string(data), "=")[1] return state, err } // Toggle the screen by simulating a press of the power button func (d *Device) ToggleScreen() error { _, err := d.RunCommand("shell input keyevent 26") return err } // Tap on a location on the screen func (d *Device) TapScreen(x int, y int) error { _, err := d.RunCommand(fmt.Sprintf("shell input tap %d %d", x, y)) return err } // Get the resolution of the screen func (d *Device) GetScreenResolution() (int, int, error) { //Not yet set on the device so use adb to retrieve it if d.Screenx == 0 || d.Screeny == 0 { data, err := d.RunCommand("shell wm size") res := strings.Split(string(data), ":")[1] dimensions := strings.Split(res, "x") if len(dimensions) != 2 { return -1, -1, fmt.Errorf("invalid dimensions for screen") } x, err := strconv.Atoi(strings.TrimSpace(dimensions[0])) if err != nil { return -1, -1, err } y, err := strconv.Atoi(strings.TrimSpace(dimensions[1])) if err != nil { return -1, -1, err } d.Screenx = x d.Screeny = y return x, y, err } return d.Screenx, d.Screeny, nil } func (d *Device) GetScreenText(xStart int, xEnd int, yStart int, yEnd int, whitelist string, multiline bool) (string, error) { // Take screenshot in memory and get bytes data, err := d.RunCommand("exec-out screencap -p") if err != nil { return "", fmt.Errorf("failed to take screenshot: %w", err) } // Decode image from bytes img, err := png.Decode(bytes.NewReader(data)) if err != nil { return "", fmt.Errorf("failed to decode PNG: %w", err) } // Crop region rect := image.Rect(0, 0, xEnd-xStart, yEnd-yStart) cropped := image.NewRGBA(rect) for y := yStart; y < yEnd; y++ { for x := xStart; x < xEnd; x++ { cropped.Set(x-xStart, y-yStart, img.At(x, y)) } } // Convert to grayscale and compute average brightness gray := image.NewGray(rect) var totalBrightness int for y := 0; y < rect.Dy(); y++ { for x := 0; x < rect.Dx(); x++ { grayVal := color.GrayModel.Convert(cropped.At(x, y)).(color.Gray) gray.Set(x, y, grayVal) totalBrightness += int(grayVal.Y) } } avgBrightness := totalBrightness / (rect.Dx() * rect.Dy()) // Binarize based on brightness polarity binarized := image.NewGray(rect) for y := 0; y < rect.Dy(); y++ { for x := 0; x < rect.Dx(); x++ { grayVal := gray.GrayAt(x, y).Y if avgBrightness < 128 { // Dark background, light text if int(grayVal) > avgBrightness+10 { binarized.SetGray(x, y, color.Gray{Y: 0}) } else { binarized.SetGray(x, y, color.Gray{Y: 255}) } } else { // Light background, dark text if int(grayVal) < avgBrightness-10 { binarized.SetGray(x, y, color.Gray{Y: 0}) } else { binarized.SetGray(x, y, color.Gray{Y: 255}) } } } } // Encode binarized image to PNG var buf bytes.Buffer if err := png.Encode(&buf, binarized); err != nil { return "", fmt.Errorf("failed to encode image: %w", err) } // Call Tesseract client := gosseract.NewClient() defer client.Close() client.SetImageFromBytes(buf.Bytes()) client.SetWhitelist(whitelist) if multiline { client.SetPageSegMode(gosseract.PSM_AUTO) } else { client.SetPageSegMode(gosseract.PSM_SINGLE_BLOCK) } text, err := client.Text() if err != nil { return "", fmt.Errorf("tesseract error: %w", err) } return text, nil }