agent/computer.py

# Copyright 2025 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import abc
import pydantic
from typing import Literal, Tuple, List


class EnvState(pydantic.BaseModel):
    # The screenshot in PNG format.
    screenshot: bytes
    url: str


class Computer(abc.ABC):
    """Defines an interface for environments."""

    @abc.abstractmethod
    def screen_size(self) -> Tuple[int, int]:
        """Returns the screen size of the environment."""

    @abc.abstractmethod
    def open_web_browser(self) -> EnvState:
        """Opens the web browser."""

    @abc.abstractmethod
    def click_at(self, x: int, y: int) -> EnvState:
        """Clicks at a specific x, y  coordinate on the webpage.

        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
        """

    @abc.abstractmethod
    def hover_at(self, x: int, y: int) -> EnvState:
        """Hovers at a specific x, y coordinate on the webpage.

        May be used to explore sub-menus that appear on hover.
        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
        """

    @abc.abstractmethod
    def type_text_at(
        self,
        x: int,
        y: int,
        text: str,
        press_enter: bool,
        clear_before_typing: bool,
    ) -> EnvState:
        """Types text at a specific x, y coordinate.

        The system automatically presses ENTER after typing. To disable this, set `press_enter` to False.
        The system automatically clears any existing content before typing the specified `text`. To disable this, set `clear_before_typing` to False.
        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
        """

    @abc.abstractmethod
    def scroll_document(
        self, direction: Literal["up", "down", "left", "right"]
    ) -> EnvState:
        """Scrolls the entire webpage "up", "down", "left" or "right" based on direction."""

    @abc.abstractmethod
    def scroll_at(
        self,
        x: int,
        y: int,
        direction: Literal["up", "down", "left", "right"],
        magnitude: int,
    ) -> EnvState:
        """Scrolls up, down, right, or left at a x, y coordinate by magnitude.

        The 'x' and 'y' values are absolute values, scaled to the height and width of the screen.
        """

    @abc.abstractmethod
    def wait_5_seconds(self) -> EnvState:
        """Waits for 5 seconds to allow unfinished webpage processes to complete."""

    @abc.abstractmethod
    def go_back(self) -> EnvState:
        """Navigates back to the previous webpage in the browser history."""

    @abc.abstractmethod
    def go_forward(self) -> EnvState:
        """Navigates forward to the next webpage in the browser history."""

    @abc.abstractmethod
    def search(self) -> EnvState:
        """Directly jumps to a search engine home page.

        Used when you need to start with a search. For example, this is used when
        the current website doesn't have the information needed or because a new
        task is being started.
        """

    @abc.abstractmethod
    def navigate(self, url: str) -> EnvState:
        """Navigates directly to a specified URL."""

    @abc.abstractmethod
    def key_combination(self, keys: List[str]) -> EnvState:
        """Presses keyboard keys and combinations, such as "control+c" or "enter"."""

    @abc.abstractmethod
    def drag_and_drop(
        self, x: int, y: int, destination_x: int, destination_y: int
    ) -> EnvState:
        """Drag and drop an element from a x, y coordinate to a destination destination_y, destination_x coordinate.
        The 'x', 'y', 'destination_y' and 'destination_x' values are absolute values, scaled to the height and width of the screen.
        """

    @abc.abstractmethod
    def current_state(self) -> EnvState:
        """Returns the current state of the current webpage."""