# Copyright 2013-2018 Intranet AG and contributors
#
# guibot is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# guibot is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with guibot. If not, see <http://www.gnu.org/licenses/>.
"""
SUMMARY
------------------------------------------------------
Secondary (and more advanced) interface for generic screen regions.
The main guibot interface is just a specialized region where we could match
and work with subregions. Any region instance can also be a complete screen,
hence the increased generality of using this as an interface and calling it
directly.
INTERFACE
------------------------------------------------------
"""
import time
import os
# interconnected classes - carefully avoid circular reference
from .config import GlobalConfig
from .location import Location
from .imagelogger import ImageLogger
from .errors import *
from .target import *
from .finder import *
from .controller import *
import logging
log = logging.getLogger('guibot.region')
[docs]class Region(object):
"""
Region of the screen supporting vertex and nearby region selection,
validation of expected images, and mouse and keyboard control.
"""
[docs] def __init__(self, xpos=0, ypos=0, width=0, height=0,
dc=None, cv=None):
"""
Build a region object from upleft to downright vertex coordinates.
:param int xpos: x coordinate of the upleft vertex of the region
:param int ypos: y coordinate of the upleft vertex of the region
:param int width: width of the region (xpos+width for downright vertex x)
:param int height: height of the region (ypos+height for downright vertex y)
:param dc: DC backend used for any display control
:type dc: :py:class:`controller.Controller` or None
:param cv: CV backend used for any target finding
:type cv: :py:class:`finder.Finder` or None
:raises: :py:class:`UninitializedBackendError` if the region is empty
If any of the backends is not defined a new one will be initiated
using the parameters defined in :py:class:`config.GlobalConfig`.
If `width` or `height` remains zero, it will be set to the maximum
available within the screen space.
"""
if dc is None:
if GlobalConfig.display_control_backend == "autopy":
dc = AutoPyController()
elif GlobalConfig.display_control_backend == "xdotool":
dc = XDoToolController()
elif GlobalConfig.display_control_backend == "vncdotool":
dc = VNCDoToolController()
elif GlobalConfig.display_control_backend == "pyautogui":
dc = PyAutoGUIController()
if cv is None:
if GlobalConfig.find_backend == "autopy":
cv = AutoPyFinder()
elif GlobalConfig.find_backend == "contour":
cv = ContourFinder()
elif GlobalConfig.find_backend == "template":
cv = TemplateFinder()
elif GlobalConfig.find_backend == "feature":
cv = FeatureFinder()
elif GlobalConfig.find_backend == "cascade":
cv = CascadeFinder()
elif GlobalConfig.find_backend == "text":
cv = TextFinder()
elif GlobalConfig.find_backend == "tempfeat":
cv = TemplateFeatureFinder()
elif GlobalConfig.find_backend == "deep":
cv = DeepFinder()
elif GlobalConfig.find_backend == "hybrid":
cv = HybridFinder()
# since the backends are read/write make them public attributes
self.dc_backend = dc
self.cv_backend = cv
self.default_target_type = Image
self._last_match = None
self._xpos = xpos
self._ypos = ypos
# zero width/height implies the one of the available screen
if width == 0 and self.dc_backend.width != 0:
self._width = self.dc_backend.width
else:
self._width = width
if height == 0 and self.dc_backend.height != 0:
self._height = self.dc_backend.height
else:
self._height = height
# clipping should only be performed on initialized screen
if self.dc_backend.width != 0 and self.dc_backend.height != 0:
self._ensure_screen_clipping()
mouse_map = self.dc_backend.mousemap
for mouse_button in dir(mouse_map):
if mouse_button.endswith('_BUTTON'):
setattr(self, mouse_button, getattr(mouse_map, mouse_button))
key_map = self.dc_backend.keymap
for key in dir(key_map):
if not key.startswith('__') and key != "to_string":
setattr(self, key, getattr(key_map, key))
mod_map = self.dc_backend.modmap
for modifier_key in dir(mod_map):
if modifier_key.startswith('MOD_'):
setattr(self, modifier_key, getattr(mod_map, modifier_key))
def _ensure_screen_clipping(self):
screen_width = self.dc_backend.width
screen_height = self.dc_backend.height
if self._xpos < 0:
self._xpos = 0
if self._ypos < 0:
self._ypos = 0
if self._xpos > screen_width:
self._xpos = screen_width - 1
if self._ypos > screen_height:
self._ypos = screen_height - 1
if self._xpos + self._width > screen_width:
self._width = screen_width - self._xpos
if self._ypos + self._height > screen_height:
self._height = screen_height - self._ypos
def get_x(self):
"""
Getter for readonly attribute.
:returns: x coordinate of the upleft vertex of the region
:rtype: int
"""
return self._xpos
x = property(fget=get_x)
def get_y(self):
"""
Getter for readonly attribute.
:returns: y coordinate of the upleft vertex of the region
:rtype: int
"""
return self._ypos
y = property(fget=get_y)
def get_width(self):
"""
Getter for readonly attribute.
:returns: width of the region (xpos+width for downright vertex x)
:rtype: int
"""
return self._width
width = property(fget=get_width)
def get_height(self):
"""
Getter for readonly attribute.
:returns: height of the region (ypos+height for downright vertex y)
:rtype: int
"""
return self._height
height = property(fget=get_height)
def get_center(self):
"""
Getter for readonly attribute.
:returns: center of the region
:rtype: :py:class:`location.Location`
"""
xpos = self._xpos + int(self._width / 2)
ypos = self._ypos + int(self._height / 2)
return Location(xpos, ypos)
center = property(fget=get_center)
def get_top_left(self):
"""
Getter for readonly attribute.
:returns: upleft vertex of the region
:rtype: :py:class:`location.Location`
"""
return Location(self._xpos, self._ypos)
top_left = property(fget=get_top_left)
def get_top_right(self):
"""
Getter for readonly attribute.
:returns: upright vertex of the region
:rtype: :py:class:`location.Location`
"""
return Location(self._xpos + self._width, self._ypos)
top_right = property(fget=get_top_right)
def get_bottom_left(self):
"""
Getter for readonly attribute.
:returns: downleft vertex of the region
:rtype: :py:class:`location.Location`
"""
return Location(self._xpos, self._ypos + self._height)
bottom_left = property(fget=get_bottom_left)
def get_bottom_right(self):
"""
Getter for readonly attribute.
:returns: downright vertex of the region
:rtype: :py:class:`location.Location`
"""
return Location(self._xpos + self._width, self._ypos + self._height)
bottom_right = property(fget=get_bottom_right)
def is_empty(self):
"""
Getter for readonly attribute.
:returns: whether the region is empty, i.e. has zero size
:rtype: bool
"""
return self._width == 0 and self._height == 0
is_empty = property(fget=is_empty)
def get_last_match(self):
"""
Getter for readonly attribute.
:returns: last match obtained from finding a target within the region
:rtype: :py:class:`match.Match`
"""
return self._last_match
last_match = property(fget=get_last_match)
def get_mouse_location(self):
"""
Getter for readonly attribute.
:returns: mouse location
:rtype: :py:class:`location.Location`
"""
return self.dc_backend.mouse_location
mouse_location = property(fget=get_mouse_location)
"""Main region methods"""
[docs] def nearby(self, rrange=50):
"""
Obtain a region containing the previous one but enlarged
by a number of pixels on each side.
:param int rrange: number of pixels to add
:returns: new region enlarged by `rrange` on all sides
:rtype: :py:class:`Region`
"""
log.debug("Checking nearby the current region")
new_xpos = self._xpos - rrange
if new_xpos < 0:
new_xpos = 0
new_ypos = self._ypos - rrange
if new_ypos < 0:
new_ypos = 0
new_width = self._width + rrange + self._xpos - new_xpos
new_height = self._height + rrange + self._ypos - new_ypos
# Final clipping is done in the Region constructor
return Region(new_xpos, new_ypos, new_width, new_height,
self.dc_backend, self.cv_backend)
[docs] def above(self, rrange=0):
"""
Obtain a region containing the previous one but enlarged
by a number of pixels on the upper side.
:param int rrange: number of pixels to add
:returns: new region enlarged by `rrange` on upper side
:rtype: :py:class:`Region`
"""
log.debug("Checking above the current region")
if rrange == 0:
new_ypos = 0
new_height = self._ypos + self._height
else:
new_ypos = self._ypos - rrange
if new_ypos < 0:
new_ypos = 0
new_height = self._height + self._ypos - new_ypos
# Final clipping is done in the Region constructor
return Region(self._xpos, new_ypos, self._width, new_height,
self.dc_backend, self.cv_backend)
[docs] def below(self, rrange=0):
"""
Obtain a region containing the previous one but enlarged
by a number of pixels on the lower side.
:param int rrange: number of pixels to add
:returns: new region enlarged by `rrange` on lower side
:rtype: :py:class:`Region`
"""
log.debug("Checking below the current region")
if rrange == 0:
rrange = self.dc_backend.height
new_height = self._height + rrange
# Final clipping is done in the Region constructor
return Region(self._xpos, self._ypos, self._width, new_height,
self.dc_backend, self.cv_backend)
[docs] def left(self, rrange=0):
"""
Obtain a region containing the previous one but enlarged
by a number of pixels on the left side.
:param int rrange: number of pixels to add
:returns: new region enlarged by `rrange` on left side
:rtype: :py:class:`Region`
"""
log.debug("Checking left of the current region")
if rrange == 0:
new_xpos = 0
new_width = self._xpos + self._width
else:
new_xpos = self._xpos - rrange
if new_xpos < 0:
new_xpos = 0
new_width = self._width + self._xpos - new_xpos
# Final clipping is done in the Region constructor
return Region(new_xpos, self._ypos, new_width, self._height,
self.dc_backend, self.cv_backend)
[docs] def right(self, rrange=0):
"""
Obtain a region containing the previous one but enlarged
by a number of pixels on the right side.
:param int rrange: number of pixels to add
:returns: new region enlarged by `rrange` on right side
:rtype: :py:class:`Region`
"""
log.debug("Checking right of the current region")
if rrange == 0:
rrange = self.dc_backend.width
new_width = self._width + rrange
# Final clipping is done in the Region constructor
return Region(self._xpos, self._ypos, new_width, self._height,
self.dc_backend, self.cv_backend)
"""Image expect methods"""
[docs] def find(self, target, timeout=10):
"""
Find a target on the screen.
:param target: target to look for
:type target: str or :py:class:`target.Target`
:param int timeout: timeout before giving up
:returns: match obtained from finding the target within the region
:rtype: :py:class:`match.Match`
This method is the main entrance to all our target finding capabilities
and is the milestone for all target expect methods.
"""
matches = self.find_all(target, timeout=timeout, allow_zero=False)
return matches[0]
[docs] def find_all(self, target, timeout=10, allow_zero=False):
"""
Find multiples of a target on the screen.
:param target: target to look for
:type target: str or :py:class:`target.Target`
:param int timeout: timeout before giving up
:param bool allow_zero: whether to allow zero matches or raise error
:returns: matches obtained from finding the target within the region
:rtype: [:py:class:`match.Match`]
:raises: :py:class:`errors.FindError` if no matches are found
and zero matches are not allowed
This method is similar the one above but allows for more than one match.
"""
if isinstance(target, str):
target = self._target_from_string(target)
log.debug("Looking for targets %s", target)
cv_backend = self._determine_cv_backend(target)
dc_backend = self.dc_backend
# TODO: decide about updating the last_match attribute
last_matches = []
moving_targets = True
timeout_limit = time.time() + timeout
while True:
screen_capture = dc_backend.capture_screen(self)
relative_matches = cv_backend.find(target, screen_capture)
if len(relative_matches) > 0:
from .match import Match
for i, match in enumerate(relative_matches):
absolute_x, absolute_y = match.x + self.x, match.y + self.y
new_match = Match(absolute_x, absolute_y,
match.width, match.height, match.dx, match.dy,
match.similarity, dc=dc_backend, cv=cv_backend)
if len(last_matches) > i:
if last_matches[i].x == absolute_x and last_matches[i].y == absolute_y:
moving_targets = False
last_matches[i] = new_match
else:
# disappearing or appearing (teleporting) targets count as moving targets
moving_targets = True
last_matches.append(new_match)
self._last_match = last_matches[-1]
if not GlobalConfig.wait_for_animations or not moving_targets:
return last_matches
elif time.time() > timeout_limit:
if allow_zero:
return last_matches
else:
if GlobalConfig.save_needle_on_error:
if not os.path.exists(ImageLogger.logging_destination):
os.mkdir(ImageLogger.logging_destination)
dump_path = GlobalConfig.image_logging_destination
hdump_path = os.path.join(dump_path, "last_finderror_haystack.png")
ndump_path = os.path.join(dump_path, "last_finderror_needle.png")
screen_capture.save(hdump_path)
target.save(ndump_path)
raise FindError(target)
else:
# don't hog the CPU
time.sleep(GlobalConfig.rescan_speed_on_find)
def _target_from_string(self, target_str):
# handle some specific target types
try:
# guess from a match file has the highest precedence
return Target.from_match_file(target_str)
except (IOError, FileNotFoundError) as ex:
log.debug(ex)
try:
# if a match file does not exist but a data file exists
return Target.from_data_file(target_str)
except (IncompatibleTargetFileError, FileNotFoundError) as ex:
log.debug(ex)
# if anything else goes wrong fail on the default type
return self.default_target_type(target_str)
def _determine_cv_backend(self, target):
if target.use_own_settings:
log.debug("Using special settings to match %s", target)
return target.match_settings
if isinstance(target, Text) and not isinstance(self.cv_backend, TextFinder):
raise IncompatibleTargetError("Need text matcher for matching text")
if isinstance(target, Pattern) and not (isinstance(self.cv_backend, CascadeFinder)
or isinstance(self.cv_backend, DeepFinder)):
raise IncompatibleTargetError("Need pattern matcher for matching patterns")
if isinstance(target, Chain) and not isinstance(self.cv_backend, HybridFinder):
raise IncompatibleTargetError("Need hybrid matcher for matching chain targets")
target.match_settings = self.cv_backend
return self.cv_backend
[docs] def sample(self, target):
"""
Sample the similarity between a target and the screen,
i.e. an empirical probability that the target is on the screen.
:param target: target to look for
:type target: str or :py:class:`target.Target`
:returns: similarity with best match on the screen
:rtype: float
.. note:: Not all matchers support a 'similarity' value. The ones that don't
will return zero similarity (similarly to the target logging case).
"""
log.debug("Looking for target %s", target)
if isinstance(target, str):
target = Image(target)
if not target.use_own_settings:
target.match_settings = self.cv_backend
target.use_own_settings = True
target = target.with_similarity(0.0)
match = self.find(target)
similarity = match.similarity
return similarity
[docs] def exists(self, target, timeout=0):
"""
Check if a target exists on the screen using the matching
success as a threshold for the existence.
:param target: target to look for
:type target: str or :py:class:`target.Target`
:param int timeout: timeout before giving up
:returns: match obtained from finding the target within the region
or nothing if no match is found
:rtype: :py:class:`match.Match` or None
"""
log.info("Checking if %s is present", target)
try:
return self.find(target, timeout)
except FindError:
log.info("%s is not present", target)
return None
[docs] def wait(self, target, timeout=30):
"""
Wait for a target to appear (be matched) with a given timeout
as failing tolerance.
:param target: target to look for
:type target: str or :py:class:`target.Target`
:param int timeout: timeout before giving up
:returns: match obtained from finding the target within the region
:rtype: :py:class:`match.Match`
:raises: :py:class:`errors.FindError` if no match is found
"""
log.info("Waiting for %s", target)
return self.find(target, timeout)
[docs] def wait_vanish(self, target, timeout=30):
"""
Wait for a target to disappear (be unmatched, i.e. matched
without success) with a given timeout as failing tolerance.
:param target: target to look for
:type target: str or :py:class:`target.Target`
:param int timeout: timeout before giving up
:returns: self
:rtype: :py:class:`Region`
:raises: :py:class:`errors.NotFindError` if match is still found
"""
log.info("Waiting for %s to vanish", target)
expires = time.time() + timeout
while time.time() < expires:
if self.exists(target, 0) is None:
return self
# don't hog the CPU (as rescan within find will check the inverse)
time.sleep(GlobalConfig.rescan_speed_on_find)
# target is still there
raise NotFindError(target)
[docs] def idle(self, timeout):
"""
Wait for a number of seconds and continue the nested call chain.
:param int timeout: timeout to wait for
:returns: self
:rtype: :py:class:`Region`
This method can be used as both a way to compactly wait for some time
while not breaking the call chain. e.g.::
aregion.hover('abox').idle(1).click('aboxwithinthebox')
and as a way to conveniently perform timeout in between actions.
"""
log.debug("Waiting for %ss", timeout)
time.sleep(timeout)
return self
"""Mouse methods"""
[docs] def hover(self, target_or_location):
"""
Hover the mouse over a target or location.
:param target_or_location: target or location to hover to
:type target_or_location: :py:class:`match.Match` or :py:class:`location.Location` or
str or :py:class:`target.Target`
:returns: match from finding the target or nothing if hovering over a known location
:rtype: :py:class:`match.Match` or None
"""
log.info("Hovering over %s", target_or_location)
smooth = GlobalConfig.smooth_mouse_drag
# Handle Match
from .match import Match
if isinstance(target_or_location, Match):
self.dc_backend.mouse_move(target_or_location.target, smooth)
return None
# Handle Location
if isinstance(target_or_location, Location):
self.dc_backend.mouse_move(target_or_location, smooth)
return None
# Find target (image, text, pattern) or str
match = self.find(target_or_location)
self.dc_backend.mouse_move(match.target, smooth)
return match
[docs] def click(self, target_or_location, modifiers=None):
"""
Click on a target or location using the left mouse button and
optionally holding special keys.
:param target_or_location: target or location to click on
:type target_or_location: :py:class:`match.Match` or :py:class:`location.Location` or
str or :py:class:`target.Target`
:param modifiers: special keys to hold during clicking
(see :py:class:`inputmap.KeyModifier` for extensive list)
:type modifiers: [str]
:returns: match from finding the target or nothing if clicking on a known location
:rtype: :py:class:`match.Match` or None
The special keys refer to a list of key modifiers, e.g.::
self.click('my_target', [KeyModifier.MOD_CTRL, 'x']).
"""
match = self.hover(target_or_location)
log.info("Clicking at %s", target_or_location)
if modifiers is not None:
log.info("Holding the modifiers %s", " ".join(modifiers))
self.dc_backend.mouse_click(self.LEFT_BUTTON, 1, modifiers)
return match
[docs] def right_click(self, target_or_location, modifiers=None):
"""
Click on a target or location using the right mouse button and
optionally holding special keys.
Arguments and return values are analogical to :py:func:`Region.click`.
"""
match = self.hover(target_or_location)
log.info("Right clicking at %s", target_or_location)
if modifiers is not None:
log.info("Holding the modifiers %s", " ".join(modifiers))
self.dc_backend.mouse_click(self.RIGHT_BUTTON, 1, modifiers)
return match
[docs] def middle_click(self, target_or_location, modifiers=None):
"""
Click on a target or location using the middle mouse button and
optionally holding special keys.
Arguments and return values are analogical to :py:func:`Region.click`.
"""
match = self.hover(target_or_location)
log.info("Right clicking at %s", target_or_location)
if modifiers is not None:
log.info("Holding the modifiers %s", " ".join(modifiers))
self.dc_backend.mouse_click(self.CENTER_BUTTON, 1, modifiers)
return match
[docs] def double_click(self, target_or_location, modifiers=None):
"""
Double click on a target or location using the left mouse button
and optionally holding special keys.
Arguments and return values are analogical to :py:func:`Region.click`.
"""
match = self.hover(target_or_location)
log.info("Double clicking at %s", target_or_location)
if modifiers is not None:
log.info("Holding the modifiers %s", " ".join(modifiers))
self.dc_backend.mouse_click(self.LEFT_BUTTON, 2, modifiers)
return match
[docs] def multi_click(self, target_or_location, count=3, modifiers=None):
"""
Click N times on a target or location using the left mouse button
and optionally holding special keys.
Arguments and return values are analogical to :py:func:`Region.click`.
"""
match = self.hover(target_or_location)
log.info("Clicking %s times at %s", count, target_or_location)
if modifiers is not None:
log.info("Holding the modifiers %s", " ".join(modifiers))
self.dc_backend.mouse_click(self.LEFT_BUTTON, count, modifiers)
return match
[docs] def click_expect(self, click_image_or_location, expect_target,
modifiers=None, timeout=60, retries=3):
"""
Click on an image or location and wait for another one to appear.
:param click_image_or_location: image or location to click on
:type click_image_or_location: Image or Location
:param expect_target: target to wait for
:type expect_target: :type target: str or :py:class:`target.Target`
:param modifiers: key modifiers when clicking
:type modifiers: [Key] or None
:param int timeout: time in seconds to wait for
:param int retries: number of retries to reach expected target behavior
:returns: match obtained from finding the second target within the region
:rtype: :py:class:`match.Match`
"""
for i in range(retries):
if i > 0:
log.info("Retrying the mouse click (%s of %s)", i+1, retries)
self.click(click_image_or_location, modifiers=modifiers)
try:
return self.wait(expect_target, timeout)
except FindError as error:
if i == retries - 1:
raise error
[docs] def click_vanish(self, click_image_or_location, expect_target,
modifiers=None, timeout=60, retries=3):
"""
Click on an image or location and wait for another one to disappear.
:param click_image_or_location: image or location to click on
:type click_image_or_location: Image or Location
:param expect_target: target to wait for
:type expect_target: :type target: str or :py:class:`target.Target`
:param modifiers: key modifiers when clicking
:type modifiers: [Key] or None
:param int timeout: time in seconds to wait for
:param int retries: number of retries to reach expected target behavior
:returns: self
:rtype: :py:class:`Region`
"""
for i in range(retries):
if i > 0:
log.info("Retrying the mouse click (%s of %s)", i+1, retries)
self.click(click_image_or_location, modifiers=modifiers)
try:
return self.wait_vanish(expect_target, timeout)
except NotFindError as error:
if i == retries - 1:
raise error
[docs] def click_at_index(self, anchor, index=0, find_number=3, timeout=10):
"""
Find all instances of an anchor image and click on the one with the
desired index given that they are horizontally then vertically sorted.
:param anchor: image to find all matches of
:type anchor: str or :py:class:`target.Target`
:param int index: index of the match to click on (assuming >=1 matches),
sorted according to their (x,y) coordinates
:param int find_number: expected number of matches which is necessary
for fast failure in case some elements are not visualized and/or
proper matching result
:param int timeout: timeout before which the number of matches should be found
:returns: match from finding the target of the desired index
:rtype: :py:class:`match.Match`
.. note:: This method is a good replacement of a number of coincident
limitations regarding the Windows version of autopy and PyRO and
therefore the (Windows) virtual user:
* autopy has an old BUG regarding capturing the screen at a region
with boundaries, different than the entire screen -> subregioning which
is the main way to deal with any kind of highly repeating and homogeneous
interface, is totally unavailable here.
* PyRO cannot serialize generators, so this is an implementation of a
"generator step" involving clicking on consecutive matches.
* The serialized virtual user now returns a list of proxified matches
when calling find_all, but they are all essentially useless as they
don't proxify their returned objects and cannot be sent back as arguments.
The special proxy interface of the virtual user was implemented only to
handle the most basic case - serialize the objects returned by the main
shared class by proxifying them (turning them into remote objects as well,
which already have a well-defined serialization method) and nothing more.
"""
matched = False
for _ in range(timeout):
targets = self.find_all(anchor)
if len(targets) == find_number:
matched = True
break
if not matched:
# raise an error without redundant imports
self.find(anchor)
sorted_targets = sorted(targets, key=lambda x: (x.x, x.y))
logging.debug("Totally %s clicking matches found: %s", len(sorted_targets),
["(%s, %s)" % (x.x, x.y) for x in sorted_targets])
self.click(sorted_targets[index])
return sorted_targets[index]
[docs] def mouse_down(self, target_or_location, button=None):
"""
Hold down an arbitrary mouse button on a target or location.
:param target_or_location: target or location to toggle on
:type target_or_location: :py:class:`match.Match` or :py:class:`location.Location` or
str or :py:class:`target.Target`
:param button: button index depending on backend (default is left button)
(see :py:class:`inputmap.MouseButton` for extensive list)
:type button: int or None
:returns: match from finding the target or nothing if toggling on a known location
:rtype: :py:class:`match.Match` or None
"""
if button is None:
button = self.LEFT_BUTTON
match = self.hover(target_or_location)
log.debug("Holding down the mouse at %s", target_or_location)
self.dc_backend.mouse_down(button)
return match
[docs] def mouse_up(self, target_or_location, button=None):
"""
Release an arbitrary mouse button on a target or location.
:param target_or_location: target or location to toggle on
:type target_or_location: :py:class:`match.Match` or :py:class:`location.Location` or
str or :py:class:`target.Target`
:param button: button index depending on backend (default is left button)
(see :py:class:`inputmap.MouseButton` for extensive list)
:type button: int or None
:returns: match from finding the target or nothing if toggling on a known location
:rtype: :py:class:`match.Match` or None
"""
if button is None:
button = self.LEFT_BUTTON
match = self.hover(target_or_location)
log.debug("Holding up the mouse at %s", target_or_location)
self.dc_backend.mouse_up(button)
return match
[docs] def drag_drop(self, src_target_or_location, dst_target_or_location, modifiers=None):
"""
Drag from and drop at a target or location optionally holding special keys.
:param src_target_or_location: target or location to drag from
:type src_target_or_location: :py:class:`match.Match` or :py:class:`location.Location` or
str or :py:class:`target.Target`
:param dst_target_or_location: target or location to drop at
:type dst_target_or_location: :py:class:`match.Match` or :py:class:`location.Location` or
str or :py:class:`target.Target`
:param modifiers: special keys to hold during dragging and dropping
(see :py:class:`inputmap.KeyModifier` for extensive list)
:type modifiers: [str]
:returns: match from finding the target or nothing if dropping at a known location
:rtype: :py:class:`match.Match` or None
"""
self.drag_from(src_target_or_location, modifiers)
match = self.drop_at(dst_target_or_location, modifiers)
return match
[docs] def drag_from(self, target_or_location, modifiers=None):
"""
Drag from a target or location optionally holding special keys.
Arguments and return values are analogical to :py:func:`Region.drag_drop`
but with `target_or_location` as `src_target_or_location`.
"""
match = self.hover(target_or_location)
time.sleep(0.2)
if modifiers is not None:
log.info("Holding the modifiers %s", " ".join(modifiers))
self.dc_backend.keys_toggle(modifiers, True)
#self.dc_backend.keys_toggle(["Ctrl"], True)
log.info("Dragging %s", target_or_location)
self.dc_backend.mouse_down(self.LEFT_BUTTON)
time.sleep(GlobalConfig.delay_after_drag)
return match
[docs] def drop_at(self, target_or_location, modifiers=None):
"""
Drop at a target or location optionally holding special keys.
Arguments and return values are analogical to :py:func:`Region.drag_drop`
but with `target_or_location` as `dst_target_or_location`.
"""
match = self.hover(target_or_location)
time.sleep(GlobalConfig.delay_before_drop)
log.info("Dropping at %s", target_or_location)
self.dc_backend.mouse_up(self.LEFT_BUTTON)
time.sleep(0.5)
if modifiers is not None:
log.info("Holding the modifiers %s", " ".join(modifiers))
self.dc_backend.keys_toggle(modifiers, False)
return match
"""Keyboard methods"""
[docs] def press_keys(self, keys):
"""
Press a single key or a list of keys simultaneously.
:param keys: characters or special keys depending on the backend
(see :py:class:`inputmap.Key` for extensive list)
:type keys: [str] or str (possibly special keys in both cases)
:returns: self
:rtype: :py:class:`Region`
Thus, the line ``self.press_keys([Key.ENTER])`` is equivalent to
the line ``self.press_keys(Key.ENTER)``. Other examples are::
self.press_keys([Key.CTRL, 'X'])
self.press_keys(['a', 'b', 3])
"""
keys_list = self._parse_keys(keys)
time.sleep(GlobalConfig.delay_before_keys)
self.dc_backend.keys_press(keys_list)
return self
[docs] def press_at(self, keys, target_or_location):
"""
Press a single key or a list of keys simultaneously
at a specified target or location.
This method is similar to :py:func:`Region.press_keys` but
with an extra argument like :py:func:`Region.click`.
"""
keys_list = self._parse_keys(keys, target_or_location)
match = self.click(target_or_location)
time.sleep(GlobalConfig.delay_before_keys)
self.dc_backend.keys_press(keys_list)
return match
def _parse_keys(self, keys, target_or_location=None):
at_str = " at %s" % target_or_location if target_or_location else ""
keys_list = []
if isinstance(keys, list):
key_strings = []
for key in keys:
try:
key_strings.append(self.dc_backend.keymap.to_string(key))
except KeyError:
if isinstance(key, int):
key = str(key)
elif len(key) > 1:
raise # a key cannot be a string (text)
key_strings.append(key)
keys_list.append(key)
log.info("Pressing together keys '%s'%s",
"'+'".join(keystr for keystr in key_strings),
at_str)
else:
# if not a list (i.e. if a single key)
key = keys
try:
log.info("Pressing key '%s'%s", self.dc_backend.keymap.to_string(key), at_str)
# if not a special key (i.e. if a character key)
except KeyError:
if isinstance(key, int):
key = str(key)
elif len(key) > 1:
raise # only left keys are chars
log.info("Pressing key '%s'%s", key, at_str)
keys_list.append(key)
return keys_list
[docs] def press_expect(self, keys, expect_target, timeout=60, retries=3):
"""
Press a key and wait for a target to appear.
:param keys: characters or special keys depending on the backend
(see :py:class:`inputmap.Key` for extensive list)
:type keys: [str] or str (possibly special keys in both cases)
:param expect_target: target to wait for
:type expect_target: :type target: str or :py:class:`target.Target`
:param modifiers: key modifiers when clicking
:type modifiers: [Key] or None
:param int timeout: time in seconds to wait for
:param int retries: number of retries to reach expected target behavior
:returns: match obtained from finding the second target within the region
:rtype: :py:class:`match.Match`
"""
for i in range(retries):
if i > 0:
log.info("Retrying the key press (%s of %s)", i+1, retries)
self.press_keys(keys)
try:
return self.wait(expect_target, timeout)
except FindError as error:
if i == retries - 1:
raise error
[docs] def press_vanish(self, keys, expect_target, timeout=60, retries=3):
"""
Press a key and wait for a target to disappear.
:param keys: characters or special keys depending on the backend
(see :py:class:`inputmap.Key` for extensive list)
:type keys: [str] or str (possibly special keys in both cases)
:param expect_target: target to wait for
:type expect_target: :type target: str or :py:class:`target.Target`
:param modifiers: key modifiers when clicking
:type modifiers: [Key] or None
:param int timeout: time in seconds to wait for
:param int retries: number of retries to reach expected target behavior
:returns: self
:rtype: :py:class:`Region`
"""
for i in range(retries):
if i > 0:
log.info("Retrying the key press (%s of %s)", i+1, retries)
self.press_keys(keys)
try:
return self.wait_vanish(expect_target, timeout)
except NotFindError as error:
if i == retries - 1:
raise error
[docs] def type_text(self, text, modifiers=None):
"""
Type a list of consecutive character keys (without special keys).
:param text: characters or strings (independent of the backend)
:type text: [str] or str (no special keys in both cases)
:param modifiers: special keys to hold during typing
(see :py:class:`inputmap.KeyModifier` for extensive list)
:type modifiers: [str]
:returns: self
:rtype: :py:class:`Region`
Thus, the line ``self.type_text(['hello'])`` is equivalent to
the line ``self.type_text('hello')``. Other examples are::
self.type_text('ab3') # compare with press_keys()
self.type_text(['Hello', ' ', 'user3614']) # in cases with appending
Special keys are only allowed as modifiers here - simply call
:py:func:`Region.press_keys` multiple times for consecutively
typing special keys.
"""
text_list = self._parse_text(text)
time.sleep(GlobalConfig.delay_before_keys)
if modifiers is not None:
if isinstance(modifiers, str):
modifiers = [modifiers]
log.info("Holding the modifiers '%s'", "'+'".join(modifiers))
self.dc_backend.keys_type(text_list, modifiers)
return self
[docs] def type_at(self, text, target_or_location, modifiers=None):
"""
Type a list of consecutive character keys (without special keys)
at a specified target or location.
This method is similar to :py:func:`Region.type_text` but
with an extra argument like :py:func:`Region.click`.
"""
text_list = self._parse_text(text, target_or_location)
match = None
if target_or_location is not None:
match = self.click(target_or_location)
time.sleep(GlobalConfig.delay_before_keys)
if modifiers is not None:
if isinstance(modifiers, str):
modifiers = [modifiers]
log.info("Holding the modifiers '%s'", "'+'".join(modifiers))
self.dc_backend.keys_type(text_list, modifiers)
return match
def _parse_text(self, text, target_or_location=None):
at_str = " at %s" % target_or_location if target_or_location else ""
text_list = []
if isinstance(text, str):
log.info("Typing text '%s'%s", text, at_str)
text_list = [text]
else:
for part in text:
if isinstance(part, str):
log.info("Typing text '%s'%s", part, at_str)
text_list.append(part)
elif isinstance(part, int):
log.info("Typing '%i'%s", part, at_str)
text_list.append(str(part))
else:
raise ValueError("Unknown text character" % part)
return text_list
"""Mixed (form) methods"""
[docs] def click_at(self, anchor, dx, dy, count=1):
"""
Clicks on a relative location using a displacement from an anchor.
:param anchor: target of reference for relative location
:type anchor: :py:class:`Match` or :py:class:`Location` or :py:class:`Target` or str
:param int dx: displacement from the anchor in the x direction
:param int dy: displacement from the anchor in the y direction
:param int count: 0, 1, 2, ... clicks on the relative location
:returns: self
:rtype: :py:class:`Region`
:raises: :py:class:`exceptions.ValueError` if `count` is not acceptable value
"""
from .match import Match
if isinstance(anchor, Match):
start_loc = anchor.target
elif isinstance(anchor, Location):
start_loc = anchor
else:
start_loc = self.hover(anchor).target
loc = Location(start_loc.x + dx, start_loc.y + dy)
self.multi_click(loc, count=count)
return self
[docs] def fill_at(self, anchor, text, dx, dy,
del_flag=True, esc_flag=True,
mark_clicks=1):
"""
Fills a new text at a text box using a displacement from an anchor.
:param anchor: target of reference for the input field
:type anchor: :py:class:`Match` or :py:class:`Location` or :py:class:`Target` or str
:param str text: text to fill in
:param int dx: displacement from the anchor in the x direction
:param int dy: displacement from the anchor in the y direction
:param bool del_flag: whether to delete the highlighted text
:param bool esc_flag: whether to escape any possible fill suggestions
:param int mark_clicks: 0, 1, 2, ... clicks to highlight previous text
:returns: self
:rtype: :py:class:`Region`
:raises: :py:class:`exceptions.ValueError` if `mark_click` is not acceptable value
If the delete flag is set the previous content will be deleted or
otherwise the new text will be added in the end of the current text.
If the escape flag is set an escape will be pressed after typing
in order to avoid any entry suggestions from a dropdown list that
could cover important image matching areas.
Since different interfaces behave differently, one might need a
single, double or triple click to mark the already present text that
has to be replaced.
"""
# NOTE: handle cases of empty value no filling anything
if not text:
return
self.click_at(anchor, dx, dy, count=mark_clicks)
# make sure any highlighting is given enough time
self.idle(1)
if isinstance(text, str):
text = [text]
if del_flag:
text.insert(0, self.DELETE)
else:
text.insert(0, self.RIGHT)
if esc_flag:
text.append(self.ESC)
for part in text:
try:
_key_str = self.dc_backend.keymap.to_string(part)
self.press_keys(part)
except KeyError:
self.type_text(part)
return self
[docs] def select_at(self, anchor, image_or_index, dx, dy,
dw=0, dh=0, ret_flag=True,
mark_clicks=1):
"""
Select an option at a dropdown list using either an integer index
or an option image if the order cannot be easily inferred.
:param anchor: target of reference for the input dropdown menu
:type anchor: :py:class:`Match` or :py:class:`Location` or :py:class:`Target` or str
:param image_or_index: item image or item index
:type image_or_index: str or int
:param int dx: displacement from the anchor in the x direction
:param int dy: displacement from the anchor in the y direction
:param int dw: width to add to the displacement for an image search area
:param int dh: height to add to the displacement for an image search area
:param bool ret_flag: whether to press Enter after selecting
:param int mark_clicks: 0, 1, 2, ... clicks to highlight previous text
:returns: self
:rtype: :py:class:`Region`
It uses an anchor image which is rather constant and a displacement
to locate the dropdown location. It moves down to the option if
index is used where index 0 represents the current selection.
To avoid the limitations of the index method, an image of the option
can be provided and will be matched in the area with and under the
dropdown list. This also handles cases where the option coincides
with the previously selected option. For more details see the really
cool note in the end of this method.
"""
# NOTE: handle cases of empty value no filling anything
if not image_or_index:
return
self.click_at(anchor, dx, dy, count=mark_clicks)
# make sure the dropdown options appear
self.idle(1)
if isinstance(image_or_index, int):
move_key = self.UP if image_or_index < 0 else self.DOWN
for _ in range(abs(image_or_index)):
# TODO: multiple DOWN in a single press call doesn't work
self.press_keys([move_key])
if ret_flag:
self.press_keys([self.ENTER])
else:
# NOTE: By definition, the dropdown list will be below and centered
# at the clicking location that we obtain from the anchor image.
# Therefore, we need to center the generated dropdown haystack around
# the x of 'loc'. However, it is possible that the option is
# already selected and cannot be matched among the dropdown options.
# This can be handled if that option is matched at the dropdown box (by
# the clicking location) where the already selected option is shown again in
# a normal way. Therefore, we need to make sure that the entire dropdown
# is in the haystack area. Since the height of any of the options varies,
# we need to take the worst case scenario, i.e. the largest y distance
# that we have to cover so that we don't cut away the dropdown box from
# the dropdown haystack. This worst case is a minimal number of options
# which is 0, implying empty space repeated in the dropdown box and the
# list, therefore a total of 2 option heights spanning the haystack height.
# The haystack y displacement relative to 'loc' is then 1/2*1/2*dh
loc = self.get_mouse_location()
dropdown_haystack = Region(xpos=int(loc.x - dw / 2),
ypos=int(loc.y - dh / 4),
width=dw, height=dh,
dc=self.dc_backend, cv=self.cv_backend)
dropdown_haystack.click(image_or_index)
return self