Source code for guibot.target

# Copyright 2013-2018 Intranet AG and contributors
#
# guibot is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# guibot is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with guibot.  If not, see <http://www.gnu.org/licenses/>.

"""

SUMMARY
------------------------------------------------------
Classes and functionality related to sought targets on screen.


INTERFACE
------------------------------------------------------

"""

import copy
import os
import re
import PIL.Image

from .config import GlobalConfig
from .location import Location
from .fileresolver import FileResolver
from .finder import *
from .errors import *


__all__ = ['Target', 'Image', 'Text', 'Pattern', 'Chain']


[docs] class Target(object): """ Target used to obtain screen location for clicking, typing, validation of expected visual output, etc. """
[docs] @staticmethod def from_data_file(filename): """ Read the target type from the extension of the target filename. :param str filename: data filename for the target :returns: target of type determined from its data filename extension :rtype: :py:class:`target.Target` :raises: :py:class:`errors.IncompatibleTargetFileError` if the data file if of unknown type """ if not os.path.exists(filename): filename = FileResolver().search(filename) basename = os.path.basename(filename) name, extension = os.path.splitext(basename) if extension in (".png", ".jpg"): target = Image(filename) elif extension == ".txt": target = Text(name) elif extension in (".xml", ".csv"): target = Pattern(filename) elif extension == ".steps": target = Chain(name) else: raise IncompatibleTargetFileError("The target file %s is not among any of the known types" % filename) return target
[docs] @staticmethod def from_match_file(filename): """ Read the target type and configuration from a match file with the given filename. :param str filename: match filename for the configuration :returns: target of type determined from its parsed (and generated) settings :rtype: :py:class:`target.Target` """ if not os.path.exists(filename): filename = FileResolver().search(filename) name = os.path.splitext(os.path.basename(filename))[0] match_filename = os.path.splitext(filename)[0] + ".match" finder = Finder.from_match_file(match_filename) if finder.params["find"]["backend"] in ("autopy", "contour", "template", "feature", "tempfeat"): target = Image(filename, match_settings=finder) elif finder.params["find"]["backend"] == "text": target = Text(name, match_settings=finder) elif finder.params["find"]["backend"] in ("cascade", "deep"): target = Pattern(filename, match_settings=finder) elif finder.params["find"]["backend"] == "hybrid": target = Chain(name, match_settings=finder) else: raise RuntimeError("Could not detect the target type from the find backend") return target
def __init__(self, match_settings=None): """ Build a target object. :param match_settings: predefined configuration for the CV backend if any :type match_settings: :py:class:`finder.Finder` or None """ self.match_settings = match_settings if self.match_settings is not None: self.use_own_settings = True else: if GlobalConfig.find_backend == "autopy": self.match_settings = AutoPyFinder() elif GlobalConfig.find_backend == "contour": self.match_settings = ContourFinder() elif GlobalConfig.find_backend == "template": self.match_settings = TemplateFinder() elif GlobalConfig.find_backend == "feature": self.match_settings = FeatureFinder() elif GlobalConfig.find_backend == "cascade": self.match_settings = CascadeFinder() elif GlobalConfig.find_backend == "text": self.match_settings = TextFinder() elif GlobalConfig.find_backend == "tempfeat": self.match_settings = TemplateFeatureFinder() elif GlobalConfig.find_backend == "deep": self.match_settings = DeepFinder() elif GlobalConfig.find_backend == "hybrid": self.match_settings = HybridFinder() self.use_own_settings = False self._center_offset = Location(0, 0) def __str__(self): """Provide a constant name 'target'.""" return "target" def get_similarity(self): """ Getter for readonly attribute. :returns: similarity required for the image to be matched :rtype: float """ return self.match_settings.params["find"]["similarity"].value similarity = property(fget=get_similarity) def get_center_offset(self): """ Getter for readonly attribute. :returns: offset with respect to the target center (used for clicking) :rtype: :py:class:`location.Location` This clicking location is set in the target in order to be customizable, it is then taken when matching to produce a clicking target for a match. """ return self._center_offset center_offset = property(fget=get_center_offset)
[docs] def load(self, filename, **kwargs): """ Load target from a file. :param str filename: name for the target file If no local file is found, we will perform search in the previously added paths. """ if not os.path.exists(filename): filename = FileResolver().search(filename) match_filename = os.path.splitext(filename)[0] + ".match" if os.path.exists(match_filename): self.match_settings = Finder.from_match_file(match_filename) try: self.match_settings.synchronize() except UnsupportedBackendError: # some finders don't support synchronization pass self.use_own_settings = True
[docs] def save(self, filename): """ Save target to a file. :param str filename: name for the target file """ match_filename = os.path.splitext(filename)[0] + ".match" if self.use_own_settings: Finder.to_match_file(self.match_settings, match_filename)
[docs] def copy(self): """ Perform a copy of the target data and match settings. :returns: copy of the current target (with settings) :rtype: :py:class:`target.Target` """ selfcopy = copy.copy(self) copy_settings = self.match_settings.copy() selfcopy.match_settings = copy_settings return selfcopy
[docs] def with_center_offset(self, xpos, ypos): """ Perform a copy of the target data with new match settings and with a newly defined center offset. :param int xpos: new offset in the x direction :param int ypos: new offset in the y direction :returns: copy of the current target with new center offset :rtype: :py:class:`target.Target` """ new_target = self.copy() new_target._center_offset = Location(xpos, ypos) return new_target
[docs] def with_similarity(self, new_similarity): """ Perform a copy of the target data with new match settings and with a newly defined required similarity. :param float new_similarity: new required similarity :returns: copy of the current target with new similarity :rtype: :py:class:`target.Target` """ new_target = self.copy() new_target.match_settings.params["find"]["similarity"].value = new_similarity return new_target
[docs] class Image(Target): """ Container for image data supporting caching, clicking target, file operations, and preprocessing. """ _cache = {} def __init__(self, image_filename=None, pil_image=None, match_settings=None, use_cache=True): """ Build an image object. :param image_filename: name of the image file if any :type image_filename: str or None :param pil_image: image data - use cache or recreate if none :type pil_image: :py:class:`PIL.Image` or None :param match_settings: predefined configuration for the CV backend if any :type match_settings: :py:class:`finder.Finder` or None :param bool use_cache: whether to cache image data for better performance """ super(Image, self).__init__(match_settings) self._filename = image_filename self._pil_image = None self._width = 0 self._height = 0 if self._filename is not None: self.load(self._filename, use_cache) # per instance pil image has the final word if pil_image is not None: self._pil_image = pil_image # per instance match settings have the final word if match_settings is not None: self.match_settings = match_settings self.use_own_settings = True if self._pil_image: self._width = self._pil_image.size[0] self._height = self._pil_image.size[1] def __str__(self): """Provide the image filename.""" return "noname" if self._filename is None else os.path.splitext(os.path.basename(self._filename))[0] def get_filename(self): """ Getter for readonly attribute. :returns: filename of the image :rtype: str """ return self._filename filename = property(fget=get_filename) def get_width(self): """ Getter for readonly attribute. :returns: width of the image :rtype: int """ return self._width width = property(fget=get_width) def get_height(self): """ Getter for readonly attribute. :returns: height of the image :rtype: int """ return self._height height = property(fget=get_height) def get_pil_image(self): """ Getter for readonly attribute. :returns: image data of the image :rtype: :py:class:`PIL.Image` """ return self._pil_image pil_image = property(fget=get_pil_image)
[docs] def load(self, filename, use_cache=True, **kwargs): """ Load image from a file. :param str filename: name for the target file :param bool use_cache: whether to cache image data for better performance """ super(Image, self).load(filename) if not os.path.exists(filename): filename = FileResolver().search(filename) # TODO: check if mtime of the file changed -> cache dirty? if use_cache and filename in self._cache: self._pil_image = self._cache[filename] else: # load and cache image self._pil_image = PIL.Image.open(filename).convert('RGB') if use_cache: self._cache[filename] = self._pil_image self._filename = filename
[docs] def save(self, filename): """ Save image to a file. :param str filename: name for the target file :returns: copy of the current image with the new filename :rtype: :py:class:`target.Image` The image is compressed upon saving with a PNG compression setting specified by :py:func:`config.GlobalConfig.image_quality`. """ super(Image, self).save(filename) filename += ".png" if os.path.splitext(filename)[-1] != ".png" else "" self.pil_image.save(filename, compress_level=GlobalConfig.image_quality) new_image = self.copy() new_image._filename = filename return new_image
[docs] class Text(Target): """ Container for text data which is visually identified using OCR or general text detection methods. """ def __init__(self, value=None, text_filename=None, match_settings=None): """ Build a text object. :param str value: text value to search for :param str text_filename: custom filename to read the text from :param match_settings: predefined configuration for the CV backend if any :type match_settings: :py:class:`finder.Finder` or None """ super(Text, self).__init__(match_settings) self.value = value self.filename = text_filename try: filename = self.filename if self.filename else str(self) + ".txt" self.load(filename) self.filename = filename except FileNotFoundError: # text generated on the fly is also acceptable pass def __str__(self): """Provide a part of the text value.""" return self.value[:30].replace('/', '').replace('\\', '')
[docs] def load(self, filename, **kwargs): """ Load text from a file. :param str filename: name for the target file """ super(Text, self).load(filename) if not os.path.exists(filename): filename = FileResolver().search(filename) with open(filename) as f: self.value = f.read()
[docs] def save(self, filename): """ Save text to a file. :param str filename: name for the target file """ super(Text, self).save(filename) filename += ".txt" if os.path.splitext(filename)[-1] != ".txt" else "" with open(filename, "w") as f: f.write(self.value)
[docs] def distance_to(self, str2): """ Approximate Hungarian distance. :param str str2: string to compare to :returns: string distance value :rtype: float """ str1 = self.value import numpy M = numpy.empty((len(str1) + 1, len(str2) + 1), int) for a in range(0, len(str1)+1): M[a, 0] = a for b in range(0, len(str2)+1): M[0, b] = b for a in range(1, len(str1)+1): # (size_t a = 1; a <= NA; ++a): for b in range(1, len(str2)+1): # (size_t b = 1; b <= NB; ++b) z = M[a-1, b-1] + (0 if str1[a-1] == str2[b-1] else 1) M[a, b] = min(min(M[a-1, b] + 1, M[a, b-1] + 1), z) return M[len(str1), len(str2)]
[docs] class Pattern(Target): """ Container for abstracted data which is obtained from training of a classifier in order to recognize a target. """ def __init__(self, id, match_settings=None): """ Build a pattern object. :param str id: alphanumeric id of logit or label for the given pattern :param match_settings: predefined configuration for the CV backend if any :type match_settings: :py:class:`finder.Finder` or None """ super(Pattern, self).__init__(match_settings) self.id = id self.data_file = None try: # base file name can be used as an ID for some finders like cascade base_name = str(self.id) if "." in str(self.id) else str(self.id) + ".csv" filename = FileResolver().search(base_name) self.load(filename) except FileNotFoundError: # pattern as a label from a reusable model is also acceptable pass # per instance match settings have the final word if match_settings is not None: self.match_settings = match_settings self.use_own_settings = True def __str__(self): """Provide the data filename.""" return self.id
[docs] def load(self, filename, **kwargs): """ Load pattern from a file. :param str filename: name for the target file """ super(Pattern, self).load(filename) if not os.path.exists(filename): filename = FileResolver().search(filename) # loading the actual data is backend specific so only register its path self.data_file = filename
[docs] def save(self, filename): """ Save pattern to a file. :param str filename: name for the target file """ super(Pattern, self).save(filename) filename += ".csv" if "." not in str(self.id) else "" with open(filename, "wb") as fo: if self.data_file is not None: with open(self.data_file, "rb") as fi: fo.write(fi.read())
[docs] class Chain(Target): """ Container for multiple configurations representing the same target. The simplest version of a chain is a sequence of the same match configuration steps performed on a sequence of images until one of them succeeds. Every next step in this chain is a fallback case if the previous step did not succeed. """ def __init__(self, target_name, match_settings=None): """ Build an chain object. :param str target_name: name of the target for all steps :param match_settings: predefined configuration for the CV backend if any :type match_settings: :py:class:`finder.Finder` or None """ super(Chain, self).__init__(match_settings) self.target_name = target_name self._steps = [] self.load(self.target_name) def __str__(self): """Provide the target name.""" return self.target_name def __iter__(self): """Provide an interator over the steps.""" return self._steps.__iter__()
[docs] def load(self, steps_filename, **kwargs): """ Load steps from a sequence definition file. :param str steps_filename: names for the sequence definition file :raises: :py:class:`errors.UnsupportedBackendError` if a chain step is of unknown type :raises: :py:class:`IOError` if an chain step line cannot be parsed """ def resolve_stepsfile(filename): """ Try to find a valid steps file from a given file name. :param str filename: full or partial name of the file to find :returns: valid path to a steps file :rtype: str """ if not filename.endswith(".steps"): filename += ".steps" if not os.path.exists(filename): filename = FileResolver().search(filename) return filename # make sure we have the correct file steps_filename = resolve_stepsfile(steps_filename) stepsfiles_seen = [steps_filename] with open(steps_filename) as f: lines = f.readlines() while lines: step = lines.pop(0) dataconfig = re.split(r'\t+', step.rstrip('\t\n')) # read a nested steps file and append to this chain if dataconfig[0].endswith(".steps"): nested_steps_filename = resolve_stepsfile(dataconfig[0]) # avoid infinite loops if nested_steps_filename not in stepsfiles_seen: stepsfiles_seen.append(nested_steps_filename) with open(nested_steps_filename) as f: lines = f.readlines() + lines continue if len(dataconfig) != 2: raise IOError("Invalid chain step line '%s'" % dataconfig[0]) data, config = dataconfig super(Chain, self).load(config) self.use_own_settings = False step_backend = self.match_settings.params["find"]["backend"] if step_backend in ["autopy", "contour", "template", "feature", "tempfeat"]: data_and_config = Image(data, match_settings=self.match_settings) elif step_backend in ["cascade", "deep"]: data_and_config = Pattern(data, match_settings=self.match_settings) elif step_backend == "text": if data.endswith(".txt"): data_and_config = Text(text_filename=data, match_settings=self.match_settings) else: data_and_config = Text(value=data, match_settings=self.match_settings) else: # in particular, we cannot have a chain within the chain since it is not useful raise UnsupportedBackendError("No target step type for '%s' backend" % step_backend) self._steps.append(data_and_config) # now define own match configuration super(Chain, self).load(steps_filename)
[docs] def save(self, steps_filename): """ Save steps to a sequence definition file. :param str steps_filename: names for the sequence definition file """ super(Chain, self).save(self.target_name) save_lines = [] for data_and_config in self._steps: config = data_and_config.match_settings step_backend = config.params["find"]["backend"] if step_backend in ["autopy", "contour", "template", "feature", "tempfeat"]: data = data_and_config.filename elif step_backend in ["cascade", "deep"]: # special case - dynamic pattern without a filename # save only the matchfile and add the corresponding line if not data_and_config.data_file: matchfile = str(data_and_config) + ".match" Target.save(data_and_config, matchfile) save_lines.append(data_and_config.id + "\t" + matchfile + "\n") continue data = data_and_config.data_file elif step_backend == "text": # special case - dynamic text without a filename # save only the matchfile and add the corresponding line if not data_and_config.filename: matchfile = str(data_and_config) + ".match" Target.save(data_and_config, matchfile) save_lines.append(data_and_config.value + "\t" + matchfile + "\n") continue data = data_and_config.filename else: # in particular, we cannot have a chain within the chain since it is not useful raise UnsupportedBackendError("No target step type for '%s' backend" % step_backend) data_and_config.save(data) save_lines.append(data + "\t" + os.path.splitext(data)[0] + ".match\n") with open(steps_filename, "w") as f: f.writelines(save_lines)